From 372c61397636e54d8ce2a3a49e9e1fde1d2a4277 Mon Sep 17 00:00:00 2001 From: Mihir Harshad Khatri Date: Mon, 16 Nov 2026 18:59:09 +0530 Subject: [PATCH] fix(ai): track failed OpenAI Responses stream events Prepared with OpenAI Codex assistance. --- .../openai-responses-failed-stream-event.md | 5 + posthog/ai/openai/_streaming.py | 362 ++--- posthog/ai/openai/openai.py | 1282 ++++++++-------- posthog/ai/openai/openai_async.py | 1290 ++++++++--------- .../test/ai/test_stream_midstream_error.py | 1146 ++++++++------- 5 files changed, 2091 insertions(+), 1994 deletions(-) create mode 100644 .sampo/changesets/openai-responses-failed-stream-event.md diff --git a/.sampo/changesets/openai-responses-failed-stream-event.md b/.sampo/changesets/openai-responses-failed-stream-event.md new file mode 100644 index 000000000..c647e49e6 --- /dev/null +++ b/.sampo/changesets/openai-responses-failed-stream-event.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: patch +--- + +Record failed OpenAI Responses API stream events as generation errors in the sync and async clients. The original event still reaches the caller, and privacy mode redacts the provider's error message. diff --git a/posthog/ai/openai/_streaming.py b/posthog/ai/openai/_streaming.py index 49234bee0..bba603ca0 100644 --- a/posthog/ai/openai/_streaming.py +++ b/posthog/ai/openai/_streaming.py @@ -1,176 +1,186 @@ -"""Sync-neutral state accumulation for OpenAI streaming endpoints.""" - -from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional, Union - -from ..types import FormattedMessage, StreamingEventData, TokenUsage -from ..utils import merge_usage_stats, _responses_stop_reason -from .openai_converter import ( - accumulate_openai_tool_calls, - extract_openai_content_from_chunk, - extract_openai_tool_calls_from_chunk, - extract_openai_usage_from_chunk, - format_openai_streaming_output, -) - - -@dataclass -class _ResponsesStreamState: - """Accumulates state specific to a Responses API stream.""" - - usage_stats: TokenUsage = field(default_factory=lambda: TokenUsage()) - output: List[Any] = field(default_factory=list) - model: Optional[str] = None - stop_reason: Optional[str] = None - service_tier: Optional[str] = None - - def process_chunk(self, chunk: Any) -> None: - response = getattr(chunk, "response", None) - self._record_response_metadata(response) - - chunk_usage = extract_openai_usage_from_chunk(chunk, "responses") - if chunk_usage: - merge_usage_stats(self.usage_stats, chunk_usage) - - content = extract_openai_content_from_chunk(chunk, "responses") - if content is not None: - self.output.extend(content) - - # A stream can end on response.completed, response.incomplete, or - # response.failed; any terminal response names the stop reason. - if response: - stop_reason = _responses_stop_reason(response) - if stop_reason is not None: - self.stop_reason = stop_reason - - def _record_response_metadata(self, response: Any) -> None: - if response and self.model is None and hasattr(response, "model"): - self.model = response.model - if response and self.service_tier is None: - self.service_tier = getattr(response, "service_tier", None) - - -@dataclass -class _ChoiceStream: - """One choice's view of a chunk, so the per-chunk helpers read that choice.""" - - choices: List[Any] - - -@dataclass -class _ChatCompletionsStreamState: - """Accumulates state specific to a Chat Completions stream. - - With ``n > 1`` each chunk carries the delta of one choice, named by its - ``index``, so content and tool calls are kept per choice. - """ - - usage_stats: TokenUsage = field(default_factory=lambda: TokenUsage()) - _outputs: Dict[int, List[Any]] = field(default_factory=dict) - _tool_calls: Dict[int, Dict[Union[int, str], Dict[str, Any]]] = field( - default_factory=dict - ) - model: Optional[str] = None - stop_reason: Optional[str] = None - service_tier: Optional[str] = None - - def process_chunk(self, chunk: Any) -> None: - if self.model is None and hasattr(chunk, "model"): - self.model = chunk.model - if self.service_tier is None: - self.service_tier = getattr(chunk, "service_tier", None) - - chunk_usage = extract_openai_usage_from_chunk(chunk, "chat") - if chunk_usage: - merge_usage_stats(self.usage_stats, chunk_usage) - - for choice in getattr(chunk, "choices", None) or []: - self._process_choice(choice) - - def _process_choice(self, choice: Any) -> None: - index = getattr(choice, "index", None) - if not isinstance(index, int): - index = 0 - view = _ChoiceStream([choice]) - - content = extract_openai_content_from_chunk(view, "chat") - if content is not None: - self._outputs.setdefault(index, []).append(content) - - chunk_tool_calls = extract_openai_tool_calls_from_chunk(view) - if chunk_tool_calls: - accumulate_openai_tool_calls( - self._tool_calls.setdefault(index, {}), chunk_tool_calls - ) - - # Like a non-streamed completion, the stop reason is the first choice's. - finish_reason = getattr(choice, "finish_reason", None) - if finish_reason is not None and index == 0: - self.stop_reason = finish_reason - - @property - def tool_calls(self) -> Optional[List[Dict[str, Any]]]: - calls = [ - call - for index in sorted(self._tool_calls) - for call in self._tool_calls[index].values() - ] - return calls or None - - def _indexes(self) -> List[int]: - return sorted(set(self._outputs) | set(self._tool_calls)) - - def formatted_output(self) -> List[FormattedMessage]: - """One assistant message with each choice's content in index order, the - shape a non-streamed completion is captured with.""" - content: List[Any] = [] - for index in self._indexes(): - tool_calls = self._tool_calls.get(index) - content.extend( - format_openai_streaming_output( - self._outputs.get(index, []), - "chat", - list(tool_calls.values()) if tool_calls else None, - )[0]["content"] - ) - return [{"role": "assistant", "content": content}] - - -def _build_streaming_event_data( - *, - base_url: Any, - kwargs: Dict[str, Any], - formatted_input: Any, - formatted_output: Any, - usage_stats: TokenUsage, - latency: float, - distinct_id: Optional[str], - trace_id: Optional[str], - properties: Optional[Dict[str, Any]], - privacy_mode: bool, - groups: Optional[Dict[str, Any]], - model_from_response: Optional[str], - stop_reason: Optional[str], - service_tier: Optional[str] = None, - error: Optional[BaseException] = None, -) -> StreamingEventData: - """Build the fields shared by both OpenAI streaming endpoint events.""" - - return StreamingEventData( - provider="openai", - model=kwargs.get("model") or model_from_response or "unknown", - base_url=str(base_url), - kwargs=kwargs, - formatted_input=formatted_input, - formatted_output=formatted_output, - usage_stats=usage_stats, - latency=latency, - distinct_id=distinct_id, - trace_id=trace_id, - properties=properties, - privacy_mode=privacy_mode, - groups=groups, - stop_reason=stop_reason, - service_tier=service_tier, - error=error, - ) +"""Sync-neutral state accumulation for OpenAI streaming endpoints.""" + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Union + +from ..types import FormattedMessage, StreamingEventData, TokenUsage +from ..utils import merge_usage_stats, _responses_stop_reason +from .openai_converter import ( + accumulate_openai_tool_calls, + extract_openai_content_from_chunk, + extract_openai_tool_calls_from_chunk, + extract_openai_usage_from_chunk, + format_openai_streaming_output, +) + + +@dataclass +class _ResponsesStreamState: + """Accumulates state specific to a Responses API stream.""" + + usage_stats: TokenUsage = field(default_factory=lambda: TokenUsage()) + output: List[Any] = field(default_factory=list) + model: Optional[str] = None + stop_reason: Optional[str] = None + service_tier: Optional[str] = None + failure: Optional[RuntimeError] = None + + def process_chunk(self, chunk: Any) -> None: + response = getattr(chunk, "response", None) + self._record_response_metadata(response) + + if getattr(chunk, "type", None) == "response.failed": + error = getattr(response, "error", None) + message = getattr(error, "message", None) + self.failure = RuntimeError( + message + if isinstance(message, str) and message + else "OpenAI response failed" + ) + + chunk_usage = extract_openai_usage_from_chunk(chunk, "responses") + if chunk_usage: + merge_usage_stats(self.usage_stats, chunk_usage) + + content = extract_openai_content_from_chunk(chunk, "responses") + if content is not None: + self.output.extend(content) + + # A stream can end on response.completed, response.incomplete, or + # response.failed; any terminal response names the stop reason. + if response: + stop_reason = _responses_stop_reason(response) + if stop_reason is not None: + self.stop_reason = stop_reason + + def _record_response_metadata(self, response: Any) -> None: + if response and self.model is None and hasattr(response, "model"): + self.model = response.model + if response and self.service_tier is None: + self.service_tier = getattr(response, "service_tier", None) + + +@dataclass +class _ChoiceStream: + """One choice's view of a chunk, so the per-chunk helpers read that choice.""" + + choices: List[Any] + + +@dataclass +class _ChatCompletionsStreamState: + """Accumulates state specific to a Chat Completions stream. + + With ``n > 1`` each chunk carries the delta of one choice, named by its + ``index``, so content and tool calls are kept per choice. + """ + + usage_stats: TokenUsage = field(default_factory=lambda: TokenUsage()) + _outputs: Dict[int, List[Any]] = field(default_factory=dict) + _tool_calls: Dict[int, Dict[Union[int, str], Dict[str, Any]]] = field( + default_factory=dict + ) + model: Optional[str] = None + stop_reason: Optional[str] = None + service_tier: Optional[str] = None + + def process_chunk(self, chunk: Any) -> None: + if self.model is None and hasattr(chunk, "model"): + self.model = chunk.model + if self.service_tier is None: + self.service_tier = getattr(chunk, "service_tier", None) + + chunk_usage = extract_openai_usage_from_chunk(chunk, "chat") + if chunk_usage: + merge_usage_stats(self.usage_stats, chunk_usage) + + for choice in getattr(chunk, "choices", None) or []: + self._process_choice(choice) + + def _process_choice(self, choice: Any) -> None: + index = getattr(choice, "index", None) + if not isinstance(index, int): + index = 0 + view = _ChoiceStream([choice]) + + content = extract_openai_content_from_chunk(view, "chat") + if content is not None: + self._outputs.setdefault(index, []).append(content) + + chunk_tool_calls = extract_openai_tool_calls_from_chunk(view) + if chunk_tool_calls: + accumulate_openai_tool_calls( + self._tool_calls.setdefault(index, {}), chunk_tool_calls + ) + + # Like a non-streamed completion, the stop reason is the first choice's. + finish_reason = getattr(choice, "finish_reason", None) + if finish_reason is not None and index == 0: + self.stop_reason = finish_reason + + @property + def tool_calls(self) -> Optional[List[Dict[str, Any]]]: + calls = [ + call + for index in sorted(self._tool_calls) + for call in self._tool_calls[index].values() + ] + return calls or None + + def _indexes(self) -> List[int]: + return sorted(set(self._outputs) | set(self._tool_calls)) + + def formatted_output(self) -> List[FormattedMessage]: + """One assistant message with each choice's content in index order, the + shape a non-streamed completion is captured with.""" + content: List[Any] = [] + for index in self._indexes(): + tool_calls = self._tool_calls.get(index) + content.extend( + format_openai_streaming_output( + self._outputs.get(index, []), + "chat", + list(tool_calls.values()) if tool_calls else None, + )[0]["content"] + ) + return [{"role": "assistant", "content": content}] + + +def _build_streaming_event_data( + *, + base_url: Any, + kwargs: Dict[str, Any], + formatted_input: Any, + formatted_output: Any, + usage_stats: TokenUsage, + latency: float, + distinct_id: Optional[str], + trace_id: Optional[str], + properties: Optional[Dict[str, Any]], + privacy_mode: bool, + groups: Optional[Dict[str, Any]], + model_from_response: Optional[str], + stop_reason: Optional[str], + service_tier: Optional[str] = None, + error: Optional[BaseException] = None, +) -> StreamingEventData: + """Build the fields shared by both OpenAI streaming endpoint events.""" + + return StreamingEventData( + provider="openai", + model=kwargs.get("model") or model_from_response or "unknown", + base_url=str(base_url), + kwargs=kwargs, + formatted_input=formatted_input, + formatted_output=formatted_output, + usage_stats=usage_stats, + latency=latency, + distinct_id=distinct_id, + trace_id=trace_id, + properties=properties, + privacy_mode=privacy_mode, + groups=groups, + stop_reason=stop_reason, + service_tier=service_tier, + error=error, + ) diff --git a/posthog/ai/openai/openai.py b/posthog/ai/openai/openai.py index e00a2d3c0..dc34925e4 100644 --- a/posthog/ai/openai/openai.py +++ b/posthog/ai/openai/openai.py @@ -1,641 +1,641 @@ -import logging -import time -import uuid -from typing import TYPE_CHECKING as _TYPE_CHECKING, Any, Dict, Optional - -from posthog.ai.types import TokenUsage as TokenUsage - -try: - import openai -except ImportError: - raise ModuleNotFoundError( - "Please install the OpenAI SDK to use this feature: 'pip install openai'" - ) - -from posthog.ai.utils import ( - call_llm_and_track_usage, - extract_available_tool_calls as extract_available_tool_calls, - finalize_ai_content as finalize_ai_content, - merge_usage_stats as merge_usage_stats, - with_privacy_mode as with_privacy_mode, -) -from posthog.ai.openai.openai_converter import ( - accumulate_openai_tool_calls as accumulate_openai_tool_calls, - extract_openai_content_from_chunk as extract_openai_content_from_chunk, - extract_openai_tool_calls_from_chunk as extract_openai_tool_calls_from_chunk, - extract_openai_usage_from_chunk as extract_openai_usage_from_chunk, - format_openai_streaming_input as _format_openai_streaming_input, - format_openai_streaming_output as _format_openai_streaming_output, -) -from posthog.client import Client as PostHogClient -from posthog import setup -from posthog.ai.openai._streaming import ( - _ChatCompletionsStreamState, - _ResponsesStreamState, - _build_streaming_event_data, -) -from ..stream import _StreamWrapper -from ._embeddings import _capture_embedding_event -from .wrapper_utils import ( - _OpenAIWrapperResource, - _wrap_openai_resources, - merge_provider_override, -) - -_log = logging.getLogger("posthog") - - -class OpenAI(openai.OpenAI): - """ - A wrapper around the OpenAI SDK that automatically sends LLM usage events to PostHog. - """ - - _ph_client: PostHogClient - - if _TYPE_CHECKING: - chat: "WrappedChat" - embeddings: "WrappedEmbeddings" - beta: "WrappedBeta" - responses: "WrappedResponses" - - def __init__(self, posthog_client: Optional[PostHogClient] = None, **kwargs): - """ - Args: - posthog_client: If provided, events will be captured via this client - instead of the global ``posthog`` client. - **kwargs: Arguments passed to ``openai.OpenAI`` such as ``api_key`` - or ``organization``. - """ - - super().__init__(**kwargs) - self._ph_client = posthog_client or setup() - - _wrap_openai_resources(self, _SYNC_RESOURCE_WRAPPERS) - - -def _parse_and_track( - wrapper, - posthog_distinct_id: Optional[str], - posthog_trace_id: Optional[str], - posthog_properties: Optional[Dict[str, Any]], - posthog_privacy_mode: bool, - posthog_groups: Optional[Dict[str, Any]], - posthog_provider_override: Optional[str] = None, - **kwargs: Any, -): - return call_llm_and_track_usage( - posthog_distinct_id, - wrapper._client._ph_client, - "openai", - posthog_trace_id, - merge_provider_override(posthog_properties, posthog_provider_override), - posthog_privacy_mode, - posthog_groups, - wrapper._client.base_url, - wrapper._original.parse, - **kwargs, - ) - - -class WrappedResponses(_OpenAIWrapperResource): - """Wrapper for OpenAI responses that tracks usage in PostHog.""" - - def create( - self, - posthog_distinct_id: Optional[str] = None, - posthog_trace_id: Optional[str] = None, - posthog_properties: Optional[Dict[str, Any]] = None, - posthog_privacy_mode: bool = False, - posthog_groups: Optional[Dict[str, Any]] = None, - posthog_provider_override: Optional[str] = None, - **kwargs: Any, - ): - """ - Create an OpenAI Responses API response while tracking usage in PostHog. - - Args: - posthog_distinct_id: Optional distinct ID to associate with the usage event. - posthog_trace_id: Optional trace ID. Generated automatically when omitted. - posthog_properties: Additional properties to include with the usage event. - posthog_privacy_mode: Whether to redact captured input and output. - posthog_groups: Optional PostHog groups to associate with the event. - posthog_provider_override: Optional override for the ``$ai_provider`` - reported on the usage event. Useful when this client is pointed at - an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom - ``base_url``, so cost attribution matches the real provider. - Defaults to ``"openai"`` when omitted. - **kwargs: Arguments passed to OpenAI's ``responses.create`` API. - - Returns: - The OpenAI response, or a streaming iterator when ``stream=True``. - """ - if posthog_trace_id is None: - posthog_trace_id = str(uuid.uuid4()) - - posthog_properties = merge_provider_override( - posthog_properties, posthog_provider_override - ) - - if kwargs.get("stream", False): - return self._create_streaming( - posthog_distinct_id, - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - **kwargs, - ) - - return call_llm_and_track_usage( - posthog_distinct_id, - self._client._ph_client, - "openai", - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - self._client.base_url, - self._original.create, - **kwargs, - ) - - def _create_streaming( - self, - posthog_distinct_id: Optional[str], - posthog_trace_id: Optional[str], - posthog_properties: Optional[Dict[str, Any]], - posthog_privacy_mode: bool, - posthog_groups: Optional[Dict[str, Any]], - **kwargs: Any, - ): - start_time = time.time() - state = _ResponsesStreamState() - try: - response = self._original.create(**kwargs) - except Exception as exc: - try: - self._capture_streaming_event( - posthog_distinct_id, - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - kwargs, - state, - time.time() - start_time, - error=exc, - ) - except Exception: - # Telemetry must never replace the provider's own error, and the - # fixed text avoids echoing prompt or error content into logs. - _log.warning("[PostHog] Failed to capture streaming connect error") - raise - - def generator(): - stream_error: Optional[BaseException] = None - try: - for chunk in response: - state.process_chunk(chunk) - yield chunk - except Exception as exc: - stream_error = exc - raise - finally: - self._capture_streaming_event( - posthog_distinct_id, - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - kwargs, - state, - time.time() - start_time, - error=stream_error, - ) - - return _StreamWrapper(generator(), stream=response) - - def _capture_streaming_event( - self, - posthog_distinct_id: Optional[str], - posthog_trace_id: Optional[str], - posthog_properties: Optional[Dict[str, Any]], - posthog_privacy_mode: bool, - posthog_groups: Optional[Dict[str, Any]], - kwargs: Dict[str, Any], - state: _ResponsesStreamState, - latency: float, - error: Optional[BaseException] = None, - ): - from posthog.ai.utils import capture_streaming_event - - event_data = _build_streaming_event_data( - base_url=self._client.base_url, - kwargs=kwargs, - formatted_input=_format_openai_streaming_input(kwargs, "responses"), - formatted_output=_format_openai_streaming_output(state.output, "responses"), - usage_stats=state.usage_stats, - latency=latency, - distinct_id=posthog_distinct_id, - trace_id=posthog_trace_id, - properties=posthog_properties, - privacy_mode=posthog_privacy_mode, - groups=posthog_groups, - model_from_response=state.model, - stop_reason=state.stop_reason, - service_tier=state.service_tier, - error=error, - ) - capture_streaming_event(self._client._ph_client, event_data) - - def parse( - self, - posthog_distinct_id: Optional[str] = None, - posthog_trace_id: Optional[str] = None, - posthog_properties: Optional[Dict[str, Any]] = None, - posthog_privacy_mode: bool = False, - posthog_groups: Optional[Dict[str, Any]] = None, - posthog_provider_override: Optional[str] = None, - **kwargs: Any, - ): - """ - Parse structured output using OpenAI's 'responses.parse' method, but also track usage in PostHog. - - Args: - posthog_distinct_id: Optional ID to associate with the usage event. - posthog_trace_id: Optional trace UUID for linking events. - posthog_properties: Optional dictionary of extra properties to include in the event. - posthog_privacy_mode: Whether to anonymize the input and output. - posthog_groups: Optional dictionary of groups to associate with the event. - posthog_provider_override: Optional override for the ``$ai_provider`` - reported on the usage event. Useful when this client is pointed at - an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom - ``base_url``, so cost attribution matches the real provider. - Defaults to ``"openai"`` when omitted. - **kwargs: Any additional parameters for the OpenAI Responses Parse API. - - Returns: - The response from OpenAI's responses.parse call. - """ - return _parse_and_track( - self, - posthog_distinct_id, - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - posthog_provider_override, - **kwargs, - ) - - -class WrappedChat(_OpenAIWrapperResource): - """Wrapper for OpenAI chat that tracks usage in PostHog.""" - - @property - def completions(self): - """Access chat completions with PostHog usage tracking.""" - return WrappedCompletions(self._client, self._original.completions) - - -class WrappedCompletions(_OpenAIWrapperResource): - """Wrapper for OpenAI chat completions that tracks usage in PostHog.""" - - def parse( - self, - posthog_distinct_id: Optional[str] = None, - posthog_trace_id: Optional[str] = None, - posthog_properties: Optional[Dict[str, Any]] = None, - posthog_privacy_mode: bool = False, - posthog_groups: Optional[Dict[str, Any]] = None, - posthog_provider_override: Optional[str] = None, - **kwargs: Any, - ): - """ - Parse an OpenAI chat completion while tracking usage in PostHog. - - Args: - posthog_distinct_id: Optional distinct ID to associate with the usage event. - posthog_trace_id: Optional trace ID. Generated automatically when omitted. - posthog_properties: Additional properties to include with the usage event. - posthog_privacy_mode: Whether to redact captured input and output. - posthog_groups: Optional PostHog groups to associate with the event. - posthog_provider_override: Optional override for the ``$ai_provider`` - reported on the usage event. Useful when this client is pointed at - an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom - ``base_url``, so cost attribution matches the real provider. - Defaults to ``"openai"`` when omitted. - **kwargs: Arguments passed to OpenAI's ``chat.completions.parse`` API. - - Returns: - The parsed response from OpenAI. - """ - return _parse_and_track( - self, - posthog_distinct_id, - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - posthog_provider_override, - **kwargs, - ) - - def create( - self, - posthog_distinct_id: Optional[str] = None, - posthog_trace_id: Optional[str] = None, - posthog_properties: Optional[Dict[str, Any]] = None, - posthog_privacy_mode: bool = False, - posthog_groups: Optional[Dict[str, Any]] = None, - posthog_provider_override: Optional[str] = None, - **kwargs: Any, - ): - """ - Create an OpenAI chat completion while tracking usage in PostHog. - - Args: - posthog_distinct_id: Optional distinct ID to associate with the usage event. - posthog_trace_id: Optional trace ID. Generated automatically when omitted. - posthog_properties: Additional properties to include with the usage event. - posthog_privacy_mode: Whether to redact captured input and output. - posthog_groups: Optional PostHog groups to associate with the event. - posthog_provider_override: Optional override for the ``$ai_provider`` - reported on the usage event. Useful when this client is pointed at - an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom - ``base_url``, so cost attribution matches the real provider. - Defaults to ``"openai"`` when omitted. - **kwargs: Arguments passed to OpenAI's ``chat.completions.create`` API. - - Returns: - The OpenAI chat completion, or a streaming iterator when ``stream=True``. - """ - if posthog_trace_id is None: - posthog_trace_id = str(uuid.uuid4()) - - posthog_properties = merge_provider_override( - posthog_properties, posthog_provider_override - ) - - if kwargs.get("stream", False): - return self._create_streaming( - posthog_distinct_id, - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - **kwargs, - ) - - return call_llm_and_track_usage( - posthog_distinct_id, - self._client._ph_client, - "openai", - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - self._client.base_url, - self._original.create, - **kwargs, - ) - - def _create_streaming( - self, - posthog_distinct_id: Optional[str], - posthog_trace_id: Optional[str], - posthog_properties: Optional[Dict[str, Any]], - posthog_privacy_mode: bool, - posthog_groups: Optional[Dict[str, Any]], - **kwargs: Any, - ): - start_time = time.time() - state = _ChatCompletionsStreamState() - # The caller's dict is left alone, and None / NOT_GIVEN mean "no options" - stream_options = kwargs.get("stream_options") - kwargs["stream_options"] = { - **(stream_options if isinstance(stream_options, dict) else {}), - "include_usage": True, - } - try: - response = self._original.create(**kwargs) - except Exception as exc: - try: - self._capture_streaming_event( - posthog_distinct_id, - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - kwargs, - state, - time.time() - start_time, - error=exc, - ) - except Exception: - # Telemetry must never replace the provider's own error, and the - # fixed text avoids echoing prompt or error content into logs. - _log.warning("[PostHog] Failed to capture streaming connect error") - raise - - def generator(): - stream_error: Optional[BaseException] = None - try: - for chunk in response: - state.process_chunk(chunk) - yield chunk - except Exception as exc: - stream_error = exc - raise - finally: - self._capture_streaming_event( - posthog_distinct_id, - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - kwargs, - state, - time.time() - start_time, - error=stream_error, - ) - - return _StreamWrapper(generator(), stream=response) - - def _capture_streaming_event( - self, - posthog_distinct_id: Optional[str], - posthog_trace_id: Optional[str], - posthog_properties: Optional[Dict[str, Any]], - posthog_privacy_mode: bool, - posthog_groups: Optional[Dict[str, Any]], - kwargs: Dict[str, Any], - state: _ChatCompletionsStreamState, - latency: float, - error: Optional[BaseException] = None, - ): - from posthog.ai.utils import capture_streaming_event - - event_data = _build_streaming_event_data( - base_url=self._client.base_url, - kwargs=kwargs, - formatted_input=_format_openai_streaming_input(kwargs, "chat"), - formatted_output=state.formatted_output(), - usage_stats=state.usage_stats, - latency=latency, - distinct_id=posthog_distinct_id, - trace_id=posthog_trace_id, - properties=posthog_properties, - privacy_mode=posthog_privacy_mode, - groups=posthog_groups, - model_from_response=state.model, - stop_reason=state.stop_reason, - service_tier=state.service_tier, - error=error, - ) - capture_streaming_event(self._client._ph_client, event_data) - - -class WrappedEmbeddings(_OpenAIWrapperResource): - """Wrapper for OpenAI embeddings that tracks usage in PostHog.""" - - def create( - self, - posthog_distinct_id: Optional[str] = None, - posthog_trace_id: Optional[str] = None, - posthog_properties: Optional[Dict[str, Any]] = None, - posthog_privacy_mode: bool = False, - posthog_groups: Optional[Dict[str, Any]] = None, - posthog_provider_override: Optional[str] = None, - **kwargs: Any, - ): - """ - Create an embedding using OpenAI's 'embeddings.create' method, but also track usage in PostHog. - - Args: - posthog_distinct_id: Optional ID to associate with the usage event. - posthog_trace_id: Optional trace UUID for linking events. - posthog_properties: Optional dictionary of extra properties to include in the event. - posthog_privacy_mode: Whether to anonymize the input and output. - posthog_groups: Optional dictionary of groups to associate with the event. - posthog_provider_override: Optional override for the ``$ai_provider`` - reported on the usage event. Useful when this client is pointed at - an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom - ``base_url``, so cost attribution matches the real provider. - Defaults to ``"openai"`` when omitted. - **kwargs: Any additional parameters for the OpenAI Embeddings API. - - Returns: - The response from OpenAI's embeddings.create call. - """ - - if posthog_trace_id is None: - posthog_trace_id = str(uuid.uuid4()) - - posthog_properties = merge_provider_override( - posthog_properties, posthog_provider_override - ) - - start_time = time.time() - try: - response = self._original.create(**kwargs) - except Exception as exc: - _capture_embedding_event( - posthog_client=self._client._ph_client, - base_url=self._client.base_url, - response=None, - request_kwargs=kwargs, - latency=time.time() - start_time, - distinct_id=posthog_distinct_id, - trace_id=posthog_trace_id, - properties=posthog_properties, - privacy_mode=posthog_privacy_mode, - groups=posthog_groups, - error=exc, - ) - raise - end_time = time.time() - - _capture_embedding_event( - posthog_client=self._client._ph_client, - base_url=self._client.base_url, - response=response, - request_kwargs=kwargs, - latency=end_time - start_time, - distinct_id=posthog_distinct_id, - trace_id=posthog_trace_id, - properties=posthog_properties, - privacy_mode=posthog_privacy_mode, - groups=posthog_groups, - ) - return response - - -class WrappedBeta(_OpenAIWrapperResource): - """Wrapper for OpenAI beta features that tracks usage in PostHog.""" - - @property - def chat(self): - """Access beta chat APIs with PostHog usage tracking.""" - return WrappedBetaChat(self._client, self._original.chat) - - -class WrappedBetaChat(_OpenAIWrapperResource): - """Wrapper for OpenAI beta chat that tracks usage in PostHog.""" - - @property - def completions(self): - """Access beta chat completions with PostHog usage tracking.""" - return WrappedBetaCompletions(self._client, self._original.completions) - - -class WrappedBetaCompletions(_OpenAIWrapperResource): - """Wrapper for OpenAI beta chat completions that tracks usage in PostHog.""" - - def parse( - self, - posthog_distinct_id: Optional[str] = None, - posthog_trace_id: Optional[str] = None, - posthog_properties: Optional[Dict[str, Any]] = None, - posthog_privacy_mode: bool = False, - posthog_groups: Optional[Dict[str, Any]] = None, - posthog_provider_override: Optional[str] = None, - **kwargs: Any, - ): - """ - Parse an OpenAI beta chat completion while tracking usage in PostHog. - - Args: - posthog_distinct_id: Optional distinct ID to associate with the usage event. - posthog_trace_id: Optional trace ID. Generated automatically when omitted. - posthog_properties: Additional properties to include with the usage event. - posthog_privacy_mode: Whether to redact captured input and output. - posthog_groups: Optional PostHog groups to associate with the event. - posthog_provider_override: Optional override for the ``$ai_provider`` - reported on the usage event. Useful when this client is pointed at - an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom - ``base_url``, so cost attribution matches the real provider. - Defaults to ``"openai"`` when omitted. - **kwargs: Arguments passed to OpenAI's beta ``chat.completions.parse`` API. - - Returns: - The parsed response from OpenAI. - """ - return _parse_and_track( - self, - posthog_distinct_id, - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - posthog_provider_override, - **kwargs, - ) - - -_SYNC_RESOURCE_WRAPPERS = { - "chat": WrappedChat, - "embeddings": WrappedEmbeddings, - "beta": WrappedBeta, - "responses": WrappedResponses, -} +import logging +import time +import uuid +from typing import TYPE_CHECKING as _TYPE_CHECKING, Any, Dict, Optional + +from posthog.ai.types import TokenUsage as TokenUsage + +try: + import openai +except ImportError: + raise ModuleNotFoundError( + "Please install the OpenAI SDK to use this feature: 'pip install openai'" + ) + +from posthog.ai.utils import ( + call_llm_and_track_usage, + extract_available_tool_calls as extract_available_tool_calls, + finalize_ai_content as finalize_ai_content, + merge_usage_stats as merge_usage_stats, + with_privacy_mode as with_privacy_mode, +) +from posthog.ai.openai.openai_converter import ( + accumulate_openai_tool_calls as accumulate_openai_tool_calls, + extract_openai_content_from_chunk as extract_openai_content_from_chunk, + extract_openai_tool_calls_from_chunk as extract_openai_tool_calls_from_chunk, + extract_openai_usage_from_chunk as extract_openai_usage_from_chunk, + format_openai_streaming_input as _format_openai_streaming_input, + format_openai_streaming_output as _format_openai_streaming_output, +) +from posthog.client import Client as PostHogClient +from posthog import setup +from posthog.ai.openai._streaming import ( + _ChatCompletionsStreamState, + _ResponsesStreamState, + _build_streaming_event_data, +) +from ..stream import _StreamWrapper +from ._embeddings import _capture_embedding_event +from .wrapper_utils import ( + _OpenAIWrapperResource, + _wrap_openai_resources, + merge_provider_override, +) + +_log = logging.getLogger("posthog") + + +class OpenAI(openai.OpenAI): + """ + A wrapper around the OpenAI SDK that automatically sends LLM usage events to PostHog. + """ + + _ph_client: PostHogClient + + if _TYPE_CHECKING: + chat: "WrappedChat" + embeddings: "WrappedEmbeddings" + beta: "WrappedBeta" + responses: "WrappedResponses" + + def __init__(self, posthog_client: Optional[PostHogClient] = None, **kwargs): + """ + Args: + posthog_client: If provided, events will be captured via this client + instead of the global ``posthog`` client. + **kwargs: Arguments passed to ``openai.OpenAI`` such as ``api_key`` + or ``organization``. + """ + + super().__init__(**kwargs) + self._ph_client = posthog_client or setup() + + _wrap_openai_resources(self, _SYNC_RESOURCE_WRAPPERS) + + +def _parse_and_track( + wrapper, + posthog_distinct_id: Optional[str], + posthog_trace_id: Optional[str], + posthog_properties: Optional[Dict[str, Any]], + posthog_privacy_mode: bool, + posthog_groups: Optional[Dict[str, Any]], + posthog_provider_override: Optional[str] = None, + **kwargs: Any, +): + return call_llm_and_track_usage( + posthog_distinct_id, + wrapper._client._ph_client, + "openai", + posthog_trace_id, + merge_provider_override(posthog_properties, posthog_provider_override), + posthog_privacy_mode, + posthog_groups, + wrapper._client.base_url, + wrapper._original.parse, + **kwargs, + ) + + +class WrappedResponses(_OpenAIWrapperResource): + """Wrapper for OpenAI responses that tracks usage in PostHog.""" + + def create( + self, + posthog_distinct_id: Optional[str] = None, + posthog_trace_id: Optional[str] = None, + posthog_properties: Optional[Dict[str, Any]] = None, + posthog_privacy_mode: bool = False, + posthog_groups: Optional[Dict[str, Any]] = None, + posthog_provider_override: Optional[str] = None, + **kwargs: Any, + ): + """ + Create an OpenAI Responses API response while tracking usage in PostHog. + + Args: + posthog_distinct_id: Optional distinct ID to associate with the usage event. + posthog_trace_id: Optional trace ID. Generated automatically when omitted. + posthog_properties: Additional properties to include with the usage event. + posthog_privacy_mode: Whether to redact captured input and output. + posthog_groups: Optional PostHog groups to associate with the event. + posthog_provider_override: Optional override for the ``$ai_provider`` + reported on the usage event. Useful when this client is pointed at + an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom + ``base_url``, so cost attribution matches the real provider. + Defaults to ``"openai"`` when omitted. + **kwargs: Arguments passed to OpenAI's ``responses.create`` API. + + Returns: + The OpenAI response, or a streaming iterator when ``stream=True``. + """ + if posthog_trace_id is None: + posthog_trace_id = str(uuid.uuid4()) + + posthog_properties = merge_provider_override( + posthog_properties, posthog_provider_override + ) + + if kwargs.get("stream", False): + return self._create_streaming( + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + **kwargs, + ) + + return call_llm_and_track_usage( + posthog_distinct_id, + self._client._ph_client, + "openai", + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + self._client.base_url, + self._original.create, + **kwargs, + ) + + def _create_streaming( + self, + posthog_distinct_id: Optional[str], + posthog_trace_id: Optional[str], + posthog_properties: Optional[Dict[str, Any]], + posthog_privacy_mode: bool, + posthog_groups: Optional[Dict[str, Any]], + **kwargs: Any, + ): + start_time = time.time() + state = _ResponsesStreamState() + try: + response = self._original.create(**kwargs) + except Exception as exc: + try: + self._capture_streaming_event( + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + kwargs, + state, + time.time() - start_time, + error=exc, + ) + except Exception: + # Telemetry must never replace the provider's own error, and the + # fixed text avoids echoing prompt or error content into logs. + _log.warning("[PostHog] Failed to capture streaming connect error") + raise + + def generator(): + stream_error: Optional[BaseException] = None + try: + for chunk in response: + state.process_chunk(chunk) + yield chunk + except Exception as exc: + stream_error = exc + raise + finally: + self._capture_streaming_event( + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + kwargs, + state, + time.time() - start_time, + error=stream_error, + ) + + return _StreamWrapper(generator(), stream=response) + + def _capture_streaming_event( + self, + posthog_distinct_id: Optional[str], + posthog_trace_id: Optional[str], + posthog_properties: Optional[Dict[str, Any]], + posthog_privacy_mode: bool, + posthog_groups: Optional[Dict[str, Any]], + kwargs: Dict[str, Any], + state: _ResponsesStreamState, + latency: float, + error: Optional[BaseException] = None, + ): + from posthog.ai.utils import capture_streaming_event + + event_data = _build_streaming_event_data( + base_url=self._client.base_url, + kwargs=kwargs, + formatted_input=_format_openai_streaming_input(kwargs, "responses"), + formatted_output=_format_openai_streaming_output(state.output, "responses"), + usage_stats=state.usage_stats, + latency=latency, + distinct_id=posthog_distinct_id, + trace_id=posthog_trace_id, + properties=posthog_properties, + privacy_mode=posthog_privacy_mode, + groups=posthog_groups, + model_from_response=state.model, + stop_reason=state.stop_reason, + service_tier=state.service_tier, + error=error if error is not None else state.failure, + ) + capture_streaming_event(self._client._ph_client, event_data) + + def parse( + self, + posthog_distinct_id: Optional[str] = None, + posthog_trace_id: Optional[str] = None, + posthog_properties: Optional[Dict[str, Any]] = None, + posthog_privacy_mode: bool = False, + posthog_groups: Optional[Dict[str, Any]] = None, + posthog_provider_override: Optional[str] = None, + **kwargs: Any, + ): + """ + Parse structured output using OpenAI's 'responses.parse' method, but also track usage in PostHog. + + Args: + posthog_distinct_id: Optional ID to associate with the usage event. + posthog_trace_id: Optional trace UUID for linking events. + posthog_properties: Optional dictionary of extra properties to include in the event. + posthog_privacy_mode: Whether to anonymize the input and output. + posthog_groups: Optional dictionary of groups to associate with the event. + posthog_provider_override: Optional override for the ``$ai_provider`` + reported on the usage event. Useful when this client is pointed at + an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom + ``base_url``, so cost attribution matches the real provider. + Defaults to ``"openai"`` when omitted. + **kwargs: Any additional parameters for the OpenAI Responses Parse API. + + Returns: + The response from OpenAI's responses.parse call. + """ + return _parse_and_track( + self, + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + posthog_provider_override, + **kwargs, + ) + + +class WrappedChat(_OpenAIWrapperResource): + """Wrapper for OpenAI chat that tracks usage in PostHog.""" + + @property + def completions(self): + """Access chat completions with PostHog usage tracking.""" + return WrappedCompletions(self._client, self._original.completions) + + +class WrappedCompletions(_OpenAIWrapperResource): + """Wrapper for OpenAI chat completions that tracks usage in PostHog.""" + + def parse( + self, + posthog_distinct_id: Optional[str] = None, + posthog_trace_id: Optional[str] = None, + posthog_properties: Optional[Dict[str, Any]] = None, + posthog_privacy_mode: bool = False, + posthog_groups: Optional[Dict[str, Any]] = None, + posthog_provider_override: Optional[str] = None, + **kwargs: Any, + ): + """ + Parse an OpenAI chat completion while tracking usage in PostHog. + + Args: + posthog_distinct_id: Optional distinct ID to associate with the usage event. + posthog_trace_id: Optional trace ID. Generated automatically when omitted. + posthog_properties: Additional properties to include with the usage event. + posthog_privacy_mode: Whether to redact captured input and output. + posthog_groups: Optional PostHog groups to associate with the event. + posthog_provider_override: Optional override for the ``$ai_provider`` + reported on the usage event. Useful when this client is pointed at + an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom + ``base_url``, so cost attribution matches the real provider. + Defaults to ``"openai"`` when omitted. + **kwargs: Arguments passed to OpenAI's ``chat.completions.parse`` API. + + Returns: + The parsed response from OpenAI. + """ + return _parse_and_track( + self, + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + posthog_provider_override, + **kwargs, + ) + + def create( + self, + posthog_distinct_id: Optional[str] = None, + posthog_trace_id: Optional[str] = None, + posthog_properties: Optional[Dict[str, Any]] = None, + posthog_privacy_mode: bool = False, + posthog_groups: Optional[Dict[str, Any]] = None, + posthog_provider_override: Optional[str] = None, + **kwargs: Any, + ): + """ + Create an OpenAI chat completion while tracking usage in PostHog. + + Args: + posthog_distinct_id: Optional distinct ID to associate with the usage event. + posthog_trace_id: Optional trace ID. Generated automatically when omitted. + posthog_properties: Additional properties to include with the usage event. + posthog_privacy_mode: Whether to redact captured input and output. + posthog_groups: Optional PostHog groups to associate with the event. + posthog_provider_override: Optional override for the ``$ai_provider`` + reported on the usage event. Useful when this client is pointed at + an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom + ``base_url``, so cost attribution matches the real provider. + Defaults to ``"openai"`` when omitted. + **kwargs: Arguments passed to OpenAI's ``chat.completions.create`` API. + + Returns: + The OpenAI chat completion, or a streaming iterator when ``stream=True``. + """ + if posthog_trace_id is None: + posthog_trace_id = str(uuid.uuid4()) + + posthog_properties = merge_provider_override( + posthog_properties, posthog_provider_override + ) + + if kwargs.get("stream", False): + return self._create_streaming( + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + **kwargs, + ) + + return call_llm_and_track_usage( + posthog_distinct_id, + self._client._ph_client, + "openai", + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + self._client.base_url, + self._original.create, + **kwargs, + ) + + def _create_streaming( + self, + posthog_distinct_id: Optional[str], + posthog_trace_id: Optional[str], + posthog_properties: Optional[Dict[str, Any]], + posthog_privacy_mode: bool, + posthog_groups: Optional[Dict[str, Any]], + **kwargs: Any, + ): + start_time = time.time() + state = _ChatCompletionsStreamState() + # The caller's dict is left alone, and None / NOT_GIVEN mean "no options" + stream_options = kwargs.get("stream_options") + kwargs["stream_options"] = { + **(stream_options if isinstance(stream_options, dict) else {}), + "include_usage": True, + } + try: + response = self._original.create(**kwargs) + except Exception as exc: + try: + self._capture_streaming_event( + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + kwargs, + state, + time.time() - start_time, + error=exc, + ) + except Exception: + # Telemetry must never replace the provider's own error, and the + # fixed text avoids echoing prompt or error content into logs. + _log.warning("[PostHog] Failed to capture streaming connect error") + raise + + def generator(): + stream_error: Optional[BaseException] = None + try: + for chunk in response: + state.process_chunk(chunk) + yield chunk + except Exception as exc: + stream_error = exc + raise + finally: + self._capture_streaming_event( + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + kwargs, + state, + time.time() - start_time, + error=stream_error, + ) + + return _StreamWrapper(generator(), stream=response) + + def _capture_streaming_event( + self, + posthog_distinct_id: Optional[str], + posthog_trace_id: Optional[str], + posthog_properties: Optional[Dict[str, Any]], + posthog_privacy_mode: bool, + posthog_groups: Optional[Dict[str, Any]], + kwargs: Dict[str, Any], + state: _ChatCompletionsStreamState, + latency: float, + error: Optional[BaseException] = None, + ): + from posthog.ai.utils import capture_streaming_event + + event_data = _build_streaming_event_data( + base_url=self._client.base_url, + kwargs=kwargs, + formatted_input=_format_openai_streaming_input(kwargs, "chat"), + formatted_output=state.formatted_output(), + usage_stats=state.usage_stats, + latency=latency, + distinct_id=posthog_distinct_id, + trace_id=posthog_trace_id, + properties=posthog_properties, + privacy_mode=posthog_privacy_mode, + groups=posthog_groups, + model_from_response=state.model, + stop_reason=state.stop_reason, + service_tier=state.service_tier, + error=error, + ) + capture_streaming_event(self._client._ph_client, event_data) + + +class WrappedEmbeddings(_OpenAIWrapperResource): + """Wrapper for OpenAI embeddings that tracks usage in PostHog.""" + + def create( + self, + posthog_distinct_id: Optional[str] = None, + posthog_trace_id: Optional[str] = None, + posthog_properties: Optional[Dict[str, Any]] = None, + posthog_privacy_mode: bool = False, + posthog_groups: Optional[Dict[str, Any]] = None, + posthog_provider_override: Optional[str] = None, + **kwargs: Any, + ): + """ + Create an embedding using OpenAI's 'embeddings.create' method, but also track usage in PostHog. + + Args: + posthog_distinct_id: Optional ID to associate with the usage event. + posthog_trace_id: Optional trace UUID for linking events. + posthog_properties: Optional dictionary of extra properties to include in the event. + posthog_privacy_mode: Whether to anonymize the input and output. + posthog_groups: Optional dictionary of groups to associate with the event. + posthog_provider_override: Optional override for the ``$ai_provider`` + reported on the usage event. Useful when this client is pointed at + an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom + ``base_url``, so cost attribution matches the real provider. + Defaults to ``"openai"`` when omitted. + **kwargs: Any additional parameters for the OpenAI Embeddings API. + + Returns: + The response from OpenAI's embeddings.create call. + """ + + if posthog_trace_id is None: + posthog_trace_id = str(uuid.uuid4()) + + posthog_properties = merge_provider_override( + posthog_properties, posthog_provider_override + ) + + start_time = time.time() + try: + response = self._original.create(**kwargs) + except Exception as exc: + _capture_embedding_event( + posthog_client=self._client._ph_client, + base_url=self._client.base_url, + response=None, + request_kwargs=kwargs, + latency=time.time() - start_time, + distinct_id=posthog_distinct_id, + trace_id=posthog_trace_id, + properties=posthog_properties, + privacy_mode=posthog_privacy_mode, + groups=posthog_groups, + error=exc, + ) + raise + end_time = time.time() + + _capture_embedding_event( + posthog_client=self._client._ph_client, + base_url=self._client.base_url, + response=response, + request_kwargs=kwargs, + latency=end_time - start_time, + distinct_id=posthog_distinct_id, + trace_id=posthog_trace_id, + properties=posthog_properties, + privacy_mode=posthog_privacy_mode, + groups=posthog_groups, + ) + return response + + +class WrappedBeta(_OpenAIWrapperResource): + """Wrapper for OpenAI beta features that tracks usage in PostHog.""" + + @property + def chat(self): + """Access beta chat APIs with PostHog usage tracking.""" + return WrappedBetaChat(self._client, self._original.chat) + + +class WrappedBetaChat(_OpenAIWrapperResource): + """Wrapper for OpenAI beta chat that tracks usage in PostHog.""" + + @property + def completions(self): + """Access beta chat completions with PostHog usage tracking.""" + return WrappedBetaCompletions(self._client, self._original.completions) + + +class WrappedBetaCompletions(_OpenAIWrapperResource): + """Wrapper for OpenAI beta chat completions that tracks usage in PostHog.""" + + def parse( + self, + posthog_distinct_id: Optional[str] = None, + posthog_trace_id: Optional[str] = None, + posthog_properties: Optional[Dict[str, Any]] = None, + posthog_privacy_mode: bool = False, + posthog_groups: Optional[Dict[str, Any]] = None, + posthog_provider_override: Optional[str] = None, + **kwargs: Any, + ): + """ + Parse an OpenAI beta chat completion while tracking usage in PostHog. + + Args: + posthog_distinct_id: Optional distinct ID to associate with the usage event. + posthog_trace_id: Optional trace ID. Generated automatically when omitted. + posthog_properties: Additional properties to include with the usage event. + posthog_privacy_mode: Whether to redact captured input and output. + posthog_groups: Optional PostHog groups to associate with the event. + posthog_provider_override: Optional override for the ``$ai_provider`` + reported on the usage event. Useful when this client is pointed at + an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom + ``base_url``, so cost attribution matches the real provider. + Defaults to ``"openai"`` when omitted. + **kwargs: Arguments passed to OpenAI's beta ``chat.completions.parse`` API. + + Returns: + The parsed response from OpenAI. + """ + return _parse_and_track( + self, + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + posthog_provider_override, + **kwargs, + ) + + +_SYNC_RESOURCE_WRAPPERS = { + "chat": WrappedChat, + "embeddings": WrappedEmbeddings, + "beta": WrappedBeta, + "responses": WrappedResponses, +} diff --git a/posthog/ai/openai/openai_async.py b/posthog/ai/openai/openai_async.py index 3d105f7a2..8863756bf 100644 --- a/posthog/ai/openai/openai_async.py +++ b/posthog/ai/openai/openai_async.py @@ -1,645 +1,645 @@ -import logging -import time -import uuid -from typing import TYPE_CHECKING as _TYPE_CHECKING, Any, Dict, Optional - -from posthog.ai.stream import AsyncStreamWrapper -from posthog.ai.types import TokenUsage as TokenUsage - -try: - import openai -except ImportError: - raise ModuleNotFoundError( - "Please install the OpenAI SDK to use this feature: 'pip install openai'" - ) - -from posthog import setup -from posthog.ai.utils import ( - call_llm_and_track_usage_async, - extract_available_tool_calls as extract_available_tool_calls, - finalize_ai_content as finalize_ai_content, - get_model_params as get_model_params, - merge_usage_stats as merge_usage_stats, - with_privacy_mode as with_privacy_mode, -) -from posthog.ai.openai.openai_converter import ( - accumulate_openai_tool_calls as accumulate_openai_tool_calls, - extract_openai_content_from_chunk as extract_openai_content_from_chunk, - extract_openai_tool_calls_from_chunk as extract_openai_tool_calls_from_chunk, - extract_openai_usage_from_chunk as extract_openai_usage_from_chunk, - format_openai_streaming_input, - format_openai_streaming_output, -) -from posthog.client import Client as PostHogClient -from posthog.ai.openai._streaming import ( - _ChatCompletionsStreamState, - _ResponsesStreamState, - _build_streaming_event_data, -) -from ._embeddings import _capture_embedding_event -from .wrapper_utils import ( - _OpenAIWrapperResource, - _wrap_openai_resources, - merge_provider_override, -) - -_log = logging.getLogger("posthog") - - -class AsyncOpenAI(openai.AsyncOpenAI): - """ - An async wrapper around the OpenAI SDK that automatically sends LLM usage events to PostHog. - """ - - _ph_client: PostHogClient - - if _TYPE_CHECKING: - chat: "WrappedChat" - embeddings: "WrappedEmbeddings" - beta: "WrappedBeta" - responses: "WrappedResponses" - - def __init__(self, posthog_client: Optional[PostHogClient] = None, **kwargs): - """ - Args: - posthog_client: If provided, events will be captured via this client - instead of the global ``posthog`` client. - **kwargs: Arguments passed to ``openai.AsyncOpenAI`` such as - ``api_key`` or ``organization``. - """ - - super().__init__(**kwargs) - self._ph_client = posthog_client or setup() - - _wrap_openai_resources(self, _ASYNC_RESOURCE_WRAPPERS) - - -async def _parse_and_track( - wrapper, - posthog_distinct_id: Optional[str], - posthog_trace_id: Optional[str], - posthog_properties: Optional[Dict[str, Any]], - posthog_privacy_mode: bool, - posthog_groups: Optional[Dict[str, Any]], - posthog_provider_override: Optional[str] = None, - **kwargs: Any, -): - return await call_llm_and_track_usage_async( - posthog_distinct_id, - wrapper._client._ph_client, - "openai", - posthog_trace_id, - merge_provider_override(posthog_properties, posthog_provider_override), - posthog_privacy_mode, - posthog_groups, - wrapper._client.base_url, - wrapper._original.parse, - **kwargs, - ) - - -class WrappedResponses(_OpenAIWrapperResource): - """Async wrapper for OpenAI responses that tracks usage in PostHog.""" - - async def create( - self, - posthog_distinct_id: Optional[str] = None, - posthog_trace_id: Optional[str] = None, - posthog_properties: Optional[Dict[str, Any]] = None, - posthog_privacy_mode: bool = False, - posthog_groups: Optional[Dict[str, Any]] = None, - posthog_provider_override: Optional[str] = None, - **kwargs: Any, - ): - """ - Create an OpenAI Responses API response while tracking usage in PostHog. - - Args: - posthog_distinct_id: Optional distinct ID to associate with the usage event. - posthog_trace_id: Optional trace ID. Generated automatically when omitted. - posthog_properties: Additional properties to include with the usage event. - posthog_privacy_mode: Whether to redact captured input and output. - posthog_groups: Optional PostHog groups to associate with the event. - posthog_provider_override: Optional override for the ``$ai_provider`` - reported on the usage event. Useful when this client is pointed at - an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom - ``base_url``, so cost attribution matches the real provider. - Defaults to ``"openai"`` when omitted. - **kwargs: Arguments passed to OpenAI's async ``responses.create`` API. - - Returns: - The OpenAI response, or an async streaming iterator when ``stream=True``. - """ - if posthog_trace_id is None: - posthog_trace_id = str(uuid.uuid4()) - - posthog_properties = merge_provider_override( - posthog_properties, posthog_provider_override - ) - - if kwargs.get("stream", False): - return await self._create_streaming( - posthog_distinct_id, - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - **kwargs, - ) - - return await call_llm_and_track_usage_async( - posthog_distinct_id, - self._client._ph_client, - "openai", - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - self._client.base_url, - self._original.create, - **kwargs, - ) - - async def _create_streaming( - self, - posthog_distinct_id: Optional[str], - posthog_trace_id: Optional[str], - posthog_properties: Optional[Dict[str, Any]], - posthog_privacy_mode: bool, - posthog_groups: Optional[Dict[str, Any]], - **kwargs: Any, - ): - start_time = time.time() - state = _ResponsesStreamState() - try: - response = await self._original.create(**kwargs) - except Exception as exc: - try: - await self._capture_streaming_event( - posthog_distinct_id, - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - kwargs, - state, - time.time() - start_time, - error=exc, - ) - except Exception: - # Telemetry must never replace the provider's own error, and the - # fixed text avoids echoing prompt or error content into logs. - _log.warning("[PostHog] Failed to capture streaming connect error") - raise - - async def async_generator(): - stream_error: Optional[BaseException] = None - try: - async for chunk in response: - state.process_chunk(chunk) - yield chunk - except Exception as exc: - stream_error = exc - raise - finally: - await self._capture_streaming_event( - posthog_distinct_id, - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - kwargs, - state, - time.time() - start_time, - error=stream_error, - ) - - return AsyncStreamWrapper(async_generator(), stream=response) - - async def _capture_streaming_event( - self, - posthog_distinct_id: Optional[str], - posthog_trace_id: Optional[str], - posthog_properties: Optional[Dict[str, Any]], - posthog_privacy_mode: bool, - posthog_groups: Optional[Dict[str, Any]], - kwargs: Dict[str, Any], - state: _ResponsesStreamState, - latency: float, - error: Optional[BaseException] = None, - ): - from posthog.ai.utils import capture_streaming_event - - event_data = _build_streaming_event_data( - base_url=self._client.base_url, - kwargs=kwargs, - formatted_input=format_openai_streaming_input(kwargs, "responses"), - formatted_output=format_openai_streaming_output(state.output, "responses"), - usage_stats=state.usage_stats, - latency=latency, - distinct_id=posthog_distinct_id, - trace_id=posthog_trace_id, - properties=posthog_properties, - privacy_mode=posthog_privacy_mode, - groups=posthog_groups, - model_from_response=state.model, - stop_reason=state.stop_reason, - service_tier=state.service_tier, - error=error, - ) - capture_streaming_event(self._client._ph_client, event_data) - - async def parse( - self, - posthog_distinct_id: Optional[str] = None, - posthog_trace_id: Optional[str] = None, - posthog_properties: Optional[Dict[str, Any]] = None, - posthog_privacy_mode: bool = False, - posthog_groups: Optional[Dict[str, Any]] = None, - posthog_provider_override: Optional[str] = None, - **kwargs: Any, - ): - """ - Parse structured output using OpenAI's 'responses.parse' method, but also track usage in PostHog. - - Args: - posthog_distinct_id: Optional ID to associate with the usage event. - posthog_trace_id: Optional trace UUID for linking events. - posthog_properties: Optional dictionary of extra properties to include in the event. - posthog_privacy_mode: Whether to anonymize the input and output. - posthog_groups: Optional dictionary of groups to associate with the event. - posthog_provider_override: Optional override for the ``$ai_provider`` - reported on the usage event. Useful when this client is pointed at - an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom - ``base_url``, so cost attribution matches the real provider. - Defaults to ``"openai"`` when omitted. - **kwargs: Any additional parameters for the OpenAI Responses Parse API. - - Returns: - The response from OpenAI's responses.parse call. - """ - return await _parse_and_track( - self, - posthog_distinct_id, - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - posthog_provider_override, - **kwargs, - ) - - -class WrappedChat(_OpenAIWrapperResource): - """Async wrapper for OpenAI chat that tracks usage in PostHog.""" - - @property - def completions(self): - """Access async chat completions with PostHog usage tracking.""" - return WrappedCompletions(self._client, self._original.completions) - - -class WrappedCompletions(_OpenAIWrapperResource): - """Async wrapper for OpenAI chat completions that tracks usage in PostHog.""" - - async def parse( - self, - posthog_distinct_id: Optional[str] = None, - posthog_trace_id: Optional[str] = None, - posthog_properties: Optional[Dict[str, Any]] = None, - posthog_privacy_mode: bool = False, - posthog_groups: Optional[Dict[str, Any]] = None, - posthog_provider_override: Optional[str] = None, - **kwargs: Any, - ): - """ - Parse an OpenAI chat completion while tracking usage in PostHog. - - Args: - posthog_distinct_id: Optional distinct ID to associate with the usage event. - posthog_trace_id: Optional trace ID. Generated automatically when omitted. - posthog_properties: Additional properties to include with the usage event. - posthog_privacy_mode: Whether to redact captured input and output. - posthog_groups: Optional PostHog groups to associate with the event. - posthog_provider_override: Optional override for the ``$ai_provider`` - reported on the usage event. Useful when this client is pointed at - an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom - ``base_url``, so cost attribution matches the real provider. - Defaults to ``"openai"`` when omitted. - **kwargs: Arguments passed to OpenAI's async ``chat.completions.parse`` API. - - Returns: - The parsed response from OpenAI. - """ - return await _parse_and_track( - self, - posthog_distinct_id, - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - posthog_provider_override, - **kwargs, - ) - - async def create( - self, - posthog_distinct_id: Optional[str] = None, - posthog_trace_id: Optional[str] = None, - posthog_properties: Optional[Dict[str, Any]] = None, - posthog_privacy_mode: bool = False, - posthog_groups: Optional[Dict[str, Any]] = None, - posthog_provider_override: Optional[str] = None, - **kwargs: Any, - ): - """ - Create an OpenAI chat completion while tracking usage in PostHog. - - Args: - posthog_distinct_id: Optional distinct ID to associate with the usage event. - posthog_trace_id: Optional trace ID. Generated automatically when omitted. - posthog_properties: Additional properties to include with the usage event. - posthog_privacy_mode: Whether to redact captured input and output. - posthog_groups: Optional PostHog groups to associate with the event. - posthog_provider_override: Optional override for the ``$ai_provider`` - reported on the usage event. Useful when this client is pointed at - an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom - ``base_url``, so cost attribution matches the real provider. - Defaults to ``"openai"`` when omitted. - **kwargs: Arguments passed to OpenAI's async ``chat.completions.create`` API. - - Returns: - The OpenAI chat completion, or an async streaming iterator when ``stream=True``. - """ - if posthog_trace_id is None: - posthog_trace_id = str(uuid.uuid4()) - - posthog_properties = merge_provider_override( - posthog_properties, posthog_provider_override - ) - - # If streaming, handle streaming specifically - if kwargs.get("stream", False): - return await self._create_streaming( - posthog_distinct_id, - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - **kwargs, - ) - - response = await call_llm_and_track_usage_async( - posthog_distinct_id, - self._client._ph_client, - "openai", - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - self._client.base_url, - self._original.create, - **kwargs, - ) - return response - - async def _create_streaming( - self, - posthog_distinct_id: Optional[str], - posthog_trace_id: Optional[str], - posthog_properties: Optional[Dict[str, Any]], - posthog_privacy_mode: bool, - posthog_groups: Optional[Dict[str, Any]], - **kwargs: Any, - ): - start_time = time.time() - state = _ChatCompletionsStreamState() - - # The caller's dict is left alone, and None / NOT_GIVEN mean "no options" - stream_options = kwargs.get("stream_options") - kwargs["stream_options"] = { - **(stream_options if isinstance(stream_options, dict) else {}), - "include_usage": True, - } - try: - response = await self._original.create(**kwargs) - except Exception as exc: - try: - await self._capture_streaming_event( - posthog_distinct_id, - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - kwargs, - state, - time.time() - start_time, - error=exc, - ) - except Exception: - # Telemetry must never replace the provider's own error, and the - # fixed text avoids echoing prompt or error content into logs. - _log.warning("[PostHog] Failed to capture streaming connect error") - raise - - async def async_generator(): - stream_error: Optional[BaseException] = None - try: - async for chunk in response: - state.process_chunk(chunk) - yield chunk - except Exception as exc: - stream_error = exc - raise - finally: - await self._capture_streaming_event( - posthog_distinct_id, - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - kwargs, - state, - time.time() - start_time, - error=stream_error, - ) - - return AsyncStreamWrapper(async_generator(), stream=response) - - async def _capture_streaming_event( - self, - posthog_distinct_id: Optional[str], - posthog_trace_id: Optional[str], - posthog_properties: Optional[Dict[str, Any]], - posthog_privacy_mode: bool, - posthog_groups: Optional[Dict[str, Any]], - kwargs: Dict[str, Any], - state: _ChatCompletionsStreamState, - latency: float, - error: Optional[BaseException] = None, - ): - from posthog.ai.utils import capture_streaming_event - - event_data = _build_streaming_event_data( - base_url=self._client.base_url, - kwargs=kwargs, - formatted_input=format_openai_streaming_input(kwargs, "chat"), - formatted_output=state.formatted_output(), - usage_stats=state.usage_stats, - latency=latency, - distinct_id=posthog_distinct_id, - trace_id=posthog_trace_id, - properties=posthog_properties, - privacy_mode=posthog_privacy_mode, - groups=posthog_groups, - model_from_response=state.model, - stop_reason=state.stop_reason, - service_tier=state.service_tier, - error=error, - ) - capture_streaming_event(self._client._ph_client, event_data) - - -class WrappedEmbeddings(_OpenAIWrapperResource): - """Async wrapper for OpenAI embeddings that tracks usage in PostHog.""" - - async def create( - self, - posthog_distinct_id: Optional[str] = None, - posthog_trace_id: Optional[str] = None, - posthog_properties: Optional[Dict[str, Any]] = None, - posthog_privacy_mode: bool = False, - posthog_groups: Optional[Dict[str, Any]] = None, - posthog_provider_override: Optional[str] = None, - **kwargs: Any, - ): - """ - Create an embedding using OpenAI's 'embeddings.create' method, but also track usage in PostHog. - - Args: - posthog_distinct_id: Optional ID to associate with the usage event. - posthog_trace_id: Optional trace UUID for linking events. - posthog_properties: Optional dictionary of extra properties to include in the event. - posthog_privacy_mode: Whether to anonymize the input and output. - posthog_groups: Optional dictionary of groups to associate with the event. - posthog_provider_override: Optional override for the ``$ai_provider`` - reported on the usage event. Useful when this client is pointed at - an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom - ``base_url``, so cost attribution matches the real provider. - Defaults to ``"openai"`` when omitted. - **kwargs: Any additional parameters for the OpenAI Embeddings API. - - Returns: - The response from OpenAI's embeddings.create call. - """ - - if posthog_trace_id is None: - posthog_trace_id = str(uuid.uuid4()) - - posthog_properties = merge_provider_override( - posthog_properties, posthog_provider_override - ) - - start_time = time.time() - try: - response = await self._original.create(**kwargs) - except Exception as exc: - _capture_embedding_event( - posthog_client=self._client._ph_client, - base_url=self._client.base_url, - response=None, - request_kwargs=kwargs, - latency=time.time() - start_time, - distinct_id=posthog_distinct_id, - trace_id=posthog_trace_id, - properties=posthog_properties, - privacy_mode=posthog_privacy_mode, - groups=posthog_groups, - error=exc, - ) - raise - end_time = time.time() - - _capture_embedding_event( - posthog_client=self._client._ph_client, - base_url=self._client.base_url, - response=response, - request_kwargs=kwargs, - latency=end_time - start_time, - distinct_id=posthog_distinct_id, - trace_id=posthog_trace_id, - properties=posthog_properties, - privacy_mode=posthog_privacy_mode, - groups=posthog_groups, - ) - return response - - -class WrappedBeta(_OpenAIWrapperResource): - """Async wrapper for OpenAI beta features that tracks usage in PostHog.""" - - @property - def chat(self): - """Access async beta chat APIs with PostHog usage tracking.""" - return WrappedBetaChat(self._client, self._original.chat) - - -class WrappedBetaChat(_OpenAIWrapperResource): - """Async wrapper for OpenAI beta chat that tracks usage in PostHog.""" - - @property - def completions(self): - """Access async beta chat completions with PostHog usage tracking.""" - return WrappedBetaCompletions(self._client, self._original.completions) - - -class WrappedBetaCompletions(_OpenAIWrapperResource): - """Async wrapper for OpenAI beta chat completions that tracks usage in PostHog.""" - - async def parse( - self, - posthog_distinct_id: Optional[str] = None, - posthog_trace_id: Optional[str] = None, - posthog_properties: Optional[Dict[str, Any]] = None, - posthog_privacy_mode: bool = False, - posthog_groups: Optional[Dict[str, Any]] = None, - posthog_provider_override: Optional[str] = None, - **kwargs: Any, - ): - """ - Parse an OpenAI beta chat completion while tracking usage in PostHog. - - Args: - posthog_distinct_id: Optional distinct ID to associate with the usage event. - posthog_trace_id: Optional trace ID. Generated automatically when omitted. - posthog_properties: Additional properties to include with the usage event. - posthog_privacy_mode: Whether to redact captured input and output. - posthog_groups: Optional PostHog groups to associate with the event. - posthog_provider_override: Optional override for the ``$ai_provider`` - reported on the usage event. Useful when this client is pointed at - an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom - ``base_url``, so cost attribution matches the real provider. - Defaults to ``"openai"`` when omitted. - **kwargs: Arguments passed to OpenAI's async beta ``chat.completions.parse`` API. - - Returns: - The parsed response from OpenAI. - """ - return await _parse_and_track( - self, - posthog_distinct_id, - posthog_trace_id, - posthog_properties, - posthog_privacy_mode, - posthog_groups, - posthog_provider_override, - **kwargs, - ) - - -_ASYNC_RESOURCE_WRAPPERS = { - "chat": WrappedChat, - "embeddings": WrappedEmbeddings, - "beta": WrappedBeta, - "responses": WrappedResponses, -} +import logging +import time +import uuid +from typing import TYPE_CHECKING as _TYPE_CHECKING, Any, Dict, Optional + +from posthog.ai.stream import AsyncStreamWrapper +from posthog.ai.types import TokenUsage as TokenUsage + +try: + import openai +except ImportError: + raise ModuleNotFoundError( + "Please install the OpenAI SDK to use this feature: 'pip install openai'" + ) + +from posthog import setup +from posthog.ai.utils import ( + call_llm_and_track_usage_async, + extract_available_tool_calls as extract_available_tool_calls, + finalize_ai_content as finalize_ai_content, + get_model_params as get_model_params, + merge_usage_stats as merge_usage_stats, + with_privacy_mode as with_privacy_mode, +) +from posthog.ai.openai.openai_converter import ( + accumulate_openai_tool_calls as accumulate_openai_tool_calls, + extract_openai_content_from_chunk as extract_openai_content_from_chunk, + extract_openai_tool_calls_from_chunk as extract_openai_tool_calls_from_chunk, + extract_openai_usage_from_chunk as extract_openai_usage_from_chunk, + format_openai_streaming_input, + format_openai_streaming_output, +) +from posthog.client import Client as PostHogClient +from posthog.ai.openai._streaming import ( + _ChatCompletionsStreamState, + _ResponsesStreamState, + _build_streaming_event_data, +) +from ._embeddings import _capture_embedding_event +from .wrapper_utils import ( + _OpenAIWrapperResource, + _wrap_openai_resources, + merge_provider_override, +) + +_log = logging.getLogger("posthog") + + +class AsyncOpenAI(openai.AsyncOpenAI): + """ + An async wrapper around the OpenAI SDK that automatically sends LLM usage events to PostHog. + """ + + _ph_client: PostHogClient + + if _TYPE_CHECKING: + chat: "WrappedChat" + embeddings: "WrappedEmbeddings" + beta: "WrappedBeta" + responses: "WrappedResponses" + + def __init__(self, posthog_client: Optional[PostHogClient] = None, **kwargs): + """ + Args: + posthog_client: If provided, events will be captured via this client + instead of the global ``posthog`` client. + **kwargs: Arguments passed to ``openai.AsyncOpenAI`` such as + ``api_key`` or ``organization``. + """ + + super().__init__(**kwargs) + self._ph_client = posthog_client or setup() + + _wrap_openai_resources(self, _ASYNC_RESOURCE_WRAPPERS) + + +async def _parse_and_track( + wrapper, + posthog_distinct_id: Optional[str], + posthog_trace_id: Optional[str], + posthog_properties: Optional[Dict[str, Any]], + posthog_privacy_mode: bool, + posthog_groups: Optional[Dict[str, Any]], + posthog_provider_override: Optional[str] = None, + **kwargs: Any, +): + return await call_llm_and_track_usage_async( + posthog_distinct_id, + wrapper._client._ph_client, + "openai", + posthog_trace_id, + merge_provider_override(posthog_properties, posthog_provider_override), + posthog_privacy_mode, + posthog_groups, + wrapper._client.base_url, + wrapper._original.parse, + **kwargs, + ) + + +class WrappedResponses(_OpenAIWrapperResource): + """Async wrapper for OpenAI responses that tracks usage in PostHog.""" + + async def create( + self, + posthog_distinct_id: Optional[str] = None, + posthog_trace_id: Optional[str] = None, + posthog_properties: Optional[Dict[str, Any]] = None, + posthog_privacy_mode: bool = False, + posthog_groups: Optional[Dict[str, Any]] = None, + posthog_provider_override: Optional[str] = None, + **kwargs: Any, + ): + """ + Create an OpenAI Responses API response while tracking usage in PostHog. + + Args: + posthog_distinct_id: Optional distinct ID to associate with the usage event. + posthog_trace_id: Optional trace ID. Generated automatically when omitted. + posthog_properties: Additional properties to include with the usage event. + posthog_privacy_mode: Whether to redact captured input and output. + posthog_groups: Optional PostHog groups to associate with the event. + posthog_provider_override: Optional override for the ``$ai_provider`` + reported on the usage event. Useful when this client is pointed at + an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom + ``base_url``, so cost attribution matches the real provider. + Defaults to ``"openai"`` when omitted. + **kwargs: Arguments passed to OpenAI's async ``responses.create`` API. + + Returns: + The OpenAI response, or an async streaming iterator when ``stream=True``. + """ + if posthog_trace_id is None: + posthog_trace_id = str(uuid.uuid4()) + + posthog_properties = merge_provider_override( + posthog_properties, posthog_provider_override + ) + + if kwargs.get("stream", False): + return await self._create_streaming( + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + **kwargs, + ) + + return await call_llm_and_track_usage_async( + posthog_distinct_id, + self._client._ph_client, + "openai", + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + self._client.base_url, + self._original.create, + **kwargs, + ) + + async def _create_streaming( + self, + posthog_distinct_id: Optional[str], + posthog_trace_id: Optional[str], + posthog_properties: Optional[Dict[str, Any]], + posthog_privacy_mode: bool, + posthog_groups: Optional[Dict[str, Any]], + **kwargs: Any, + ): + start_time = time.time() + state = _ResponsesStreamState() + try: + response = await self._original.create(**kwargs) + except Exception as exc: + try: + await self._capture_streaming_event( + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + kwargs, + state, + time.time() - start_time, + error=exc, + ) + except Exception: + # Telemetry must never replace the provider's own error, and the + # fixed text avoids echoing prompt or error content into logs. + _log.warning("[PostHog] Failed to capture streaming connect error") + raise + + async def async_generator(): + stream_error: Optional[BaseException] = None + try: + async for chunk in response: + state.process_chunk(chunk) + yield chunk + except Exception as exc: + stream_error = exc + raise + finally: + await self._capture_streaming_event( + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + kwargs, + state, + time.time() - start_time, + error=stream_error, + ) + + return AsyncStreamWrapper(async_generator(), stream=response) + + async def _capture_streaming_event( + self, + posthog_distinct_id: Optional[str], + posthog_trace_id: Optional[str], + posthog_properties: Optional[Dict[str, Any]], + posthog_privacy_mode: bool, + posthog_groups: Optional[Dict[str, Any]], + kwargs: Dict[str, Any], + state: _ResponsesStreamState, + latency: float, + error: Optional[BaseException] = None, + ): + from posthog.ai.utils import capture_streaming_event + + event_data = _build_streaming_event_data( + base_url=self._client.base_url, + kwargs=kwargs, + formatted_input=format_openai_streaming_input(kwargs, "responses"), + formatted_output=format_openai_streaming_output(state.output, "responses"), + usage_stats=state.usage_stats, + latency=latency, + distinct_id=posthog_distinct_id, + trace_id=posthog_trace_id, + properties=posthog_properties, + privacy_mode=posthog_privacy_mode, + groups=posthog_groups, + model_from_response=state.model, + stop_reason=state.stop_reason, + service_tier=state.service_tier, + error=error if error is not None else state.failure, + ) + capture_streaming_event(self._client._ph_client, event_data) + + async def parse( + self, + posthog_distinct_id: Optional[str] = None, + posthog_trace_id: Optional[str] = None, + posthog_properties: Optional[Dict[str, Any]] = None, + posthog_privacy_mode: bool = False, + posthog_groups: Optional[Dict[str, Any]] = None, + posthog_provider_override: Optional[str] = None, + **kwargs: Any, + ): + """ + Parse structured output using OpenAI's 'responses.parse' method, but also track usage in PostHog. + + Args: + posthog_distinct_id: Optional ID to associate with the usage event. + posthog_trace_id: Optional trace UUID for linking events. + posthog_properties: Optional dictionary of extra properties to include in the event. + posthog_privacy_mode: Whether to anonymize the input and output. + posthog_groups: Optional dictionary of groups to associate with the event. + posthog_provider_override: Optional override for the ``$ai_provider`` + reported on the usage event. Useful when this client is pointed at + an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom + ``base_url``, so cost attribution matches the real provider. + Defaults to ``"openai"`` when omitted. + **kwargs: Any additional parameters for the OpenAI Responses Parse API. + + Returns: + The response from OpenAI's responses.parse call. + """ + return await _parse_and_track( + self, + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + posthog_provider_override, + **kwargs, + ) + + +class WrappedChat(_OpenAIWrapperResource): + """Async wrapper for OpenAI chat that tracks usage in PostHog.""" + + @property + def completions(self): + """Access async chat completions with PostHog usage tracking.""" + return WrappedCompletions(self._client, self._original.completions) + + +class WrappedCompletions(_OpenAIWrapperResource): + """Async wrapper for OpenAI chat completions that tracks usage in PostHog.""" + + async def parse( + self, + posthog_distinct_id: Optional[str] = None, + posthog_trace_id: Optional[str] = None, + posthog_properties: Optional[Dict[str, Any]] = None, + posthog_privacy_mode: bool = False, + posthog_groups: Optional[Dict[str, Any]] = None, + posthog_provider_override: Optional[str] = None, + **kwargs: Any, + ): + """ + Parse an OpenAI chat completion while tracking usage in PostHog. + + Args: + posthog_distinct_id: Optional distinct ID to associate with the usage event. + posthog_trace_id: Optional trace ID. Generated automatically when omitted. + posthog_properties: Additional properties to include with the usage event. + posthog_privacy_mode: Whether to redact captured input and output. + posthog_groups: Optional PostHog groups to associate with the event. + posthog_provider_override: Optional override for the ``$ai_provider`` + reported on the usage event. Useful when this client is pointed at + an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom + ``base_url``, so cost attribution matches the real provider. + Defaults to ``"openai"`` when omitted. + **kwargs: Arguments passed to OpenAI's async ``chat.completions.parse`` API. + + Returns: + The parsed response from OpenAI. + """ + return await _parse_and_track( + self, + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + posthog_provider_override, + **kwargs, + ) + + async def create( + self, + posthog_distinct_id: Optional[str] = None, + posthog_trace_id: Optional[str] = None, + posthog_properties: Optional[Dict[str, Any]] = None, + posthog_privacy_mode: bool = False, + posthog_groups: Optional[Dict[str, Any]] = None, + posthog_provider_override: Optional[str] = None, + **kwargs: Any, + ): + """ + Create an OpenAI chat completion while tracking usage in PostHog. + + Args: + posthog_distinct_id: Optional distinct ID to associate with the usage event. + posthog_trace_id: Optional trace ID. Generated automatically when omitted. + posthog_properties: Additional properties to include with the usage event. + posthog_privacy_mode: Whether to redact captured input and output. + posthog_groups: Optional PostHog groups to associate with the event. + posthog_provider_override: Optional override for the ``$ai_provider`` + reported on the usage event. Useful when this client is pointed at + an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom + ``base_url``, so cost attribution matches the real provider. + Defaults to ``"openai"`` when omitted. + **kwargs: Arguments passed to OpenAI's async ``chat.completions.create`` API. + + Returns: + The OpenAI chat completion, or an async streaming iterator when ``stream=True``. + """ + if posthog_trace_id is None: + posthog_trace_id = str(uuid.uuid4()) + + posthog_properties = merge_provider_override( + posthog_properties, posthog_provider_override + ) + + # If streaming, handle streaming specifically + if kwargs.get("stream", False): + return await self._create_streaming( + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + **kwargs, + ) + + response = await call_llm_and_track_usage_async( + posthog_distinct_id, + self._client._ph_client, + "openai", + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + self._client.base_url, + self._original.create, + **kwargs, + ) + return response + + async def _create_streaming( + self, + posthog_distinct_id: Optional[str], + posthog_trace_id: Optional[str], + posthog_properties: Optional[Dict[str, Any]], + posthog_privacy_mode: bool, + posthog_groups: Optional[Dict[str, Any]], + **kwargs: Any, + ): + start_time = time.time() + state = _ChatCompletionsStreamState() + + # The caller's dict is left alone, and None / NOT_GIVEN mean "no options" + stream_options = kwargs.get("stream_options") + kwargs["stream_options"] = { + **(stream_options if isinstance(stream_options, dict) else {}), + "include_usage": True, + } + try: + response = await self._original.create(**kwargs) + except Exception as exc: + try: + await self._capture_streaming_event( + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + kwargs, + state, + time.time() - start_time, + error=exc, + ) + except Exception: + # Telemetry must never replace the provider's own error, and the + # fixed text avoids echoing prompt or error content into logs. + _log.warning("[PostHog] Failed to capture streaming connect error") + raise + + async def async_generator(): + stream_error: Optional[BaseException] = None + try: + async for chunk in response: + state.process_chunk(chunk) + yield chunk + except Exception as exc: + stream_error = exc + raise + finally: + await self._capture_streaming_event( + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + kwargs, + state, + time.time() - start_time, + error=stream_error, + ) + + return AsyncStreamWrapper(async_generator(), stream=response) + + async def _capture_streaming_event( + self, + posthog_distinct_id: Optional[str], + posthog_trace_id: Optional[str], + posthog_properties: Optional[Dict[str, Any]], + posthog_privacy_mode: bool, + posthog_groups: Optional[Dict[str, Any]], + kwargs: Dict[str, Any], + state: _ChatCompletionsStreamState, + latency: float, + error: Optional[BaseException] = None, + ): + from posthog.ai.utils import capture_streaming_event + + event_data = _build_streaming_event_data( + base_url=self._client.base_url, + kwargs=kwargs, + formatted_input=format_openai_streaming_input(kwargs, "chat"), + formatted_output=state.formatted_output(), + usage_stats=state.usage_stats, + latency=latency, + distinct_id=posthog_distinct_id, + trace_id=posthog_trace_id, + properties=posthog_properties, + privacy_mode=posthog_privacy_mode, + groups=posthog_groups, + model_from_response=state.model, + stop_reason=state.stop_reason, + service_tier=state.service_tier, + error=error, + ) + capture_streaming_event(self._client._ph_client, event_data) + + +class WrappedEmbeddings(_OpenAIWrapperResource): + """Async wrapper for OpenAI embeddings that tracks usage in PostHog.""" + + async def create( + self, + posthog_distinct_id: Optional[str] = None, + posthog_trace_id: Optional[str] = None, + posthog_properties: Optional[Dict[str, Any]] = None, + posthog_privacy_mode: bool = False, + posthog_groups: Optional[Dict[str, Any]] = None, + posthog_provider_override: Optional[str] = None, + **kwargs: Any, + ): + """ + Create an embedding using OpenAI's 'embeddings.create' method, but also track usage in PostHog. + + Args: + posthog_distinct_id: Optional ID to associate with the usage event. + posthog_trace_id: Optional trace UUID for linking events. + posthog_properties: Optional dictionary of extra properties to include in the event. + posthog_privacy_mode: Whether to anonymize the input and output. + posthog_groups: Optional dictionary of groups to associate with the event. + posthog_provider_override: Optional override for the ``$ai_provider`` + reported on the usage event. Useful when this client is pointed at + an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom + ``base_url``, so cost attribution matches the real provider. + Defaults to ``"openai"`` when omitted. + **kwargs: Any additional parameters for the OpenAI Embeddings API. + + Returns: + The response from OpenAI's embeddings.create call. + """ + + if posthog_trace_id is None: + posthog_trace_id = str(uuid.uuid4()) + + posthog_properties = merge_provider_override( + posthog_properties, posthog_provider_override + ) + + start_time = time.time() + try: + response = await self._original.create(**kwargs) + except Exception as exc: + _capture_embedding_event( + posthog_client=self._client._ph_client, + base_url=self._client.base_url, + response=None, + request_kwargs=kwargs, + latency=time.time() - start_time, + distinct_id=posthog_distinct_id, + trace_id=posthog_trace_id, + properties=posthog_properties, + privacy_mode=posthog_privacy_mode, + groups=posthog_groups, + error=exc, + ) + raise + end_time = time.time() + + _capture_embedding_event( + posthog_client=self._client._ph_client, + base_url=self._client.base_url, + response=response, + request_kwargs=kwargs, + latency=end_time - start_time, + distinct_id=posthog_distinct_id, + trace_id=posthog_trace_id, + properties=posthog_properties, + privacy_mode=posthog_privacy_mode, + groups=posthog_groups, + ) + return response + + +class WrappedBeta(_OpenAIWrapperResource): + """Async wrapper for OpenAI beta features that tracks usage in PostHog.""" + + @property + def chat(self): + """Access async beta chat APIs with PostHog usage tracking.""" + return WrappedBetaChat(self._client, self._original.chat) + + +class WrappedBetaChat(_OpenAIWrapperResource): + """Async wrapper for OpenAI beta chat that tracks usage in PostHog.""" + + @property + def completions(self): + """Access async beta chat completions with PostHog usage tracking.""" + return WrappedBetaCompletions(self._client, self._original.completions) + + +class WrappedBetaCompletions(_OpenAIWrapperResource): + """Async wrapper for OpenAI beta chat completions that tracks usage in PostHog.""" + + async def parse( + self, + posthog_distinct_id: Optional[str] = None, + posthog_trace_id: Optional[str] = None, + posthog_properties: Optional[Dict[str, Any]] = None, + posthog_privacy_mode: bool = False, + posthog_groups: Optional[Dict[str, Any]] = None, + posthog_provider_override: Optional[str] = None, + **kwargs: Any, + ): + """ + Parse an OpenAI beta chat completion while tracking usage in PostHog. + + Args: + posthog_distinct_id: Optional distinct ID to associate with the usage event. + posthog_trace_id: Optional trace ID. Generated automatically when omitted. + posthog_properties: Additional properties to include with the usage event. + posthog_privacy_mode: Whether to redact captured input and output. + posthog_groups: Optional PostHog groups to associate with the event. + posthog_provider_override: Optional override for the ``$ai_provider`` + reported on the usage event. Useful when this client is pointed at + an OpenAI-compatible endpoint (e.g. DeepSeek, Groq) via a custom + ``base_url``, so cost attribution matches the real provider. + Defaults to ``"openai"`` when omitted. + **kwargs: Arguments passed to OpenAI's async beta ``chat.completions.parse`` API. + + Returns: + The parsed response from OpenAI. + """ + return await _parse_and_track( + self, + posthog_distinct_id, + posthog_trace_id, + posthog_properties, + posthog_privacy_mode, + posthog_groups, + posthog_provider_override, + **kwargs, + ) + + +_ASYNC_RESOURCE_WRAPPERS = { + "chat": WrappedChat, + "embeddings": WrappedEmbeddings, + "beta": WrappedBeta, + "responses": WrappedResponses, +} diff --git a/posthog/test/ai/test_stream_midstream_error.py b/posthog/test/ai/test_stream_midstream_error.py index 620b5990c..b73e060e0 100644 --- a/posthog/test/ai/test_stream_midstream_error.py +++ b/posthog/test/ai/test_stream_midstream_error.py @@ -1,532 +1,614 @@ -"""A stream that raises mid-flight must be captured as an error, not as a 200.""" - -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest - -from posthog.ai.types import StreamingEventData -from posthog.ai.utils import capture_streaming_event - - -@pytest.fixture -def mock_client(): - with patch("posthog.client.Client") as mock_client: - mock_client.privacy_mode = False - yield mock_client - - -def _props(mock_client): - assert mock_client.capture.call_count == 1 - return mock_client.capture.call_args[1]["properties"] - - -class _StatusError(RuntimeError): - status_code = 529 - - -def _event_data(**overrides) -> StreamingEventData: - data = StreamingEventData( - provider="openai", - model="gpt-4", - base_url="https://api.openai.com/v1", - kwargs={}, - formatted_input=[], - formatted_output=[], - usage_stats={}, - latency=0.1, - distinct_id="id", - trace_id="trace", - properties=None, - privacy_mode=False, - groups=None, - stop_reason=None, - ) - data.update(overrides) - return data - - -def test_capture_streaming_event_without_error_is_ok(mock_client): - capture_streaming_event(mock_client, _event_data()) - props = _props(mock_client) - assert props["$ai_http_status"] == 200 - assert "$ai_is_error" not in props - - -@pytest.mark.parametrize( - "error, status", [(RuntimeError("boom"), 0), (_StatusError("overloaded"), 529)] -) -def test_capture_streaming_event_with_error(mock_client, error, status): - capture_streaming_event(mock_client, _event_data(error=error)) - props = _props(mock_client) - assert props["$ai_http_status"] == status - assert props["$ai_is_error"] is True - assert props["$ai_error"] == str(error) - - -def test_openai_chat_stream_error(mock_client): - from openai.types.chat import ChatCompletionChunk - from openai.types.chat.chat_completion_chunk import Choice, ChoiceDelta - - from posthog.ai.openai import OpenAI - - chunk = ChatCompletionChunk( - id="c1", - created=1, - model="gpt-4", - object="chat.completion.chunk", - choices=[Choice(index=0, delta=ChoiceDelta(content="partial"))], - ) - - def stream(): - yield chunk - raise RuntimeError("connection reset") - - with patch("openai.resources.chat.completions.Completions.create") as create: - create.return_value = stream() - client = OpenAI(api_key="k", posthog_client=mock_client) - with pytest.raises(RuntimeError, match="connection reset"): - list( - client.chat.completions.create( - model="gpt-4", - messages=[{"role": "user", "content": "hi"}], - stream=True, - ) - ) - - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_error"] == "connection reset" - assert props["$ai_http_status"] == 0 - - -@pytest.mark.asyncio -async def test_openai_chat_stream_error_async(mock_client): - from openai.types.chat import ChatCompletionChunk - from openai.types.chat.chat_completion_chunk import Choice, ChoiceDelta - - from posthog.ai.openai import AsyncOpenAI - - chunk = ChatCompletionChunk( - id="c1", - created=1, - model="gpt-4", - object="chat.completion.chunk", - choices=[Choice(index=0, delta=ChoiceDelta(content="partial"))], - ) - - async def stream(): - yield chunk - raise RuntimeError("connection reset") - - async def create(self, **kwargs): - return stream() - - with patch("openai.resources.chat.completions.AsyncCompletions.create", new=create): - client = AsyncOpenAI(api_key="k", posthog_client=mock_client) - response = await client.chat.completions.create( - model="gpt-4", messages=[{"role": "user", "content": "hi"}], stream=True - ) - with pytest.raises(RuntimeError, match="connection reset"): - async for _ in response: - pass - - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_http_status"] == 0 - - -def test_openai_responses_stream_error(mock_client): - from posthog.ai.openai import OpenAI - - def stream(): - yield MagicMock(type="response.created", response=None) - raise RuntimeError("connection reset") - - with patch("openai.resources.responses.Responses.create") as create: - create.return_value = stream() - client = OpenAI(api_key="k", posthog_client=mock_client) - with pytest.raises(RuntimeError, match="connection reset"): - list(client.responses.create(model="gpt-4", input="hi", stream=True)) - - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_error"] == "connection reset" - - -def test_anthropic_stream_error(mock_client): - from posthog.ai.anthropic import Anthropic - - def stream(): - yield MagicMock(type="message_start", message=None) - raise _StatusError("overloaded") - - with patch("anthropic.resources.messages.Messages.create") as create: - create.return_value = stream() - client = Anthropic(api_key="k", posthog_client=mock_client) - with pytest.raises(_StatusError): - list( - client.messages.create( - model="claude-3-opus-20240229", - messages=[{"role": "user", "content": "hi"}], - max_tokens=5, - stream=True, - ) - ) - - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_http_status"] == 529 - - -@pytest.mark.asyncio -async def test_anthropic_stream_error_async(mock_client): - from posthog.ai.anthropic import AsyncAnthropic - - async def stream(): - yield MagicMock(type="message_start", message=None) - raise _StatusError("overloaded") - - async def create(**kwargs): - return stream() - - with patch("anthropic.resources.messages.AsyncMessages.create", side_effect=create): - client = AsyncAnthropic(api_key="k", posthog_client=mock_client) - response = await client.messages.create( - model="claude-3-opus-20240229", - messages=[{"role": "user", "content": "hi"}], - max_tokens=5, - stream=True, - ) - with pytest.raises(_StatusError): - async for _ in response: - pass - - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_http_status"] == 529 - - -def _gemini_chunk(): - chunk = MagicMock() - chunk.text = "partial" - chunk.usage_metadata = None - return chunk - - -def test_gemini_stream_error(mock_client): - pytest.importorskip("google.genai") - from google import genai as google_genai - - from posthog.ai.gemini import Client - - def stream(): - yield _gemini_chunk() - raise RuntimeError("503 overloaded mid-stream") - - with patch.object(google_genai, "Client") as cls: - inst = MagicMock() - cls.return_value = inst - inst.models.generate_content_stream.return_value = stream() - client = Client(api_key="k", posthog_client=mock_client) - with pytest.raises(RuntimeError, match="503"): - list( - client.models.generate_content_stream( - model="gemini-2.0-flash", contents=["hi"] - ) - ) - - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_error"] == "503 overloaded mid-stream" - assert props["$ai_http_status"] == 0 - - -@pytest.mark.asyncio -async def test_gemini_stream_error_async(mock_client): - pytest.importorskip("google.genai") - from google import genai as google_genai - - from posthog.ai.gemini import AsyncClient - - async def stream(): - yield _gemini_chunk() - raise RuntimeError("503 overloaded mid-stream") - - with patch.object(google_genai, "Client") as cls: - inst = MagicMock() - cls.return_value = inst - inst.aio.models.generate_content_stream = AsyncMock(return_value=stream()) - client = AsyncClient(api_key="k", posthog_client=mock_client) - response = await client.models.generate_content_stream( - model="gemini-2.0-flash", contents=["hi"] - ) - with pytest.raises(RuntimeError, match="503"): - async for _ in response: - pass - - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_http_status"] == 0 - - -class _ConnectError(RuntimeError): - status_code = 401 - - -def test_openai_chat_stream_connect_error(mock_client): - from posthog.ai.openai import OpenAI - - with patch("openai.resources.chat.completions.Completions.create") as create: - create.side_effect = _ConnectError("bad key") - client = OpenAI(api_key="k", posthog_client=mock_client) - with pytest.raises(_ConnectError): - client.chat.completions.create( - model="gpt-4", - messages=[{"role": "user", "content": "hi"}], - stream=True, - ) - - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_http_status"] == 401 - - -@pytest.mark.asyncio -async def test_openai_chat_stream_connect_error_async(mock_client): - from posthog.ai.openai import AsyncOpenAI - - async def create(**kwargs): - raise _ConnectError("bad key") - - with patch( - "openai.resources.chat.completions.AsyncCompletions.create", side_effect=create - ): - client = AsyncOpenAI(api_key="k", posthog_client=mock_client) - with pytest.raises(_ConnectError): - await client.chat.completions.create( - model="gpt-4", - messages=[{"role": "user", "content": "hi"}], - stream=True, - ) - - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_http_status"] == 401 - - -def test_openai_responses_stream_connect_error(mock_client): - from posthog.ai.openai import OpenAI - - with patch("openai.resources.responses.Responses.create") as create: - create.side_effect = _ConnectError("bad key") - client = OpenAI(api_key="k", posthog_client=mock_client) - with pytest.raises(_ConnectError): - client.responses.create(model="gpt-4", input="hi", stream=True) - - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_http_status"] == 401 - - -@pytest.mark.asyncio -async def test_openai_responses_stream_connect_error_async(mock_client): - from posthog.ai.openai import AsyncOpenAI - - async def create(**kwargs): - raise _ConnectError("bad key") - - with patch("openai.resources.responses.AsyncResponses.create", side_effect=create): - client = AsyncOpenAI(api_key="k", posthog_client=mock_client) - with pytest.raises(_ConnectError): - await client.responses.create(model="gpt-4", input="hi", stream=True) - - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_http_status"] == 401 - - -def test_anthropic_stream_connect_error(mock_client): - from posthog.ai.anthropic import Anthropic - - with patch("anthropic.resources.messages.Messages.create") as create: - create.side_effect = _ConnectError("bad key") - client = Anthropic(api_key="k", posthog_client=mock_client) - with pytest.raises(_ConnectError): - client.messages.create( - model="claude-3-opus-20240229", - messages=[{"role": "user", "content": "hi"}], - max_tokens=5, - stream=True, - ) - - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_http_status"] == 401 - - -@pytest.mark.asyncio -async def test_anthropic_stream_connect_error_async(mock_client): - from posthog.ai.anthropic import AsyncAnthropic - - async def create(**kwargs): - raise _ConnectError("bad key") - - with patch("anthropic.resources.messages.AsyncMessages.create", side_effect=create): - client = AsyncAnthropic(api_key="k", posthog_client=mock_client) - with pytest.raises(_ConnectError): - await client.messages.create( - model="claude-3-opus-20240229", - messages=[{"role": "user", "content": "hi"}], - max_tokens=5, - stream=True, - ) - - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_http_status"] == 401 - - -def test_gemini_stream_connect_error(mock_client): - pytest.importorskip("google.genai") - from google import genai as google_genai - - from posthog.ai.gemini import Client - - with patch.object(google_genai, "Client") as cls: - inst = MagicMock() - cls.return_value = inst - inst.models.generate_content_stream.side_effect = _ConnectError("bad key") - client = Client(api_key="k", posthog_client=mock_client) - with pytest.raises(_ConnectError): - client.models.generate_content_stream( - model="gemini-2.0-flash", contents=["hi"] - ) - - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_http_status"] == 401 - - -@pytest.mark.asyncio -async def test_gemini_stream_connect_error_async(mock_client): - pytest.importorskip("google.genai") - from google import genai as google_genai - - from posthog.ai.gemini import AsyncClient - - with patch.object(google_genai, "Client") as cls: - inst = MagicMock() - cls.return_value = inst - inst.aio.models.generate_content_stream = AsyncMock( - side_effect=_ConnectError("bad key") - ) - client = AsyncClient(api_key="k", posthog_client=mock_client) - with pytest.raises(_ConnectError): - await client.models.generate_content_stream( - model="gemini-2.0-flash", contents=["hi"] - ) - - props = _props(mock_client) - assert props["$ai_is_error"] is True - assert props["$ai_http_status"] == 401 - - -@pytest.mark.parametrize( - ("cls_name", "code"), [("ClientError", 429), ("ServerError", 503)] -) -def test_capture_streaming_event_records_google_genai_error_status( - mock_client, cls_name, code -): - errors = pytest.importorskip("google.genai.errors") - error = getattr(errors, cls_name)( - code, {"error": {"code": code, "message": "nope", "status": "ERROR"}} - ) - capture_streaming_event(mock_client, _event_data(error=error)) - props = _props(mock_client) - assert props["$ai_http_status"] == code - assert props["$ai_is_error"] is True - - -def test_anthropic_stream_connect_error_survives_unformattable_input(mock_client): - from posthog.ai.anthropic import Anthropic - - with patch("anthropic.resources.messages.Messages.create") as create: - create.side_effect = _ConnectError("bad request") - client = Anthropic(api_key="k", posthog_client=mock_client) - with pytest.raises(_ConnectError): - client.messages.create( - model="claude-3-opus-20240229", - messages=[None], - max_tokens=5, - stream=True, - ) - - -@pytest.mark.asyncio -async def test_anthropic_stream_connect_error_survives_unformattable_input_async( - mock_client, -): - from posthog.ai.anthropic import AsyncAnthropic - - async def create(**kwargs): - raise _ConnectError("bad request") - - with patch("anthropic.resources.messages.AsyncMessages.create", side_effect=create): - client = AsyncAnthropic(api_key="k", posthog_client=mock_client) - with pytest.raises(_ConnectError): - await client.messages.create( - model="claude-3-opus-20240229", - messages=[None], - max_tokens=5, - stream=True, - ) - - -def test_openai_chat_stream_connect_error_survives_unformattable_input(mock_client): - from posthog.ai.openai import OpenAI - - with patch("openai.resources.chat.completions.Completions.create") as create: - create.side_effect = _ConnectError("bad request") - client = OpenAI(api_key="k", posthog_client=mock_client) - with pytest.raises(_ConnectError): - client.chat.completions.create(model="gpt-4", messages=123, stream=True) - - -@pytest.mark.asyncio -async def test_openai_chat_stream_connect_error_survives_unformattable_input_async( - mock_client, -): - from posthog.ai.openai import AsyncOpenAI - - async def create(**kwargs): - raise _ConnectError("bad request") - - with patch( - "openai.resources.chat.completions.AsyncCompletions.create", side_effect=create - ): - client = AsyncOpenAI(api_key="k", posthog_client=mock_client) - with pytest.raises(_ConnectError): - await client.chat.completions.create( - model="gpt-4", messages=123, stream=True - ) - - -def test_anthropic_connect_error_fallback_does_not_log_error_text(mock_client, caplog): - from posthog.ai.anthropic import Anthropic - - with patch("anthropic.resources.messages.Messages.create") as create: - create.side_effect = _ConnectError("rejected PRIVATE_CANARY") - client = Anthropic(api_key="k", posthog_client=mock_client) - with caplog.at_level("DEBUG", logger="posthog"): - with pytest.raises(_ConnectError): - client.messages.create( - model="claude-3-opus-20240229", - messages=[None], - max_tokens=5, - stream=True, - posthog_privacy_mode=True, - ) - - assert "Failed to capture streaming connect error" in caplog.text - assert "PRIVATE_CANARY" not in caplog.text - assert all(r.exc_info is None for r in caplog.records) +"""A stream that raises mid-flight must be captured as an error, not as a 200.""" + +import json +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +from posthog.ai.types import StreamingEventData +from posthog.ai.utils import capture_streaming_event + + +@pytest.fixture +def mock_client(): + with patch("posthog.client.Client") as mock_client: + mock_client.privacy_mode = False + yield mock_client + + +def _props(mock_client): + assert mock_client.capture.call_count == 1 + return mock_client.capture.call_args[1]["properties"] + + +class _StatusError(RuntimeError): + status_code = 529 + + +def _event_data(**overrides) -> StreamingEventData: + data = StreamingEventData( + provider="openai", + model="gpt-4", + base_url="https://api.openai.com/v1", + kwargs={}, + formatted_input=[], + formatted_output=[], + usage_stats={}, + latency=0.1, + distinct_id="id", + trace_id="trace", + properties=None, + privacy_mode=False, + groups=None, + stop_reason=None, + ) + data.update(overrides) + return data + + +def test_capture_streaming_event_without_error_is_ok(mock_client): + capture_streaming_event(mock_client, _event_data()) + props = _props(mock_client) + assert props["$ai_http_status"] == 200 + assert "$ai_is_error" not in props + + +@pytest.mark.parametrize( + "error, status", [(RuntimeError("boom"), 0), (_StatusError("overloaded"), 529)] +) +def test_capture_streaming_event_with_error(mock_client, error, status): + capture_streaming_event(mock_client, _event_data(error=error)) + props = _props(mock_client) + assert props["$ai_http_status"] == status + assert props["$ai_is_error"] is True + assert props["$ai_error"] == str(error) + + +def test_openai_chat_stream_error(mock_client): + from openai.types.chat import ChatCompletionChunk + from openai.types.chat.chat_completion_chunk import Choice, ChoiceDelta + + from posthog.ai.openai import OpenAI + + chunk = ChatCompletionChunk( + id="c1", + created=1, + model="gpt-4", + object="chat.completion.chunk", + choices=[Choice(index=0, delta=ChoiceDelta(content="partial"))], + ) + + def stream(): + yield chunk + raise RuntimeError("connection reset") + + with patch("openai.resources.chat.completions.Completions.create") as create: + create.return_value = stream() + client = OpenAI(api_key="k", posthog_client=mock_client) + with pytest.raises(RuntimeError, match="connection reset"): + list( + client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + stream=True, + ) + ) + + props = _props(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_error"] == "connection reset" + assert props["$ai_http_status"] == 0 + + +@pytest.mark.asyncio +async def test_openai_chat_stream_error_async(mock_client): + from openai.types.chat import ChatCompletionChunk + from openai.types.chat.chat_completion_chunk import Choice, ChoiceDelta + + from posthog.ai.openai import AsyncOpenAI + + chunk = ChatCompletionChunk( + id="c1", + created=1, + model="gpt-4", + object="chat.completion.chunk", + choices=[Choice(index=0, delta=ChoiceDelta(content="partial"))], + ) + + async def stream(): + yield chunk + raise RuntimeError("connection reset") + + async def create(self, **kwargs): + return stream() + + with patch("openai.resources.chat.completions.AsyncCompletions.create", new=create): + client = AsyncOpenAI(api_key="k", posthog_client=mock_client) + response = await client.chat.completions.create( + model="gpt-4", messages=[{"role": "user", "content": "hi"}], stream=True + ) + with pytest.raises(RuntimeError, match="connection reset"): + async for _ in response: + pass + + props = _props(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_http_status"] == 0 + + +def test_openai_responses_stream_error(mock_client): + from posthog.ai.openai import OpenAI + + def stream(): + yield MagicMock(type="response.created", response=None) + raise RuntimeError("connection reset") + + with patch("openai.resources.responses.Responses.create") as create: + create.return_value = stream() + client = OpenAI(api_key="k", posthog_client=mock_client) + with pytest.raises(RuntimeError, match="connection reset"): + list(client.responses.create(model="gpt-4", input="hi", stream=True)) + + props = _props(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_error"] == "connection reset" + + +@pytest.fixture +def failed_response_transport(): + event = { + "type": "response.failed", + "sequence_number": 1, + "response": { + "id": "resp_failed", + "status": "failed", + "model": "gpt-4", + "output": [], + "error": { + "code": "server_error", + "message": "provider failed for this input", + }, + }, + } + + def respond(request): + return httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + text=f"data: {json.dumps(event)}\n\ndata: [DONE]\n\n", + ) + + return httpx.MockTransport(respond) + + +@pytest.mark.parametrize("privacy_mode", [False, True]) +def test_openai_responses_failed_event( + mock_client, failed_response_transport, privacy_mode +): + from posthog.ai.openai import OpenAI + + with OpenAI( + api_key="test", + posthog_client=mock_client, + http_client=httpx.Client(transport=failed_response_transport), + ) as client: + events = list( + client.responses.create( + model="gpt-4", + input="hi", + stream=True, + posthog_privacy_mode=privacy_mode, + ) + ) + + assert [event.type for event in events] == ["response.failed"] + props = _props(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_http_status"] == 0 + assert props["$ai_stop_reason"] == "failed" + if privacy_mode: + assert "provider failed for this input" not in props["$ai_error"] + else: + assert props["$ai_error"] == "provider failed for this input" + + +@pytest.mark.asyncio +async def test_openai_responses_failed_event_async( + mock_client, failed_response_transport +): + from posthog.ai.openai import AsyncOpenAI + + async with AsyncOpenAI( + api_key="test", + posthog_client=mock_client, + http_client=httpx.AsyncClient(transport=failed_response_transport), + ) as client: + stream = await client.responses.create(model="gpt-4", input="hi", stream=True) + events = [event async for event in stream] + + assert [event.type for event in events] == ["response.failed"] + props = _props(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_http_status"] == 0 + assert props["$ai_error"] == "provider failed for this input" + assert props["$ai_stop_reason"] == "failed" + + +def test_anthropic_stream_error(mock_client): + from posthog.ai.anthropic import Anthropic + + def stream(): + yield MagicMock(type="message_start", message=None) + raise _StatusError("overloaded") + + with patch("anthropic.resources.messages.Messages.create") as create: + create.return_value = stream() + client = Anthropic(api_key="k", posthog_client=mock_client) + with pytest.raises(_StatusError): + list( + client.messages.create( + model="claude-3-opus-20240229", + messages=[{"role": "user", "content": "hi"}], + max_tokens=5, + stream=True, + ) + ) + + props = _props(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_http_status"] == 529 + + +@pytest.mark.asyncio +async def test_anthropic_stream_error_async(mock_client): + from posthog.ai.anthropic import AsyncAnthropic + + async def stream(): + yield MagicMock(type="message_start", message=None) + raise _StatusError("overloaded") + + async def create(**kwargs): + return stream() + + with patch("anthropic.resources.messages.AsyncMessages.create", side_effect=create): + client = AsyncAnthropic(api_key="k", posthog_client=mock_client) + response = await client.messages.create( + model="claude-3-opus-20240229", + messages=[{"role": "user", "content": "hi"}], + max_tokens=5, + stream=True, + ) + with pytest.raises(_StatusError): + async for _ in response: + pass + + props = _props(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_http_status"] == 529 + + +def _gemini_chunk(): + chunk = MagicMock() + chunk.text = "partial" + chunk.usage_metadata = None + return chunk + + +def test_gemini_stream_error(mock_client): + pytest.importorskip("google.genai") + from google import genai as google_genai + + from posthog.ai.gemini import Client + + def stream(): + yield _gemini_chunk() + raise RuntimeError("503 overloaded mid-stream") + + with patch.object(google_genai, "Client") as cls: + inst = MagicMock() + cls.return_value = inst + inst.models.generate_content_stream.return_value = stream() + client = Client(api_key="k", posthog_client=mock_client) + with pytest.raises(RuntimeError, match="503"): + list( + client.models.generate_content_stream( + model="gemini-2.0-flash", contents=["hi"] + ) + ) + + props = _props(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_error"] == "503 overloaded mid-stream" + assert props["$ai_http_status"] == 0 + + +@pytest.mark.asyncio +async def test_gemini_stream_error_async(mock_client): + pytest.importorskip("google.genai") + from google import genai as google_genai + + from posthog.ai.gemini import AsyncClient + + async def stream(): + yield _gemini_chunk() + raise RuntimeError("503 overloaded mid-stream") + + with patch.object(google_genai, "Client") as cls: + inst = MagicMock() + cls.return_value = inst + inst.aio.models.generate_content_stream = AsyncMock(return_value=stream()) + client = AsyncClient(api_key="k", posthog_client=mock_client) + response = await client.models.generate_content_stream( + model="gemini-2.0-flash", contents=["hi"] + ) + with pytest.raises(RuntimeError, match="503"): + async for _ in response: + pass + + props = _props(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_http_status"] == 0 + + +class _ConnectError(RuntimeError): + status_code = 401 + + +def test_openai_chat_stream_connect_error(mock_client): + from posthog.ai.openai import OpenAI + + with patch("openai.resources.chat.completions.Completions.create") as create: + create.side_effect = _ConnectError("bad key") + client = OpenAI(api_key="k", posthog_client=mock_client) + with pytest.raises(_ConnectError): + client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + stream=True, + ) + + props = _props(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_http_status"] == 401 + + +@pytest.mark.asyncio +async def test_openai_chat_stream_connect_error_async(mock_client): + from posthog.ai.openai import AsyncOpenAI + + async def create(**kwargs): + raise _ConnectError("bad key") + + with patch( + "openai.resources.chat.completions.AsyncCompletions.create", side_effect=create + ): + client = AsyncOpenAI(api_key="k", posthog_client=mock_client) + with pytest.raises(_ConnectError): + await client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": "hi"}], + stream=True, + ) + + props = _props(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_http_status"] == 401 + + +def test_openai_responses_stream_connect_error(mock_client): + from posthog.ai.openai import OpenAI + + with patch("openai.resources.responses.Responses.create") as create: + create.side_effect = _ConnectError("bad key") + client = OpenAI(api_key="k", posthog_client=mock_client) + with pytest.raises(_ConnectError): + client.responses.create(model="gpt-4", input="hi", stream=True) + + props = _props(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_http_status"] == 401 + + +@pytest.mark.asyncio +async def test_openai_responses_stream_connect_error_async(mock_client): + from posthog.ai.openai import AsyncOpenAI + + async def create(**kwargs): + raise _ConnectError("bad key") + + with patch("openai.resources.responses.AsyncResponses.create", side_effect=create): + client = AsyncOpenAI(api_key="k", posthog_client=mock_client) + with pytest.raises(_ConnectError): + await client.responses.create(model="gpt-4", input="hi", stream=True) + + props = _props(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_http_status"] == 401 + + +def test_anthropic_stream_connect_error(mock_client): + from posthog.ai.anthropic import Anthropic + + with patch("anthropic.resources.messages.Messages.create") as create: + create.side_effect = _ConnectError("bad key") + client = Anthropic(api_key="k", posthog_client=mock_client) + with pytest.raises(_ConnectError): + client.messages.create( + model="claude-3-opus-20240229", + messages=[{"role": "user", "content": "hi"}], + max_tokens=5, + stream=True, + ) + + props = _props(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_http_status"] == 401 + + +@pytest.mark.asyncio +async def test_anthropic_stream_connect_error_async(mock_client): + from posthog.ai.anthropic import AsyncAnthropic + + async def create(**kwargs): + raise _ConnectError("bad key") + + with patch("anthropic.resources.messages.AsyncMessages.create", side_effect=create): + client = AsyncAnthropic(api_key="k", posthog_client=mock_client) + with pytest.raises(_ConnectError): + await client.messages.create( + model="claude-3-opus-20240229", + messages=[{"role": "user", "content": "hi"}], + max_tokens=5, + stream=True, + ) + + props = _props(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_http_status"] == 401 + + +def test_gemini_stream_connect_error(mock_client): + pytest.importorskip("google.genai") + from google import genai as google_genai + + from posthog.ai.gemini import Client + + with patch.object(google_genai, "Client") as cls: + inst = MagicMock() + cls.return_value = inst + inst.models.generate_content_stream.side_effect = _ConnectError("bad key") + client = Client(api_key="k", posthog_client=mock_client) + with pytest.raises(_ConnectError): + client.models.generate_content_stream( + model="gemini-2.0-flash", contents=["hi"] + ) + + props = _props(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_http_status"] == 401 + + +@pytest.mark.asyncio +async def test_gemini_stream_connect_error_async(mock_client): + pytest.importorskip("google.genai") + from google import genai as google_genai + + from posthog.ai.gemini import AsyncClient + + with patch.object(google_genai, "Client") as cls: + inst = MagicMock() + cls.return_value = inst + inst.aio.models.generate_content_stream = AsyncMock( + side_effect=_ConnectError("bad key") + ) + client = AsyncClient(api_key="k", posthog_client=mock_client) + with pytest.raises(_ConnectError): + await client.models.generate_content_stream( + model="gemini-2.0-flash", contents=["hi"] + ) + + props = _props(mock_client) + assert props["$ai_is_error"] is True + assert props["$ai_http_status"] == 401 + + +@pytest.mark.parametrize( + ("cls_name", "code"), [("ClientError", 429), ("ServerError", 503)] +) +def test_capture_streaming_event_records_google_genai_error_status( + mock_client, cls_name, code +): + errors = pytest.importorskip("google.genai.errors") + error = getattr(errors, cls_name)( + code, {"error": {"code": code, "message": "nope", "status": "ERROR"}} + ) + capture_streaming_event(mock_client, _event_data(error=error)) + props = _props(mock_client) + assert props["$ai_http_status"] == code + assert props["$ai_is_error"] is True + + +def test_anthropic_stream_connect_error_survives_unformattable_input(mock_client): + from posthog.ai.anthropic import Anthropic + + with patch("anthropic.resources.messages.Messages.create") as create: + create.side_effect = _ConnectError("bad request") + client = Anthropic(api_key="k", posthog_client=mock_client) + with pytest.raises(_ConnectError): + client.messages.create( + model="claude-3-opus-20240229", + messages=[None], + max_tokens=5, + stream=True, + ) + + +@pytest.mark.asyncio +async def test_anthropic_stream_connect_error_survives_unformattable_input_async( + mock_client, +): + from posthog.ai.anthropic import AsyncAnthropic + + async def create(**kwargs): + raise _ConnectError("bad request") + + with patch("anthropic.resources.messages.AsyncMessages.create", side_effect=create): + client = AsyncAnthropic(api_key="k", posthog_client=mock_client) + with pytest.raises(_ConnectError): + await client.messages.create( + model="claude-3-opus-20240229", + messages=[None], + max_tokens=5, + stream=True, + ) + + +def test_openai_chat_stream_connect_error_survives_unformattable_input(mock_client): + from posthog.ai.openai import OpenAI + + with patch("openai.resources.chat.completions.Completions.create") as create: + create.side_effect = _ConnectError("bad request") + client = OpenAI(api_key="k", posthog_client=mock_client) + with pytest.raises(_ConnectError): + client.chat.completions.create(model="gpt-4", messages=123, stream=True) + + +@pytest.mark.asyncio +async def test_openai_chat_stream_connect_error_survives_unformattable_input_async( + mock_client, +): + from posthog.ai.openai import AsyncOpenAI + + async def create(**kwargs): + raise _ConnectError("bad request") + + with patch( + "openai.resources.chat.completions.AsyncCompletions.create", side_effect=create + ): + client = AsyncOpenAI(api_key="k", posthog_client=mock_client) + with pytest.raises(_ConnectError): + await client.chat.completions.create( + model="gpt-4", messages=123, stream=True + ) + + +def test_anthropic_connect_error_fallback_does_not_log_error_text(mock_client, caplog): + from posthog.ai.anthropic import Anthropic + + with patch("anthropic.resources.messages.Messages.create") as create: + create.side_effect = _ConnectError("rejected PRIVATE_CANARY") + client = Anthropic(api_key="k", posthog_client=mock_client) + with caplog.at_level("DEBUG", logger="posthog"): + with pytest.raises(_ConnectError): + client.messages.create( + model="claude-3-opus-20240229", + messages=[None], + max_tokens=5, + stream=True, + posthog_privacy_mode=True, + ) + + assert "Failed to capture streaming connect error" in caplog.text + assert "PRIVATE_CANARY" not in caplog.text + assert all(r.exc_info is None for r in caplog.records)