diff --git a/.sampo/changesets/anthropic-stream-error-event-status.md b/.sampo/changesets/anthropic-stream-error-event-status.md new file mode 100644 index 000000000..d55761ea8 --- /dev/null +++ b/.sampo/changesets/anthropic-stream-error-event-status.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: patch +--- + +An Anthropic error that arrives as an `error` event inside a stream, such as `overloaded_error`, is no longer recorded with `$ai_http_status: 200`. The anthropic SDK raises it with the stream's 200 status, so `$ai_http_status` now comes from the error type (529 for `overloaded_error`, 429 for `rate_limit_error`), or is 0 for an unknown type. This applies to the LangChain callback handler and to every capture that reads the status through `_get_http_status`. diff --git a/posthog/ai/langchain/callbacks.py b/posthog/ai/langchain/callbacks.py index 0589f472c..27b2eb6aa 100644 --- a/posthog/ai/langchain/callbacks.py +++ b/posthog/ai/langchain/callbacks.py @@ -47,6 +47,7 @@ from posthog.ai.utils import ( _capture_ai_event, _extract_cache_creation_ttl_breakdown, + _stream_error_event_status, finalize_ai_content, _format_ai_error, get_model_params, @@ -1046,6 +1047,8 @@ def _get_http_status(error: BaseException) -> int: for attribute in ("status_code", "code"): value = getattr(error, attribute, None) if isinstance(value, int) and not isinstance(value, bool): + if attribute == "status_code": + return _stream_error_event_status(error, value) return value return 0 diff --git a/posthog/ai/utils.py b/posthog/ai/utils.py index dfc4c8c90..17dc0500d 100644 --- a/posthog/ai/utils.py +++ b/posthog/ai/utils.py @@ -436,6 +436,39 @@ def _merge_openai_system_prompt( return messages +# https://docs.anthropic.com/en/api/errors +_ANTHROPIC_ERROR_TYPE_STATUS = { + "invalid_request_error": 400, + "authentication_error": 401, + "billing_error": 402, + "permission_error": 403, + "not_found_error": 404, + "request_too_large": 413, + "rate_limit_error": 429, + "api_error": 500, + "timeout_error": 504, + "overloaded_error": 529, +} + + +def _stream_error_event_status(exc: BaseException, status: int) -> int: + """Status of an error that arrived as an `error` event inside a stream. + + Anthropic sends errors that happen after a stream starts (such as + overloaded_error) as an SSE `error` event, and its SDK raises them with the + stream's own 200 status. Use the status that the error type stands for, or + 0 when it is unknown, so the event is not recorded as a 200. + """ + if status < 400: + body = getattr(exc, "body", None) + error = body.get("error") if isinstance(body, dict) else None + error_type = error.get("type") if isinstance(error, dict) else None + if isinstance(error_type, str): + return _ANTHROPIC_ERROR_TYPE_STATUS.get(error_type, 0) + return 0 + return status + + def _get_http_status(exc: BaseException) -> int: """HTTP status of an SDK exception, 0 when it has none. @@ -444,7 +477,7 @@ def _get_http_status(exc: BaseException) -> int: """ status = getattr(exc, "status_code", None) if isinstance(status, int): - return status + return _stream_error_event_status(exc, status) code = getattr(exc, "code", None) if isinstance(code, int) and not isinstance(code, bool) and 100 <= code <= 599: return code diff --git a/posthog/test/ai/test_http_status.py b/posthog/test/ai/test_http_status.py index 1f79fa276..8f464079a 100644 --- a/posthog/test/ai/test_http_status.py +++ b/posthog/test/ai/test_http_status.py @@ -122,3 +122,36 @@ def boom(**kwargs): ) assert mock_client.capture.call_args[1]["properties"]["$ai_http_status"] == 503 + + +def _anthropic_stream_error_event(error_type): + """The error the anthropic SDK raises for an SSE `error` event: it carries + the status of the stream response, which is 200.""" + import anthropic + from unittest.mock import MagicMock + + response = MagicMock(status_code=200, headers={}) + body = {"type": "error", "error": {"type": error_type, "message": "x"}} + return anthropic.Anthropic(api_key="k")._make_status_error( + str(body), body=body, response=response + ) + + +@pytest.mark.parametrize( + "error_type, status", + [("overloaded_error", 529), ("rate_limit_error", 429), ("api_error", 500)], +) +def test_anthropic_stream_error_event_status(error_type, status): + from posthog.ai.langchain.callbacks import ( + _get_http_status as _langchain_http_status, + ) + + error = _anthropic_stream_error_event(error_type) + assert error.status_code == 200 + assert _get_http_status(error) == status + assert _langchain_http_status(error) == status + + +def test_anthropic_stream_error_event_of_unknown_type_is_not_a_200(): + error = _anthropic_stream_error_event("some_new_error") + assert _get_http_status(error) == 0