From 3d307f7c10aa6acb9084fde06881d0f1d119623a Mon Sep 17 00:00:00 2001 From: jaeminbird Date: Wed, 4 Nov 2026 19:40:49 -0500 Subject: [PATCH] fix(ai): record the status of an Anthropic stream error event instead of 200 Anthropic sends errors that happen after a stream starts, such as overloaded_error, as an SSE error event, and the anthropic SDK raises them with the stream response's 200 status. Map the error type to its documented HTTP status (0 when unknown) so these failures are not recorded as a 200. --- .../anthropic-stream-error-event-status.md | 5 +++ posthog/ai/langchain/callbacks.py | 3 ++ posthog/ai/utils.py | 35 ++++++++++++++++++- posthog/test/ai/test_http_status.py | 33 +++++++++++++++++ 4 files changed, 75 insertions(+), 1 deletion(-) create mode 100644 .sampo/changesets/anthropic-stream-error-event-status.md diff --git a/.sampo/changesets/anthropic-stream-error-event-status.md b/.sampo/changesets/anthropic-stream-error-event-status.md new file mode 100644 index 000000000..d55761ea8 --- /dev/null +++ b/.sampo/changesets/anthropic-stream-error-event-status.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: patch +--- + +An Anthropic error that arrives as an `error` event inside a stream, such as `overloaded_error`, is no longer recorded with `$ai_http_status: 200`. The anthropic SDK raises it with the stream's 200 status, so `$ai_http_status` now comes from the error type (529 for `overloaded_error`, 429 for `rate_limit_error`), or is 0 for an unknown type. This applies to the LangChain callback handler and to every capture that reads the status through `_get_http_status`. diff --git a/posthog/ai/langchain/callbacks.py b/posthog/ai/langchain/callbacks.py index 0589f472c..27b2eb6aa 100644 --- a/posthog/ai/langchain/callbacks.py +++ b/posthog/ai/langchain/callbacks.py @@ -47,6 +47,7 @@ from posthog.ai.utils import ( _capture_ai_event, _extract_cache_creation_ttl_breakdown, + _stream_error_event_status, finalize_ai_content, _format_ai_error, get_model_params, @@ -1046,6 +1047,8 @@ def _get_http_status(error: BaseException) -> int: for attribute in ("status_code", "code"): value = getattr(error, attribute, None) if isinstance(value, int) and not isinstance(value, bool): + if attribute == "status_code": + return _stream_error_event_status(error, value) return value return 0 diff --git a/posthog/ai/utils.py b/posthog/ai/utils.py index dfc4c8c90..17dc0500d 100644 --- a/posthog/ai/utils.py +++ b/posthog/ai/utils.py @@ -436,6 +436,39 @@ def _merge_openai_system_prompt( return messages +# https://docs.anthropic.com/en/api/errors +_ANTHROPIC_ERROR_TYPE_STATUS = { + "invalid_request_error": 400, + "authentication_error": 401, + "billing_error": 402, + "permission_error": 403, + "not_found_error": 404, + "request_too_large": 413, + "rate_limit_error": 429, + "api_error": 500, + "timeout_error": 504, + "overloaded_error": 529, +} + + +def _stream_error_event_status(exc: BaseException, status: int) -> int: + """Status of an error that arrived as an `error` event inside a stream. + + Anthropic sends errors that happen after a stream starts (such as + overloaded_error) as an SSE `error` event, and its SDK raises them with the + stream's own 200 status. Use the status that the error type stands for, or + 0 when it is unknown, so the event is not recorded as a 200. + """ + if status < 400: + body = getattr(exc, "body", None) + error = body.get("error") if isinstance(body, dict) else None + error_type = error.get("type") if isinstance(error, dict) else None + if isinstance(error_type, str): + return _ANTHROPIC_ERROR_TYPE_STATUS.get(error_type, 0) + return 0 + return status + + def _get_http_status(exc: BaseException) -> int: """HTTP status of an SDK exception, 0 when it has none. @@ -444,7 +477,7 @@ def _get_http_status(exc: BaseException) -> int: """ status = getattr(exc, "status_code", None) if isinstance(status, int): - return status + return _stream_error_event_status(exc, status) code = getattr(exc, "code", None) if isinstance(code, int) and not isinstance(code, bool) and 100 <= code <= 599: return code diff --git a/posthog/test/ai/test_http_status.py b/posthog/test/ai/test_http_status.py index 1f79fa276..8f464079a 100644 --- a/posthog/test/ai/test_http_status.py +++ b/posthog/test/ai/test_http_status.py @@ -122,3 +122,36 @@ def boom(**kwargs): ) assert mock_client.capture.call_args[1]["properties"]["$ai_http_status"] == 503 + + +def _anthropic_stream_error_event(error_type): + """The error the anthropic SDK raises for an SSE `error` event: it carries + the status of the stream response, which is 200.""" + import anthropic + from unittest.mock import MagicMock + + response = MagicMock(status_code=200, headers={}) + body = {"type": "error", "error": {"type": error_type, "message": "x"}} + return anthropic.Anthropic(api_key="k")._make_status_error( + str(body), body=body, response=response + ) + + +@pytest.mark.parametrize( + "error_type, status", + [("overloaded_error", 529), ("rate_limit_error", 429), ("api_error", 500)], +) +def test_anthropic_stream_error_event_status(error_type, status): + from posthog.ai.langchain.callbacks import ( + _get_http_status as _langchain_http_status, + ) + + error = _anthropic_stream_error_event(error_type) + assert error.status_code == 200 + assert _get_http_status(error) == status + assert _langchain_http_status(error) == status + + +def test_anthropic_stream_error_event_of_unknown_type_is_not_a_200(): + error = _anthropic_stream_error_event("some_new_error") + assert _get_http_status(error) == 0