Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .sampo/changesets/anthropic-stream-error-event-status.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
pypi/posthog: patch
---

An Anthropic error that arrives as an `error` event inside a stream, such as `overloaded_error`, is no longer recorded with `$ai_http_status: 200`. The anthropic SDK raises it with the stream's 200 status, so `$ai_http_status` now comes from the error type (529 for `overloaded_error`, 429 for `rate_limit_error`), or is 0 for an unknown type. This applies to the LangChain callback handler and to every capture that reads the status through `_get_http_status`.
3 changes: 3 additions & 0 deletions posthog/ai/langchain/callbacks.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
from posthog.ai.utils import (
_capture_ai_event,
_extract_cache_creation_ttl_breakdown,
_stream_error_event_status,
finalize_ai_content,
_format_ai_error,
get_model_params,
Expand Down Expand Up @@ -1046,6 +1047,8 @@ def _get_http_status(error: BaseException) -> int:
for attribute in ("status_code", "code"):
value = getattr(error, attribute, None)
if isinstance(value, int) and not isinstance(value, bool):
if attribute == "status_code":
return _stream_error_event_status(error, value)
return value
return 0

Expand Down
35 changes: 34 additions & 1 deletion posthog/ai/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -436,6 +436,39 @@ def _merge_openai_system_prompt(
return messages


# https://docs.anthropic.com/en/api/errors
_ANTHROPIC_ERROR_TYPE_STATUS = {
"invalid_request_error": 400,
"authentication_error": 401,
"billing_error": 402,
"permission_error": 403,
"not_found_error": 404,
"request_too_large": 413,
"rate_limit_error": 429,
"api_error": 500,
"timeout_error": 504,
"overloaded_error": 529,
}


def _stream_error_event_status(exc: BaseException, status: int) -> int:
"""Status of an error that arrived as an `error` event inside a stream.

Anthropic sends errors that happen after a stream starts (such as
overloaded_error) as an SSE `error` event, and its SDK raises them with the
stream's own 200 status. Use the status that the error type stands for, or
0 when it is unknown, so the event is not recorded as a 200.
"""
if status < 400:
body = getattr(exc, "body", None)
error = body.get("error") if isinstance(body, dict) else None
error_type = error.get("type") if isinstance(error, dict) else None
if isinstance(error_type, str):
return _ANTHROPIC_ERROR_TYPE_STATUS.get(error_type, 0)
return 0
return status


def _get_http_status(exc: BaseException) -> int:
"""HTTP status of an SDK exception, 0 when it has none.

Expand All @@ -444,7 +477,7 @@ def _get_http_status(exc: BaseException) -> int:
"""
status = getattr(exc, "status_code", None)
if isinstance(status, int):
return status
return _stream_error_event_status(exc, status)
code = getattr(exc, "code", None)
if isinstance(code, int) and not isinstance(code, bool) and 100 <= code <= 599:
return code
Expand Down
33 changes: 33 additions & 0 deletions posthog/test/ai/test_http_status.py
Original file line number Diff line number Diff line change
Expand Up @@ -122,3 +122,36 @@ def boom(**kwargs):
)

assert mock_client.capture.call_args[1]["properties"]["$ai_http_status"] == 503


def _anthropic_stream_error_event(error_type):
"""The error the anthropic SDK raises for an SSE `error` event: it carries
the status of the stream response, which is 200."""
import anthropic
from unittest.mock import MagicMock

response = MagicMock(status_code=200, headers={})
body = {"type": "error", "error": {"type": error_type, "message": "x"}}
return anthropic.Anthropic(api_key="k")._make_status_error(
str(body), body=body, response=response
)


@pytest.mark.parametrize(
"error_type, status",
[("overloaded_error", 529), ("rate_limit_error", 429), ("api_error", 500)],
)
def test_anthropic_stream_error_event_status(error_type, status):
from posthog.ai.langchain.callbacks import (
_get_http_status as _langchain_http_status,
)

error = _anthropic_stream_error_event(error_type)
assert error.status_code == 200
assert _get_http_status(error) == status
assert _langchain_http_status(error) == status


def test_anthropic_stream_error_event_of_unknown_type_is_not_a_200():
error = _anthropic_stream_error_event("some_new_error")
assert _get_http_status(error) == 0
Loading