Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/agent_env/agents/agents/claude-code/a2a_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -508,7 +508,7 @@ async def _turn(self, request: TaskRequest[ClaudeCodeConfig], cfg: ClaudeCodeCon
data["cli_attempts"] = result.attempts
builder = TaskResult.builder()
if result.error:
builder.failed("cli_error", result.error[:_ERROR_MESSAGE_LIMIT])
builder.failed("cli_error", result.error[:_ERROR_MESSAGE_LIMIT], error_type=result.error_type or "agent_error")
else:
builder.succeeded()
return (
Expand Down
26 changes: 25 additions & 1 deletion src/agent_env/agents/agents/claude-code/claude_bridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from pathlib import Path
from typing import Literal

from agentenv_protocol.a2a_agent import TaskErrorType
from pydantic import BaseModel, ConfigDict, model_validator

from kit import in_thread_to_its_end
Expand Down Expand Up @@ -148,6 +149,8 @@ class ClaudeResult:
structured_output: dict | None = None
# How many `claude -p` runs the turn took; more than one means it was resumed.
attempts: int = 1
# The failure's type for agent-env, when the CLI's terminal `result` names a cause that isn't the agent's.
error_type: TaskErrorType | None = None


def _attribution_headers(project_id: str | None, task_id: str | None) -> dict[str, str]:
Expand Down Expand Up @@ -279,6 +282,25 @@ def _resumable_api_failure(result: ClaudeResult) -> str | None:
return text if any(marker in text.lower() for marker in _RESUMABLE_API_ERROR_MARKERS) else None


# The CLI copies the endpoint's error body `type` into `api_error_type`: LiteLLM sends budget_exceeded and auth_error,
# Anthropic authentication_error and permission_error.
_KEY_AUTH_ERROR_TYPES = frozenset({"auth_error", "authentication_error", "permission_error"})
_KEY_AUTH_STATUSES = frozenset({401, 403})


def _key_error_type(trajectory: list[dict]) -> TaskErrorType | None:
"""model_key_budget or model_key_auth when the endpoint refused the run's key, else None."""
event = _terminal_result(trajectory)
if event is None or not event.get("is_error"):
return None
kind = event.get("api_error_type")
if kind == "budget_exceeded":
return "model_key_budget"
if kind in _KEY_AUTH_ERROR_TYPES or _status(event.get("api_error_status")) in _KEY_AUTH_STATUSES:
return "model_key_auth"
return None


def _endpoint_hint(trajectory: list[dict]) -> str | None:
"""What a 404 from the model endpoint likely means."""
event = _terminal_result(trajectory)
Expand Down Expand Up @@ -756,7 +778,9 @@ def on_line(line: str) -> None:

err: str | None = None
detail: str | None = None
error_type: TaskErrorType | None = None
if proc.returncode != 0:
error_type = _key_error_type(trajectory)
err = (f"claude CLI was killed by signal {-proc.returncode}" if proc.returncode < 0
else f"claude CLI exited with code {proc.returncode}")
detail = _cli_error_text(trajectory) or _stderr_tail(stderr.decode(errors="replace"))
Expand All @@ -770,7 +794,7 @@ def on_line(line: str) -> None:

return ClaudeResult(response=final_response or "", tool_call_count=tool_count,
session_id=cli_session_id or session_id, trajectory=trajectory, error=err,
error_detail=detail, structured_output=structured_output)
error_detail=detail, structured_output=structured_output, error_type=error_type)


_TAGS_HEADER = "x-litellm-tags"
Expand Down
57 changes: 57 additions & 0 deletions tst/unit/agents/claude_code/test_model_key_errors.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
"""A turn the model endpoint refused for the run's key fails as model_key_budget or model_key_auth, not agent_error.

The CLI exits 1 and copies the endpoint's status and error body `type` into its terminal `result` event. The events
below are trimmed from captures of CLI 2.1.325 against a local endpoint answering with each body.
"""
import asyncio
import json

import pytest

from tst.unit.agents.claude_code.agent import claude_bridge
from tst.unit.agents.claude_code.support import FakeCLI, send, terminal


def _result(status: int, kind: str, text: str, terminal_reason: str = "api_error") -> dict:
return {
"type": "result", "subtype": "success", "is_error": True, "session_id": "sess-1", "num_turns": 1,
"terminal_reason": terminal_reason, "api_error_status": status, "api_error_type": kind, "result": text,
}


@pytest.fixture
def cli(monkeypatch):
"""Every `claude -p` prints the init event and ``event``, then exits 1."""
monkeypatch.setattr(claude_bridge, "_RETRY_BACKOFF_S", (0, 0))

def _script(event: dict) -> None:
stdout = "".join(json.dumps(e) + "\n" for e in (
{"type": "system", "subtype": "init", "session_id": "sess-1"}, event)).encode()

async def _exec(*cmd, **kwargs):
return FakeCLI(stdout, 1)

monkeypatch.setattr(asyncio, "create_subprocess_exec", _exec)

return _script


@pytest.mark.parametrize(("event", "error_type"), [
(_result(400, "budget_exceeded", "API Error: 400 Budget has been exceeded! Current cost: 10.1, Max budget: 10.0"),
"model_key_budget"),
(_result(401, "auth_error",
"Failed to authenticate. API Error: 401 Authentication Error, Invalid proxy server token passed."),
"model_key_auth"),
(_result(401, "authentication_error", "Invalid API key · Fix external API key"), "model_key_auth"),
(_result(403, "permission_error", "Failed to authenticate. API Error: 403 forbidden"), "model_key_auth"),
(_result(429, "rate_limit_error", "API Error: Request rejected (429) · slow down"), "agent_error"),
(_result(400, "invalid_request_error", "Prompt is too long", terminal_reason="prompt_too_long"), "agent_error"),
], ids=["litellm-budget", "litellm-auth", "anthropic-auth", "anthropic-permission", "rate-limit", "prompt-too-long"])
def test_a_failed_turn_says_whether_the_endpoint_refused_the_key(cli, client, event, error_type):
cli(event)

state, _, data = terminal(send(client, "hello"))

assert state == "failed"
assert (data[-1]["error_type"], data[-1]["error_code"]) == (error_type, "cli_error")
assert event["result"] in data[-1]["error_message"]
Loading