diff --git a/src/agent_env/agents/agents/claude-code/a2a_server.py b/src/agent_env/agents/agents/claude-code/a2a_server.py index 7dc27625e..8cae4b165 100644 --- a/src/agent_env/agents/agents/claude-code/a2a_server.py +++ b/src/agent_env/agents/agents/claude-code/a2a_server.py @@ -508,7 +508,7 @@ async def _turn(self, request: TaskRequest[ClaudeCodeConfig], cfg: ClaudeCodeCon data["cli_attempts"] = result.attempts builder = TaskResult.builder() if result.error: - builder.failed("cli_error", result.error[:_ERROR_MESSAGE_LIMIT]) + builder.failed("cli_error", result.error[:_ERROR_MESSAGE_LIMIT], error_type=result.error_type or "agent_error") else: builder.succeeded() return ( diff --git a/src/agent_env/agents/agents/claude-code/claude_bridge.py b/src/agent_env/agents/agents/claude-code/claude_bridge.py index e8fcfc651..35be97726 100644 --- a/src/agent_env/agents/agents/claude-code/claude_bridge.py +++ b/src/agent_env/agents/agents/claude-code/claude_bridge.py @@ -18,6 +18,7 @@ from pathlib import Path from typing import Literal +from agentenv_protocol.a2a_agent import TaskErrorType from pydantic import BaseModel, ConfigDict, model_validator from kit import in_thread_to_its_end @@ -148,6 +149,8 @@ class ClaudeResult: structured_output: dict | None = None # How many `claude -p` runs the turn took; more than one means it was resumed. attempts: int = 1 + # The failure's type for agent-env, when the CLI's terminal `result` names a cause that isn't the agent's. + error_type: TaskErrorType | None = None def _attribution_headers(project_id: str | None, task_id: str | None) -> dict[str, str]: @@ -279,6 +282,25 @@ def _resumable_api_failure(result: ClaudeResult) -> str | None: return text if any(marker in text.lower() for marker in _RESUMABLE_API_ERROR_MARKERS) else None +# The CLI copies the endpoint's error body `type` into `api_error_type`: LiteLLM sends budget_exceeded and auth_error, +# Anthropic authentication_error and permission_error. +_KEY_AUTH_ERROR_TYPES = frozenset({"auth_error", "authentication_error", "permission_error"}) +_KEY_AUTH_STATUSES = frozenset({401, 403}) + + +def _key_error_type(trajectory: list[dict]) -> TaskErrorType | None: + """model_key_budget or model_key_auth when the endpoint refused the run's key, else None.""" + event = _terminal_result(trajectory) + if event is None or not event.get("is_error"): + return None + kind = event.get("api_error_type") + if kind == "budget_exceeded": + return "model_key_budget" + if kind in _KEY_AUTH_ERROR_TYPES or _status(event.get("api_error_status")) in _KEY_AUTH_STATUSES: + return "model_key_auth" + return None + + def _endpoint_hint(trajectory: list[dict]) -> str | None: """What a 404 from the model endpoint likely means.""" event = _terminal_result(trajectory) @@ -756,7 +778,9 @@ def on_line(line: str) -> None: err: str | None = None detail: str | None = None + error_type: TaskErrorType | None = None if proc.returncode != 0: + error_type = _key_error_type(trajectory) err = (f"claude CLI was killed by signal {-proc.returncode}" if proc.returncode < 0 else f"claude CLI exited with code {proc.returncode}") detail = _cli_error_text(trajectory) or _stderr_tail(stderr.decode(errors="replace")) @@ -770,7 +794,7 @@ def on_line(line: str) -> None: return ClaudeResult(response=final_response or "", tool_call_count=tool_count, session_id=cli_session_id or session_id, trajectory=trajectory, error=err, - error_detail=detail, structured_output=structured_output) + error_detail=detail, structured_output=structured_output, error_type=error_type) _TAGS_HEADER = "x-litellm-tags" diff --git a/tst/unit/agents/claude_code/test_model_key_errors.py b/tst/unit/agents/claude_code/test_model_key_errors.py new file mode 100644 index 000000000..aba958eb3 --- /dev/null +++ b/tst/unit/agents/claude_code/test_model_key_errors.py @@ -0,0 +1,57 @@ +"""A turn the model endpoint refused for the run's key fails as model_key_budget or model_key_auth, not agent_error. + +The CLI exits 1 and copies the endpoint's status and error body `type` into its terminal `result` event. The events +below are trimmed from captures of CLI 2.1.325 against a local endpoint answering with each body. +""" +import asyncio +import json + +import pytest + +from tst.unit.agents.claude_code.agent import claude_bridge +from tst.unit.agents.claude_code.support import FakeCLI, send, terminal + + +def _result(status: int, kind: str, text: str, terminal_reason: str = "api_error") -> dict: + return { + "type": "result", "subtype": "success", "is_error": True, "session_id": "sess-1", "num_turns": 1, + "terminal_reason": terminal_reason, "api_error_status": status, "api_error_type": kind, "result": text, + } + + +@pytest.fixture +def cli(monkeypatch): + """Every `claude -p` prints the init event and ``event``, then exits 1.""" + monkeypatch.setattr(claude_bridge, "_RETRY_BACKOFF_S", (0, 0)) + + def _script(event: dict) -> None: + stdout = "".join(json.dumps(e) + "\n" for e in ( + {"type": "system", "subtype": "init", "session_id": "sess-1"}, event)).encode() + + async def _exec(*cmd, **kwargs): + return FakeCLI(stdout, 1) + + monkeypatch.setattr(asyncio, "create_subprocess_exec", _exec) + + return _script + + +@pytest.mark.parametrize(("event", "error_type"), [ + (_result(400, "budget_exceeded", "API Error: 400 Budget has been exceeded! Current cost: 10.1, Max budget: 10.0"), + "model_key_budget"), + (_result(401, "auth_error", + "Failed to authenticate. API Error: 401 Authentication Error, Invalid proxy server token passed."), + "model_key_auth"), + (_result(401, "authentication_error", "Invalid API key · Fix external API key"), "model_key_auth"), + (_result(403, "permission_error", "Failed to authenticate. API Error: 403 forbidden"), "model_key_auth"), + (_result(429, "rate_limit_error", "API Error: Request rejected (429) · slow down"), "agent_error"), + (_result(400, "invalid_request_error", "Prompt is too long", terminal_reason="prompt_too_long"), "agent_error"), +], ids=["litellm-budget", "litellm-auth", "anthropic-auth", "anthropic-permission", "rate-limit", "prompt-too-long"]) +def test_a_failed_turn_says_whether_the_endpoint_refused_the_key(cli, client, event, error_type): + cli(event) + + state, _, data = terminal(send(client, "hello")) + + assert state == "failed" + assert (data[-1]["error_type"], data[-1]["error_code"]) == (error_type, "cli_error") + assert event["result"] in data[-1]["error_message"]