Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion backend/app/services/agent_runtime/model_step_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -740,7 +740,18 @@ def _model_message_content(raw: Mapping[str, object], build: RuntimeContextBuild
resumed_content = payload.get("content")
if isinstance(resumed_content, (str, list)):
return parse_multimodal_content(resumed_content)
return _message_content(content)
model_content = _message_content(content)
status = raw.get("execution_status")
if raw.get("role") != "tool" or status not in {"failed", "unknown"}:
return model_content
if not isinstance(model_content, str):
return model_content
label = "Tool failed" if status == "failed" else "Tool outcome is unknown"
result = f"{label}: {model_content}"
remediation = raw.get("safe_remediation")
if isinstance(remediation, str) and remediation.strip():
result += f"\n\nSuggested correction: {remediation.strip()}"
return result


def _prompt_messages(
Expand Down Expand Up @@ -841,6 +852,10 @@ def append_history(raw: Mapping[str, object]) -> None:
content=_model_message_content(raw, build),
tool_calls=provider_tool_calls,
tool_call_id=provider_tool_call_id,
is_error=(
role == "tool"
and raw.get("execution_status") in {"failed", "unknown"}
),
reasoning_content=(
cast(str, raw.get("reasoning_content")) if isinstance(raw.get("reasoning_content"), str) else None
),
Expand Down
2 changes: 2 additions & 0 deletions backend/app/services/agent_runtime/node_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -778,6 +778,8 @@ async def _model(
repair_limit = (
WRITE_FILE_PROTOCOL_REPAIR_LIMIT
if is_write_file_repair
else 10
if repair_code == "invalid_tool_call"
else 1
)
repair_counter_key = (
Expand Down
2 changes: 1 addition & 1 deletion backend/app/services/agent_runtime/tool_execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
"reconcile",
]
ToolSideEffectState = Literal["none", "confirmed", "possible", "unknown"]
SAFE_READ_MAX_ATTEMPTS = 3
SAFE_READ_MAX_ATTEMPTS = 10

# These tools dispatch an external image-generation request and can therefore
# leave the provider outcome uncertain after a response timeout. Direct Chat
Expand Down
2 changes: 1 addition & 1 deletion backend/app/services/agent_runtime/tool_repair_budget.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from app.services.agent_runtime.state import JsonObject

SAME_FINGERPRINT_FAILURE_LIMIT = 10
TOOL_EPISODE_FAILURE_LIMIT = 20
TOOL_EPISODE_FAILURE_LIMIT = 10
_REPAIRABLE_MODEL_ACTIONS = frozenset(
{"repair_arguments", "choose_other_tool"}
)
Expand Down
4 changes: 2 additions & 2 deletions backend/app/services/llm/caller.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ async def execute_tool(*args, **kwargs):
"send_message_to_agent", "send_feishu_message", "send_email"
})

WRITE_FILE_PROTOCOL_REPAIR_LIMIT = 3
WRITE_FILE_PROTOCOL_REPAIR_LIMIT = 10
WRITE_FILE_PROTOCOL_REPAIR_COUNTER_KEY = "invalid_tool_call:write_file"
WRITE_FILE_PROTOCOL_REPAIR_INSTRUCTION = (
"Your previous `write_file` call was not executed because `function.arguments` "
Expand Down Expand Up @@ -788,7 +788,7 @@ async def _buffer_chunk(_text: str) -> None:
repair_limit = (
WRITE_FILE_PROTOCOL_REPAIR_LIMIT
if retry_tool_name == "write_file"
else 1
else 10
)
repair_counter_key = (
WRITE_FILE_PROTOCOL_REPAIR_COUNTER_KEY
Expand Down
16 changes: 9 additions & 7 deletions backend/app/services/llm/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,7 @@ class LLMMessage:
content: str | list | None = None
tool_calls: list[dict] | None = None
tool_call_id: str | None = None
is_error: bool = False
reasoning_content: str | None = None
reasoning_signature: str | None = None
dynamic_content: str | None = None
Expand Down Expand Up @@ -303,6 +304,7 @@ def to_anthropic_format(self) -> dict | None:
"type": "tool_result",
"tool_use_id": self.tool_call_id,
"content": result_content,
"is_error": self.is_error,
}
]
}
Expand Down Expand Up @@ -1671,16 +1673,16 @@ def _build_payload(
if isinstance(response_content, str):
try:
parsed = json.loads(response_content)
if isinstance(parsed, dict):
response_obj: dict[str, Any] = parsed
else:
response_obj = {"result": parsed}
response_value: Any = parsed
except json.JSONDecodeError:
response_obj = {"result": response_content}
response_value = response_content
elif isinstance(response_content, dict):
response_obj = response_content
response_value = response_content
else:
response_obj = {"result": str(response_content)}
response_value = str(response_content)
response_obj = {
"error" if msg.is_error else "output": response_value,
}

contents.append({
"role": "user",
Expand Down
56 changes: 55 additions & 1 deletion backend/tests/test_agent_runtime_model_step_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -351,6 +351,60 @@ def test_prompt_messages_restore_provider_tool_call_pairing() -> None:
assert tool.tool_call_id == "provider-call-1"


@pytest.mark.parametrize(
("status", "label"),
(("failed", "Tool failed"), ("unknown", "Tool outcome is unknown")),
)
def test_prompt_messages_make_tool_failure_actionable_for_the_model(
status: str,
label: str,
) -> None:
build = _build(
current_run={"run_id": str(uuid.uuid4()), "goal": "Write"},
recent_session_messages_snapshot=(),
recent_thread_messages=(
{
"id": "assistant-1",
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "call-instance-1",
"type": "function",
"function": {
"name": "write_file",
"arguments": "{}",
},
}
],
},
{
"id": "tool-result-1",
"role": "tool",
"tool_call_id": "call-instance-1",
"content": "$.path is required",
"execution_status": status,
"safe_remediation": "Provide a non-empty path.",
},
),
initial_input={"input_content": "Continue"},
)

messages = _prompt_messages(
static_prompt="Static",
dynamic_prompt="Dynamic",
build=build,
)

tool = next(message for message in messages if message.role == "tool")
assert tool.tool_call_id == "call-instance-1"
assert tool.is_error is True
assert tool.content == (
f"{label}: $.path is required\n\n"
"Suggested correction: Provide a non-empty path."
)


def test_message_budget_does_not_treat_large_base64_as_text_tokens() -> None:
padded_png = base64.b64encode(
base64.b64decode(_TINY_PNG_BASE64) + b"x" * (1024 * 1024)
Expand Down Expand Up @@ -598,7 +652,7 @@ async def complete(model_arg, _messages, **_kwargs):


@pytest.mark.asyncio
async def test_invalid_write_file_arguments_request_three_protocol_repairs() -> None:
async def test_invalid_write_file_arguments_request_ten_protocol_repairs() -> None:
tenant_id = uuid.uuid4()
model = _model(tenant_id)
agent = _agent(tenant_id)
Expand Down
35 changes: 17 additions & 18 deletions backend/tests/test_agent_runtime_node_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -1351,15 +1351,16 @@ async def test_empty_output_is_repaired_once_then_fails_explicitly() -> None:

@pytest.mark.asyncio
@pytest.mark.parametrize(
("repair_code", "instruction"),
("repair_code", "instruction", "repair_limit"),
[
("invalid_finish", "Retry finish with valid content."),
("invalid_tool_call", "Retry with valid JSON tool arguments."),
("invalid_finish", "Retry finish with valid content.", 1),
("invalid_tool_call", "Retry with valid JSON tool arguments.", 10),
],
)
async def test_repeated_model_tool_protocol_repair_code_fails_explicitly(
repair_code: str,
instruction: str,
repair_limit: int,
) -> None:
run_id = uuid.uuid4()
repair = ModelStepResult(
Expand All @@ -1368,7 +1369,7 @@ async def test_repeated_model_tool_protocol_repair_code_fails_explicitly(
repair_instruction=instruction,
repair_code=repair_code,
)
model = ModelService(repair, repair)
model = ModelService(*([repair] * (repair_limit + 1)))
executor = _executor(model)

result = await _invoke(run_id, executor, model_turn_limit=50)
Expand All @@ -1377,13 +1378,13 @@ async def test_repeated_model_tool_protocol_repair_code_fails_explicitly(
assert lifecycle["status"] == "failed"
assert lifecycle["reason"] == "model_tool_protocol_violation"
assert lifecycle["error"]["code"] == "model_tool_protocol_violation"
assert lifecycle["model_protocol_repairs"] == {repair_code: 1}
assert lifecycle["model_step_count"] == 2
assert model.calls == 2
assert lifecycle["model_protocol_repairs"] == {repair_code: repair_limit}
assert lifecycle["model_step_count"] == repair_limit + 1
assert model.calls == repair_limit + 1


@pytest.mark.asyncio
async def test_write_file_protocol_repair_uses_three_attempts_then_guides_user() -> None:
async def test_write_file_protocol_repair_uses_ten_attempts_then_guides_user() -> None:
run_id = uuid.uuid4()
repair = ModelStepResult(
intent="text",
Expand All @@ -1392,7 +1393,7 @@ async def test_write_file_protocol_repair_uses_three_attempts_then_guides_user()
repair_code="invalid_tool_call",
repair_tool_name="write_file",
)
model = ModelService(repair, repair, repair, repair)
model = ModelService(*([repair] * 11))
executor = _executor(model)

result = await _invoke(run_id, executor, model_turn_limit=50)
Expand All @@ -1408,14 +1409,14 @@ async def test_write_file_protocol_repair_uses_three_attempts_then_guides_user()
),
}
assert lifecycle["model_protocol_repairs"] == {
"invalid_tool_call:write_file": 3,
"invalid_tool_call:write_file": 10,
}
assert lifecycle["model_step_count"] == 4
assert model.calls == 4
assert lifecycle["model_step_count"] == 11
assert model.calls == 11


@pytest.mark.asyncio
async def test_write_file_protocol_can_recover_on_the_third_repair() -> None:
async def test_write_file_protocol_can_recover_on_the_tenth_repair() -> None:
run_id = uuid.uuid4()
repair = ModelStepResult(
intent="text",
Expand All @@ -1424,9 +1425,7 @@ async def test_write_file_protocol_can_recover_on_the_third_repair() -> None:
repair_tool_name="write_file",
)
model = ModelService(
repair,
repair,
repair,
*([repair] * 10),
ModelStepResult(intent="finish", finish_content="Recovered"),
)
executor = _executor(model)
Expand All @@ -1435,9 +1434,9 @@ async def test_write_file_protocol_can_recover_on_the_third_repair() -> None:

assert result["lifecycle"]["status"] == "completed"
assert result["lifecycle"]["model_protocol_repairs"] == {
"invalid_tool_call:write_file": 3,
"invalid_tool_call:write_file": 10,
}
assert model.calls == 4
assert model.calls == 11


@pytest.mark.asyncio
Expand Down
4 changes: 2 additions & 2 deletions backend/tests/test_agent_runtime_tool_repair_budget.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ def test_tenth_consecutive_fingerprint_pauses_without_off_by_one() -> None:
assert _episode(state)["total_failures"] == 10


def test_twentieth_tool_failure_pauses_even_when_fingerprint_changes() -> None:
def test_tenth_tool_failure_pauses_even_when_fingerprint_changes() -> None:
state: dict = {}
transition = None
for model_step in range(1, TOOL_EPISODE_FAILURE_LIMIT + 1):
Expand All @@ -63,7 +63,7 @@ def test_twentieth_tool_failure_pauses_even_when_fingerprint_changes() -> None:

assert transition is not None
assert transition.pause_reason == "tool_repair_episode_limit_reached"
assert _episode(state)["total_failures"] == 20
assert _episode(state)["total_failures"] == 10
assert _episode(state)["same_fingerprint_failures"] == 1


Expand Down
4 changes: 2 additions & 2 deletions backend/tests/test_agent_runtime_tool_step_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -2977,7 +2977,7 @@ async def test_retryable_read_exhaustion_returns_one_non_retryable_result(
"call-read-exhausted",
"read_file",
)
execution.attempt_count = 3
execution.attempt_count = 10

async def reserve(db, **kwargs):
del db
Expand Down Expand Up @@ -3017,7 +3017,7 @@ async def mark_failed(db, **kwargs):
assert "Do not repeat the identical tool call unchanged" in result.messages[0][
"content"
]
assert execution.result_metadata["runtime_attempt_count"] == 3
assert execution.result_metadata["runtime_attempt_count"] == 10
assert execution.result_metadata["runtime_retry_exhausted"] is True
assert execution.result_metadata["last_error_code"] == "temporary_read_failure"

Expand Down
10 changes: 5 additions & 5 deletions backend/tests/test_finish_protocol.py
Original file line number Diff line number Diff line change
Expand Up @@ -811,7 +811,7 @@ async def test_repeated_invalid_tool_json_is_bounded_by_protocol_code(monkeypatc
}
],
)
fake_client = FakeStreamClient([invalid, invalid])
fake_client = FakeStreamClient([invalid] * 11)
monkeypatch.setattr(caller, "_get_agent_config", lambda _agent_id: _async_return((50, None)))
monkeypatch.setattr(caller, "_get_user_name", lambda _user_id: _async_return("Ray"))
monkeypatch.setattr(
Expand Down Expand Up @@ -841,12 +841,12 @@ async def test_repeated_invalid_tool_json_is_bounded_by_protocol_code(monkeypatc
)

assert result.startswith("[Error] invalid_tool_call_protocol_violation:")
assert len(fake_client.messages_seen) == 2
assert len(fake_client.messages_seen) == 11
assert fake_client.closed is True


@pytest.mark.asyncio
async def test_invalid_write_file_json_gets_three_bounded_repairs(monkeypatch):
async def test_invalid_write_file_json_gets_ten_bounded_repairs(monkeypatch):
from app.services.llm import caller
from app.services.llm.client import LLMResponse

Expand All @@ -863,7 +863,7 @@ async def test_invalid_write_file_json_gets_three_bounded_repairs(monkeypatch):
}
],
)
fake_client = FakeStreamClient([invalid, invalid, invalid, invalid])
fake_client = FakeStreamClient([invalid] * 11)
monkeypatch.setattr(caller, "_get_agent_config", lambda _agent_id: _async_return((50, None)))
monkeypatch.setattr(caller, "_get_user_name", lambda _user_id: _async_return("Ray"))
monkeypatch.setattr(
Expand Down Expand Up @@ -897,7 +897,7 @@ async def test_invalid_write_file_json_gets_three_bounded_repairs(monkeypatch):
"本次文件生成未完成:write_file 工具参数无效或被截断,连续重试后仍无法执行。"
"请回复「重新生成」,我会基于当前对话重新尝试。"
)
assert len(fake_client.messages_seen) == 4
assert len(fake_client.messages_seen) == 11
assert fake_client.closed is True


Expand Down
Loading