From af05201b57f8b84c3ec56e94263d5c8dd03ddddf Mon Sep 17 00:00:00 2001 From: westey <164392973+westey-m@users.noreply.github.com> Date: Mon, 27 Jul 2026 14:34:23 +0000 Subject: [PATCH] Add support for maximum function loop iterations to harness --- python/packages/core/AGENTS.md | 2 +- .../core/agent_framework/_harness/_agent.py | 58 ++++++++++++----- .../core/agent_framework/_harness/_agent.pyi | 1 + .../core/test_function_invocation_logic.py | 1 + .../core/tests/core/test_harness_agent.py | 62 +++++++++++++++++++ 5 files changed, 109 insertions(+), 15 deletions(-) diff --git a/python/packages/core/AGENTS.md b/python/packages/core/AGENTS.md index 7ebfaa59f44..06df1d44de7 100644 --- a/python/packages/core/AGENTS.md +++ b/python/packages/core/AGENTS.md @@ -152,7 +152,7 @@ agent_framework/ - **`todos_remaining(*, looping_modes=None)`** / **`todos_remaining_message`** - Helper factories for todo-driven loops (the Python counterpart of .NET's `TodoCompletionLoopEvaluator`), designed for `create_harness_agent` but usable with any agent that registers a `TodoProvider` via `context_providers`. They resolve the `TodoProvider`/`AgentModeProvider` from the *running agent* (`agent.context_providers`, via `_resolve_context_provider`) rather than taking the provider as an argument, so they can be wired directly into `loop_should_continue`/`loop_next_message`. `todos_remaining` returns a `should_continue` predicate that loops while any todo is open; pass `looping_modes=[...]` to gate looping to specific operating modes (case-insensitive; honors the `AgentModeProvider`'s `source_id`/`available_modes`), `looping_modes=None` (default) applies in every mode, and an empty sequence raises `ValueError`. `todos_remaining_message` is a `next_message` callable that lists the still-open todo titles and tells the agent to finish them, returning `None` when the session/agent/provider is unavailable or nothing is open (in which case the middleware's default `None` handling applies: reuse the previous iteration's messages verbatim under the default `fresh_context=False`, or `DEFAULT_NEXT_MESSAGE` only when `fresh_context=True`). - **`background_tasks_running()`** / **`background_tasks_running_message`** - Helper factories for background-agent-driven loops, mirroring the `todos_remaining` pair. They resolve the `BackgroundAgentsProvider` from the *running agent* (`agent.context_providers`, via `_resolve_context_provider`) rather than taking the provider as an argument, so they can be wired directly into `create_harness_agent`'s `loop_should_continue`/`loop_next_message`. `background_tasks_running` returns a `should_continue` predicate that loops while the provider's persisted state shows any task with `status == RUNNING` (pair it with `max_iterations` so the loop is bounded even if a task's persisted status is never refreshed). `background_tasks_running_message` is a `next_message` callable that lists the still-running tasks (`# (): `) and tells the agent to wait for them to finish and retrieve their results, returning `None` when the session/agent/provider is unavailable or no task is running. - **Approval escape hatch** - `_has_pending_approval_request(result)` checks whether an iteration's response carries a pending tool-approval request (any content with `type == "function_approval_request"`). Both the streaming and non-streaming loops stop and return that response to the caller *before* evaluating `should_continue`/`max_iterations` or injecting `next_message`, so the loop is HITL-safe even when wrapped outermost around a `ToolApprovalMiddleware` (mirrors the C# `LoopAgent`'s `HasPendingApprovalRequests`). - - **Harness integration** - `create_harness_agent` enables the loop when a `loop_should_continue` callable is passed; it prepends `AgentLoopMiddleware(loop_should_continue, max_iterations=loop_max_iterations, next_message=loop_next_message)` ahead of `ToolApprovalMiddleware` so the loop is the outermost middleware (each iteration is a full agent run including tool approval, and the escape hatch hands pending approvals back to the caller). `loop_next_message` and `loop_max_iterations` only take effect together with `loop_should_continue` (with no `loop_should_continue` there is no loop, so they are ignored); `loop_max_iterations` defaults to the loop's default cap (`None` → unbounded). + - **Harness integration** - `create_harness_agent` enables the loop when a `loop_should_continue` callable is passed; it prepends `AgentLoopMiddleware(loop_should_continue, max_iterations=loop_max_iterations, next_message=loop_next_message)` ahead of `ToolApprovalMiddleware` so the loop is the outermost middleware (each iteration is a full agent run including tool approval, and the escape hatch hands pending approvals back to the caller). `loop_next_message` and `loop_max_iterations` only take effect together with `loop_should_continue` (with no `loop_should_continue` there is no loop, so they are ignored); `loop_max_iterations` defaults to the loop's default cap (`None` → unbounded). Note the `loop_*` params control this **agent re-run loop** and are distinct from `function_loop_max_iterations`, which caps the **per-request function/tool-calling loop** by setting `client.function_invocation_configuration["max_iterations"]` on the passed-in chat client (Python counterpart of the .NET harness's `MaximumIterationsPerRequest`). `function_loop_max_iterations` must be ≥ 1 when set, defaults to `None` (leaves the client's config untouched; built-in default is 40), and logs a warning if the client does not expose `function_invocation_configuration`. ### Workflows (`_workflows/`) diff --git a/python/packages/core/agent_framework/_harness/_agent.py b/python/packages/core/agent_framework/_harness/_agent.py index 7e17a3d3509..e85399b4211 100644 --- a/python/packages/core/agent_framework/_harness/_agent.py +++ b/python/packages/core/agent_framework/_harness/_agent.py @@ -336,6 +336,7 @@ def create_harness_agent( loop_should_continue: ShouldContinueCallable | None = None, loop_next_message: NextMessageCallable | None = None, loop_max_iterations: int | None = DEFAULT_MAX_ITERATIONS, + function_loop_max_iterations: int | None = None, otel_provider_name: str | None = None, context_providers: Sequence[ContextProvider] | None = None, middleware: Sequence[MiddlewareTypes] | None = None, @@ -500,19 +501,31 @@ def create_harness_agent( content and returns ``True`` to approve it. Rules are evaluated after standing rules (derived from prior user approvals) but before prompting the user. Only used when ``disable_tool_auto_approval`` is False. - loop_should_continue: Optional predicate that enables the looping middleware. When provided, the - agent is re-run in a loop (via :class:`~agent_framework.AgentLoopMiddleware`, wired as - the outermost middleware so each iteration is a full agent run including tool approval) - for as long as the predicate returns ``True``, up to ``loop_max_iterations``. If an - iteration returns a pending tool-approval request, the loop stops and returns it so the - caller can approve before continuing. When None (default), no loop is added. - loop_next_message: Optional callable controlling the input for the next loop iteration. - Only takes effect when ``loop_should_continue`` is set (otherwise no loop is added and - this is ignored). - loop_max_iterations: Safety cap on the number of loop iterations. ``None`` means unbounded; - a positive integer caps the loop (defaults to the loop middleware's default cap). Only - takes effect when ``loop_should_continue`` is set (otherwise no loop is added and this - is ignored). + loop_should_continue: Optional predicate that enables the agent-level **looping** + middleware. This loop **re-runs the entire agent** (via + :class:`~agent_framework.AgentLoopMiddleware`, wired as the outermost middleware so + each iteration is a full agent run including tool approval) for as long as the + predicate returns ``True``, up to ``loop_max_iterations``. It is distinct from the + per-request function/tool-calling loop capped by ``function_loop_max_iterations``. If + an iteration returns a pending tool-approval request, the loop stops and returns it so + the caller can approve before continuing. When None (default), no loop is added. + loop_next_message: Optional callable controlling the input for the next **agent re-run** + iteration of the ``AgentLoopMiddleware`` loop. Only takes effect when + ``loop_should_continue`` is set (otherwise no loop is added and this is ignored). + loop_max_iterations: Safety cap on the number of **agent re-run** iterations performed by + the ``AgentLoopMiddleware`` loop (this is *not* the function/tool-calling loop; see + ``function_loop_max_iterations`` for that). ``None`` means unbounded; a positive + integer caps the loop (defaults to the loop middleware's default cap). Only takes + effect when ``loop_should_continue`` is set (otherwise no loop is added and this is + ignored). + function_loop_max_iterations: Maximum number of **function/tool-calling loop iterations + per request** (LLM round-trips) the chat client performs while resolving tool calls + for a single model request. When set, it is applied to the client's + ``function_invocation_configuration["max_iterations"]``. This is unrelated to the + agent-level ``loop_*`` parameters, which re-run the whole agent. Must be at least 1 + when provided. When None (default), the client's existing configuration is left + unchanged (the built-in default is 40). A warning is logged if the supplied client + does not expose ``function_invocation_configuration``. otel_provider_name: Custom OpenTelemetry provider/source name for telemetry. context_providers: Additional context providers to include after the built-in ones. middleware: Additional middleware to include. @@ -524,7 +537,8 @@ def create_harness_agent( Raises: ValueError: If max_context_window_tokens is provided and <= 0, or max_output_tokens is provided and <= 0, or max_output_tokens >= - max_context_window_tokens when both are provided. + max_context_window_tokens when both are provided, or + function_loop_max_iterations is provided and < 1. """ if max_context_window_tokens is not None and max_context_window_tokens <= 0: raise ValueError("max_context_window_tokens must be positive.") @@ -536,6 +550,22 @@ def create_harness_agent( and max_output_tokens >= max_context_window_tokens ): raise ValueError("max_output_tokens must be less than max_context_window_tokens.") + if function_loop_max_iterations is not None and function_loop_max_iterations < 1: + raise ValueError("function_loop_max_iterations must be at least 1.") + + # Apply the per-request function/tool-calling loop cap to the chat client. This is the + # Python counterpart of the .NET harness's MaximumIterationsPerRequest option. When None, + # the client's existing function_invocation_configuration is left untouched. + if function_loop_max_iterations is not None: + function_invocation_configuration = getattr(client, "function_invocation_configuration", None) + if function_invocation_configuration is None: + logger.warning( + "function_loop_max_iterations was set but client %r does not expose " + "function_invocation_configuration; the setting will be ignored.", + type(client).__name__, + ) + else: + function_invocation_configuration["max_iterations"] = function_loop_max_iterations # Warn when opting into harness features that are still experimental. create_harness_agent # itself is released, but background agents, file access, and looping remain experimental, diff --git a/python/packages/core/agent_framework/_harness/_agent.pyi b/python/packages/core/agent_framework/_harness/_agent.pyi index 26a57288a74..82d0860f6c3 100644 --- a/python/packages/core/agent_framework/_harness/_agent.pyi +++ b/python/packages/core/agent_framework/_harness/_agent.pyi @@ -85,6 +85,7 @@ def create_harness_agent( loop_should_continue: ShouldContinueCallable | None = None, loop_next_message: NextMessageCallable | None = None, loop_max_iterations: int | None = DEFAULT_MAX_ITERATIONS, + function_loop_max_iterations: int | None = None, otel_provider_name: str | None = None, context_providers: Sequence[ContextProvider] | None = None, middleware: Sequence[MiddlewareTypes] | None = None, diff --git a/python/packages/core/tests/core/test_function_invocation_logic.py b/python/packages/core/tests/core/test_function_invocation_logic.py index ae01d82a226..b950aab0f06 100644 --- a/python/packages/core/tests/core/test_function_invocation_logic.py +++ b/python/packages/core/tests/core/test_function_invocation_logic.py @@ -2374,6 +2374,7 @@ def test_replace_approval_contents_with_results_allows_reused_call_id_after_comp ("call_reused", "second output"), ] + def test_replace_approval_contents_with_results_uses_result_call_ids_for_placeholders() -> None: from agent_framework._tools import _collect_approval_responses, _replace_approval_contents_with_results diff --git a/python/packages/core/tests/core/test_harness_agent.py b/python/packages/core/tests/core/test_harness_agent.py index fb8cab0506e..5a398769545 100644 --- a/python/packages/core/tests/core/test_harness_agent.py +++ b/python/packages/core/tests/core/test_harness_agent.py @@ -40,6 +40,8 @@ from agent_framework._sessions import ContextProvider, PerServiceCallHistoryPersistingMiddleware from agent_framework._tools import FunctionInvocationLayer +from .conftest import MockBaseChatClient + class _FakeChatClient(BaseChatClient[ChatOptions[Any]]): """Minimal chat client stub for testing assembly.""" @@ -1430,3 +1432,63 @@ def test_create_harness_agent_shell_dedup_does_not_suppress_harness_warning() -> disable_file_memory=True, background_agents=[bg_agent], # type: ignore[list-item] # pyrefly: ignore[bad-argument-type] # ty: ignore[invalid-argument-type] ) + + +# --- function_loop_max_iterations Tests --- + + +def test_create_harness_agent_sets_function_loop_max_iterations() -> None: + """function_loop_max_iterations should update the client's function-invocation loop cap.""" + from agent_framework._tools import DEFAULT_MAX_ITERATIONS + + client = MockBaseChatClient() + assert client.function_invocation_configuration["max_iterations"] == DEFAULT_MAX_ITERATIONS + + create_harness_agent( + client=client, + disable_web_search=True, + disable_file_memory=True, + function_loop_max_iterations=7, + ) + assert client.function_invocation_configuration["max_iterations"] == 7 + + +def test_create_harness_agent_function_loop_max_iterations_default_leaves_config() -> None: + """When omitted, the client's function-invocation configuration is left unchanged.""" + from agent_framework._tools import DEFAULT_MAX_ITERATIONS + + client = MockBaseChatClient() + create_harness_agent( + client=client, + disable_web_search=True, + disable_file_memory=True, + ) + assert client.function_invocation_configuration["max_iterations"] == DEFAULT_MAX_ITERATIONS + + +def test_create_harness_agent_function_loop_max_iterations_rejects_below_one() -> None: + """A value below 1 should raise ValueError.""" + with pytest.raises(ValueError, match="function_loop_max_iterations must be at least 1"): + create_harness_agent( + client=MockBaseChatClient(), + disable_web_search=True, + disable_file_memory=True, + function_loop_max_iterations=0, + ) + + +def test_create_harness_agent_function_loop_max_iterations_warns_when_unsupported( + caplog: pytest.LogCaptureFixture, +) -> None: + """A client without function_invocation_configuration should warn and be ignored.""" + client = _FakeChatClient() + assert not hasattr(client, "function_invocation_configuration") + + with caplog.at_level("WARNING"): + create_harness_agent( + client=client, + disable_web_search=True, + disable_file_memory=True, + function_loop_max_iterations=5, + ) + assert any("function_invocation_configuration" in record.message for record in caplog.records)