tool_trajectory_avg_score fails for confirmation-gated tools because the framework adk_request_confirmation call is counted as an agent tool call #8025

"""Offline repro: a confirmation-gated tool breaks tool_trajectory_avg_score.

No API key needed: a scripted BaseLlm plays the model.

The same agent, the same scripted model and the same eval case are run twice:
- require_confirmation=False -> trajectory [get_order, cancel_order] -> PASS
- require_confirmation=True  -> ADK adds its own `adk_request_confirmation`
  function call to the actual trajectory -> FAIL (EXACT, ignore_args=True)

Run:
  pytest -q test_confirmation_trajectory.py
"""

import os
from typing import AsyncGenerator

from google.adk.agents import LlmAgent
from google.adk.evaluation.agent_evaluator import AgentEvaluator
from google.adk.models.base_llm import BaseLlm
from google.adk.models.llm_request import LlmRequest
from google.adk.models.llm_response import LlmResponse
from google.adk.tools.function_tool import FunctionTool
from google.genai import types
import pytest

HERE = os.path.dirname(os.path.abspath(__file__))


class ScriptedModel(BaseLlm):
  """Returns pre-written responses in order."""

  model: str = "scripted"
  script: list[types.Content]
  calls: int = 0

  async def generate_content_async(
      self, llm_request: LlmRequest, stream: bool = False
  ) -> AsyncGenerator[LlmResponse, None]:
    content = self.script[min(self.calls, len(self.script) - 1)]
    self.calls += 1
    yield LlmResponse(content=content)


def _call(name: str, args: dict) -> types.Content:
  return types.Content(
      role="model",
      parts=[types.Part(function_call=types.FunctionCall(name=name, args=args))],
  )


def get_order(order_id: str) -> dict:
  """Looks up an order."""
  return {"id": order_id, "status": "processing"}


def cancel_order(order_id: str) -> dict:
  """Cancels an order."""
  return {"id": order_id, "status": "cancelled"}


def _build_agent(require_confirmation: bool) -> LlmAgent:
  model = ScriptedModel(
      script=[
          _call("get_order", {"order_id": "A-100"}),
          _call("cancel_order", {"order_id": "A-100"}),
          types.Content(role="model", parts=[types.Part(text="Order A-100 is cancelled.")]),
      ]
  )
  return LlmAgent(
      name="orders_agent",
      model=model,
      instruction="Help with orders.",
      tools=[
          get_order,
          FunctionTool(cancel_order, require_confirmation=require_confirmation),
      ],
  )


@pytest.mark.asyncio
@pytest.mark.parametrize("require_confirmation", [False, True])
async def test_trajectory_with_and_without_confirmation(require_confirmation):
  # Expected trajectory in data/cancel.test.json: get_order, cancel_order.
  # test_config.json: tool_trajectory_avg_score EXACT, ignore_args=True.
  await AgentEvaluator.evaluate(
      agent_module=_build_agent(require_confirmation),
      eval_dataset_file_path_or_dir=os.path.join(HERE, "data", "cancel.test.json"),
      num_runs=1,
  )

  "eval_set_id": "confirmation_trajectory_repro",
  "eval_cases": [
    {
      "eval_id": "cancel_order",
      "conversation": [
        {
          "invocation_id": "inv-1",
          "user_content": {"role": "user", "parts": [{"text": "Cancel order A-100."}]},
          "final_response": {"role": "model", "parts": [{"text": "Order A-100 is cancelled."}]},
          "intermediate_data": {
            "tool_uses": [
              {"name": "get_order", "args": {"order_id": "A-100"}},
              {"name": "cancel_order", "args": {"order_id": "A-100"}}
            ],
            "intermediate_responses": []
          }
        }
      ],
      "session_input": {"app_name": "orders_agent", "user_id": "user", "state": {}}
    }
  ]
}

{
  "criteria": {
    "tool_trajectory_avg_score": {"threshold": 1.0, "match_type": "EXACT", "ignore_args": true}
  }
}