From d2795f5ce096c47b76ae92e5630da9a161e3e926 Mon Sep 17 00:00:00 2001 From: YusefSyed <211442445+YusefSyed@users.noreply.github.com> Date: Wed, 11 Nov 2026 04:20:58 -0500 Subject: [PATCH] fix(eval): label a sub-agent's reply with its own name for the judge The multi-turn trajectory metric shows the judge each turn's final reply under the name of the first agent that acted in the turn. After a transfer that is the agent that handed off, not the one that answered, so a billing_agent reply was shown as "AGENT (router)" and a rubric about which agent answers was judged on the wrong name. Label the reply with the author of the event that carried it. That event stays in the invocation events with its content moved to final_response, so it is the last agent event left without content; when there is none, the last agent event is used. --- .../adk/evaluation/llm_as_judge_utils.py | 32 ++++++ ...c_based_multi_turn_trajectory_evaluator.py | 9 +- .../evaluation/test_llm_as_judge_utils.py | 106 ++++++++++++++++++ ...c_based_multi_turn_trajectory_evaluator.py | 46 ++++++++ 4 files changed, 186 insertions(+), 7 deletions(-) diff --git a/src/google/adk/evaluation/llm_as_judge_utils.py b/src/google/adk/evaluation/llm_as_judge_utils.py index 22277be9df1..42a3f02cac7 100644 --- a/src/google/adk/evaluation/llm_as_judge_utils.py +++ b/src/google/adk/evaluation/llm_as_judge_utils.py @@ -100,6 +100,38 @@ def get_text_from_content( return None +def get_final_response_author(invocation: Invocation) -> Optional[str]: + """Returns the author of the event that carried the final response. + + `EvaluationGenerator.convert_events_to_eval_invocations` keeps that event + among the invocation events and moves its content to `final_response`, so it + is the last agent event left without content. An event after it that has + content, such as a tool call another agent made later in the turn, is not + it. + + An invocation that was put together some other way may have no such event, + and the last agent event is used instead. + + Args: + invocation: The invocation whose final response is being attributed. + + Returns: + The author, or None if the invocation has no agent events. + """ + intermediate_data = invocation.intermediate_data + if not isinstance(intermediate_data, InvocationEvents): + return None + agent_events = [ + event + for event in intermediate_data.invocation_events + if not (event.author and event.author.lower() == "user") + ] + for event in reversed(agent_events): + if not (event.content and event.content.parts): + return event.author + return agent_events[-1].author if agent_events else None + + def get_eval_status(score: Optional[float], threshold: float) -> EvalStatus: if score is None: return EvalStatus.NOT_EVALUATED diff --git a/src/google/adk/evaluation/rubric_based_multi_turn_trajectory_evaluator.py b/src/google/adk/evaluation/rubric_based_multi_turn_trajectory_evaluator.py index af9d4476bf9..50c16126516 100644 --- a/src/google/adk/evaluation/rubric_based_multi_turn_trajectory_evaluator.py +++ b/src/google/adk/evaluation/rubric_based_multi_turn_trajectory_evaluator.py @@ -31,6 +31,7 @@ from .evaluator import _validate_invocation_lengths from .evaluator import EvaluationResult from .evaluator import PerInvocationResult +from .llm_as_judge_utils import get_final_response_author from .rubric_based_evaluator import RubricBasedEvaluator logger = logging.getLogger("google_adk." + __name__) @@ -234,13 +235,7 @@ def _assemble_dialogue_history( # FINAL AGENT TURN if invocation.final_response and invocation.final_response.parts: - intermediate_data = invocation.intermediate_data - agent_name = "agent" - if ( - isinstance(intermediate_data, InvocationEvents) - and intermediate_data.invocation_events - ): - agent_name = intermediate_data.invocation_events[0].author + agent_name = get_final_response_author(invocation) or "agent" role = f"AGENT ({agent_name})" text = extract_text_from_content( invocation.final_response, splitter=" " diff --git a/tests/unittests/evaluation/test_llm_as_judge_utils.py b/tests/unittests/evaluation/test_llm_as_judge_utils.py index 9663cedbe0b..2bdaac59560 100644 --- a/tests/unittests/evaluation/test_llm_as_judge_utils.py +++ b/tests/unittests/evaluation/test_llm_as_judge_utils.py @@ -26,6 +26,7 @@ from google.adk.evaluation.evaluator import EvalStatus from google.adk.evaluation.llm_as_judge_utils import get_average_rubric_score from google.adk.evaluation.llm_as_judge_utils import get_eval_status +from google.adk.evaluation.llm_as_judge_utils import get_final_response_author from google.adk.evaluation.llm_as_judge_utils import get_grounding_metadata_as_json_str from google.adk.evaluation.llm_as_judge_utils import get_text_from_content from google.adk.evaluation.llm_as_judge_utils import get_tool_calls_and_responses_as_json_str @@ -222,6 +223,111 @@ def test_get_text_from_content_skips_thought_parts_in_intermediate_responses(): assert text == "Let me check.\nDone." +def _make_invocation_with_reply( + intermediate_data: IntermediateData | InvocationEvents | None, +) -> Invocation: + """Returns an invocation that has a final response.""" + return Invocation( + user_content=genai_types.Content( + parts=[genai_types.Part(text="How much do I owe?")] + ), + final_response=genai_types.Content( + parts=[genai_types.Part(text="You owe $42.")] + ), + intermediate_data=intermediate_data, + ) + + +def _make_tool_call_event(author: str) -> InvocationEvent: + """Returns an event in which `author` calls a tool.""" + return InvocationEvent( + author=author, + content=genai_types.Content( + parts=[ + genai_types.Part( + function_call=genai_types.FunctionCall(name="a_tool", args={}) + ) + ] + ), + ) + + +def test_get_final_response_author_after_transfer(): + """The reply belongs to the sub-agent, not the agent that handed off.""" + invocation = _make_invocation_with_reply( + InvocationEvents( + invocation_events=[ + _make_tool_call_event("router"), + # `convert_events_to_eval_invocations` keeps the final + # response's event, with its content moved to `final_response`. + InvocationEvent(author="billing_agent"), + ] + ) + ) + + assert get_final_response_author(invocation) == "billing_agent" + + +@pytest.mark.parametrize( + "later_event", + [ + InvocationEvent( + author="billing_agent", + usage_metadata=genai_types.GenerateContentResponseUsageMetadata( + total_token_count=12 + ), + ), + _make_tool_call_event("audit_agent"), + ], + ids=["usage_report", "tool_call_of_another_agent"], +) +def test_get_final_response_author_with_later_event(later_event): + """An event after the reply's own event does not take the reply over.""" + invocation = _make_invocation_with_reply( + InvocationEvents( + invocation_events=[ + _make_tool_call_event("router"), + InvocationEvent(author="billing_agent"), + later_event, + ] + ) + ) + + assert get_final_response_author(invocation) == "billing_agent" + + +def test_get_final_response_author_without_event_for_the_reply(): + """A hand-built invocation falls back to its last agent event.""" + invocation = _make_invocation_with_reply( + InvocationEvents( + invocation_events=[ + _make_tool_call_event("router"), + _make_tool_call_event("billing_agent"), + InvocationEvent( + author="user", + content=genai_types.Content( + parts=[genai_types.Part(text="Thanks.")] + ), + ), + ] + ) + ) + + assert get_final_response_author(invocation) == "billing_agent" + + +@pytest.mark.parametrize( + "intermediate_data", + [None, IntermediateData(), InvocationEvents(invocation_events=[])], + ids=["no_intermediate_data", "legacy_intermediate_data", "no_events"], +) +def test_get_final_response_author_without_agent_events(intermediate_data): + """There is no author to report when the invocation has no agent events.""" + invocation = _make_invocation_with_reply(intermediate_data) + + assert get_final_response_author(invocation) is None + + def test_get_eval_status_with_none_score(): """Tests get_eval_status returns NOT_EVALUATED for a None score.""" assert get_eval_status(score=None, threshold=0.5) == EvalStatus.NOT_EVALUATED diff --git a/tests/unittests/evaluation/test_rubric_based_multi_turn_trajectory_evaluator.py b/tests/unittests/evaluation/test_rubric_based_multi_turn_trajectory_evaluator.py index 9a9ab32cfd2..e0ee93842d9 100644 --- a/tests/unittests/evaluation/test_rubric_based_multi_turn_trajectory_evaluator.py +++ b/tests/unittests/evaluation/test_rubric_based_multi_turn_trajectory_evaluator.py @@ -378,6 +378,52 @@ async def test_invocation_without_user_content(self, evaluator): "AGENT (agent) TURN 1: Agent response." in evaluator._formatted_dialogue ) + def test_final_reply_is_attributed_to_the_agent_that_wrote_it( + self, evaluator + ): + """After a transfer, the reply is labelled with the sub-agent's name.""" + transfer_call = genai_types.Part( + function_call=genai_types.FunctionCall( + name="transfer_to_agent", args={"agent_name": "billing_agent"} + ) + ) + transfer_response = genai_types.Part( + function_response=genai_types.FunctionResponse( + name="transfer_to_agent", response={"result": None} + ) + ) + invocation = _make_invocation( + user_text="How much do I owe?", + agent_text="You owe $42.", + intermediate_data=InvocationEvents( + invocation_events=[ + InvocationEvent( + author="router", + content=genai_types.Content(parts=[transfer_call]), + ), + InvocationEvent( + author="router", + content=genai_types.Content(parts=[transfer_response]), + ), + # `convert_events_to_eval_invocations` keeps the final + # response's event, with its content moved to + # `final_response`. + InvocationEvent(author="billing_agent"), + ] + ), + ) + + evaluator._assemble_dialogue_history([invocation]) + + assert evaluator._formatted_dialogue == ( + "USER TURN 1: How much do I owe?\n" + "AGENT (router) TURN 1 (tool call):" + ' transfer_to_agent({"agent_name": "billing_agent"})\n' + "AGENT (router) TURN 1 (tool output):" + ' transfer_to_agent -> {"result": null}\n' + "AGENT (billing_agent) TURN 1: You owe $42." + ) + def test_thought_parts_are_left_out_of_dialogue(self, evaluator): """Agent thought parts are left out of the judge's dialogue.""" invocation = Invocation(