diff --git a/src/google/adk/evaluation/llm_as_judge_utils.py b/src/google/adk/evaluation/llm_as_judge_utils.py index 22277be9df1..42a3f02cac7 100644 --- a/src/google/adk/evaluation/llm_as_judge_utils.py +++ b/src/google/adk/evaluation/llm_as_judge_utils.py @@ -100,6 +100,38 @@ def get_text_from_content( return None +def get_final_response_author(invocation: Invocation) -> Optional[str]: + """Returns the author of the event that carried the final response. + + `EvaluationGenerator.convert_events_to_eval_invocations` keeps that event + among the invocation events and moves its content to `final_response`, so it + is the last agent event left without content. An event after it that has + content, such as a tool call another agent made later in the turn, is not + it. + + An invocation that was put together some other way may have no such event, + and the last agent event is used instead. + + Args: + invocation: The invocation whose final response is being attributed. + + Returns: + The author, or None if the invocation has no agent events. + """ + intermediate_data = invocation.intermediate_data + if not isinstance(intermediate_data, InvocationEvents): + return None + agent_events = [ + event + for event in intermediate_data.invocation_events + if not (event.author and event.author.lower() == "user") + ] + for event in reversed(agent_events): + if not (event.content and event.content.parts): + return event.author + return agent_events[-1].author if agent_events else None + + def get_eval_status(score: Optional[float], threshold: float) -> EvalStatus: if score is None: return EvalStatus.NOT_EVALUATED diff --git a/src/google/adk/evaluation/rubric_based_multi_turn_trajectory_evaluator.py b/src/google/adk/evaluation/rubric_based_multi_turn_trajectory_evaluator.py index af9d4476bf9..50c16126516 100644 --- a/src/google/adk/evaluation/rubric_based_multi_turn_trajectory_evaluator.py +++ b/src/google/adk/evaluation/rubric_based_multi_turn_trajectory_evaluator.py @@ -31,6 +31,7 @@ from .evaluator import _validate_invocation_lengths from .evaluator import EvaluationResult from .evaluator import PerInvocationResult +from .llm_as_judge_utils import get_final_response_author from .rubric_based_evaluator import RubricBasedEvaluator logger = logging.getLogger("google_adk." + __name__) @@ -234,13 +235,7 @@ def _assemble_dialogue_history( # FINAL AGENT TURN if invocation.final_response and invocation.final_response.parts: - intermediate_data = invocation.intermediate_data - agent_name = "agent" - if ( - isinstance(intermediate_data, InvocationEvents) - and intermediate_data.invocation_events - ): - agent_name = intermediate_data.invocation_events[0].author + agent_name = get_final_response_author(invocation) or "agent" role = f"AGENT ({agent_name})" text = extract_text_from_content( invocation.final_response, splitter=" " diff --git a/tests/unittests/evaluation/test_llm_as_judge_utils.py b/tests/unittests/evaluation/test_llm_as_judge_utils.py index 9663cedbe0b..2bdaac59560 100644 --- a/tests/unittests/evaluation/test_llm_as_judge_utils.py +++ b/tests/unittests/evaluation/test_llm_as_judge_utils.py @@ -26,6 +26,7 @@ from google.adk.evaluation.evaluator import EvalStatus from google.adk.evaluation.llm_as_judge_utils import get_average_rubric_score from google.adk.evaluation.llm_as_judge_utils import get_eval_status +from google.adk.evaluation.llm_as_judge_utils import get_final_response_author from google.adk.evaluation.llm_as_judge_utils import get_grounding_metadata_as_json_str from google.adk.evaluation.llm_as_judge_utils import get_text_from_content from google.adk.evaluation.llm_as_judge_utils import get_tool_calls_and_responses_as_json_str @@ -222,6 +223,111 @@ def test_get_text_from_content_skips_thought_parts_in_intermediate_responses(): assert text == "Let me check.\nDone." +def _make_invocation_with_reply( + intermediate_data: IntermediateData | InvocationEvents | None, +) -> Invocation: + """Returns an invocation that has a final response.""" + return Invocation( + user_content=genai_types.Content( + parts=[genai_types.Part(text="How much do I owe?")] + ), + final_response=genai_types.Content( + parts=[genai_types.Part(text="You owe $42.")] + ), + intermediate_data=intermediate_data, + ) + + +def _make_tool_call_event(author: str) -> InvocationEvent: + """Returns an event in which `author` calls a tool.""" + return InvocationEvent( + author=author, + content=genai_types.Content( + parts=[ + genai_types.Part( + function_call=genai_types.FunctionCall(name="a_tool", args={}) + ) + ] + ), + ) + + +def test_get_final_response_author_after_transfer(): + """The reply belongs to the sub-agent, not the agent that handed off.""" + invocation = _make_invocation_with_reply( + InvocationEvents( + invocation_events=[ + _make_tool_call_event("router"), + # `convert_events_to_eval_invocations` keeps the final + # response's event, with its content moved to `final_response`. + InvocationEvent(author="billing_agent"), + ] + ) + ) + + assert get_final_response_author(invocation) == "billing_agent" + + +@pytest.mark.parametrize( + "later_event", + [ + InvocationEvent( + author="billing_agent", + usage_metadata=genai_types.GenerateContentResponseUsageMetadata( + total_token_count=12 + ), + ), + _make_tool_call_event("audit_agent"), + ], + ids=["usage_report", "tool_call_of_another_agent"], +) +def test_get_final_response_author_with_later_event(later_event): + """An event after the reply's own event does not take the reply over.""" + invocation = _make_invocation_with_reply( + InvocationEvents( + invocation_events=[ + _make_tool_call_event("router"), + InvocationEvent(author="billing_agent"), + later_event, + ] + ) + ) + + assert get_final_response_author(invocation) == "billing_agent" + + +def test_get_final_response_author_without_event_for_the_reply(): + """A hand-built invocation falls back to its last agent event.""" + invocation = _make_invocation_with_reply( + InvocationEvents( + invocation_events=[ + _make_tool_call_event("router"), + _make_tool_call_event("billing_agent"), + InvocationEvent( + author="user", + content=genai_types.Content( + parts=[genai_types.Part(text="Thanks.")] + ), + ), + ] + ) + ) + + assert get_final_response_author(invocation) == "billing_agent" + + +@pytest.mark.parametrize( + "intermediate_data", + [None, IntermediateData(), InvocationEvents(invocation_events=[])], + ids=["no_intermediate_data", "legacy_intermediate_data", "no_events"], +) +def test_get_final_response_author_without_agent_events(intermediate_data): + """There is no author to report when the invocation has no agent events.""" + invocation = _make_invocation_with_reply(intermediate_data) + + assert get_final_response_author(invocation) is None + + def test_get_eval_status_with_none_score(): """Tests get_eval_status returns NOT_EVALUATED for a None score.""" assert get_eval_status(score=None, threshold=0.5) == EvalStatus.NOT_EVALUATED diff --git a/tests/unittests/evaluation/test_rubric_based_multi_turn_trajectory_evaluator.py b/tests/unittests/evaluation/test_rubric_based_multi_turn_trajectory_evaluator.py index 9a9ab32cfd2..e0ee93842d9 100644 --- a/tests/unittests/evaluation/test_rubric_based_multi_turn_trajectory_evaluator.py +++ b/tests/unittests/evaluation/test_rubric_based_multi_turn_trajectory_evaluator.py @@ -378,6 +378,52 @@ async def test_invocation_without_user_content(self, evaluator): "AGENT (agent) TURN 1: Agent response." in evaluator._formatted_dialogue ) + def test_final_reply_is_attributed_to_the_agent_that_wrote_it( + self, evaluator + ): + """After a transfer, the reply is labelled with the sub-agent's name.""" + transfer_call = genai_types.Part( + function_call=genai_types.FunctionCall( + name="transfer_to_agent", args={"agent_name": "billing_agent"} + ) + ) + transfer_response = genai_types.Part( + function_response=genai_types.FunctionResponse( + name="transfer_to_agent", response={"result": None} + ) + ) + invocation = _make_invocation( + user_text="How much do I owe?", + agent_text="You owe $42.", + intermediate_data=InvocationEvents( + invocation_events=[ + InvocationEvent( + author="router", + content=genai_types.Content(parts=[transfer_call]), + ), + InvocationEvent( + author="router", + content=genai_types.Content(parts=[transfer_response]), + ), + # `convert_events_to_eval_invocations` keeps the final + # response's event, with its content moved to + # `final_response`. + InvocationEvent(author="billing_agent"), + ] + ), + ) + + evaluator._assemble_dialogue_history([invocation]) + + assert evaluator._formatted_dialogue == ( + "USER TURN 1: How much do I owe?\n" + "AGENT (router) TURN 1 (tool call):" + ' transfer_to_agent({"agent_name": "billing_agent"})\n' + "AGENT (router) TURN 1 (tool output):" + ' transfer_to_agent -> {"result": null}\n' + "AGENT (billing_agent) TURN 1: You owe $42." + ) + def test_thought_parts_are_left_out_of_dialogue(self, evaluator): """Agent thought parts are left out of the judge's dialogue.""" invocation = Invocation(