diff --git a/src/google/adk/evaluation/final_response_match_v1.py b/src/google/adk/evaluation/final_response_match_v1.py index 63559b26a9c..49aadb89265 100644 --- a/src/google/adk/evaluation/final_response_match_v1.py +++ b/src/google/adk/evaluation/final_response_match_v1.py @@ -91,7 +91,9 @@ def evaluate_invocations( def _get_text_from_content(content: Optional[genai_types.Content]) -> str: if content and content.parts: - return "\n".join([part.text for part in content.parts if part.text]) + return "\n".join( + [part.text for part in content.parts if part.text and not part.thought] + ) return "" diff --git a/tests/unittests/evaluation/test_final_response_match_v1.py b/tests/unittests/evaluation/test_final_response_match_v1.py index cd7128f2aa3..ce50f371bee 100644 --- a/tests/unittests/evaluation/test_final_response_match_v1.py +++ b/tests/unittests/evaluation/test_final_response_match_v1.py @@ -19,7 +19,6 @@ from google.adk.evaluation.eval_case import Invocation from google.adk.evaluation.eval_metrics import BaseCriterion from google.adk.evaluation.eval_metrics import EvalMetric -from google.adk.evaluation.eval_metrics import PrebuiltMetrics from google.adk.evaluation.evaluator import EvalStatus from google.adk.evaluation.final_response_match_v1 import _calculate_rouge_1_scores from google.adk.evaluation.final_response_match_v1 import _is_cjk @@ -59,6 +58,62 @@ def _create_test_invocations( ) +@pytest.mark.parametrize( + "actual_parts, expected_parts, score", + [ + ( + [ + genai_types.Part( + text="Consider the capital of France.", thought=True + ), + genai_types.Part(text="Paris"), + ], + [genai_types.Part(text="Paris")], + 1.0, + ), + ( + [genai_types.Part(text="Paris", thought=False)], + [ + genai_types.Part( + text="Consider the capital of France.", thought=True + ), + genai_types.Part(text="Paris"), + ], + 1.0, + ), + ( + [genai_types.Part(text="Paris", thought=True)], + [genai_types.Part(text="Paris")], + 0.0, + ), + ( + [ + genai_types.Part(text="Paris", thought=True), + genai_types.Part(text="London"), + ], + [genai_types.Part(text="Paris")], + 0.0, + ), + ], +) +def test_response_match_scores_visible_text_only( + actual_parts, expected_parts, score +): + """Thought summaries neither dilute correct answers nor credit wrong ones.""" + actual, expected = _create_test_invocations("", "") + actual.final_response.parts = actual_parts + expected.final_response.parts = expected_parts + evaluator = _create_test_rouge_evaluator(threshold=0.5) + + result = evaluator.evaluate_invocations([actual], [expected]) + + assert result.overall_score == pytest.approx(score) + assert result.per_invocation_results[0].score == pytest.approx(score) + assert result.overall_eval_status == ( + EvalStatus.PASSED if score == 1.0 else EvalStatus.FAILED + ) + + def test_calculate_rouge_1_scores_empty_candidate_and_reference(): candidate = "" reference = ""