From 6256828b6f3d5cd3375613bab216220790771eb0 Mon Sep 17 00:00:00 2001 From: Jiang Wu <245355730+JiangLLM@users.noreply.github.com> Date: Sat, 19 Sep 2026 13:53:17 -0700 Subject: [PATCH 1/4] fix(eval): classify PrefEval judge answers like the reference benchmark classify_error_type compared judge answers with exact 'Yes'/'No' strings and sent every unmatched case to the 'Personalized Response' bucket, including empty answers from failed judge calls and responses that were both violating and unhelpful. Move the classifier to evaluation/scripts/utils/pref_classify.py, mirror the amazon-science/PrefEval semantics, report unrecognized answers as 'Judge Failure', and add unit tests. --- evaluation/scripts/PrefEval/pref_eval.py | 23 ++--- evaluation/scripts/utils/pref_classify.py | 71 +++++++++++++++ tests/evaluation/test_pref_classify.py | 103 ++++++++++++++++++++++ 3 files changed, 179 insertions(+), 18 deletions(-) create mode 100644 evaluation/scripts/utils/pref_classify.py create mode 100644 tests/evaluation/test_pref_classify.py diff --git a/evaluation/scripts/PrefEval/pref_eval.py b/evaluation/scripts/PrefEval/pref_eval.py index 9da3c9438..0f782fae7 100644 --- a/evaluation/scripts/PrefEval/pref_eval.py +++ b/evaluation/scripts/PrefEval/pref_eval.py @@ -3,6 +3,7 @@ import json import os import re +import sys from collections import Counter from typing import Any @@ -14,6 +15,10 @@ from tqdm.asyncio import tqdm +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from utils.pref_classify import classify_error_type + + load_dotenv() API_KEY = os.getenv("OPENAI_API_KEY") @@ -178,24 +183,6 @@ async def evaluate_helpful_response_async( } -def classify_error_type(evaluation_results: dict[str, Any]) -> str: - violate = evaluation_results["violate_preference"]["answer"] - acknowledge = evaluation_results["acknowledge_preference"]["answer"] - hallucinate = evaluation_results["hallucinate_preference"]["answer"] - helpful = evaluation_results["helpful_response"]["answer"] - - if violate == "Yes" and acknowledge == "No" and helpful == "Yes": - return "Preference-Unaware Violation" - elif violate == "Yes" and acknowledge == "Yes" and hallucinate == "Yes" and helpful == "Yes": - return "Preference Hallucination Violation" - elif violate == "Yes" and acknowledge == "Yes" and hallucinate == "No" and helpful == "Yes": - return "Inconsistency Violation" - elif violate == "No" and helpful == "No": - return "Unhelpful Response" - else: - return "Personalized Response" - - async def process_line(line: str, client: OpenAI, semaphore: asyncio.Semaphore) -> dict[str, Any]: async with semaphore: data = json.loads(line.strip()) diff --git a/evaluation/scripts/utils/pref_classify.py b/evaluation/scripts/utils/pref_classify.py new file mode 100644 index 000000000..7e587de70 --- /dev/null +++ b/evaluation/scripts/utils/pref_classify.py @@ -0,0 +1,71 @@ +"""Error-type classification for PrefEval judge answers. + +Kept free of third-party imports so it can be unit tested without the +evaluation dependency group. +""" + +from typing import Any + + +JUDGE_FAILURE = "Judge Failure" +PERSONALIZED_RESPONSE = "Personalized Response" +UNHELPFUL_RESPONSE = "Unhelpful Response" +PREFERENCE_UNAWARE_VIOLATION = "Preference-Unaware Violation" +PREFERENCE_HALLUCINATION_VIOLATION = "Preference Hallucination Violation" +INCONSISTENCY_VIOLATION = "Inconsistency Violation" + +_JUDGE_KEYS = ( + "violate_preference", + "acknowledge_preference", + "hallucinate_preference", + "helpful_response", +) + + +def parse_yes_no(answer: str | None) -> bool | None: + """Map a judge answer to True (yes) or False (no). + + Returns None when the answer is missing or is neither yes nor no, for + example the empty string that the judge call returns after an API error. + """ + if answer is None: + return None + normalized = answer.strip().lower() + if normalized.startswith("yes"): + return True + if normalized.startswith("no"): + return False + return None + + +def classify_error_type(evaluation_results: dict[str, Any]) -> str: + """Classify one PrefEval sample from the four judge answers. + + Follows the reference implementation in amazon-science/PrefEval + (generation_task/get_preference_following_accuracy_generation_task.py): + an unhelpful response is an error on its own, the three violation + buckets require a helpful response, and a hallucinated preference only + counts when the preference was acknowledged. A missing or unrecognized + judge answer is reported as a judge failure instead of being counted as a + personalized response. + """ + answers = { + key: parse_yes_no(evaluation_results.get(key, {}).get("answer")) for key in _JUDGE_KEYS + } + if any(value is None for value in answers.values()): + return JUDGE_FAILURE + + violate = answers["violate_preference"] + acknowledge = answers["acknowledge_preference"] + hallucinate = acknowledge and answers["hallucinate_preference"] + unhelpful = not answers["helpful_response"] + + if unhelpful: + return UNHELPFUL_RESPONSE + if violate and not acknowledge: + return PREFERENCE_UNAWARE_VIOLATION + if violate and hallucinate: + return PREFERENCE_HALLUCINATION_VIOLATION + if violate: + return INCONSISTENCY_VIOLATION + return PERSONALIZED_RESPONSE diff --git a/tests/evaluation/test_pref_classify.py b/tests/evaluation/test_pref_classify.py new file mode 100644 index 000000000..45b885044 --- /dev/null +++ b/tests/evaluation/test_pref_classify.py @@ -0,0 +1,103 @@ +import importlib.util +import itertools + +from pathlib import Path + +import pytest + + +MODULE_PATH = ( + Path(__file__).resolve().parents[2] / "evaluation" / "scripts" / "utils" / "pref_classify.py" +) + + +def _load_module(): + spec = importlib.util.spec_from_file_location("pref_classify", MODULE_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +pref_classify = _load_module() + + +def _judge(violate, acknowledge, hallucinate, helpful): + return { + "violate_preference": {"answer": violate}, + "acknowledge_preference": {"answer": acknowledge}, + "hallucinate_preference": {"answer": hallucinate}, + "helpful_response": {"answer": helpful}, + } + + +def _reference_bucket(violate, acknowledge, hallucinate, helpful): + """The bucket implied by amazon-science/PrefEval analyze_errors().""" + is_ack = "yes" in acknowledge.lower() + is_hal = is_ack and "yes" in hallucinate.lower() + is_vio = "yes" in violate.lower() + is_unhelpful = "no" in helpful.lower() + if is_unhelpful: + return pref_classify.UNHELPFUL_RESPONSE + if is_ack and not is_hal and is_vio: + return pref_classify.INCONSISTENCY_VIOLATION + if is_ack and is_hal and is_vio: + return pref_classify.PREFERENCE_HALLUCINATION_VIOLATION + if not is_ack and is_vio: + return pref_classify.PREFERENCE_UNAWARE_VIOLATION + return pref_classify.PERSONALIZED_RESPONSE + + +@pytest.mark.parametrize("answers", list(itertools.product(["Yes", "No"], repeat=4))) +def test_matches_reference_on_well_formed_answers(answers): + assert pref_classify.classify_error_type(_judge(*answers)) == _reference_bucket(*answers) + + +@pytest.mark.parametrize( + ("answers", "expected"), + [ + (("yes", "no", "No", "yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION), + (("Yes.", "No", "No", "Yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION), + ((" YES ", "No", "No", "Yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION), + (("Yes", "No", "No", "No"), pref_classify.UNHELPFUL_RESPONSE), + (("Yes", "Yes", "Yes", "No"), pref_classify.UNHELPFUL_RESPONSE), + (("Yes", "No", "Yes", "Yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION), + (("No", "Yes", "Yes", "Yes"), pref_classify.PERSONALIZED_RESPONSE), + ], +) +def test_case_and_precedence(answers, expected): + assert pref_classify.classify_error_type(_judge(*answers)) == expected + + +@pytest.mark.parametrize( + "answers", + [ + ("", "", "", ""), + ("Yes", "No", "No", ""), + ("Maybe", "No", "No", "Yes"), + ("Yes", "No", "No", None), + ], +) +def test_unrecognized_answers_are_judge_failures(answers): + assert pref_classify.classify_error_type(_judge(*answers)) == pref_classify.JUDGE_FAILURE + + +def test_missing_judge_key_is_a_failure(): + partial = {"violate_preference": {"answer": "Yes"}} + assert pref_classify.classify_error_type(partial) == pref_classify.JUDGE_FAILURE + + +@pytest.mark.parametrize( + ("answer", "expected"), + [ + ("Yes", True), + ("yes.", True), + ("No", False), + (" no", False), + ("", None), + (None, None), + ("Unknown", None), + ], +) +def test_parse_yes_no(answer, expected): + assert pref_classify.parse_yes_no(answer) is expected From 3e9559f6e296b24df7fee9570598a216dcb2df36 Mon Sep 17 00:00:00 2001 From: Jiang Wu <245355730+JiangLLM@users.noreply.github.com> Date: Sat, 24 Oct 2026 11:18:11 -0700 Subject: [PATCH 2/4] fix(eval): require a whole Yes or No answer word --- evaluation/scripts/utils/pref_classify.py | 16 +++++++--------- tests/evaluation/test_pref_classify.py | 8 ++++++++ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/evaluation/scripts/utils/pref_classify.py b/evaluation/scripts/utils/pref_classify.py index 7e587de70..71427aceb 100644 --- a/evaluation/scripts/utils/pref_classify.py +++ b/evaluation/scripts/utils/pref_classify.py @@ -4,6 +4,8 @@ evaluation dependency group. """ +import re + from typing import Any @@ -23,19 +25,15 @@ def parse_yes_no(answer: str | None) -> bool | None: - """Map a judge answer to True (yes) or False (no). + """Map a leading Yes or No word to True or False. - Returns None when the answer is missing or is neither yes nor no, for - example the empty string that the judge call returns after an API error. + Allow punctuation or an explanation after that word. Return None for a + missing answer or other words, such as "Not sure" and "Yesterday". """ if answer is None: return None - normalized = answer.strip().lower() - if normalized.startswith("yes"): - return True - if normalized.startswith("no"): - return False - return None + match = re.match(r"(yes|no)\b", answer.strip(), re.IGNORECASE) + return match.group(1).lower() == "yes" if match else None def classify_error_type(evaluation_results: dict[str, Any]) -> str: diff --git a/tests/evaluation/test_pref_classify.py b/tests/evaluation/test_pref_classify.py index 45b885044..847a70a95 100644 --- a/tests/evaluation/test_pref_classify.py +++ b/tests/evaluation/test_pref_classify.py @@ -75,6 +75,9 @@ def test_case_and_precedence(answers, expected): ("", "", "", ""), ("Yes", "No", "No", ""), ("Maybe", "No", "No", "Yes"), + ("Not sure", "No", "No", "Yes"), + ("None", "No", "No", "Yes"), + ("Yesterday", "No", "No", "Yes"), ("Yes", "No", "No", None), ], ) @@ -92,11 +95,16 @@ def test_missing_judge_key_is_a_failure(): [ ("Yes", True), ("yes.", True), + ("Yes, it follows the preference.", True), ("No", False), (" no", False), + ("No: it does not follow the preference.", False), ("", None), (None, None), ("Unknown", None), + ("Not sure", None), + ("None", None), + ("Yesterday", None), ], ) def test_parse_yes_no(answer, expected): From 392277ffda33fdc53926108b8efb1c444818ed0d Mon Sep 17 00:00:00 2001 From: Jiang Wu <245355730+JiangLLM@users.noreply.github.com> Date: Sat, 24 Oct 2026 12:11:52 -0700 Subject: [PATCH 3/4] fix(eval): address PrefEval import and report errors --- evaluation/scripts/PrefEval/pref_eval.py | 14 +- tests/evaluation/test_pref_eval.py | 157 +++++++++++++++++++++++ 2 files changed, 165 insertions(+), 6 deletions(-) create mode 100644 tests/evaluation/test_pref_eval.py diff --git a/evaluation/scripts/PrefEval/pref_eval.py b/evaluation/scripts/PrefEval/pref_eval.py index 0f782fae7..2d0c81d7e 100644 --- a/evaluation/scripts/PrefEval/pref_eval.py +++ b/evaluation/scripts/PrefEval/pref_eval.py @@ -15,8 +15,8 @@ from tqdm.asyncio import tqdm -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from utils.pref_classify import classify_error_type +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from utils.pref_classify import JUDGE_FAILURE, classify_error_type load_dotenv() @@ -255,6 +255,7 @@ def get_pct(key): inconsistency_pct = get_pct("Inconsistency Violation") unhelpful_pct = get_pct("Unhelpful Response") personalized_pct = get_pct("Personalized Response") + judge_failure_pct = get_pct(JUDGE_FAILURE) data = { "Model": [model_name], @@ -263,6 +264,7 @@ def get_pct(key): "Inconsistency\n意识到偏好但给出了不一致的回答": [inconsistency_pct / 100], "Unhelpful Response\n没帮助的回答": [unhelpful_pct / 100], "Personalized Response\n个性化回答": [personalized_pct / 100], + "Judge Failure\n评判失败": [judge_failure_pct / 100], "context token": [avg_context_tokens], "Time添加": [f"{avg_add_time:.2f}s"], "Time搜索": [f"{avg_search_time:.2f}s"], @@ -280,10 +282,10 @@ def get_pct(key): float_format = workbook.add_format({"num_format": "0.00"}) wrap_format = workbook.add_format({"text_wrap": True, "align": "center", "valign": "top"}) - worksheet.set_column("B:F", 18, pct_format) - worksheet.set_column("G:G", 12, float_format) - worksheet.set_column("H:I", 15) - worksheet.set_column("A:I", None, wrap_format) + worksheet.set_column("A:J", None, wrap_format) + worksheet.set_column("B:G", 18, pct_format) + worksheet.set_column("H:H", 12, float_format) + worksheet.set_column("I:J", 15) worksheet.set_row(0, 45) bold_pct_format = workbook.add_format({"num_format": "0.0%", "bold": True}) worksheet.set_column("F:F", 18, bold_pct_format) diff --git a/tests/evaluation/test_pref_eval.py b/tests/evaluation/test_pref_eval.py new file mode 100644 index 000000000..33a8cf6d0 --- /dev/null +++ b/tests/evaluation/test_pref_eval.py @@ -0,0 +1,157 @@ +import importlib.util +import os +import subprocess +import sys +import xml.etree.ElementTree as ET + +from collections import Counter +from pathlib import Path +from zipfile import ZipFile + +import pytest + + +MODULE_PATH = Path(__file__).resolve().parents[2] / "evaluation/scripts/PrefEval/pref_eval.py" +CATEGORIES = ( + "Preference-Unaware Violation", + "Preference Hallucination Violation", + "Inconsistency Violation", + "Unhelpful Response", + "Personalized Response", + "Judge Failure", +) +XML_NS = {"s": "http://schemas.openxmlformats.org/spreadsheetml/2006/main"} + + +@pytest.fixture +def evaluation_dependencies(): + for dependency in ("pandas", "xlsxwriter", "dotenv", "openai", "tqdm"): + pytest.importorskip(dependency, reason="PrefEval requires optional evaluation dependencies") + + +@pytest.fixture +def pref_eval(evaluation_dependencies, monkeypatch): + import dotenv + + monkeypatch.setattr(dotenv, "load_dotenv", lambda: False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.syspath_prepend(str(MODULE_PATH.parents[1])) + previous_utils = { + name: module + for name, module in sys.modules.items() + if name == "utils" or name.startswith("utils.") + } + for name in previous_utils: + del sys.modules[name] + try: + spec = importlib.util.spec_from_file_location("pref_eval_under_test", MODULE_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + yield module + finally: + for name in list(sys.modules): + if name == "utils" or name.startswith("utils."): + del sys.modules[name] + sys.modules.update(previous_utils) + + +@pytest.mark.parametrize("conflict", ["module", "package"]) +def test_cli_help_uses_local_utils(evaluation_dependencies, tmp_path, conflict): + if conflict == "module": + unrelated_utils = tmp_path / "utils.py" + else: + (tmp_path / "utils").mkdir() + unrelated_utils = tmp_path / "utils/__init__.py" + unrelated_utils.write_text('raise RuntimeError("Unrelated utils was imported")\n') + env = { + **os.environ, + "PYTHONPATH": str(tmp_path), + "PYTHON_DOTENV_DISABLED": "1", + } + env.pop("OPENAI_API_KEY", None) + env.pop("OPENAI_API_BASE", None) + + result = subprocess.run( + [sys.executable, str(MODULE_PATH), "--help"], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert "--input" in result.stdout + assert "--concurrency-limit" in result.stdout + + +def _read_workbook(path): + with ZipFile(path) as workbook: + shared = ET.fromstring(workbook.read("xl/sharedStrings.xml")) + sheet = ET.fromstring(workbook.read("xl/worksheets/sheet1.xml")) + styles = ET.fromstring(workbook.read("xl/styles.xml")) + strings = ["".join(item.itertext()) for item in shared.findall("s:si", XML_NS)] + number_formats = {0: "General", 2: "0.00"} + number_formats.update( + { + int(item.attrib["numFmtId"]): item.attrib["formatCode"] + for item in styles.findall("s:numFmts/s:numFmt", XML_NS) + } + ) + cell_styles = styles.findall("s:cellXfs/s:xf", XML_NS) + fonts = styles.findall("s:fonts/s:font", XML_NS) + values, formats, bold_cells = {}, {}, set() + for cell in sheet.findall("s:sheetData/s:row/s:c", XML_NS): + address = cell.attrib["r"] + value = cell.find("s:v", XML_NS).text + values[address] = strings[int(value)] if cell.get("t") == "s" else float(value) + style = cell_styles[int(cell.get("s", "0"))] + formats[address] = number_formats[int(style.attrib["numFmtId"])] + if fonts[int(style.attrib["fontId"])].find("s:b", XML_NS) is not None: + bold_cells.add(address) + return values, formats, bold_cells + + +@pytest.mark.parametrize( + "counts", + [ + pytest.param((10, 15, 20, 5, 30, 20), id="mixed"), + pytest.param((1, 2, 3, 4, 10, 0), id="no-judge-failures"), + pytest.param((0, 0, 0, 0, 0, 3), id="all-judge-failures"), + pytest.param((0, 0, 0, 0, 0, 0), id="empty"), + ], +) +def test_excel_summary_preserves_all_categories_and_metrics(pref_eval, tmp_path, counts): + total = sum(counts) + counter = Counter( + {category: count for category, count in zip(CATEGORIES, counts, strict=True) if count} + ) + summary = pref_eval.log_summary(counter, total) + output = tmp_path / "summary.xlsx" + + pref_eval.generate_excel_summary( + summary, + avg_search_time=2.345, + avg_context_tokens=123.5, + avg_add_time=1.234, + output_excel_file=str(output), + model_name="test-model", + ) + + values, formats, bold_cells = _read_workbook(output) + assert values["G1"] == "Judge Failure\n评判失败" + assert values["A2"] == "test-model" + percentages = [values[f"{column}2"] for column in "BCDEFG"] + expected = [count / total if total else 0 for count in counts] + assert percentages == pytest.approx(expected) + assert sum(percentages) == pytest.approx(1 if total else 0) + assert all(formats[f"{column}2"] == "0.0%" for column in "BCDEFG") + assert "F2" in bold_cells + assert [values[f"{column}1"] for column in "HIJ"] == ["context token", "Time添加", "Time搜索"] + assert values["H2"] == 123.5 + assert formats["H2"] == "0.00" + assert values["I2"] == "1.23s" + assert values["J2"] == "2.35s" From ca25fdc6dd8ff350be68ba8e28b469c8c6324a40 Mon Sep 17 00:00:00 2001 From: Jiang Wu <245355730+JiangLLM@users.noreply.github.com> Date: Wed, 28 Oct 2026 18:01:38 -0700 Subject: [PATCH 4/4] fix(eval): keep PrefEval report headers readable Signed-off-by: Jiang Wu <245355730+JiangLLM@users.noreply.github.com> --- evaluation/scripts/PrefEval/pref_eval.py | 1 + tests/evaluation/test_pref_eval.py | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/evaluation/scripts/PrefEval/pref_eval.py b/evaluation/scripts/PrefEval/pref_eval.py index 2d0c81d7e..43a98fead 100644 --- a/evaluation/scripts/PrefEval/pref_eval.py +++ b/evaluation/scripts/PrefEval/pref_eval.py @@ -282,6 +282,7 @@ def get_pct(key): float_format = workbook.add_format({"num_format": "0.00"}) wrap_format = workbook.add_format({"text_wrap": True, "align": "center", "valign": "top"}) + worksheet.write_row(0, 0, df.columns, wrap_format) worksheet.set_column("A:J", None, wrap_format) worksheet.set_column("B:G", 18, pct_format) worksheet.set_column("H:H", 12, float_format) diff --git a/tests/evaluation/test_pref_eval.py b/tests/evaluation/test_pref_eval.py index 33a8cf6d0..4be5a55d4 100644 --- a/tests/evaluation/test_pref_eval.py +++ b/tests/evaluation/test_pref_eval.py @@ -103,7 +103,7 @@ def _read_workbook(path): ) cell_styles = styles.findall("s:cellXfs/s:xf", XML_NS) fonts = styles.findall("s:fonts/s:font", XML_NS) - values, formats, bold_cells = {}, {}, set() + values, formats, bold_cells, wrapped_cells = {}, {}, set(), set() for cell in sheet.findall("s:sheetData/s:row/s:c", XML_NS): address = cell.attrib["r"] value = cell.find("s:v", XML_NS).text @@ -112,7 +112,10 @@ def _read_workbook(path): formats[address] = number_formats[int(style.attrib["numFmtId"])] if fonts[int(style.attrib["fontId"])].find("s:b", XML_NS) is not None: bold_cells.add(address) - return values, formats, bold_cells + alignment = style.find("s:alignment", XML_NS) + if alignment is not None and alignment.get("wrapText") == "1": + wrapped_cells.add(address) + return values, formats, bold_cells, wrapped_cells @pytest.mark.parametrize( @@ -141,7 +144,8 @@ def test_excel_summary_preserves_all_categories_and_metrics(pref_eval, tmp_path, model_name="test-model", ) - values, formats, bold_cells = _read_workbook(output) + values, formats, bold_cells, wrapped_cells = _read_workbook(output) + assert all(f"{column}1" in wrapped_cells for column in "ABCDEFGHIJ") assert values["G1"] == "Judge Failure\n评判失败" assert values["A2"] == "test-model" percentages = [values[f"{column}2"] for column in "BCDEFG"]