diff --git a/evaluation/scripts/PrefEval/pref_eval.py b/evaluation/scripts/PrefEval/pref_eval.py index 9da3c9438..43a98fead 100644 --- a/evaluation/scripts/PrefEval/pref_eval.py +++ b/evaluation/scripts/PrefEval/pref_eval.py @@ -3,6 +3,7 @@ import json import os import re +import sys from collections import Counter from typing import Any @@ -14,6 +15,10 @@ from tqdm.asyncio import tqdm +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from utils.pref_classify import JUDGE_FAILURE, classify_error_type + + load_dotenv() API_KEY = os.getenv("OPENAI_API_KEY") @@ -178,24 +183,6 @@ async def evaluate_helpful_response_async( } -def classify_error_type(evaluation_results: dict[str, Any]) -> str: - violate = evaluation_results["violate_preference"]["answer"] - acknowledge = evaluation_results["acknowledge_preference"]["answer"] - hallucinate = evaluation_results["hallucinate_preference"]["answer"] - helpful = evaluation_results["helpful_response"]["answer"] - - if violate == "Yes" and acknowledge == "No" and helpful == "Yes": - return "Preference-Unaware Violation" - elif violate == "Yes" and acknowledge == "Yes" and hallucinate == "Yes" and helpful == "Yes": - return "Preference Hallucination Violation" - elif violate == "Yes" and acknowledge == "Yes" and hallucinate == "No" and helpful == "Yes": - return "Inconsistency Violation" - elif violate == "No" and helpful == "No": - return "Unhelpful Response" - else: - return "Personalized Response" - - async def process_line(line: str, client: OpenAI, semaphore: asyncio.Semaphore) -> dict[str, Any]: async with semaphore: data = json.loads(line.strip()) @@ -268,6 +255,7 @@ def get_pct(key): inconsistency_pct = get_pct("Inconsistency Violation") unhelpful_pct = get_pct("Unhelpful Response") personalized_pct = get_pct("Personalized Response") + judge_failure_pct = get_pct(JUDGE_FAILURE) data = { "Model": [model_name], @@ -276,6 +264,7 @@ def get_pct(key): "Inconsistency\n意识到偏好但给出了不一致的回答": [inconsistency_pct / 100], "Unhelpful Response\n没帮助的回答": [unhelpful_pct / 100], "Personalized Response\n个性化回答": [personalized_pct / 100], + "Judge Failure\n评判失败": [judge_failure_pct / 100], "context token": [avg_context_tokens], "Time添加": [f"{avg_add_time:.2f}s"], "Time搜索": [f"{avg_search_time:.2f}s"], @@ -293,10 +282,11 @@ def get_pct(key): float_format = workbook.add_format({"num_format": "0.00"}) wrap_format = workbook.add_format({"text_wrap": True, "align": "center", "valign": "top"}) - worksheet.set_column("B:F", 18, pct_format) - worksheet.set_column("G:G", 12, float_format) - worksheet.set_column("H:I", 15) - worksheet.set_column("A:I", None, wrap_format) + worksheet.write_row(0, 0, df.columns, wrap_format) + worksheet.set_column("A:J", None, wrap_format) + worksheet.set_column("B:G", 18, pct_format) + worksheet.set_column("H:H", 12, float_format) + worksheet.set_column("I:J", 15) worksheet.set_row(0, 45) bold_pct_format = workbook.add_format({"num_format": "0.0%", "bold": True}) worksheet.set_column("F:F", 18, bold_pct_format) diff --git a/evaluation/scripts/utils/pref_classify.py b/evaluation/scripts/utils/pref_classify.py new file mode 100644 index 000000000..71427aceb --- /dev/null +++ b/evaluation/scripts/utils/pref_classify.py @@ -0,0 +1,69 @@ +"""Error-type classification for PrefEval judge answers. + +Kept free of third-party imports so it can be unit tested without the +evaluation dependency group. +""" + +import re + +from typing import Any + + +JUDGE_FAILURE = "Judge Failure" +PERSONALIZED_RESPONSE = "Personalized Response" +UNHELPFUL_RESPONSE = "Unhelpful Response" +PREFERENCE_UNAWARE_VIOLATION = "Preference-Unaware Violation" +PREFERENCE_HALLUCINATION_VIOLATION = "Preference Hallucination Violation" +INCONSISTENCY_VIOLATION = "Inconsistency Violation" + +_JUDGE_KEYS = ( + "violate_preference", + "acknowledge_preference", + "hallucinate_preference", + "helpful_response", +) + + +def parse_yes_no(answer: str | None) -> bool | None: + """Map a leading Yes or No word to True or False. + + Allow punctuation or an explanation after that word. Return None for a + missing answer or other words, such as "Not sure" and "Yesterday". + """ + if answer is None: + return None + match = re.match(r"(yes|no)\b", answer.strip(), re.IGNORECASE) + return match.group(1).lower() == "yes" if match else None + + +def classify_error_type(evaluation_results: dict[str, Any]) -> str: + """Classify one PrefEval sample from the four judge answers. + + Follows the reference implementation in amazon-science/PrefEval + (generation_task/get_preference_following_accuracy_generation_task.py): + an unhelpful response is an error on its own, the three violation + buckets require a helpful response, and a hallucinated preference only + counts when the preference was acknowledged. A missing or unrecognized + judge answer is reported as a judge failure instead of being counted as a + personalized response. + """ + answers = { + key: parse_yes_no(evaluation_results.get(key, {}).get("answer")) for key in _JUDGE_KEYS + } + if any(value is None for value in answers.values()): + return JUDGE_FAILURE + + violate = answers["violate_preference"] + acknowledge = answers["acknowledge_preference"] + hallucinate = acknowledge and answers["hallucinate_preference"] + unhelpful = not answers["helpful_response"] + + if unhelpful: + return UNHELPFUL_RESPONSE + if violate and not acknowledge: + return PREFERENCE_UNAWARE_VIOLATION + if violate and hallucinate: + return PREFERENCE_HALLUCINATION_VIOLATION + if violate: + return INCONSISTENCY_VIOLATION + return PERSONALIZED_RESPONSE diff --git a/tests/evaluation/test_pref_classify.py b/tests/evaluation/test_pref_classify.py new file mode 100644 index 000000000..847a70a95 --- /dev/null +++ b/tests/evaluation/test_pref_classify.py @@ -0,0 +1,111 @@ +import importlib.util +import itertools + +from pathlib import Path + +import pytest + + +MODULE_PATH = ( + Path(__file__).resolve().parents[2] / "evaluation" / "scripts" / "utils" / "pref_classify.py" +) + + +def _load_module(): + spec = importlib.util.spec_from_file_location("pref_classify", MODULE_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +pref_classify = _load_module() + + +def _judge(violate, acknowledge, hallucinate, helpful): + return { + "violate_preference": {"answer": violate}, + "acknowledge_preference": {"answer": acknowledge}, + "hallucinate_preference": {"answer": hallucinate}, + "helpful_response": {"answer": helpful}, + } + + +def _reference_bucket(violate, acknowledge, hallucinate, helpful): + """The bucket implied by amazon-science/PrefEval analyze_errors().""" + is_ack = "yes" in acknowledge.lower() + is_hal = is_ack and "yes" in hallucinate.lower() + is_vio = "yes" in violate.lower() + is_unhelpful = "no" in helpful.lower() + if is_unhelpful: + return pref_classify.UNHELPFUL_RESPONSE + if is_ack and not is_hal and is_vio: + return pref_classify.INCONSISTENCY_VIOLATION + if is_ack and is_hal and is_vio: + return pref_classify.PREFERENCE_HALLUCINATION_VIOLATION + if not is_ack and is_vio: + return pref_classify.PREFERENCE_UNAWARE_VIOLATION + return pref_classify.PERSONALIZED_RESPONSE + + +@pytest.mark.parametrize("answers", list(itertools.product(["Yes", "No"], repeat=4))) +def test_matches_reference_on_well_formed_answers(answers): + assert pref_classify.classify_error_type(_judge(*answers)) == _reference_bucket(*answers) + + +@pytest.mark.parametrize( + ("answers", "expected"), + [ + (("yes", "no", "No", "yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION), + (("Yes.", "No", "No", "Yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION), + ((" YES ", "No", "No", "Yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION), + (("Yes", "No", "No", "No"), pref_classify.UNHELPFUL_RESPONSE), + (("Yes", "Yes", "Yes", "No"), pref_classify.UNHELPFUL_RESPONSE), + (("Yes", "No", "Yes", "Yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION), + (("No", "Yes", "Yes", "Yes"), pref_classify.PERSONALIZED_RESPONSE), + ], +) +def test_case_and_precedence(answers, expected): + assert pref_classify.classify_error_type(_judge(*answers)) == expected + + +@pytest.mark.parametrize( + "answers", + [ + ("", "", "", ""), + ("Yes", "No", "No", ""), + ("Maybe", "No", "No", "Yes"), + ("Not sure", "No", "No", "Yes"), + ("None", "No", "No", "Yes"), + ("Yesterday", "No", "No", "Yes"), + ("Yes", "No", "No", None), + ], +) +def test_unrecognized_answers_are_judge_failures(answers): + assert pref_classify.classify_error_type(_judge(*answers)) == pref_classify.JUDGE_FAILURE + + +def test_missing_judge_key_is_a_failure(): + partial = {"violate_preference": {"answer": "Yes"}} + assert pref_classify.classify_error_type(partial) == pref_classify.JUDGE_FAILURE + + +@pytest.mark.parametrize( + ("answer", "expected"), + [ + ("Yes", True), + ("yes.", True), + ("Yes, it follows the preference.", True), + ("No", False), + (" no", False), + ("No: it does not follow the preference.", False), + ("", None), + (None, None), + ("Unknown", None), + ("Not sure", None), + ("None", None), + ("Yesterday", None), + ], +) +def test_parse_yes_no(answer, expected): + assert pref_classify.parse_yes_no(answer) is expected diff --git a/tests/evaluation/test_pref_eval.py b/tests/evaluation/test_pref_eval.py new file mode 100644 index 000000000..4be5a55d4 --- /dev/null +++ b/tests/evaluation/test_pref_eval.py @@ -0,0 +1,161 @@ +import importlib.util +import os +import subprocess +import sys +import xml.etree.ElementTree as ET + +from collections import Counter +from pathlib import Path +from zipfile import ZipFile + +import pytest + + +MODULE_PATH = Path(__file__).resolve().parents[2] / "evaluation/scripts/PrefEval/pref_eval.py" +CATEGORIES = ( + "Preference-Unaware Violation", + "Preference Hallucination Violation", + "Inconsistency Violation", + "Unhelpful Response", + "Personalized Response", + "Judge Failure", +) +XML_NS = {"s": "http://schemas.openxmlformats.org/spreadsheetml/2006/main"} + + +@pytest.fixture +def evaluation_dependencies(): + for dependency in ("pandas", "xlsxwriter", "dotenv", "openai", "tqdm"): + pytest.importorskip(dependency, reason="PrefEval requires optional evaluation dependencies") + + +@pytest.fixture +def pref_eval(evaluation_dependencies, monkeypatch): + import dotenv + + monkeypatch.setattr(dotenv, "load_dotenv", lambda: False) + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.syspath_prepend(str(MODULE_PATH.parents[1])) + previous_utils = { + name: module + for name, module in sys.modules.items() + if name == "utils" or name.startswith("utils.") + } + for name in previous_utils: + del sys.modules[name] + try: + spec = importlib.util.spec_from_file_location("pref_eval_under_test", MODULE_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + yield module + finally: + for name in list(sys.modules): + if name == "utils" or name.startswith("utils."): + del sys.modules[name] + sys.modules.update(previous_utils) + + +@pytest.mark.parametrize("conflict", ["module", "package"]) +def test_cli_help_uses_local_utils(evaluation_dependencies, tmp_path, conflict): + if conflict == "module": + unrelated_utils = tmp_path / "utils.py" + else: + (tmp_path / "utils").mkdir() + unrelated_utils = tmp_path / "utils/__init__.py" + unrelated_utils.write_text('raise RuntimeError("Unrelated utils was imported")\n') + env = { + **os.environ, + "PYTHONPATH": str(tmp_path), + "PYTHON_DOTENV_DISABLED": "1", + } + env.pop("OPENAI_API_KEY", None) + env.pop("OPENAI_API_BASE", None) + + result = subprocess.run( + [sys.executable, str(MODULE_PATH), "--help"], + cwd=tmp_path, + env=env, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert "--input" in result.stdout + assert "--concurrency-limit" in result.stdout + + +def _read_workbook(path): + with ZipFile(path) as workbook: + shared = ET.fromstring(workbook.read("xl/sharedStrings.xml")) + sheet = ET.fromstring(workbook.read("xl/worksheets/sheet1.xml")) + styles = ET.fromstring(workbook.read("xl/styles.xml")) + strings = ["".join(item.itertext()) for item in shared.findall("s:si", XML_NS)] + number_formats = {0: "General", 2: "0.00"} + number_formats.update( + { + int(item.attrib["numFmtId"]): item.attrib["formatCode"] + for item in styles.findall("s:numFmts/s:numFmt", XML_NS) + } + ) + cell_styles = styles.findall("s:cellXfs/s:xf", XML_NS) + fonts = styles.findall("s:fonts/s:font", XML_NS) + values, formats, bold_cells, wrapped_cells = {}, {}, set(), set() + for cell in sheet.findall("s:sheetData/s:row/s:c", XML_NS): + address = cell.attrib["r"] + value = cell.find("s:v", XML_NS).text + values[address] = strings[int(value)] if cell.get("t") == "s" else float(value) + style = cell_styles[int(cell.get("s", "0"))] + formats[address] = number_formats[int(style.attrib["numFmtId"])] + if fonts[int(style.attrib["fontId"])].find("s:b", XML_NS) is not None: + bold_cells.add(address) + alignment = style.find("s:alignment", XML_NS) + if alignment is not None and alignment.get("wrapText") == "1": + wrapped_cells.add(address) + return values, formats, bold_cells, wrapped_cells + + +@pytest.mark.parametrize( + "counts", + [ + pytest.param((10, 15, 20, 5, 30, 20), id="mixed"), + pytest.param((1, 2, 3, 4, 10, 0), id="no-judge-failures"), + pytest.param((0, 0, 0, 0, 0, 3), id="all-judge-failures"), + pytest.param((0, 0, 0, 0, 0, 0), id="empty"), + ], +) +def test_excel_summary_preserves_all_categories_and_metrics(pref_eval, tmp_path, counts): + total = sum(counts) + counter = Counter( + {category: count for category, count in zip(CATEGORIES, counts, strict=True) if count} + ) + summary = pref_eval.log_summary(counter, total) + output = tmp_path / "summary.xlsx" + + pref_eval.generate_excel_summary( + summary, + avg_search_time=2.345, + avg_context_tokens=123.5, + avg_add_time=1.234, + output_excel_file=str(output), + model_name="test-model", + ) + + values, formats, bold_cells, wrapped_cells = _read_workbook(output) + assert all(f"{column}1" in wrapped_cells for column in "ABCDEFGHIJ") + assert values["G1"] == "Judge Failure\n评判失败" + assert values["A2"] == "test-model" + percentages = [values[f"{column}2"] for column in "BCDEFG"] + expected = [count / total if total else 0 for count in counts] + assert percentages == pytest.approx(expected) + assert sum(percentages) == pytest.approx(1 if total else 0) + assert all(formats[f"{column}2"] == "0.0%" for column in "BCDEFG") + assert "F2" in bold_cells + assert [values[f"{column}1"] for column in "HIJ"] == ["context token", "Time添加", "Time搜索"] + assert values["H2"] == 123.5 + assert formats["H2"] == "0.00" + assert values["I2"] == "1.23s" + assert values["J2"] == "2.35s"