Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 12 additions & 22 deletions evaluation/scripts/PrefEval/pref_eval.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import json
import os
import re
import sys

from collections import Counter
from typing import Any
Expand All @@ -14,6 +15,10 @@
from tqdm.asyncio import tqdm


sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from utils.pref_classify import JUDGE_FAILURE, classify_error_type


load_dotenv()

API_KEY = os.getenv("OPENAI_API_KEY")
Expand Down Expand Up @@ -178,24 +183,6 @@ async def evaluate_helpful_response_async(
}


def classify_error_type(evaluation_results: dict[str, Any]) -> str:
violate = evaluation_results["violate_preference"]["answer"]
acknowledge = evaluation_results["acknowledge_preference"]["answer"]
hallucinate = evaluation_results["hallucinate_preference"]["answer"]
helpful = evaluation_results["helpful_response"]["answer"]

if violate == "Yes" and acknowledge == "No" and helpful == "Yes":
return "Preference-Unaware Violation"
elif violate == "Yes" and acknowledge == "Yes" and hallucinate == "Yes" and helpful == "Yes":
return "Preference Hallucination Violation"
elif violate == "Yes" and acknowledge == "Yes" and hallucinate == "No" and helpful == "Yes":
return "Inconsistency Violation"
elif violate == "No" and helpful == "No":
return "Unhelpful Response"
else:
return "Personalized Response"


async def process_line(line: str, client: OpenAI, semaphore: asyncio.Semaphore) -> dict[str, Any]:
async with semaphore:
data = json.loads(line.strip())
Expand Down Expand Up @@ -268,6 +255,7 @@ def get_pct(key):
inconsistency_pct = get_pct("Inconsistency Violation")
unhelpful_pct = get_pct("Unhelpful Response")
personalized_pct = get_pct("Personalized Response")
judge_failure_pct = get_pct(JUDGE_FAILURE)

data = {
"Model": [model_name],
Expand All @@ -276,6 +264,7 @@ def get_pct(key):
"Inconsistency\n意识到偏好但给出了不一致的回答": [inconsistency_pct / 100],
"Unhelpful Response\n没帮助的回答": [unhelpful_pct / 100],
"Personalized Response\n个性化回答": [personalized_pct / 100],
"Judge Failure\n评判失败": [judge_failure_pct / 100],
"context token": [avg_context_tokens],
"Time添加": [f"{avg_add_time:.2f}s"],
"Time搜索": [f"{avg_search_time:.2f}s"],
Expand All @@ -293,10 +282,11 @@ def get_pct(key):
float_format = workbook.add_format({"num_format": "0.00"})
wrap_format = workbook.add_format({"text_wrap": True, "align": "center", "valign": "top"})

worksheet.set_column("B:F", 18, pct_format)
worksheet.set_column("G:G", 12, float_format)
worksheet.set_column("H:I", 15)
worksheet.set_column("A:I", None, wrap_format)
worksheet.write_row(0, 0, df.columns, wrap_format)
worksheet.set_column("A:J", None, wrap_format)
worksheet.set_column("B:G", 18, pct_format)
worksheet.set_column("H:H", 12, float_format)
worksheet.set_column("I:J", 15)
worksheet.set_row(0, 45)
bold_pct_format = workbook.add_format({"num_format": "0.0%", "bold": True})
worksheet.set_column("F:F", 18, bold_pct_format)
Expand Down
69 changes: 69 additions & 0 deletions evaluation/scripts/utils/pref_classify.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
"""Error-type classification for PrefEval judge answers.

Kept free of third-party imports so it can be unit tested without the
evaluation dependency group.
"""

import re

from typing import Any


JUDGE_FAILURE = "Judge Failure"
PERSONALIZED_RESPONSE = "Personalized Response"
UNHELPFUL_RESPONSE = "Unhelpful Response"
PREFERENCE_UNAWARE_VIOLATION = "Preference-Unaware Violation"
PREFERENCE_HALLUCINATION_VIOLATION = "Preference Hallucination Violation"
INCONSISTENCY_VIOLATION = "Inconsistency Violation"

_JUDGE_KEYS = (
"violate_preference",
"acknowledge_preference",
"hallucinate_preference",
"helpful_response",
)


def parse_yes_no(answer: str | None) -> bool | None:
"""Map a leading Yes or No word to True or False.

Allow punctuation or an explanation after that word. Return None for a
missing answer or other words, such as "Not sure" and "Yesterday".
"""
if answer is None:
return None
match = re.match(r"(yes|no)\b", answer.strip(), re.IGNORECASE)
return match.group(1).lower() == "yes" if match else None


def classify_error_type(evaluation_results: dict[str, Any]) -> str:
"""Classify one PrefEval sample from the four judge answers.

Follows the reference implementation in amazon-science/PrefEval
(generation_task/get_preference_following_accuracy_generation_task.py):
an unhelpful response is an error on its own, the three violation
buckets require a helpful response, and a hallucinated preference only
counts when the preference was acknowledged. A missing or unrecognized
judge answer is reported as a judge failure instead of being counted as a
personalized response.
"""
answers = {
key: parse_yes_no(evaluation_results.get(key, {}).get("answer")) for key in _JUDGE_KEYS
}
if any(value is None for value in answers.values()):
return JUDGE_FAILURE

violate = answers["violate_preference"]
acknowledge = answers["acknowledge_preference"]
hallucinate = acknowledge and answers["hallucinate_preference"]
unhelpful = not answers["helpful_response"]

if unhelpful:
return UNHELPFUL_RESPONSE
if violate and not acknowledge:
return PREFERENCE_UNAWARE_VIOLATION
if violate and hallucinate:
return PREFERENCE_HALLUCINATION_VIOLATION
if violate:
return INCONSISTENCY_VIOLATION
return PERSONALIZED_RESPONSE
111 changes: 111 additions & 0 deletions tests/evaluation/test_pref_classify.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
import importlib.util
import itertools

from pathlib import Path

import pytest


MODULE_PATH = (
Path(__file__).resolve().parents[2] / "evaluation" / "scripts" / "utils" / "pref_classify.py"
)


def _load_module():
spec = importlib.util.spec_from_file_location("pref_classify", MODULE_PATH)
assert spec is not None and spec.loader is not None
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module


pref_classify = _load_module()


def _judge(violate, acknowledge, hallucinate, helpful):
return {
"violate_preference": {"answer": violate},
"acknowledge_preference": {"answer": acknowledge},
"hallucinate_preference": {"answer": hallucinate},
"helpful_response": {"answer": helpful},
}


def _reference_bucket(violate, acknowledge, hallucinate, helpful):
"""The bucket implied by amazon-science/PrefEval analyze_errors()."""
is_ack = "yes" in acknowledge.lower()
is_hal = is_ack and "yes" in hallucinate.lower()
is_vio = "yes" in violate.lower()
is_unhelpful = "no" in helpful.lower()
if is_unhelpful:
return pref_classify.UNHELPFUL_RESPONSE
if is_ack and not is_hal and is_vio:
return pref_classify.INCONSISTENCY_VIOLATION
if is_ack and is_hal and is_vio:
return pref_classify.PREFERENCE_HALLUCINATION_VIOLATION
if not is_ack and is_vio:
return pref_classify.PREFERENCE_UNAWARE_VIOLATION
return pref_classify.PERSONALIZED_RESPONSE


@pytest.mark.parametrize("answers", list(itertools.product(["Yes", "No"], repeat=4)))
def test_matches_reference_on_well_formed_answers(answers):
assert pref_classify.classify_error_type(_judge(*answers)) == _reference_bucket(*answers)


@pytest.mark.parametrize(
("answers", "expected"),
[
(("yes", "no", "No", "yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION),
(("Yes.", "No", "No", "Yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION),
((" YES ", "No", "No", "Yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION),
(("Yes", "No", "No", "No"), pref_classify.UNHELPFUL_RESPONSE),
(("Yes", "Yes", "Yes", "No"), pref_classify.UNHELPFUL_RESPONSE),
(("Yes", "No", "Yes", "Yes"), pref_classify.PREFERENCE_UNAWARE_VIOLATION),
(("No", "Yes", "Yes", "Yes"), pref_classify.PERSONALIZED_RESPONSE),
],
)
def test_case_and_precedence(answers, expected):
assert pref_classify.classify_error_type(_judge(*answers)) == expected


@pytest.mark.parametrize(
"answers",
[
("", "", "", ""),
("Yes", "No", "No", ""),
("Maybe", "No", "No", "Yes"),
("Not sure", "No", "No", "Yes"),
("None", "No", "No", "Yes"),
("Yesterday", "No", "No", "Yes"),
("Yes", "No", "No", None),
],
)
def test_unrecognized_answers_are_judge_failures(answers):
assert pref_classify.classify_error_type(_judge(*answers)) == pref_classify.JUDGE_FAILURE


def test_missing_judge_key_is_a_failure():
partial = {"violate_preference": {"answer": "Yes"}}
assert pref_classify.classify_error_type(partial) == pref_classify.JUDGE_FAILURE


@pytest.mark.parametrize(
("answer", "expected"),
[
("Yes", True),
("yes.", True),
("Yes, it follows the preference.", True),
("No", False),
(" no", False),
("No: it does not follow the preference.", False),
("", None),
(None, None),
("Unknown", None),
("Not sure", None),
("None", None),
("Yesterday", None),
],
)
def test_parse_yes_no(answer, expected):
assert pref_classify.parse_yes_no(answer) is expected
Loading