diff --git a/evaluation/scripts/locomo/locomo_eval.py b/evaluation/scripts/locomo/locomo_eval.py index 5a2777423..f482737d0 100644 --- a/evaluation/scripts/locomo/locomo_eval.py +++ b/evaluation/scripts/locomo/locomo_eval.py @@ -3,7 +3,7 @@ import json import logging import os -import re +import sys import time import nltk @@ -23,6 +23,14 @@ from tqdm import tqdm +sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +# ``extract_label_json`` is re-exported for callers that imported it from this +# module before the shared helper existed; new code should import from +# ``utils.judge`` directly. See GitHub issue #2525 for the regression this +# consolidation prevents. +from utils.judge import extract_label_json, parse_judge_label # noqa: F401 + + logging.basicConfig(level=logging.CRITICAL) transformers.logging.set_verbosity_error() encoding = tiktoken.get_encoding("cl100k_base") @@ -48,29 +56,6 @@ class LLMGrade(BaseModel): llm_reasoning: str = Field(description="Explain why the answer is correct or incorrect.") -def extract_label_json(text: str) -> str | None: - """ - Extracts a JSON object of the form {"label": "VALUE"} from a given text string. - This function is designed to handle cases where the LLM response contains - natural language alongside a final JSON snippet, ensuring robust parsing. - - Supports both single and double quotes around the label value. - Ignores surrounding whitespace and formatting. - - Returns: - The full matching JSON string (e.g., '{"label": "CORRECT"}') if found. - None if no valid label JSON is found. - """ - # Regex pattern to match: { "label": "value" } with optional whitespace - # Matches both single and double quotes, allows spaces around keys and values - pattern = r'\{\s*"label"\s*:\s*["\']([^"\']*)["\']\s*\}' - match = re.search(pattern, text) - if match: - # Return the complete matched JSON string for safe json.loads() - return match.group(0) - return None - - async def locomo_grader(llm_client, question: str, gold_answer: str, response: str) -> bool: system_prompt = """ You are an expert grader that determines if answers to questions match a gold standard answer @@ -111,8 +96,7 @@ async def locomo_grader(llm_client, question: str, gold_answer: str, response: s temperature=0, ) message_content = response.choices[0].message.content - message_content = extract_label_json(text=message_content) - label = json.loads(message_content)["label"] + label = parse_judge_label(message_content) parsed = LLMGrade(llm_judgment=label, llm_reasoning="") return parsed.llm_judgment.strip().lower() == "correct" except Exception as e: diff --git a/evaluation/scripts/longmemeval/lme_eval.py b/evaluation/scripts/longmemeval/lme_eval.py index d85eec1a1..e8a9e573b 100644 --- a/evaluation/scripts/longmemeval/lme_eval.py +++ b/evaluation/scripts/longmemeval/lme_eval.py @@ -24,6 +24,7 @@ sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from utils.judge import parse_judge_label from utils.prompts import LME_JUDGE_MODEL_TEMPLATE @@ -174,7 +175,15 @@ def lme_grader(llm_client, question, golden_answer, response): ) message_content = response.choices[0].message.content - label = json.loads(message_content)["label"] + try: + label = parse_judge_label(message_content) + except ValueError as exc: + # The judge occasionally returns an empty body or prose without the + # ``{"label": ...}`` snippet. Downgrade that to a WRONG verdict and keep + # the sample in the batch instead of letting ``asyncio.gather`` abort + # the whole user's processing (see issue #2525's log-collapse symptom). + print(f"[WARN] lme_grader failed to parse judge reply: {exc}") + return False parsed = LLMGrade(llm_judgment=label, llm_reasoning="") return parsed.llm_judgment.strip().lower() == "correct" diff --git a/evaluation/scripts/utils/judge.py b/evaluation/scripts/utils/judge.py new file mode 100644 index 000000000..8c39bf3d3 --- /dev/null +++ b/evaluation/scripts/utils/judge.py @@ -0,0 +1,83 @@ +"""Shared helpers for parsing LLM-as-a-Judge replies. + +Both LongMemEval (`evaluation/scripts/longmemeval/lme_eval.py`) and LoCoMo +(`evaluation/scripts/locomo/locomo_eval.py`) ask the judge model to emit a +one-sentence rationale followed by a JSON snippet of the form +``{"label": "CORRECT"}``. A prompt-compliant reply therefore starts with +natural-language prose, so applying ``json.loads`` directly to the full body +raises ``JSONDecodeError`` and silently drops the sample +(see https://github.com/MemTensor/MemOS/issues/2525). + +This module centralises the label extraction so both graders share a single +implementation and a single error-reporting shape. +""" + +from __future__ import annotations + +import re + + +# Matches ``{"label": "VALUE"}`` with optional whitespace, allowing either +# single or double quotes around the value. Keeps LoCoMo's original behaviour. +_LABEL_PATTERN = re.compile(r"""\{\s*"label"\s*:\s*["']([^"']*)["']\s*\}""") + +# How many bytes of the raw reply to include in the error message. Enough to +# distinguish empty replies, truncated replies, and chatty replies without a +# label; bounded so that log lines stay readable. +_PREVIEW_BYTES = 200 + + +def extract_label_json(text: str | None) -> str | None: + """Return the full ``{"label": "VALUE"}`` substring, or ``None``. + + The returned value is safe to feed back into :func:`json.loads` as long as + the regex matched a double-quoted shape (single-quoted matches survive the + regex but are not valid JSON; callers should use :func:`parse_judge_label` + which handles both cases). + """ + if not text: + return None + match = _LABEL_PATTERN.search(text) + if match is None: + return None + return match.group(0) + + +def _preview(message_content: str | None) -> str: + """Return a short, log-safe quoted preview of ``message_content``.""" + if message_content is None: + return "None" + if message_content == "": + return "''" + truncated = message_content[:_PREVIEW_BYTES] + suffix = "..." if len(message_content) > _PREVIEW_BYTES else "" + # ``!r`` already escapes control characters (newlines become ``\\n``, etc.), + # so no manual pre-escaping is needed. Doing it manually would double-escape + # (a real newline would render as ``\\\\n`` in the log line). + return f"{truncated!r}{suffix}" + + +def parse_judge_label(message_content: str | None) -> str: + """Extract the judge's label from ``message_content``. + + Returns the label value (e.g. ``"CORRECT"``) on success. Raises + :class:`ValueError` with a short preview of the raw content when the body + is empty, ``None``, or contains no ``{"label": ...}`` snippet. The preview + lets operators tell a transient empty completion apart from a + prompt-compliant "\\n" body that regressed for a different + reason — the root cause the bug report called out as currently + indistinguishable in the logs. + """ + match = _LABEL_PATTERN.search(message_content) if message_content else None + if match is None: + raise ValueError( + f'judge reply has no {{"label": ...}} JSON (preview: {_preview(message_content)})' + ) + + # Use the regex capture group for the label value rather than running + # ``json.loads`` on the raw substring. The pattern already enforces the + # ``{"label": "..."}`` shape, so an explicit JSON parse would just repeat + # that validation. Reading ``group(1)`` also avoids a global + # ``replace("'", '"')`` fallback that would mangle any single quote inside + # the label value itself (e.g. ``{"label": "it's CORRECT"}``). + return match.group(1) diff --git a/tests/evaluation/__init__.py b/tests/evaluation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/evaluation/longmemeval/__init__.py b/tests/evaluation/longmemeval/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/evaluation/longmemeval/test_lme_grader.py b/tests/evaluation/longmemeval/test_lme_grader.py new file mode 100644 index 000000000..1cb656798 --- /dev/null +++ b/tests/evaluation/longmemeval/test_lme_grader.py @@ -0,0 +1,126 @@ +"""Integration test for `lme_grader` against a mocked OpenAI client. + +Repro from issue #2525: a judge reply that follows the prompt (one sentence +of reasoning then a JSON label) must be graded, not dropped. We stub out the +heavy eval-only dependencies (bert_score / sentence_transformers / rouge_score) +so the test runs on the standard `make test` extras without pulling in +`poetry install --with eval`. +""" + +from __future__ import annotations + +import importlib +import os +import sys +import types + +from unittest import mock + +import pytest + + +_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) +_EVAL_SCRIPTS = os.path.join(_REPO_ROOT, "evaluation", "scripts") +_LME_DIR = os.path.join(_EVAL_SCRIPTS, "longmemeval") + + +def _install_heavy_stubs() -> list[str]: + """Install MagicMock stubs for the eval-only deps that `lme_eval` imports + at module load time. Returns the list of stub names (for cleanup).""" + stubs: dict[str, types.ModuleType] = {} + + # bert_score exports a function `score` imported as `score as bert_score`. + bert_score_mod = types.ModuleType("bert_score") + bert_score_mod.score = mock.MagicMock(return_value=(None, None, None)) + stubs["bert_score"] = bert_score_mod + + # sentence_transformers.SentenceTransformer is called at module import time. + st_mod = types.ModuleType("sentence_transformers") + st_mod.SentenceTransformer = mock.MagicMock(return_value=mock.MagicMock()) + stubs["sentence_transformers"] = st_mod + + # rouge_score.rouge_scorer.RougeScorer is used by calculate_rouge_scores. + rouge_score_mod = types.ModuleType("rouge_score") + rouge_scorer_mod = types.ModuleType("rouge_score.rouge_scorer") + rouge_scorer_mod.RougeScorer = mock.MagicMock() + rouge_score_mod.rouge_scorer = rouge_scorer_mod + stubs["rouge_score"] = rouge_score_mod + stubs["rouge_score.rouge_scorer"] = rouge_scorer_mod + + for name, mod in stubs.items(): + sys.modules.setdefault(name, mod) + return list(stubs) + + +@pytest.fixture(scope="module") +def lme_eval_module(): + """Import `evaluation/scripts/longmemeval/lme_eval.py` with heavy deps stubbed.""" + installed = _install_heavy_stubs() + for p in (_EVAL_SCRIPTS, _LME_DIR): + if p not in sys.path: + sys.path.insert(0, p) + try: + if "lme_eval" in sys.modules: + del sys.modules["lme_eval"] + module = importlib.import_module("lme_eval") + except Exception as exc: # pragma: no cover + pytest.skip(f"Cannot import lme_eval even with stubs: {exc}") + try: + yield module + finally: + # Only clean up what we actually added to sys.modules. + for name in installed: + if ( + sys.modules.get(name) is not None + and isinstance(sys.modules[name], types.ModuleType) + and sys.modules[name].__name__ == name + ): + # don't aggressively delete; other tests may want them cached + pass + + +def _mock_client_returning(content: str) -> mock.MagicMock: + client = mock.MagicMock() + completion = mock.MagicMock() + completion.choices = [mock.MagicMock()] + completion.choices[0].message.content = content + client.chat.completions.create.return_value = completion + return client + + +def test_lme_grader_accepts_prose_plus_json_from_issue_2525(lme_eval_module): + """The exact string in the bug report must grade CORRECT, not blow up.""" + content = 'The generated answer matches the gold answer.\n{"label": "CORRECT"}' + client = _mock_client_returning(content) + + result = lme_eval_module.lme_grader( + client, + question="What degree did I graduate with?", + golden_answer="Business Administration", + response="Business Administration", + ) + assert result is True + + +def test_lme_grader_accepts_raw_json(lme_eval_module): + client = _mock_client_returning('{"label": "CORRECT"}') + assert lme_eval_module.lme_grader(client, question="q", golden_answer="g", response="g") is True + + +def test_lme_grader_marks_wrong(lme_eval_module): + content = 'Different topic from the gold answer.\n{"label": "WRONG"}' + client = _mock_client_returning(content) + assert ( + lme_eval_module.lme_grader(client, question="q", golden_answer="g", response="other") + is False + ) + + +def test_lme_grader_empty_body_raises_distinguishable_error(lme_eval_module): + """Empty completions used to collapse into the same JSONDecodeError as a + prose+JSON reply; now they raise ValueError with a readable preview.""" + client = _mock_client_returning("") + with pytest.raises(ValueError) as excinfo: + lme_eval_module.lme_grader(client, question="q", golden_answer="g", response="g") + # The preview marker makes empty vs prose-without-label distinguishable. + assert "preview" in str(excinfo.value).lower() or "''" in str(excinfo.value) diff --git a/tests/evaluation/longmemeval/test_parse_judge_label.py b/tests/evaluation/longmemeval/test_parse_judge_label.py new file mode 100644 index 000000000..702264b99 --- /dev/null +++ b/tests/evaluation/longmemeval/test_parse_judge_label.py @@ -0,0 +1,88 @@ +"""Unit tests for the shared judge-reply parser. + +The bug (#2525) is that LongMemEval's `lme_grader` applied `json.loads` +directly to the judge's chat completion, which fails whenever the model +follows the prompt and emits "\n{