Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 10 additions & 26 deletions evaluation/scripts/locomo/locomo_eval.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import json
import logging
import os
import re
import sys
import time

import nltk
Expand All @@ -23,6 +23,14 @@
from tqdm import tqdm


sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
# ``extract_label_json`` is re-exported for callers that imported it from this
# module before the shared helper existed; new code should import from
# ``utils.judge`` directly. See GitHub issue #2525 for the regression this
# consolidation prevents.
from utils.judge import extract_label_json, parse_judge_label # noqa: F401


logging.basicConfig(level=logging.CRITICAL)
transformers.logging.set_verbosity_error()
encoding = tiktoken.get_encoding("cl100k_base")
Expand All @@ -48,29 +56,6 @@ class LLMGrade(BaseModel):
llm_reasoning: str = Field(description="Explain why the answer is correct or incorrect.")


def extract_label_json(text: str) -> str | None:
"""
Extracts a JSON object of the form {"label": "VALUE"} from a given text string.
This function is designed to handle cases where the LLM response contains
natural language alongside a final JSON snippet, ensuring robust parsing.

Supports both single and double quotes around the label value.
Ignores surrounding whitespace and formatting.

Returns:
The full matching JSON string (e.g., '{"label": "CORRECT"}') if found.
None if no valid label JSON is found.
"""
# Regex pattern to match: { "label": "value" } with optional whitespace
# Matches both single and double quotes, allows spaces around keys and values
pattern = r'\{\s*"label"\s*:\s*["\']([^"\']*)["\']\s*\}'
match = re.search(pattern, text)
if match:
# Return the complete matched JSON string for safe json.loads()
return match.group(0)
return None


async def locomo_grader(llm_client, question: str, gold_answer: str, response: str) -> bool:
system_prompt = """
You are an expert grader that determines if answers to questions match a gold standard answer
Expand Down Expand Up @@ -111,8 +96,7 @@ async def locomo_grader(llm_client, question: str, gold_answer: str, response: s
temperature=0,
)
message_content = response.choices[0].message.content
message_content = extract_label_json(text=message_content)
label = json.loads(message_content)["label"]
label = parse_judge_label(message_content)
parsed = LLMGrade(llm_judgment=label, llm_reasoning="")
return parsed.llm_judgment.strip().lower() == "correct"
except Exception as e:
Expand Down
11 changes: 10 additions & 1 deletion evaluation/scripts/longmemeval/lme_eval.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@


sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from utils.judge import parse_judge_label
from utils.prompts import LME_JUDGE_MODEL_TEMPLATE


Expand Down Expand Up @@ -174,7 +175,15 @@ def lme_grader(llm_client, question, golden_answer, response):
)

message_content = response.choices[0].message.content
label = json.loads(message_content)["label"]
try:
label = parse_judge_label(message_content)
except ValueError as exc:
# The judge occasionally returns an empty body or prose without the
# ``{"label": ...}`` snippet. Downgrade that to a WRONG verdict and keep
# the sample in the batch instead of letting ``asyncio.gather`` abort
# the whole user's processing (see issue #2525's log-collapse symptom).
print(f"[WARN] lme_grader failed to parse judge reply: {exc}")
return False
parsed = LLMGrade(llm_judgment=label, llm_reasoning="")

return parsed.llm_judgment.strip().lower() == "correct"
Expand Down
83 changes: 83 additions & 0 deletions evaluation/scripts/utils/judge.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
"""Shared helpers for parsing LLM-as-a-Judge replies.

Both LongMemEval (`evaluation/scripts/longmemeval/lme_eval.py`) and LoCoMo
(`evaluation/scripts/locomo/locomo_eval.py`) ask the judge model to emit a
one-sentence rationale followed by a JSON snippet of the form
``{"label": "CORRECT"}``. A prompt-compliant reply therefore starts with
natural-language prose, so applying ``json.loads`` directly to the full body
raises ``JSONDecodeError`` and silently drops the sample
(see https://github.com/MemTensor/MemOS/issues/2525).

This module centralises the label extraction so both graders share a single
implementation and a single error-reporting shape.
"""

from __future__ import annotations

import re


# Matches ``{"label": "VALUE"}`` with optional whitespace, allowing either
# single or double quotes around the value. Keeps LoCoMo's original behaviour.
_LABEL_PATTERN = re.compile(r"""\{\s*"label"\s*:\s*["']([^"']*)["']\s*\}""")

# How many bytes of the raw reply to include in the error message. Enough to
# distinguish empty replies, truncated replies, and chatty replies without a
# label; bounded so that log lines stay readable.
_PREVIEW_BYTES = 200


def extract_label_json(text: str | None) -> str | None:
"""Return the full ``{"label": "VALUE"}`` substring, or ``None``.

The returned value is safe to feed back into :func:`json.loads` as long as
the regex matched a double-quoted shape (single-quoted matches survive the
regex but are not valid JSON; callers should use :func:`parse_judge_label`
which handles both cases).
"""
if not text:
return None
match = _LABEL_PATTERN.search(text)
if match is None:
return None
return match.group(0)


def _preview(message_content: str | None) -> str:
"""Return a short, log-safe quoted preview of ``message_content``."""
if message_content is None:
return "None"
if message_content == "":
return "''"
truncated = message_content[:_PREVIEW_BYTES]
suffix = "..." if len(message_content) > _PREVIEW_BYTES else ""
# ``!r`` already escapes control characters (newlines become ``\\n``, etc.),
# so no manual pre-escaping is needed. Doing it manually would double-escape
# (a real newline would render as ``\\\\n`` in the log line).
return f"{truncated!r}{suffix}"


def parse_judge_label(message_content: str | None) -> str:
"""Extract the judge's label from ``message_content``.

Returns the label value (e.g. ``"CORRECT"``) on success. Raises
:class:`ValueError` with a short preview of the raw content when the body
is empty, ``None``, or contains no ``{"label": ...}`` snippet. The preview
lets operators tell a transient empty completion apart from a
prompt-compliant "<reasoning>\\n<json>" body that regressed for a different
reason — the root cause the bug report called out as currently
indistinguishable in the logs.
"""
match = _LABEL_PATTERN.search(message_content) if message_content else None
if match is None:
raise ValueError(
f'judge reply has no {{"label": ...}} JSON (preview: {_preview(message_content)})'
)

# Use the regex capture group for the label value rather than running
# ``json.loads`` on the raw substring. The pattern already enforces the
# ``{"label": "..."}`` shape, so an explicit JSON parse would just repeat
# that validation. Reading ``group(1)`` also avoids a global
# ``replace("'", '"')`` fallback that would mangle any single quote inside
# the label value itself (e.g. ``{"label": "it's CORRECT"}``).
return match.group(1)
Empty file added tests/evaluation/__init__.py
Empty file.
Empty file.
126 changes: 126 additions & 0 deletions tests/evaluation/longmemeval/test_lme_grader.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
"""Integration test for `lme_grader` against a mocked OpenAI client.

Repro from issue #2525: a judge reply that follows the prompt (one sentence
of reasoning then a JSON label) must be graded, not dropped. We stub out the
heavy eval-only dependencies (bert_score / sentence_transformers / rouge_score)
so the test runs on the standard `make test` extras without pulling in
`poetry install --with eval`.
"""

from __future__ import annotations

import importlib
import os
import sys
import types

from unittest import mock

import pytest


_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
_EVAL_SCRIPTS = os.path.join(_REPO_ROOT, "evaluation", "scripts")
_LME_DIR = os.path.join(_EVAL_SCRIPTS, "longmemeval")


def _install_heavy_stubs() -> list[str]:
"""Install MagicMock stubs for the eval-only deps that `lme_eval` imports
at module load time. Returns the list of stub names (for cleanup)."""
stubs: dict[str, types.ModuleType] = {}

# bert_score exports a function `score` imported as `score as bert_score`.
bert_score_mod = types.ModuleType("bert_score")
bert_score_mod.score = mock.MagicMock(return_value=(None, None, None))
stubs["bert_score"] = bert_score_mod

# sentence_transformers.SentenceTransformer is called at module import time.
st_mod = types.ModuleType("sentence_transformers")
st_mod.SentenceTransformer = mock.MagicMock(return_value=mock.MagicMock())
stubs["sentence_transformers"] = st_mod

# rouge_score.rouge_scorer.RougeScorer is used by calculate_rouge_scores.
rouge_score_mod = types.ModuleType("rouge_score")
rouge_scorer_mod = types.ModuleType("rouge_score.rouge_scorer")
rouge_scorer_mod.RougeScorer = mock.MagicMock()
rouge_score_mod.rouge_scorer = rouge_scorer_mod
stubs["rouge_score"] = rouge_score_mod
stubs["rouge_score.rouge_scorer"] = rouge_scorer_mod

for name, mod in stubs.items():
sys.modules.setdefault(name, mod)
return list(stubs)


@pytest.fixture(scope="module")
def lme_eval_module():
"""Import `evaluation/scripts/longmemeval/lme_eval.py` with heavy deps stubbed."""
installed = _install_heavy_stubs()
for p in (_EVAL_SCRIPTS, _LME_DIR):
if p not in sys.path:
sys.path.insert(0, p)
try:
if "lme_eval" in sys.modules:
del sys.modules["lme_eval"]
module = importlib.import_module("lme_eval")
except Exception as exc: # pragma: no cover
pytest.skip(f"Cannot import lme_eval even with stubs: {exc}")
try:
yield module
finally:
# Only clean up what we actually added to sys.modules.
for name in installed:
if (
sys.modules.get(name) is not None
and isinstance(sys.modules[name], types.ModuleType)
and sys.modules[name].__name__ == name
):
# don't aggressively delete; other tests may want them cached
pass


def _mock_client_returning(content: str) -> mock.MagicMock:
client = mock.MagicMock()
completion = mock.MagicMock()
completion.choices = [mock.MagicMock()]
completion.choices[0].message.content = content
client.chat.completions.create.return_value = completion
return client


def test_lme_grader_accepts_prose_plus_json_from_issue_2525(lme_eval_module):
"""The exact string in the bug report must grade CORRECT, not blow up."""
content = 'The generated answer matches the gold answer.\n{"label": "CORRECT"}'
client = _mock_client_returning(content)

result = lme_eval_module.lme_grader(
client,
question="What degree did I graduate with?",
golden_answer="Business Administration",
response="Business Administration",
)
assert result is True


def test_lme_grader_accepts_raw_json(lme_eval_module):
client = _mock_client_returning('{"label": "CORRECT"}')
assert lme_eval_module.lme_grader(client, question="q", golden_answer="g", response="g") is True


def test_lme_grader_marks_wrong(lme_eval_module):
content = 'Different topic from the gold answer.\n{"label": "WRONG"}'
client = _mock_client_returning(content)
assert (
lme_eval_module.lme_grader(client, question="q", golden_answer="g", response="other")
is False
)


def test_lme_grader_empty_body_raises_distinguishable_error(lme_eval_module):
"""Empty completions used to collapse into the same JSONDecodeError as a
prose+JSON reply; now they raise ValueError with a readable preview."""
client = _mock_client_returning("")
with pytest.raises(ValueError) as excinfo:
lme_eval_module.lme_grader(client, question="q", golden_answer="g", response="g")
# The preview marker makes empty vs prose-without-label distinguishable.
assert "preview" in str(excinfo.value).lower() or "''" in str(excinfo.value)
88 changes: 88 additions & 0 deletions tests/evaluation/longmemeval/test_parse_judge_label.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
"""Unit tests for the shared judge-reply parser.

The bug (#2525) is that LongMemEval's `lme_grader` applied `json.loads`
directly to the judge's chat completion, which fails whenever the model
follows the prompt and emits "<one sentence of reasoning>\n{<label json>}".
These tests pin the behaviour of the shared `parse_judge_label` helper that
both LongMemEval and LoCoMo now consume.
"""

from __future__ import annotations

import os
import sys

import pytest


# Make `evaluation/scripts/` importable without installing the eval group.
_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
_EVAL_SCRIPTS = os.path.join(_REPO_ROOT, "evaluation", "scripts")
if _EVAL_SCRIPTS not in sys.path:
sys.path.insert(0, _EVAL_SCRIPTS)

from utils.judge import extract_label_json, parse_judge_label # noqa: E402


class TestExtractLabelJson:
def test_raw_json_object(self):
assert extract_label_json('{"label": "CORRECT"}') == '{"label": "CORRECT"}'

def test_prose_then_json_from_issue_2525(self):
"""The exact shape of the judge reply described in the bug report."""
text = 'The generated answer matches the gold answer.\n{"label": "CORRECT"}'
extracted = extract_label_json(text)
assert extracted == '{"label": "CORRECT"}'

def test_single_quoted_value_still_matches(self):
"""LoCoMo's regex requires the ``"label"`` key double-quoted but
accepts single OR double quotes around the value. Keep that behaviour."""
assert extract_label_json("{\"label\": 'correct'}") == "{\"label\": 'correct'}"

def test_empty_string_returns_none(self):
assert extract_label_json("") is None

def test_none_input_returns_none(self):
assert extract_label_json(None) is None

def test_text_without_label_returns_none(self):
assert extract_label_json("The model refused to answer.") is None


class TestParseJudgeLabel:
def test_raw_json_returns_label(self):
assert parse_judge_label('{"label": "CORRECT"}') == "CORRECT"

def test_prose_plus_json_returns_label(self):
"""The exact reproduction from issue #2525."""
text = 'The generated answer matches the gold answer.\n{"label": "CORRECT"}'
assert parse_judge_label(text) == "CORRECT"

def test_wrong_label_round_trip(self):
text = 'Different topic.\n{"label": "WRONG"}'
assert parse_judge_label(text) == "WRONG"

def test_empty_content_raises_value_error_with_preview(self):
"""Empty replies and prose-with-no-label replies must be distinguishable
in the error message, per the bug report's log-collapse complaint."""
with pytest.raises(ValueError) as excinfo:
parse_judge_label("")
# Message should mention the preview so operators can tell which input
# tripped it; empty string preview must be visible.
assert "preview" in str(excinfo.value).lower() or "''" in str(excinfo.value)

def test_none_content_raises_value_error(self):
with pytest.raises(ValueError):
parse_judge_label(None)

def test_no_label_raises_value_error_with_preview_substring(self):
with pytest.raises(ValueError) as excinfo:
parse_judge_label("The model said no answer could be given.")
assert "The model said no answer" in str(excinfo.value)

def test_long_content_preview_is_truncated(self):
"""200-byte preview cap, per the design doc."""
with pytest.raises(ValueError) as excinfo:
parse_judge_label("X" * 5000)
# Preview truncation keeps log lines bounded.
assert len(str(excinfo.value)) < 2000
Loading