From 6747039346d567fc38e3afe906dd96e227c313e2 Mon Sep 17 00:00:00 2001 From: Aymen Hammouda Date: Wed, 8 Jul 2026 18:35:11 +0100 Subject: [PATCH 1/6] agent: add benchmark corpus JSON schema Defines the shape of the v0.5.0 public benchmark corpus (docs/benchmarks/corpus.yml, once frozen by the human-led #71) per PUBLIC-BENCHMARK-METHODOLOGY.md "Corpus Design": stable id, category, python_version (single version or a version pair), prompt, answer_key, citations, expected_properties, and optional ambiguity_notes. The methodology's 15/10/15/5/5 category distribution is declared once via a custom 'x-category-distribution' extension (standard JSON Schema has no clean way to express "exactly N items with property X = Y" across an array) so benchmarks/corpus.py can read it instead of duplicating the numbers as a second hard-coded constant. Refs #94, #71, #63. --- docs/benchmarks/corpus.schema.json | 119 +++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 docs/benchmarks/corpus.schema.json diff --git a/docs/benchmarks/corpus.schema.json b/docs/benchmarks/corpus.schema.json new file mode 100644 index 0000000..6eba5b3 --- /dev/null +++ b/docs/benchmarks/corpus.schema.json @@ -0,0 +1,119 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/ayhammouda/python-docs-mcp-server/docs/benchmarks/corpus.schema.json", + "title": "Python docs MCP public benchmark corpus", + "description": "Shape of docs/benchmarks/corpus.yml, the frozen 50-question evaluation pack for the v0.5.0 public benchmark (see docs/benchmarks/PUBLIC-BENCHMARK-METHODOLOGY.md, 'Corpus Design'). This file defines the STRUCTURAL contract for one question and the corpus as a whole. It is loaded and enforced by benchmarks/corpus.py:validate_corpus(), which is invoked via `python -m benchmarks validate-corpus`. Corpus question AUTHORSHIP is out of scope for that validator and for issue #94 -- it is tracked separately on issue #71 and remains permanently maintainer-owned.", + "type": "object", + "required": [ + "questions" + ], + "additionalProperties": true, + "properties": { + "schema_version": { + "type": "integer", + "const": 1, + "description": "Reserved for future corpus schema revisions. Not yet enforced by the validator." + }, + "questions": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/question" + } + } + }, + "$defs": { + "question": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "category", + "python_version", + "prompt", + "answer_key", + "citations", + "expected_properties" + ], + "properties": { + "id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$", + "description": "Stable question ID, unique across the whole corpus. Must not change once the corpus is frozen." + }, + "category": { + "type": "string", + "enum": [ + "exact_symbol", + "concept", + "cross_version", + "pep_adjacent", + "applied" + ], + "description": "One of the five methodology categories (PUBLIC-BENCHMARK-METHODOLOGY.md 'Corpus Design')." + }, + "python_version": { + "description": "The single Python version this question targets, or the exact two-version pair being compared (required for category 'cross_version').", + "oneOf": [ + { + "type": "string", + "minLength": 1 + }, + { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 2, + "maxItems": 2 + } + ] + }, + "prompt": { + "type": "string", + "minLength": 1, + "description": "The exact prompt shown to the model under test. Must not leak the answer key or rubric." + }, + "answer_key": { + "type": "string", + "minLength": 1, + "description": "The official-docs-sourced correct answer used for correctness scoring." + }, + "citations": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + }, + "description": "Required citations or source sections (CPython docs source, generated official docs, or official What's New pages) supporting the answer key." + }, + "expected_properties": { + "type": "array", + "minItems": 1, + "items": { + "type": "string", + "minLength": 1 + }, + "description": "Properties a correct candidate answer must exhibit (e.g. version-specific wording, named symbol, required caveat)." + }, + "ambiguity_notes": { + "type": [ + "string", + "null" + ], + "description": "Known ambiguity notes, if any. Optional; null or omitted when there are none." + } + } + } + }, + "x-category-distribution-comment": "Exact per-category question counts required by PUBLIC-BENCHMARK-METHODOLOGY.md 'Corpus Design'. Not a standard JSON Schema keyword -- standard JSON Schema has no clean way to express 'exactly N items with property X = Y' across a whole array without draft2019-09+ contains/minContains/maxContains gymnastics for every category. benchmarks/corpus.py:validate_corpus() reads the 'x-category-distribution' mapping below directly so the 15/10/15/5/5 distribution is declared exactly once, here, instead of being duplicated (and risking drift) as a second hard-coded constant in the validator.", + "x-category-distribution": { + "exact_symbol": 15, + "concept": 10, + "cross_version": 15, + "pep_adjacent": 5, + "applied": 5 + } +} From deda518c279c661b4cdf6c35fc2db6e123c8455a Mon Sep 17 00:00:00 2001 From: Aymen Hammouda Date: Wed, 8 Jul 2026 18:35:19 +0100 Subject: [PATCH 2/6] agent: add benchmark corpus validator module benchmarks/corpus.py:validate_corpus() loads a corpus YAML/JSON file and docs/benchmarks/corpus.schema.json, then enforces: per-question required fields, category membership (schema enum), non-empty citations/expected_properties, python_version shape (string or exact version pair), duplicate-id detection, and the schema's declared category-count distribution. Every failure raises BenchmarkValidationError (benchmarks.runner's existing exception type) with a clean message -- no jsonschema dependency, no new third-party package. This is a deeper superset of the shallow duplicate-id/missing-prompt checks benchmarks/runner.py already performs at load time; it does not replace or weaken those. Refs #94, #71, #63. --- benchmarks/corpus.py | 336 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 336 insertions(+) create mode 100644 benchmarks/corpus.py diff --git a/benchmarks/corpus.py b/benchmarks/corpus.py new file mode 100644 index 0000000..e03fd2f --- /dev/null +++ b/benchmarks/corpus.py @@ -0,0 +1,336 @@ +"""Benchmark corpus schema validation for the v0.5.0 public benchmark. + +Validates a corpus file (``docs/benchmarks/corpus.yml`` once frozen, or any +other corpus-shaped YAML/JSON file such as a test fixture) against +``docs/benchmarks/corpus.schema.json``. See +``docs/benchmarks/PUBLIC-BENCHMARK-METHODOLOGY.md`` ("Corpus Design") for the +methodology this schema encodes. + +This module intentionally does not depend on a third-party JSON Schema +library: the schema file's ``$defs.question`` block (required fields, +``category`` enum, field types) and its ``x-category-distribution`` extension +are read directly and enforced with hand-written checks, matching the +existing house style in ``benchmarks.runner`` and ``benchmarks.model_matrix``. +This keeps the 15/10/15/5/5 methodology distribution declared exactly once +(in the schema file) while avoiding a new runtime dependency. + +Corpus question AUTHORSHIP is out of scope here and for issue #94. This +module only validates shape, uniqueness, citation presence, category +membership, and category counts -- it never generates or edits corpus +questions. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +from benchmarks.runner import BenchmarkValidationError + +_SAFE_ID_FIELD = "id" + + +@dataclass(frozen=True) +class CorpusQuestion: + """One validated corpus question.""" + + id: str + category: str + python_version: str | list[str] + prompt: str + answer_key: str + citations: list[str] + expected_properties: list[str] + ambiguity_notes: str | None + raw: dict[str, Any] + + +@dataclass(frozen=True) +class CorpusValidationResult: + """Outcome of validating a corpus file against the corpus schema.""" + + corpus_path: Path + schema_path: Path + questions: list[CorpusQuestion] + category_counts: dict[str, int] + + @property + def question_count(self) -> int: + return len(self.questions) + + +def validate_corpus(corpus_path: Path, schema_path: Path) -> CorpusValidationResult: + """Validate ``corpus_path`` against ``schema_path``. + + Raises ``BenchmarkValidationError`` with a clean, non-traceback message on + any of: a missing or malformed file, a corpus question that does not + match the schema's ``question`` shape, a duplicate question ID, missing + citations, an unsupported category, or a category whose count does not + exactly match the schema's declared distribution. + """ + schema = _load_schema(schema_path) + question_schema = _question_schema(schema) + required_fields = _required_fields(question_schema) + allowed_categories = _allowed_categories(question_schema) + expected_distribution = _category_distribution(schema, allowed_categories) + + corpus_data = _load_corpus(corpus_path) + items = corpus_data.get("questions") + if not isinstance(items, list) or not items: + raise BenchmarkValidationError( + f"corpus file must contain a non-empty 'questions' list: {corpus_path}" + ) + + questions: list[CorpusQuestion] = [] + seen_ids: set[str] = set() + category_counts: dict[str, int] = dict.fromkeys(allowed_categories, 0) + + for index, item in enumerate(items): + question = _validate_question( + item, + index=index, + required_fields=required_fields, + allowed_categories=allowed_categories, + seen_ids=seen_ids, + ) + questions.append(question) + category_counts[question.category] += 1 + + _validate_category_counts(category_counts, expected_distribution) + + return CorpusValidationResult( + corpus_path=corpus_path, + schema_path=schema_path, + questions=questions, + category_counts=category_counts, + ) + + +def _load_schema(schema_path: Path) -> dict[str, Any]: + if not schema_path.exists(): + raise BenchmarkValidationError(f"corpus schema file does not exist: {schema_path}") + try: + text = schema_path.read_text(encoding="utf-8") + except OSError as exc: + raise BenchmarkValidationError( + f"corpus schema file could not be read: {schema_path}: {exc}" + ) from exc + try: + data = json.loads(text) + except json.JSONDecodeError as exc: + raise BenchmarkValidationError( + f"corpus schema file is not valid JSON: {schema_path}: {exc}" + ) from exc + if not isinstance(data, dict): + raise BenchmarkValidationError( + f"corpus schema file must contain a JSON object: {schema_path}" + ) + return data + + +def _question_schema(schema: dict[str, Any]) -> dict[str, Any]: + defs = schema.get("$defs") + question_schema = defs.get("question") if isinstance(defs, dict) else None + if not isinstance(question_schema, dict): + raise BenchmarkValidationError( + "corpus schema file must define '$defs.question' describing one corpus question" + ) + return question_schema + + +def _required_fields(question_schema: dict[str, Any]) -> list[str]: + required = question_schema.get("required") + if ( + not isinstance(required, list) + or not required + or not all(isinstance(f, str) for f in required) + ): + raise BenchmarkValidationError( + "corpus schema '$defs.question' must declare a non-empty 'required' field list" + ) + return [str(field) for field in required] + + +def _allowed_categories(question_schema: dict[str, Any]) -> list[str]: + properties = question_schema.get("properties") + category_schema = properties.get("category") if isinstance(properties, dict) else None + categories = category_schema.get("enum") if isinstance(category_schema, dict) else None + if ( + not isinstance(categories, list) + or not categories + or not all(isinstance(c, str) for c in categories) + ): + raise BenchmarkValidationError( + "corpus schema '$defs.question.properties.category' must declare a non-empty 'enum'" + ) + return [str(category) for category in categories] + + +def _category_distribution(schema: dict[str, Any], allowed_categories: list[str]) -> dict[str, int]: + distribution = schema.get("x-category-distribution") + if not isinstance(distribution, dict) or not distribution: + raise BenchmarkValidationError( + "corpus schema must declare a non-empty 'x-category-distribution' mapping" + ) + parsed: dict[str, int] = {} + for category, count in distribution.items(): + is_valid_count = isinstance(count, int) and not isinstance(count, bool) and count >= 0 + if not isinstance(category, str) or not is_valid_count: + raise BenchmarkValidationError( + "corpus schema 'x-category-distribution' entries must map a category name to a " + f"non-negative integer count; got {category!r}: {count!r}" + ) + if category not in allowed_categories: + raise BenchmarkValidationError( + f"corpus schema 'x-category-distribution' references unsupported category " + f"{category!r}; allowed categories are {allowed_categories}" + ) + parsed[category] = count + return parsed + + +def _load_corpus(corpus_path: Path) -> dict[str, Any]: + if not corpus_path.exists(): + raise BenchmarkValidationError(f"corpus file does not exist: {corpus_path}") + try: + text = corpus_path.read_text(encoding="utf-8") + except OSError as exc: + raise BenchmarkValidationError( + f"corpus file could not be read: {corpus_path}: {exc}" + ) from exc + + if corpus_path.suffix.lower() == ".json": + try: + data = json.loads(text) + except json.JSONDecodeError as exc: + raise BenchmarkValidationError( + f"corpus file is not valid JSON: {corpus_path}: {exc}" + ) from exc + else: + try: + data = yaml.safe_load(text) + except yaml.YAMLError as exc: + raise BenchmarkValidationError( + f"corpus file is not valid YAML: {corpus_path}: {exc}" + ) from exc + + if not isinstance(data, dict): + raise BenchmarkValidationError(f"corpus file must contain a mapping: {corpus_path}") + return data + + +def _validate_question( + item: Any, + *, + index: int, + required_fields: list[str], + allowed_categories: list[str], + seen_ids: set[str], +) -> CorpusQuestion: + label = f"corpus question at index {index}" + if not isinstance(item, dict): + raise BenchmarkValidationError(f"{label} must be a mapping") + + missing = [field for field in required_fields if field not in item] + if missing: + raise BenchmarkValidationError(f"{label} is missing required field(s): {missing}") + + question_id = item.get(_SAFE_ID_FIELD) + if not isinstance(question_id, str) or not question_id.strip(): + raise BenchmarkValidationError(f"{label} must have a non-empty string 'id'") + if question_id in seen_ids: + raise BenchmarkValidationError(f"duplicate corpus question id: {question_id!r}") + seen_ids.add(question_id) + + category = item.get("category") + if not isinstance(category, str) or not category.strip(): + raise BenchmarkValidationError( + f"corpus question {question_id!r} must have a non-empty 'category'" + ) + if category not in allowed_categories: + raise BenchmarkValidationError( + f"corpus question {question_id!r} has unsupported category {category!r}; " + f"allowed categories are {allowed_categories}" + ) + + python_version = _validate_python_version(item.get("python_version"), question_id) + prompt = _require_nonempty_string(item, "prompt", question_id) + answer_key = _require_nonempty_string(item, "answer_key", question_id) + citations = _require_nonempty_string_list(item, "citations", question_id) + expected_properties = _require_nonempty_string_list(item, "expected_properties", question_id) + + ambiguity_notes = item.get("ambiguity_notes") + if ambiguity_notes is not None and not isinstance(ambiguity_notes, str): + raise BenchmarkValidationError( + f"corpus question {question_id!r} 'ambiguity_notes' must be a string or null" + ) + + return CorpusQuestion( + id=question_id, + category=category, + python_version=python_version, + prompt=prompt, + answer_key=answer_key, + citations=citations, + expected_properties=expected_properties, + ambiguity_notes=ambiguity_notes, + raw=item, + ) + + +def _validate_python_version(value: Any, question_id: str) -> str | list[str]: + if isinstance(value, str) and value.strip(): + return value + if ( + isinstance(value, list) + and len(value) == 2 + and all(isinstance(v, str) and v.strip() for v in value) + ): + return [str(v) for v in value] + raise BenchmarkValidationError( + f"corpus question {question_id!r} 'python_version' must be a non-empty string or a " + "two-item list of version strings" + ) + + +def _require_nonempty_string(item: dict[str, Any], field: str, question_id: str) -> str: + value = item.get(field) + if not isinstance(value, str) or not value.strip(): + raise BenchmarkValidationError( + f"corpus question {question_id!r} must have a non-empty string {field!r}" + ) + return value + + +def _require_nonempty_string_list(item: dict[str, Any], field: str, question_id: str) -> list[str]: + value = item.get(field) + if ( + not isinstance(value, list) + or not value + or not all(isinstance(v, str) and v.strip() for v in value) + ): + raise BenchmarkValidationError( + f"corpus question {question_id!r} must have a non-empty list of non-empty strings " + f"for {field!r}" + ) + return [str(v) for v in value] + + +def _validate_category_counts( + category_counts: dict[str, int], expected_distribution: dict[str, int] +) -> None: + mismatches = { + category: (category_counts.get(category, 0), expected_count) + for category, expected_count in expected_distribution.items() + if category_counts.get(category, 0) != expected_count + } + if mismatches: + details = ", ".join( + f"{category}: got {actual}, expected {expected}" + for category, (actual, expected) in sorted(mismatches.items()) + ) + raise BenchmarkValidationError(f"corpus has wrong category counts: {details}") From 0f31d003d17a0f577bf9b51a26c96ad515238c5a Mon Sep 17 00:00:00 2001 From: Aymen Hammouda Date: Wed, 8 Jul 2026 18:35:26 +0100 Subject: [PATCH 3/6] agent: wire validate-corpus subcommand into python -m benchmarks CLI Adds `python -m benchmarks validate-corpus --corpus [--schema ]`, dispatching to benchmarks.corpus.validate_corpus and following the same catch-BenchmarkValidationError-then-exit-2 shape already used by the `run` and `report` subcommands. --schema defaults to docs/benchmarks/corpus.schema.json. Naming: this command's prog is "python -m benchmarks", distinct from the server's own `python-docs-mcp-server validate-corpus` (src/mcp_server_python_docs/__main__.py:413), which smoke-tests a built docs *index* database and shares no code path with this corpus validator. The two cannot be invoked interchangeably; the disambiguation is documented inline in the subparser help text. Refs #94, #71, #63. --- benchmarks/__main__.py | 49 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/benchmarks/__main__.py b/benchmarks/__main__.py index 409b458..daa4bf9 100644 --- a/benchmarks/__main__.py +++ b/benchmarks/__main__.py @@ -7,11 +7,13 @@ import sys from pathlib import Path +from benchmarks.corpus import validate_corpus from benchmarks.report import generate_readme_summary, generate_report from benchmarks.runner import BenchmarkConfig, BenchmarkValidationError, run_benchmark _DEFAULT_MODEL_MATRIX = Path("docs/benchmarks/model-matrix.yml") _DEFAULT_METHODOLOGY = Path("docs/benchmarks/PUBLIC-BENCHMARK-METHODOLOGY.md") +_DEFAULT_CORPUS_SCHEMA = Path("docs/benchmarks/corpus.schema.json") def _build_parser() -> argparse.ArgumentParser: @@ -59,6 +61,33 @@ def _build_parser() -> argparse.ArgumentParser: report.add_argument( "--readme-summary-out", type=Path, help="Defaults to /README-SUMMARY.md." ) + + # Note on naming: this is `python -m benchmarks validate-corpus`, a distinct + # program (`prog="python -m benchmarks"`, set above) from the server's own + # `python-docs-mcp-server validate-corpus` (src/mcp_server_python_docs/__main__.py), + # which smoke-tests a built docs *index* database and is unrelated to the + # benchmark evaluation corpus validated here. The two commands cannot be + # invoked interchangeably and do not share any code path. + validate_corpus_parser = subparsers.add_parser( + "validate-corpus", + help=( + "Validate a benchmark evaluation corpus file against " + "docs/benchmarks/corpus.schema.json (distinct from " + "`python-docs-mcp-server validate-corpus`, which validates a built docs index)." + ), + ) + validate_corpus_parser.add_argument( + "--corpus", + required=True, + type=Path, + help="Path to the corpus YAML or JSON file to validate.", + ) + validate_corpus_parser.add_argument( + "--schema", + type=Path, + default=_DEFAULT_CORPUS_SCHEMA, + help=f"Path to the corpus JSON schema file (default: {_DEFAULT_CORPUS_SCHEMA}).", + ) return parser @@ -107,6 +136,26 @@ def main(argv: list[str] | None = None) -> int: ) return 0 + if args.command == "validate-corpus": + try: + result = validate_corpus(args.corpus, args.schema) + except BenchmarkValidationError as exc: + parser.exit(2, f"benchmark corpus validation failed: {exc}\n") + + print( + json.dumps( + { + "corpus_path": str(args.corpus), + "schema_path": str(args.schema), + "question_count": result.question_count, + "category_counts": result.category_counts, + }, + indent=2, + sort_keys=True, + ) + ) + return 0 + parser.print_help() return 2 From a4654a79281e989f9ac5118e41f0837785336bcb Mon Sep 17 00:00:00 2001 From: Aymen Hammouda Date: Wed, 8 Jul 2026 18:35:33 +0100 Subject: [PATCH 4/6] agent: add synthetic placeholder corpus fixture New tests/benchmarks/fixtures/ directory (explicitly blessed by maintainer decision D4 on issue #63/#94) holding corpus.sample.yml: a mechanically generated, obviously synthetic 50-question corpus matching the methodology's 15/10/15/5/5 category distribution. Every question references a fictional `synthstdlib` module and cites only the RFC 2606 example.invalid domain, and the file carries a SYNTHETIC/FIXTURE DATA header. It validates against docs/benchmarks/corpus.schema.json. Zero real corpus questions are authored here. The real, human-authored corpus is tracked on #71 and reserved for docs/benchmarks/corpus.yml, which this diff never creates. Refs #94, #71, #63. --- tests/benchmarks/fixtures/corpus.sample.yml | 703 ++++++++++++++++++++ 1 file changed, 703 insertions(+) create mode 100644 tests/benchmarks/fixtures/corpus.sample.yml diff --git a/tests/benchmarks/fixtures/corpus.sample.yml b/tests/benchmarks/fixtures/corpus.sample.yml new file mode 100644 index 0000000..773bb82 --- /dev/null +++ b/tests/benchmarks/fixtures/corpus.sample.yml @@ -0,0 +1,703 @@ +# SYNTHETIC/FIXTURE DATA -- NOT A REAL CORPUS. +# +# This file is a placeholder fixture for tests/benchmarks/test_corpus_validation.py +# and benchmarks/corpus.py's schema validator (issue #94, split from #71 under +# maintainer approval D4; see PLAN.md Amendment 2026-07-08, decision D4, and the +# agent-context comment on issue #94). +# +# Every question below is mechanically generated, references a fictional +# `synthstdlib` module that does not exist in any real Python standard library, +# and cites only the reserved example.invalid domain (RFC 2606). None of this +# content is a real Python documentation question, a real citation, or a real +# answer. Zero real corpus questions were authored to produce this file. +# +# The real, human-authored 50-question corpus is tracked on issue #71 and will +# live at docs/benchmarks/corpus.yml once frozen. This file must never be +# confused with that corpus and must never be copied there. +# +# Shape: validates against docs/benchmarks/corpus.schema.json, including the +# schema's declared category distribution (15 exact_symbol / 10 concept / +# 15 cross_version / 5 pep_adjacent / 5 applied = 50 total), so it also serves +# as the "valid fixture" happy-path case in test_corpus_validation.py. + +questions: +- id: SYN-EX-001 + category: exact_symbol + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_ex_1()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_ex_1()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/ex/1#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-EX-002 + category: exact_symbol + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_ex_2()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_ex_2()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/ex/2#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-EX-003 + category: exact_symbol + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_ex_3()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_ex_3()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/ex/3#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-EX-004 + category: exact_symbol + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_ex_4()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_ex_4()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/ex/4#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-EX-005 + category: exact_symbol + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_ex_5()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_ex_5()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/ex/5#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-EX-006 + category: exact_symbol + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_ex_6()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_ex_6()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/ex/6#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-EX-007 + category: exact_symbol + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_ex_7()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_ex_7()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/ex/7#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-EX-008 + category: exact_symbol + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_ex_8()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_ex_8()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/ex/8#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-EX-009 + category: exact_symbol + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_ex_9()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_ex_9()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/ex/9#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-EX-010 + category: exact_symbol + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_ex_10()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_ex_10()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/ex/10#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-EX-011 + category: exact_symbol + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_ex_11()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_ex_11()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/ex/11#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-EX-012 + category: exact_symbol + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_ex_12()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_ex_12()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/ex/12#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-EX-013 + category: exact_symbol + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_ex_13()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_ex_13()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/ex/13#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-EX-014 + category: exact_symbol + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_ex_14()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_ex_14()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/ex/14#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-EX-015 + category: exact_symbol + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_ex_15()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_ex_15()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/ex/15#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-CN-001 + category: concept + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_cn_1()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_cn_1()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/cn/1#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-CN-002 + category: concept + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_cn_2()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_cn_2()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/cn/2#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-CN-003 + category: concept + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_cn_3()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_cn_3()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/cn/3#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-CN-004 + category: concept + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_cn_4()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_cn_4()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/cn/4#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-CN-005 + category: concept + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_cn_5()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_cn_5()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/cn/5#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-CN-006 + category: concept + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_cn_6()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_cn_6()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/cn/6#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-CN-007 + category: concept + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_cn_7()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_cn_7()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/cn/7#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-CN-008 + category: concept + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_cn_8()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_cn_8()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/cn/8#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-CN-009 + category: concept + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_cn_9()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_cn_9()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/cn/9#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-CN-010 + category: concept + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_cn_10()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_cn_10()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/cn/10#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-XV-001 + category: cross_version + python_version: + - '3.12' + - '3.13' + prompt: '[SYNTHETIC FIXTURE] Between Python 3.12 and 3.13, what changed about the fictional stdlib callable + `synthstdlib.fixture_symbol_xv_1()`?' + answer_key: '[SYNTHETIC FIXTURE] In 3.13, `synthstdlib.fixture_symbol_xv_1()` gained a fictional `strict=` + keyword-only parameter; no such API exists in real CPython.' + citations: + - https://example.invalid/fixture-docs/xv/1#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-XV-002 + category: cross_version + python_version: + - '3.12' + - '3.13' + prompt: '[SYNTHETIC FIXTURE] Between Python 3.12 and 3.13, what changed about the fictional stdlib callable + `synthstdlib.fixture_symbol_xv_2()`?' + answer_key: '[SYNTHETIC FIXTURE] In 3.13, `synthstdlib.fixture_symbol_xv_2()` gained a fictional `strict=` + keyword-only parameter; no such API exists in real CPython.' + citations: + - https://example.invalid/fixture-docs/xv/2#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-XV-003 + category: cross_version + python_version: + - '3.12' + - '3.13' + prompt: '[SYNTHETIC FIXTURE] Between Python 3.12 and 3.13, what changed about the fictional stdlib callable + `synthstdlib.fixture_symbol_xv_3()`?' + answer_key: '[SYNTHETIC FIXTURE] In 3.13, `synthstdlib.fixture_symbol_xv_3()` gained a fictional `strict=` + keyword-only parameter; no such API exists in real CPython.' + citations: + - https://example.invalid/fixture-docs/xv/3#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-XV-004 + category: cross_version + python_version: + - '3.12' + - '3.13' + prompt: '[SYNTHETIC FIXTURE] Between Python 3.12 and 3.13, what changed about the fictional stdlib callable + `synthstdlib.fixture_symbol_xv_4()`?' + answer_key: '[SYNTHETIC FIXTURE] In 3.13, `synthstdlib.fixture_symbol_xv_4()` gained a fictional `strict=` + keyword-only parameter; no such API exists in real CPython.' + citations: + - https://example.invalid/fixture-docs/xv/4#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-XV-005 + category: cross_version + python_version: + - '3.12' + - '3.13' + prompt: '[SYNTHETIC FIXTURE] Between Python 3.12 and 3.13, what changed about the fictional stdlib callable + `synthstdlib.fixture_symbol_xv_5()`?' + answer_key: '[SYNTHETIC FIXTURE] In 3.13, `synthstdlib.fixture_symbol_xv_5()` gained a fictional `strict=` + keyword-only parameter; no such API exists in real CPython.' + citations: + - https://example.invalid/fixture-docs/xv/5#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-XV-006 + category: cross_version + python_version: + - '3.12' + - '3.13' + prompt: '[SYNTHETIC FIXTURE] Between Python 3.12 and 3.13, what changed about the fictional stdlib callable + `synthstdlib.fixture_symbol_xv_6()`?' + answer_key: '[SYNTHETIC FIXTURE] In 3.13, `synthstdlib.fixture_symbol_xv_6()` gained a fictional `strict=` + keyword-only parameter; no such API exists in real CPython.' + citations: + - https://example.invalid/fixture-docs/xv/6#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-XV-007 + category: cross_version + python_version: + - '3.12' + - '3.13' + prompt: '[SYNTHETIC FIXTURE] Between Python 3.12 and 3.13, what changed about the fictional stdlib callable + `synthstdlib.fixture_symbol_xv_7()`?' + answer_key: '[SYNTHETIC FIXTURE] In 3.13, `synthstdlib.fixture_symbol_xv_7()` gained a fictional `strict=` + keyword-only parameter; no such API exists in real CPython.' + citations: + - https://example.invalid/fixture-docs/xv/7#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-XV-008 + category: cross_version + python_version: + - '3.12' + - '3.13' + prompt: '[SYNTHETIC FIXTURE] Between Python 3.12 and 3.13, what changed about the fictional stdlib callable + `synthstdlib.fixture_symbol_xv_8()`?' + answer_key: '[SYNTHETIC FIXTURE] In 3.13, `synthstdlib.fixture_symbol_xv_8()` gained a fictional `strict=` + keyword-only parameter; no such API exists in real CPython.' + citations: + - https://example.invalid/fixture-docs/xv/8#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-XV-009 + category: cross_version + python_version: + - '3.12' + - '3.13' + prompt: '[SYNTHETIC FIXTURE] Between Python 3.12 and 3.13, what changed about the fictional stdlib callable + `synthstdlib.fixture_symbol_xv_9()`?' + answer_key: '[SYNTHETIC FIXTURE] In 3.13, `synthstdlib.fixture_symbol_xv_9()` gained a fictional `strict=` + keyword-only parameter; no such API exists in real CPython.' + citations: + - https://example.invalid/fixture-docs/xv/9#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-XV-010 + category: cross_version + python_version: + - '3.12' + - '3.13' + prompt: '[SYNTHETIC FIXTURE] Between Python 3.12 and 3.13, what changed about the fictional stdlib callable + `synthstdlib.fixture_symbol_xv_10()`?' + answer_key: '[SYNTHETIC FIXTURE] In 3.13, `synthstdlib.fixture_symbol_xv_10()` gained a fictional `strict=` + keyword-only parameter; no such API exists in real CPython.' + citations: + - https://example.invalid/fixture-docs/xv/10#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-XV-011 + category: cross_version + python_version: + - '3.12' + - '3.13' + prompt: '[SYNTHETIC FIXTURE] Between Python 3.12 and 3.13, what changed about the fictional stdlib callable + `synthstdlib.fixture_symbol_xv_11()`?' + answer_key: '[SYNTHETIC FIXTURE] In 3.13, `synthstdlib.fixture_symbol_xv_11()` gained a fictional `strict=` + keyword-only parameter; no such API exists in real CPython.' + citations: + - https://example.invalid/fixture-docs/xv/11#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-XV-012 + category: cross_version + python_version: + - '3.12' + - '3.13' + prompt: '[SYNTHETIC FIXTURE] Between Python 3.12 and 3.13, what changed about the fictional stdlib callable + `synthstdlib.fixture_symbol_xv_12()`?' + answer_key: '[SYNTHETIC FIXTURE] In 3.13, `synthstdlib.fixture_symbol_xv_12()` gained a fictional `strict=` + keyword-only parameter; no such API exists in real CPython.' + citations: + - https://example.invalid/fixture-docs/xv/12#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-XV-013 + category: cross_version + python_version: + - '3.12' + - '3.13' + prompt: '[SYNTHETIC FIXTURE] Between Python 3.12 and 3.13, what changed about the fictional stdlib callable + `synthstdlib.fixture_symbol_xv_13()`?' + answer_key: '[SYNTHETIC FIXTURE] In 3.13, `synthstdlib.fixture_symbol_xv_13()` gained a fictional `strict=` + keyword-only parameter; no such API exists in real CPython.' + citations: + - https://example.invalid/fixture-docs/xv/13#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-XV-014 + category: cross_version + python_version: + - '3.12' + - '3.13' + prompt: '[SYNTHETIC FIXTURE] Between Python 3.12 and 3.13, what changed about the fictional stdlib callable + `synthstdlib.fixture_symbol_xv_14()`?' + answer_key: '[SYNTHETIC FIXTURE] In 3.13, `synthstdlib.fixture_symbol_xv_14()` gained a fictional `strict=` + keyword-only parameter; no such API exists in real CPython.' + citations: + - https://example.invalid/fixture-docs/xv/14#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-XV-015 + category: cross_version + python_version: + - '3.12' + - '3.13' + prompt: '[SYNTHETIC FIXTURE] Between Python 3.12 and 3.13, what changed about the fictional stdlib callable + `synthstdlib.fixture_symbol_xv_15()`?' + answer_key: '[SYNTHETIC FIXTURE] In 3.13, `synthstdlib.fixture_symbol_xv_15()` gained a fictional `strict=` + keyword-only parameter; no such API exists in real CPython.' + citations: + - https://example.invalid/fixture-docs/xv/15#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-PEP-001 + category: pep_adjacent + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_pep_1()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_pep_1()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/pep/1#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-PEP-002 + category: pep_adjacent + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_pep_2()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_pep_2()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/pep/2#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-PEP-003 + category: pep_adjacent + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_pep_3()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_pep_3()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/pep/3#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-PEP-004 + category: pep_adjacent + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_pep_4()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_pep_4()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/pep/4#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-PEP-005 + category: pep_adjacent + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_pep_5()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_pep_5()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/pep/5#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-APP-001 + category: applied + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_app_1()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_app_1()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/app/1#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-APP-002 + category: applied + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_app_2()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_app_2()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/app/2#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-APP-003 + category: applied + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_app_3()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_app_3()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/app/3#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-APP-004 + category: applied + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_app_4()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_app_4()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/app/4#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null +- id: SYN-APP-005 + category: applied + python_version: '3.13' + prompt: '[SYNTHETIC FIXTURE] What does the fictional `synthstdlib.fixture_symbol_app_5()` do in Python + 3.13?' + answer_key: '[SYNTHETIC FIXTURE] `synthstdlib.fixture_symbol_app_5()` is a made-up placeholder API invented + only for tests/benchmarks/fixtures/corpus.sample.yml; it is not part of any real Python standard library.' + citations: + - https://example.invalid/fixture-docs/app/5#synthstdlib + expected_properties: + - mentions-synthetic-symbol + - cites-fixture-source + ambiguity_notes: null From a8656296d938699e800cc7feede4dbd17404ff5d Mon Sep 17 00:00:00 2001 From: Aymen Hammouda Date: Wed, 8 Jul 2026 18:35:40 +0100 Subject: [PATCH 5/6] agent: add corpus validation tests New tests/benchmarks/test_corpus_validation.py (pinned filename). Covers: the real placeholder fixture validates against the real schema; the schema's declared distribution matches the methodology's 15/10/15/5/5; a minimal mechanically generated 50-question corpus passes; each refusal reason fires (duplicate id, missing/empty citations, field entirely absent, wrong category counts, unsupported category); malformed corpus YAML, malformed corpus JSON, and malformed schema JSON are handled as clean BenchmarkValidationError messages rather than raw tracebacks; missing corpus/schema files; empty questions list; and the `python -m benchmarks validate-corpus` CLI both accepts the real fixture (exit 0) and exits 2 with a clean stderr message (no traceback) on an invalid corpus. Refs #94, #71, #63. --- tests/benchmarks/test_corpus_validation.py | 255 +++++++++++++++++++++ 1 file changed, 255 insertions(+) create mode 100644 tests/benchmarks/test_corpus_validation.py diff --git a/tests/benchmarks/test_corpus_validation.py b/tests/benchmarks/test_corpus_validation.py new file mode 100644 index 0000000..7117133 --- /dev/null +++ b/tests/benchmarks/test_corpus_validation.py @@ -0,0 +1,255 @@ +"""Tests for benchmarks/corpus.py and `python -m benchmarks validate-corpus`. + +Covers issue #94's acceptance criteria: the real placeholder fixture +(tests/benchmarks/fixtures/corpus.sample.yml) validates against the real +schema (docs/benchmarks/corpus.schema.json); each refusal reason (duplicate +IDs, missing citations, wrong category counts, unsupported categories) fires +with a clean BenchmarkValidationError message; and malformed YAML/JSON is +handled without a raw traceback. +""" + +from __future__ import annotations + +import copy +import json +import subprocess +import sys +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from benchmarks.corpus import validate_corpus +from benchmarks.runner import BenchmarkValidationError + +REPO_ROOT = Path(__file__).resolve().parents[2] +REAL_SCHEMA_PATH = REPO_ROOT / "docs" / "benchmarks" / "corpus.schema.json" +REAL_FIXTURE_PATH = REPO_ROOT / "tests" / "benchmarks" / "fixtures" / "corpus.sample.yml" + +# Mirrors PUBLIC-BENCHMARK-METHODOLOGY.md "Corpus Design" and the +# 'x-category-distribution' block in docs/benchmarks/corpus.schema.json. +EXPECTED_DISTRIBUTION = { + "exact_symbol": 15, + "concept": 10, + "cross_version": 15, + "pep_adjacent": 5, + "applied": 5, +} + + +def _write_yaml(path: Path, data: dict[str, Any]) -> Path: + path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8") + return path + + +def _question(**overrides: object) -> dict[str, Any]: + base: dict[str, Any] = { + "id": "Q-0000", + "category": "concept", + "python_version": "3.13", + "prompt": "[TEST FIXTURE] placeholder prompt, not a real corpus question.", + "answer_key": "[TEST FIXTURE] placeholder answer key.", + "citations": ["https://example.invalid/fixture-docs/placeholder"], + "expected_properties": ["mentions-placeholder"], + "ambiguity_notes": None, + } + base.update(overrides) + return base + + +def _minimal_valid_questions() -> list[dict[str, Any]]: + """A minimal, mechanically generated 50-question set matching the + methodology distribution -- used only to exercise the validator's + mechanics in tests. Not a real corpus and never written to + docs/benchmarks/corpus.yml. + """ + questions: list[dict[str, Any]] = [] + counter = 0 + for category, count in EXPECTED_DISTRIBUTION.items(): + for _ in range(count): + counter += 1 + python_version: str | list[str] = ( + ["3.12", "3.13"] if category == "cross_version" else "3.13" + ) + questions.append( + _question( + id=f"Q-{category}-{counter:03d}", + category=category, + python_version=python_version, + ) + ) + return questions + + +def _corpus(path: Path, questions: list[dict[str, Any]] | None = None) -> Path: + return _write_yaml( + path, + {"questions": questions if questions is not None else _minimal_valid_questions()}, + ) + + +def test_real_fixture_validates_against_real_schema() -> None: + result = validate_corpus(REAL_FIXTURE_PATH, REAL_SCHEMA_PATH) + + assert result.question_count == 50 + assert result.category_counts == EXPECTED_DISTRIBUTION + ids = [question.id for question in result.questions] + assert len(ids) == len(set(ids)), "fixture must not contain duplicate ids" + + +def test_real_schema_declares_methodology_distribution() -> None: + schema = json.loads(REAL_SCHEMA_PATH.read_text(encoding="utf-8")) + + assert schema["x-category-distribution"] == EXPECTED_DISTRIBUTION + assert sum(schema["x-category-distribution"].values()) == 50 + assert set(schema["$defs"]["question"]["properties"]["category"]["enum"]) == set( + EXPECTED_DISTRIBUTION + ) + + +def test_valid_minimal_corpus_passes(tmp_path: Path) -> None: + result = validate_corpus(_corpus(tmp_path / "corpus.yml"), REAL_SCHEMA_PATH) + + assert result.question_count == 50 + assert result.category_counts == EXPECTED_DISTRIBUTION + + +def test_duplicate_ids_rejected(tmp_path: Path) -> None: + questions = _minimal_valid_questions() + questions[1] = {**questions[1], "id": questions[0]["id"]} + + with pytest.raises(BenchmarkValidationError, match="duplicate corpus question id"): + validate_corpus(_corpus(tmp_path / "corpus.yml", questions), REAL_SCHEMA_PATH) + + +def test_missing_citations_rejected(tmp_path: Path) -> None: + questions = _minimal_valid_questions() + broken = copy.deepcopy(questions[0]) + broken["citations"] = [] + questions[0] = broken + + with pytest.raises(BenchmarkValidationError, match="citations"): + validate_corpus(_corpus(tmp_path / "corpus.yml", questions), REAL_SCHEMA_PATH) + + +def test_citations_field_entirely_missing_is_rejected(tmp_path: Path) -> None: + questions = _minimal_valid_questions() + broken = copy.deepcopy(questions[0]) + del broken["citations"] + questions[0] = broken + + with pytest.raises(BenchmarkValidationError, match="missing required field"): + validate_corpus(_corpus(tmp_path / "corpus.yml", questions), REAL_SCHEMA_PATH) + + +def test_wrong_category_counts_rejected(tmp_path: Path) -> None: + questions = _minimal_valid_questions() + # Flip one 'applied' question to 'concept': applied drops to 4, concept + # rises to 11 -- both now disagree with the schema's declared distribution. + for question in questions: + if question["category"] == "applied": + question["category"] = "concept" + break + + with pytest.raises(BenchmarkValidationError, match="wrong category counts"): + validate_corpus(_corpus(tmp_path / "corpus.yml", questions), REAL_SCHEMA_PATH) + + +def test_unsupported_category_rejected(tmp_path: Path) -> None: + questions = _minimal_valid_questions() + questions[0] = {**questions[0], "category": "not-a-real-category"} + + with pytest.raises(BenchmarkValidationError, match="unsupported category"): + validate_corpus(_corpus(tmp_path / "corpus.yml", questions), REAL_SCHEMA_PATH) + + +def test_malformed_corpus_yaml_raises_validation_error(tmp_path: Path) -> None: + corpus_path = tmp_path / "corpus.yml" + corpus_path.write_text("questions: [unterminated\n", encoding="utf-8") + + with pytest.raises(BenchmarkValidationError, match="not valid YAML"): + validate_corpus(corpus_path, REAL_SCHEMA_PATH) + + +def test_malformed_corpus_json_raises_validation_error(tmp_path: Path) -> None: + corpus_path = tmp_path / "corpus.json" + corpus_path.write_text('{"questions": [', encoding="utf-8") + + with pytest.raises(BenchmarkValidationError, match="not valid JSON"): + validate_corpus(corpus_path, REAL_SCHEMA_PATH) + + +def test_missing_corpus_file_is_rejected(tmp_path: Path) -> None: + with pytest.raises(BenchmarkValidationError, match="does not exist"): + validate_corpus(tmp_path / "missing.yml", REAL_SCHEMA_PATH) + + +def test_missing_schema_file_is_rejected(tmp_path: Path) -> None: + with pytest.raises(BenchmarkValidationError, match="does not exist"): + validate_corpus(_corpus(tmp_path / "corpus.yml"), tmp_path / "missing-schema.json") + + +def test_malformed_schema_json_raises_validation_error(tmp_path: Path) -> None: + schema_path = tmp_path / "schema.json" + schema_path.write_text("{not valid json", encoding="utf-8") + + with pytest.raises(BenchmarkValidationError, match="not valid JSON"): + validate_corpus(_corpus(tmp_path / "corpus.yml"), schema_path) + + +def test_empty_questions_list_is_rejected(tmp_path: Path) -> None: + with pytest.raises(BenchmarkValidationError, match="non-empty 'questions' list"): + validate_corpus(_corpus(tmp_path / "corpus.yml", []), REAL_SCHEMA_PATH) + + +def test_cli_validate_corpus_accepts_real_fixture() -> None: + result = subprocess.run( + [ + sys.executable, + "-m", + "benchmarks", + "validate-corpus", + "--corpus", + str(REAL_FIXTURE_PATH), + "--schema", + str(REAL_SCHEMA_PATH), + ], + capture_output=True, + text=True, + timeout=15, + cwd=REPO_ROOT, + ) + + assert result.returncode == 0, result.stderr + summary = json.loads(result.stdout) + assert summary["question_count"] == 50 + assert summary["category_counts"] == EXPECTED_DISTRIBUTION + + +def test_cli_validate_corpus_exits_nonzero_with_clean_message(tmp_path: Path) -> None: + questions = _minimal_valid_questions() + questions[1] = {**questions[1], "id": questions[0]["id"]} + corpus_path = _corpus(tmp_path / "corpus.yml", questions) + + result = subprocess.run( + [ + sys.executable, + "-m", + "benchmarks", + "validate-corpus", + "--corpus", + str(corpus_path), + "--schema", + str(REAL_SCHEMA_PATH), + ], + capture_output=True, + text=True, + timeout=15, + cwd=REPO_ROOT, + ) + + assert result.returncode == 2 + assert result.stdout == "" + assert "duplicate corpus question id" in result.stderr + assert "Traceback" not in result.stderr From 80b23da72f53d310d83e85509cdc2d9ca95d1205 Mon Sep 17 00:00:00 2001 From: Aymen Hammouda Date: Wed, 8 Jul 2026 19:06:27 +0100 Subject: [PATCH 6/6] agent: add CodeRabbit-requested regression tests for blank citations and CLI malformed-YAML path Addresses both Minor findings from the CodeRabbit review on PR #97: 1. test_blank_citation_entry_rejected (parametrized "" and " "): a citations list whose only entry is empty or whitespace-only must raise BenchmarkValidationError. No validator change needed -- verified benchmarks/corpus.py:_require_nonempty_string_list already requires every citation to be non-empty after strip; this locks that behavior in as regression cover. 2. test_cli_validate_corpus_handles_malformed_yaml_without_traceback: feeds malformed YAML through `python -m benchmarks validate-corpus` as a subprocess and asserts exit code 2, empty stdout, a "not valid YAML" message on stderr, and no Traceback -- proving the CLI path takes the catch-BenchmarkValidationError-then-exit-2 shape rather than crashing. Both additions are purely additive to the new (unmerged) test file; no existing test was modified. Refs #94, #71, #63. --- tests/benchmarks/test_corpus_validation.py | 48 ++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/benchmarks/test_corpus_validation.py b/tests/benchmarks/test_corpus_validation.py index 7117133..4dc1f46 100644 --- a/tests/benchmarks/test_corpus_validation.py +++ b/tests/benchmarks/test_corpus_validation.py @@ -143,6 +143,22 @@ def test_citations_field_entirely_missing_is_rejected(tmp_path: Path) -> None: validate_corpus(_corpus(tmp_path / "corpus.yml", questions), REAL_SCHEMA_PATH) +@pytest.mark.parametrize("blank_citation", ["", " "]) +def test_blank_citation_entry_rejected(tmp_path: Path, blank_citation: str) -> None: + # Regression cover (CodeRabbit review on PR #97): a citations list whose + # only entry is an empty or whitespace-only string must be rejected the + # same way as a missing or empty citations list. validate_corpus requires + # every citation entry to be non-empty after strip + # (benchmarks/corpus.py:_require_nonempty_string_list). + questions = _minimal_valid_questions() + broken = copy.deepcopy(questions[0]) + broken["citations"] = [blank_citation] + questions[0] = broken + + with pytest.raises(BenchmarkValidationError, match="citations"): + validate_corpus(_corpus(tmp_path / "corpus.yml", questions), REAL_SCHEMA_PATH) + + def test_wrong_category_counts_rejected(tmp_path: Path) -> None: questions = _minimal_valid_questions() # Flip one 'applied' question to 'concept': applied drops to 4, concept @@ -253,3 +269,35 @@ def test_cli_validate_corpus_exits_nonzero_with_clean_message(tmp_path: Path) -> assert result.stdout == "" assert "duplicate corpus question id" in result.stderr assert "Traceback" not in result.stderr + + +def test_cli_validate_corpus_handles_malformed_yaml_without_traceback(tmp_path: Path) -> None: + # Regression cover (CodeRabbit review on PR #97): malformed corpus input + # must be handled cleanly through the `python -m benchmarks` CLI path too, + # not only via the direct validate_corpus() library surface -- a raw + # yaml.YAMLError traceback would exit 1 and dump a stack trace instead of + # taking the catch-BenchmarkValidationError-then-exit-2 path. + corpus_path = tmp_path / "corpus.yml" + corpus_path.write_text("questions: [unterminated\n", encoding="utf-8") + + result = subprocess.run( + [ + sys.executable, + "-m", + "benchmarks", + "validate-corpus", + "--corpus", + str(corpus_path), + "--schema", + str(REAL_SCHEMA_PATH), + ], + capture_output=True, + text=True, + timeout=15, + cwd=REPO_ROOT, + ) + + assert result.returncode == 2 + assert result.stdout == "" + assert "not valid YAML" in result.stderr + assert "Traceback" not in result.stderr