diff --git a/dokimi/changelog.d/20261016_000000_judge_validity_stored_context.md b/dokimi/changelog.d/20261016_000000_judge_validity_stored_context.md new file mode 100644 index 000000000..0d948dca4 --- /dev/null +++ b/dokimi/changelog.d/20261016_000000_judge_validity_stored_context.md @@ -0,0 +1,28 @@ +### Added + +- **`eval judge-validity --context stored`** — judges each base with the arms + artifact's verified `judge_materials`, the same material the arms were scored + with. Bases without verified material are dropped before `--bases` / + `--max-bases` sampling; the summary reports `n_without_context` and per-arm + kept/dropped counts in `by_arm` (with `scored` bases in `--sequential` mode). The run exits 2 with a typed scorer or + `--mode pairwise`, on a file with no non-empty verified material, if an arm + loses all its bases, has only `padding` pairs or gets none sampled, if a `--sequential` run would pass + with none of an arm's bases scored, or if no pair is usable. + +### Changed + +- **`eval judge-validity` exits 2 when no pair is usable, in every mode** — a + file with no answer longer than the minimum (or where every degradation is a + no-op) now prints an `Error:` line and exits 2 instead of a `ValueError` + traceback (exit 1), with or without `--context`. + +- **`cross-judge --validity` matches a PASS by judge and context** — a passing + report validates a measure only when both the judge label and the context + (`stored` or `none`) agree, so a no-context PASS no longer validates a judge + that read the stored material. A FAIL still silences every measure of that + judge in any context, and a PASS never lifts it, whatever the report + order. Reports without a `context` key (written before this + change) count as `none`, and a stored measure whose scores fell back to + snippets is `none` too; one mixing stored and snippet scores is `mixed`, + which no PASS validates. `cross-judge --judges` re-scores with the + stored material. diff --git a/dokimi/changelog.d/20261021_000000_napetrov_sequential-unscored-arm.md b/dokimi/changelog.d/20261021_000000_napetrov_sequential-unscored-arm.md new file mode 100644 index 000000000..288d3e775 --- /dev/null +++ b/dokimi/changelog.d/20261021_000000_napetrov_sequential-unscored-arm.md @@ -0,0 +1,7 @@ +### Changed + +- **`eval judge-validity --context stored --sequential` exits 2 when an arm with + verified material got no scored base** — the SPRT can decide on the other arms + and stop first; the run now refuses, as fixed mode does, instead of passing + without covering that arm. A FAIL on the bases that were scored is still + reported as a FAIL (exit 1). diff --git a/dokimi/docs/benchmarking-and-comparison.md b/dokimi/docs/benchmarking-and-comparison.md index 82f76e3c4..d2b3a74f7 100644 --- a/dokimi/docs/benchmarking-and-comparison.md +++ b/dokimi/docs/benchmarking-and-comparison.md @@ -447,19 +447,30 @@ under several judges (absolute, and with `--grounded` against each question's measures, and labels each arm `CONSISTENT ±`, `LEANING ±`, `JUDGE-DEPENDENT` or `NO DETECTABLE EFFECT`, on Holm-adjusted p over the measures that vote. With `--validity` reports, a judge that failed its gate is shown but does not vote -(its grounded and checklist measures included); `--require-validated` leaves -only judges that passed. The stored scores are labelled from the file's -`judge_provider`/`judge_model` (`provider/model (stored)`), so a `--validity` -report for that judge reaches them and a verdict resting on it alone reads -`(one judge)`. Re-scoring resolves each arm's context the way the arms run does, -from the file's `judge_materials`, so a `--judges` judge sees what the stored -judge saw. A file written before `judge_materials` has only the report snippets; -with stored scores on such material, `--judges` refuses (exit 2), and -`--pairwise` or `--checklist` always do, since those judges see only the answers. -`--no-stored` compares those judges with each other. The gate itself (and `eval pairwise` / `eval checklist`) passes no -context, so it validates a judge only in the no-context setting. A label whose significant measures all rest on one judge says `(one judge)`; an escalated pairwise measure counts as its cheap judge's. Gate a judge on -answers from the same answer model it will judge: `cross-judge` warns when the -validity report was measured on another model's answers. +(its grounded and checklist measures included, in any context); +`--require-validated` leaves only judges that passed. The stored measure is +named `provider/model (stored)` from the file's `judge_provider`/`judge_model`, +and a validity report names `provider/model`. A PASS validates only measures of +the same context: a `--context stored` report for measures scored with material, +a `--context none` report (or one without `context`) for the rest. A stored file +whose scores all fell back to snippets counts as `none`; one that mixes stored and +snippet scores counts as `mixed`, which no PASS validates (a FAIL still reaches it). Re-scoring resolves each arm's +context the way the arms run does, from the file's `judge_materials`, so a `--judges` +judge sees what the stored judge saw. A file written before `judge_materials` has +only the report snippets; with stored scores on such material, `--judges` refuses +(exit 2), and `--pairwise` or `--checklist` always do, since those judges see only +the answers. `--no-stored` compares those judges with each other. The gate itself +passes no context by default (`eval pairwise` and `eval checklist` never pass +one), so it validates a judge in the no-context setting; `judge-validity --context +stored` on an arms artifact gates it with each arm's stored material instead. That +run exits 2 with a typed scorer or `--mode pairwise`, on a file with no non-empty +verified material, when an arm loses all its bases, has only `padding` pairs or gets none sampled by +`--bases`, when a `--sequential` run would pass with none of an arm's bases +scored, or when no pair is usable. A label whose +significant measures all rest on one judge says `(one judge)`; an escalated +pairwise measure counts as its cheap judge's. Gate a judge on answers from the +same answer model it will judge: `cross-judge` warns when the validity report was +measured on another model's answers. ```bash python cli.py eval cross-judge --answers run.json --questions golden.json --grounded --both \ diff --git a/dokimi/docs/evaluating-treatments.md b/dokimi/docs/evaluating-treatments.md index cfe792973..22ebea0ce 100644 --- a/dokimi/docs/evaluating-treatments.md +++ b/dokimi/docs/evaluating-treatments.md @@ -72,8 +72,14 @@ judge, 24 bases, one run), the full material raised the mean intact score from 0.674, not significant at this size), unchanged on wrong numbers. A skill arm's delta measured with the material is therefore on a higher level than one measured without it, and the difference between the two is not evidence that -the skill helps. `eval judge-validity` passes no context, so it does not -measure this setting. +the skill helps. `eval judge-validity` passes no context by default; run it on an +arms artifact with `--context stored` to gate the judge the way it scored the +arms — each answer with its arm's stored material (bases with no stored entry are +dropped and counted, except baseline with the no-context marker). It exits 2 +with a typed scorer or `--mode pairwise`, on a file with no non-empty verified +material, when an arm loses all its bases, has only `padding` pairs or gets none sampled by `--bases`, +when a `--sequential` run would pass with none of an arm's bases scored, or +when no pair is usable. Runtime plugins are a separate run dimension, not arms. They modify every arm inside the same `(model, harness, plugin_set)` cell, and the JSON/Markdown diff --git a/dokimi/dokimi/commands/evaluate.py b/dokimi/dokimi/commands/evaluate.py index 78a09b099..0618fcf63 100644 --- a/dokimi/dokimi/commands/evaluate.py +++ b/dokimi/dokimi/commands/evaluate.py @@ -19,7 +19,7 @@ JUDGE_MODEL_HELP, ) from dokimi.eval.judge import JUDGE_PROVIDERS -from dokimi.eval.judge_validity import DEFAULT_MIN_ACCURACY +from dokimi.eval.judge_validity import DEFAULT_MIN_ACCURACY, NoUsablePairs from dokimi.utils import normalize_model_ref @@ -298,14 +298,20 @@ def cmd_eval_judge_validity(args: argparse.Namespace) -> None: # the length-only one, not a judge replacement. # Built once and passed to both modes, so a new scorer kind is a change to # this block, not to each run path. + if args.scorer not in ("judge", "typed") and not args.scorer.startswith("typed:"): + print(f"Error: unknown --scorer {args.scorer!r}; use 'judge' or 'typed:'", file=sys.stderr) + sys.exit(2) + if args.context == "stored" and (args.scorer != "judge" or args.mode == "pairwise"): + # Refused before any model is loaded: neither scorer has a context slot. + what = "the typed scorer" if args.scorer != "judge" else "--mode pairwise" + print(f"Error: --context stored needs a judge that takes context; {what} does not", + file=sys.stderr) + sys.exit(2) typed = None if args.scorer == "judge": judge = Judge(model=args.judge_model, provider=args.judge_provider, temperature=args.judge_temperature) scorer = judge_scorer(judge) label = f"{args.judge_provider}/{args.judge_model}" - elif args.scorer != "typed" and not args.scorer.startswith("typed:"): - print(f"Error: unknown --scorer {args.scorer!r}; use 'judge' or 'typed:'", file=sys.stderr) - sys.exit(2) elif args.mode == "pairwise": # _score_pairs gives pair_scorer precedence, so the pairwise judge would grade # the pairs and the report would carry the typed arm's label. @@ -346,6 +352,20 @@ def cmd_eval_judge_validity(args: argparse.Namespace) -> None: # The budget is part of what was measured: one checkpoint at 512 and at # 2048 tokens is two arms. label = typed.name + (f" max_len={typed.effective_max_len}" if typed.effective_max_len else "") + store = None + if args.context == "stored": + # The gate in the setting arms runs score in: each answer with its arm's + # treatment material, from the artifact's judge_materials. + from dokimi.eval.judge_context import JudgeContextStore + + # from_artifact reads the file's judge_materials, which the Run records do not keep. + data = json.loads(Path(args.answers).read_text(encoding="utf-8")) + store = JudgeContextStore.from_artifact(data) if isinstance(data, dict) else None + if store is None or not any(store.materials().values()): + print("Error: --context stored needs an arms artifact with non-empty stored judge " + "material; this file has none (judge_materials missing, or no record cites a " + "digest that matches)", file=sys.stderr) + sys.exit(2) pair_scorer = None if args.mode == "pairwise": from dokimi.commands.judge_reliability import pairwise_judge @@ -369,13 +389,18 @@ def progress(done: int, total: int) -> None: # overstates the cost per score by the pool size. concurrency=1 if typed is not None else args.concurrency, pair_scorer=pair_scorer, + store=store, ) - if args.sequential: - from dokimi.eval.judge_sequential import run_sequential + try: + if args.sequential: + from dokimi.eval.judge_sequential import run_sequential - report = run_sequential(answers, scorer, max_bases=args.max_bases, **common) - else: - report = run_validity(answers, scorer, n_bases=args.bases, **common) + report = run_sequential(answers, scorer, max_bases=args.max_bases, **common) + else: + report = run_validity(answers, scorer, n_bases=args.bases, **common) + except NoUsablePairs as exc: # none long enough, all no-ops, none with material + print(f"Error: {exc}", file=sys.stderr) + sys.exit(2) # Written before anything is printed: a console that cannot encode the summary must not lose the run. if args.output: @@ -493,6 +518,13 @@ def register(sub, positive_int) -> None: "it PASS or FAIL (ignores --bases; see --max-bases)") validity_p.add_argument("--max-bases", type=positive_int, default=120, dest="max_bases", help="Cap on base answers in --sequential mode (default: 120)") + validity_p.add_argument("--context", choices=["none", "stored"], default="none", + help="What the judge sees with each answer: none (default), or 'stored' — " + "the arm's treatment material from the artifact's judge_materials, the " + "setting an arms run scores skill and docs arms in. Exits 2 with a typed " + "scorer or --mode pairwise, with no verified material, when an arm " + "loses all its bases, has only padding pairs or gets none sampled, when a --sequential run " + "would pass with none of an arm's bases scored, or when no pair is usable") validity_p.add_argument("--mode", choices=["absolute", "pairwise"], default="absolute", help="absolute: score each member 0-100 (the shipped judge); pairwise: show " "the judge both members, in both orders, and ask which is better") diff --git a/dokimi/dokimi/commands/judge_reliability.py b/dokimi/dokimi/commands/judge_reliability.py index 4854d019c..3c0277f78 100644 --- a/dokimi/dokimi/commands/judge_reliability.py +++ b/dokimi/dokimi/commands/judge_reliability.py @@ -189,7 +189,7 @@ def cmd_eval_judge_wording(args: argparse.Namespace) -> None: if unknown: print(f"Error: unknown wording(s): {unknown}; known: {sorted(WORDINGS)}", file=sys.stderr) sys.exit(2) - pairs = _build(answers, {}, n_bases=args.bases, seed=args.seed, variants=variants, library=library) + pairs = _build(answers, {}, n_bases=args.bases, seed=args.seed, variants=variants, library=library).pairs label = f"{args.judge_provider}/{args.judge_model}" n_cond = 1 + args.redraws + len(wordings) print(f"Judge wording sensitivity: {label}, {len(pairs)} pairs x {n_cond} conditions " @@ -366,7 +366,7 @@ def score(library, question, answer, ground_truth, context): print(f"⚠️ --validity for `{v.get('judge')}` was measured on {v['answers_model']} answers, " f"these are {answers_model}'s — the gate may not transfer", file=sys.stderr) for missed in report.apply_validity(validity): - print(f"⚠️ --validity for `{missed}` matches no measure (compare the judge labels)", file=sys.stderr) + print(f"⚠️ --validity for `{missed}` matches no measure (compare the judge labels and contexts)", file=sys.stderr) if not report.measures: print("Error: nothing to compare — give --judges, --checklist, or an answers file with evaluations", file=sys.stderr) diff --git a/dokimi/dokimi/eval/cross_judge.py b/dokimi/dokimi/eval/cross_judge.py index 238f44d91..df9c18ae2 100644 --- a/dokimi/dokimi/eval/cross_judge.py +++ b/dokimi/dokimi/eval/cross_judge.py @@ -74,6 +74,9 @@ class Measure: scores: Dict[Tuple[Union[str, int], str], Optional[float]] = field(default_factory=dict) #: "PASS" / "FAIL" from this judge's own judge-validity run; None = not validated. gate: Optional[str] = None + #: "stored" when every score saw the stored material, "none" when none did, + #: "mixed" when a stored measure's scores differ (no PASS validates it). + context: str = "none" def versus(self, arm: str, baseline: str) -> Dict[str, Any]: pairs = [ @@ -212,10 +215,17 @@ def stored_measure(data: Dict[str, Any], name: Optional[str] = None) -> Optional binary file's condition scores are not this measure. """ m = Measure(name=name or stored_label(data), kind="aggregate") + sources = set() for r in _unjoined(data, "evaluations"): for arm, s in arm_scores(r).items(): if s.aggregate is not None: m.scores[(r.question_id, arm)] = float(s.aggregate) + sources.add(s.raw.get("judge_context_source") == SOURCE_STORED) + # Per score: "stored" only when every score saw the stored material, "none" + # when none did (snippet_fallback, pre-judge_context files). A file mixing + # both is "mixed": no single-context PASS describes it, so none validates it + # (a FAIL of the judge still reaches it). + m.context = {frozenset({True}): "stored", frozenset({False}): "none"}.get(frozenset(sources), "mixed") return m if m.scores else None @@ -276,11 +286,12 @@ def score_measure( what a run before ``judge_context`` showed its judge. """ store = store if store is not None else JudgeContextStore() - jobs = [ - (r.question_id, arm, question, a.text, - store.resolve_with_source(r.question_id, arm, dict(a.raw))[0]) - for r, arm, a, question in scored_answers(records, arms) - ] + jobs = [] + stored_flags: set = set() # per answer: True if the judge saw stored material + for r, arm, a, question in scored_answers(records, arms): + context, source = store.resolve_with_source(r.question_id, arm, dict(a.raw)) + stored_flags.add(source == SOURCE_STORED) + jobs.append((r.question_id, arm, question, a.text, context)) def one(job): qid, arm, question, answer, context = job @@ -291,7 +302,15 @@ def one(job): except Exception: # an unscored answer drops out of the pairing, it does not kill the run return (qid, arm), None - m = Measure(name=name, kind="grounded" if ground_truth else "aggregate") + if not stored_flags: + context_label = "none" + elif stored_flags == {True}: + context_label = "stored" + elif stored_flags == {False}: + context_label = "none" + else: + context_label = "mixed" # some answers fell back to snippets: no single-context PASS describes it + m = Measure(name=name, kind="grounded" if ground_truth else "aggregate", context=context_label) with ThreadPoolExecutor(max_workers=max(1, concurrency)) as pool: for i, (key, value) in enumerate(pool.map(one, jobs), 1): m.scores[key] = value @@ -318,32 +337,51 @@ class CrossJudgeReport: def apply_validity(self, validity: Sequence[Dict[str, Any]]) -> List[str]: """Mark measures with their judge's gate result; returns validity labels that matched nothing. - ``validity`` holds ``eval judge-validity`` summaries. A summary marks - the absolute or pairwise measure of the same judge label PASS or FAIL. - A FAIL also reaches that judge's grounded and checklist-coverage - measures — the gate tested neither, but a judge that cannot order - damaged answers is not trusted to vote just because it was handed - reference points or a list of them. A PASS does not carry over: those - measures stay unvalidated. + ``validity`` holds ``eval judge-validity`` summaries. A PASS marks the + absolute or pairwise measure of the same judge label and the same + context (``none`` or ``stored``; a report without ``context`` counts as + ``none``): the gate only validated the setting it ran in. A FAIL reaches + every measure of that judge label, in any context, including its + grounded and checklist-coverage measures — a judge that cannot order + damaged answers in one setting is not trusted to vote in another. A + PASS does not carry over to grounded or coverage measures: those stay + unvalidated. A label that matched a measure only in another context is + returned with a note naming both contexts. A FAIL is never overwritten + by a PASS, so the result does not depend on the order of ``validity``. """ unmatched = [] for v in validity: passed = bool(v.get("passed")) + v_context = v.get("context", "none") + v_judge = v.get("judge") hits = 0 + other_contexts = set() for m in self.measures: - if m.name.removesuffix(STORED) != v.get("judge"): + m_name = m.name.removesuffix(STORED) + if m_name != v_judge: continue - if m.kind in ("aggregate", "pairwise"): - m.gate = "PASS" if passed else "FAIL" - hits += 1 - elif not passed: - # Independent of whether the same judge also has an aggregate measure - # in this report: --grounded without --both, or --checklist alone, leaves - # only these, and a FAIL has to reach them all the same. + if not passed: + # Every kind and context, independent of whether the same judge also has + # an aggregate measure here: --grounded without --both, or --checklist + # alone, leaves only those, and a FAIL has to reach them all the same. m.gate = "FAIL" hits += 1 + elif m.kind not in ("aggregate", "pairwise"): + # A PASS never validates grounded or coverage measures, so their context + # says nothing about this PASS. + continue + elif m.context != v_context: + # A PASS validates only the (label, context) it was measured in. + other_contexts.add(m.context) + elif m.kind in ("aggregate", "pairwise"): + # A FAIL is final: a PASS never lifts it, whichever report came first. + if m.gate != "FAIL": + m.gate = "PASS" + hits += 1 if not hits: - unmatched.append(str(v.get("judge"))) + unmatched.append( + f"{v_judge} (report context {v_context}; its measures are " + f"{', '.join(sorted(other_contexts))})" if other_contexts else str(v_judge)) return unmatched def summary(self) -> Dict[str, Any]: diff --git a/dokimi/dokimi/eval/degrade.py b/dokimi/dokimi/eval/degrade.py index 18b9e5272..ad184faa3 100644 --- a/dokimi/dokimi/eval/degrade.py +++ b/dokimi/dokimi/eval/degrade.py @@ -45,11 +45,14 @@ import random import re from decimal import Context, Decimal -from typing import Any, Callable, Dict, Iterable, List, Optional, Sequence +from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Optional, Sequence from dokimi.eval.question_text import require_question_text from dokimi.records import RecordLike, records +if TYPE_CHECKING: + from dokimi.eval.judge_context import JudgeContextStore + #: Minimum answer length (characters) worth degrading. Short answers have too #: little content for truncation or code-stripping to mean anything. DEFAULT_MIN_CHARS = 1500 @@ -612,12 +615,17 @@ def build_pairs( variants: Optional[Sequence[str]] = None, library: str = "the library", noop: Optional[Dict[str, int]] = None, + store: Optional["JudgeContextStore"] = None, + dropped: Optional[Dict[str, int]] = None, + by_arm: Optional[Dict[str, Dict[str, int]]] = None, ) -> List[Dict[str, Any]]: """Build (intact, degraded) pairs with a known correct ordering. One pair per (base answer x variant); ``intact`` must score higher. A degradation that left the text unchanged is dropped and counted in ``noop``. On a neutral variant (``padding``) the target is the same score instead. + + If ``store`` is provided, only answers with material in the store are used as bases. """ for name in variants or (): if name not in DEGRADATIONS: @@ -625,13 +633,32 @@ def build_pairs( pool = _answer_pool(answers) rng = random.Random(seed) rng.shuffle(pool) - usable = [row for row in pool if len(row["answer"]) > min_chars] + long_answers = [row for row in pool if len(row["answer"]) > min_chars] + long_answers_before_store = len(long_answers) + # Filter by material before sampling if store is provided + if store is not None: + # One pass: decide kept or dropped per row and count it there. + has_material = [] + for row in long_answers: + ok = store.get(row["question_id"], row["arm"]) is not None + if ok: + has_material.append(row) + if by_arm is not None: + counts = by_arm.setdefault(row["arm"], {"kept": 0, "dropped": 0, "sampled": 0}) + counts["kept" if ok else "dropped"] += 1 + long_answers = has_material + if dropped is not None: + dropped["total"] = long_answers_before_store - len(long_answers) # Refused, not skipped: a pair built without its question is graded against nothing. Every - # usable row is checked (not only the drawn bases), so the outcome does not depend on the seed. - for row in usable: + # row that can become a base is checked (not only the drawn bases), so the outcome does not + # depend on the seed; a row dropped for missing judge material is never judged. + for row in long_answers: if not row["question_text"].strip(): require_question_text(*row["_record"]) - bases = usable[:n_bases] + bases = long_answers[:n_bases] + if store is not None and by_arm is not None: + for row in bases: + by_arm[row["arm"]]["sampled"] += 1 pairs: List[Dict[str, Any]] = [] for base in bases: diff --git a/dokimi/dokimi/eval/judge_context.py b/dokimi/dokimi/eval/judge_context.py index 5cae4bc80..55be99805 100644 --- a/dokimi/dokimi/eval/judge_context.py +++ b/dokimi/dokimi/eval/judge_context.py @@ -248,7 +248,9 @@ def from_artifact(cls, data: Dict[str, Any]) -> "JudgeContextStore": if bad: logger.warning( "%d arm records cite judge material that is missing from the artifact " - "or does not match its digest; those answers fall back to snippets.", bad, + "or does not match its digest; those answers have no verified material " + "(callers that fall back use the report snippets; judge-validity " + "--context stored drops them).", bad, ) return store @@ -262,8 +264,9 @@ def get(self, question_id: Union[str, int], arm_name: str) -> Optional[str]: self._warned = True logger.warning( "No stored judge context for %s/%s (the run did not keep it, or the text " - "failed its digest), so judging falls back to 200-char report snippets, which under-scores " - "arms that cite their own material.", + "failed its digest). Callers that fall back judge the 200-char report " + "snippets, which under-scores arms that cite their own material; " + "judge-validity --context stored drops such bases instead.", question_id, arm_name, ) return hit diff --git a/dokimi/dokimi/eval/judge_sequential.py b/dokimi/dokimi/eval/judge_sequential.py index 4543c08e8..d84d86968 100644 --- a/dokimi/dokimi/eval/judge_sequential.py +++ b/dokimi/dokimi/eval/judge_sequential.py @@ -29,11 +29,14 @@ from dokimi.eval.degrade import DEFAULT_MIN_CHARS, DEGRADATIONS, NEUTRAL_VARIANTS, _answer_pool from dokimi.eval.judge_validity import ( DEFAULT_MIN_ACCURACY, + NoUsablePairs, PairScorer, Scorer, ValidityReport, _build, _score_pairs, + refuse_unpaired_arms, + require_context_scorer, ) #: Sequential mode tests H0 "accuracy = gate - delta" against H1 "accuracy = @@ -120,6 +123,7 @@ def run_sequential( progress: Optional[Callable[[int, int], None]] = None, concurrency: int = 1, pair_scorer: Optional[PairScorer] = None, + store: Optional[Any] = None, ) -> ValidityReport: """Like :func:`run_validity`, but add bases until every variant is decided. @@ -130,14 +134,25 @@ def run_sequential( is gated on them. """ sprt_bounds(min_accuracy) # refuse an undecidable gate before any pair is built or scored + if store is not None: + require_context_scorer(scorer) noop: Dict[str, int] = {} - pairs = _build(answers, noop, n_bases=max_bases, seed=seed, variants=variants, library=library) + by_arm: Dict[str, Dict[str, int]] = {} + built = _build(answers, noop, n_bases=max_bases, seed=seed, variants=variants, library=library, + store=store, by_arm=by_arm) + pairs = built.pairs + if store is not None: + refuse_unpaired_arms(pairs, by_arm, "--max-bases", variants) order: List[tuple] = list(dict.fromkeys((p["base_id"], p.get("arm", "")) for p in pairs)) graded = sorted({p["variant"] for p in pairs if p["variant"] not in NEUTRAL_VARIANTS}) decisions: Dict[str, Dict[str, Any]] = {v: {"decision": "continue", "llr": 0.0, "n": 0} for v in graded} report = ValidityReport(judge=judge, min_accuracy=min_accuracy, seed=seed, sequential=decisions) report.n_noop = noop report.variants = list(variants or DEGRADATIONS) + report.context = "stored" if store is not None else "none" + report.n_without_context = built.n_dropped + report.by_arm = by_arm + scored_bases: set = set() intact_cache: Dict[tuple, Any] = {} started = time.time() for start in range(0, len(order), batch): @@ -151,6 +166,11 @@ def run_sequential( and (p["variant"] in open_variants or (p["variant"] in NEUTRAL_VARIANTS and start < NEUTRAL_MAX_BASES)) ] + # Only a graded pair covers an arm: a neutral variant (padding) feeds no + # gate decision, so a base scored by it alone leaves the arm unchecked. + scored_bases.update( + (p["base_id"], p.get("arm", "")) for p in todo if p["variant"] not in NEUTRAL_VARIANTS + ) report.rows += _score_pairs( scorer, todo, intact_cache, concurrency=concurrency, progress=progress, done_offset=len(report.rows), total=len(pairs), pair_scorer=pair_scorer, @@ -168,10 +188,30 @@ def run_sequential( # them ran out of pool even when another variant reached the cap. With # more usable answers than the cap, a higher --max-bases adds bases to # every variant, so that is the cap even for a variant with few pairs. - usable = sum(1 for r in _answer_pool(answers) if len(r["answer"]) > DEFAULT_MIN_CHARS) + # With a store, only answers with verified material can ever be scored. + if store is not None: + usable = sum(c["kept"] for c in by_arm.values()) + else: + usable = sum(1 for r in _answer_pool(answers) if len(r["answer"]) > DEFAULT_MIN_CHARS) for variant, d in decisions.items(): if d["decision"] == "continue": bases = len({(p["base_id"], p.get("arm", "")) for p in pairs if p["variant"] == variant}) d["stopped_by"] = "cap" if bases >= max_bases or usable > max_bases else "pool" + # The pool is built up to --max-bases, but the SPRT may stop early: report + # per arm the bases that were actually scored, not the pool that was built. + for arm, counts in by_arm.items(): + counts["scored"] = sum(1 for _, a in scored_bases if a == arm) + if store is not None: + # The loop stops once every variant is decided, which can happen before + # an arm's bases are reached; the gate would then pass without covering it. + # A FAIL on the bases that were scored stands whatever the other arms hold. + unscored = sorted(a for a, c in by_arm.items() if c.get("kept", 0) > 0 and c["scored"] == 0) + if unscored and report.summary()["passed"]: + raise NoUsablePairs( + f"Arm(s) {unscored} have verified judge material but no base was scored " + "before the sequential test stopped, so the gate would not cover them. A higher " + "--max-bases does not change where it stops: run without --sequential and with " + "--bases at least the number of answers, or remove those arms from the answers file" + ) report.wall_seconds = time.time() - started return report diff --git a/dokimi/dokimi/eval/judge_validity.py b/dokimi/dokimi/eval/judge_validity.py index 2c9fbf4fc..df5b90c08 100644 --- a/dokimi/dokimi/eval/judge_validity.py +++ b/dokimi/dokimi/eval/judge_validity.py @@ -53,6 +53,7 @@ from __future__ import annotations +import inspect import time from dataclasses import dataclass, field from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple @@ -139,6 +140,13 @@ def _move(mean_drop: float) -> str: return f"{move:+.1f} ({lean}" +class NoUsablePairs(ValueError): + """The answers yield no pair to score (too short, all no-ops, no stored material). + + A user-facing condition, distinct from a ``ValueError`` raised by a bug. + """ + + @dataclass class ValidityReport: """Per-variant and overall validity of one judge configuration.""" @@ -154,6 +162,19 @@ class ValidityReport: #: Set by :func:`run_sequential`: per graded variant, its SPRT decision, #: log-likelihood ratio and the pairs it took. ``None`` for a fixed-n run. sequential: Optional[Dict[str, Dict[str, Any]]] = None + #: What the judge was shown with each answer: ``none`` (the default; the + #: gate's original setting) or ``stored`` (the arm's treatment material from + #: the artifact's ``judge_materials`` — the setting arms runs score in). + context: str = "none" + #: With ``context="stored"``, eligible answers dropped because the file held + #: no verified material for their arm, counted over the whole pool before + #: ``--bases`` sampling; scoring them without it would mix the two settings. + n_without_context: int = 0 + #: With ``context="stored"``, per arm: eligible answers kept (material + #: verified) and dropped (none), before ``--bases`` sampling, and the bases + #: sampled from the kept ones. In --sequential mode also ``scored``: the + #: bases that got rows before the SPRT stopped (``sampled`` is the pool built). + by_arm: Dict[str, Dict[str, int]] = field(default_factory=dict) @property def scored(self) -> List[ValidityRow]: @@ -287,6 +308,9 @@ def summary(self) -> Dict[str, Any]: "unmeasured_variants": unmeasured, "per_variant": per_variant, "n_noop": dict(sorted(self.n_noop.items())), + "context": self.context, + "n_without_context": self.n_without_context, + "by_arm": {a: dict(c) for a, c in sorted(self.by_arm.items())}, "wall_seconds": self.wall_seconds, } if self.sequential is not None: @@ -307,7 +331,9 @@ def _format_inconclusive(self, s: Dict[str, Any]) -> List[str]: v = s["per_variant"][variant] if "sequential" in v: why = ( - "answer pool exhausted — more answers, not a higher --max-bases, would settle it" + "answer pool exhausted — more answers" + + (" with verified material" if s.get("n_without_context") else "") + + ", not a higher --max-bases, would settle it" if v["sequential"].get("stopped_by") == "pool" else "cap reached — true accuracy is inside the gate's indifference band" ) @@ -338,6 +364,14 @@ def format_summary(self) -> str: f" length-only : {s['length_accuracy']:.3f} (len(answer) alone — not a judge)", f" tie rate : {s['tie_rate']:.3f} (saturated scale if high)", f" no-op pairs : {sum(s['n_noop'].values())} dropped (degradation left the text unchanged)", + *([" context : stored treatment material (as the arms run judged)" + + (f"; {s['n_without_context']} answers without verified material " + "(counted before sampling)" + if s["n_without_context"] else "")] if s.get("context") == "stored" else []), + *([f" per arm : {a}: {c['kept']} kept, {c['dropped']} dropped, " + + (f"{c['scored']} bases scored" if "scored" in c + else f"{c.get('sampled', 0)} bases sampled") + for a, c in s["by_arm"].items()] if s.get("context") == "stored" else []), ] if s["mode"] == "sequential": t = s["sprt"] @@ -427,16 +461,59 @@ def judge_scorer(judge: Any, *, draw: int = 0) -> Scorer: against the excerpt rather than judge the answer itself. """ - def score(library: str, question: str, answer: str) -> float: + def score(library: str, question: str, answer: str, context: Optional[str] = None) -> float: + kwargs: Dict[str, Any] = {"library_name": library, "question": question, "answer": answer} if draw: - scores = judge.score_answer(library_name=library, question=question, answer=answer, draw=draw) - else: - scores = judge.score_answer(library_name=library, question=question, answer=answer) - return float(scores["aggregate"]) + kwargs["draw"] = draw + if context is not None: + kwargs["context"] = context + return float(judge.score_answer(**kwargs)["aggregate"]) return score +def require_context_scorer(scorer: Callable[..., Any]) -> None: + """With a store, pairs are scored with ``context=``: refuse a scorer that cannot take it + before any pair is built, rather than failing every pair mid-run.""" + try: + params = inspect.signature(scorer).parameters + except (TypeError, ValueError): # a callable with no introspectable signature: let the call decide + return + takes_context = "context" in params or any( + p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values()) + if not takes_context: + raise NoUsablePairs( + "The scorer does not accept a `context=` argument, so it cannot judge stored " + "material: add a `context` parameter, or run without a store") + + +def refuse_unpaired_arms( + pairs: Sequence[Dict[str, Any]], by_arm: Dict[str, Dict[str, int]], flag: str, + variants: Optional[Sequence[str]] = None, +) -> None: + """Refuse before scoring if an arm with kept material has no pair left: the + sample can leave it out, or every one of its variants can be a no-op.""" + # Only a graded pair covers an arm: padding feeds no gate decision. A run + # that requests no graded variant is left to fail as unmeasured. + graded = [v for v in (variants or DEGRADATIONS) if v not in NEUTRAL_VARIANTS] + paired_arms = {p.get("arm", "") for p in pairs if not graded or p["variant"] not in NEUTRAL_VARIANTS} + starved = sorted(a for a, c in by_arm.items() if c.get("kept", 0) > 0 and a not in paired_arms) + if not starved: + return + # A larger sample only helps an arm whose kept bases were not all drawn yet. + unsampled = [a for a in starved if by_arm[a].get("sampled", 0) < by_arm[a]["kept"]] + if unsampled: + raise NoUsablePairs( + f"Arm(s) {unsampled} have verified judge material but no base was scored, " + f"so the gate would not cover them: raise {flag}, or remove those arms from the answers file" + ) + raise NoUsablePairs( + f"Arm(s) {starved} have every kept base sampled, and every requested graded variant " + f"({', '.join(graded or variants) if variants else 'all'}) left its text unchanged: a higher {flag} " + "cannot help; use other degradations, or remove those arms from the answers file" + ) + + def run_validity( answers: Sequence[RecordLike], scorer: Scorer, @@ -450,21 +527,64 @@ def run_validity( progress: Optional[Callable[[int, int], None]] = None, concurrency: int = 1, pair_scorer: Optional[PairScorer] = None, + store: Optional[Any] = None, ) -> ValidityReport: - """Build degradation pairs from ``answers`` and measure the judge on them.""" + """Build degradation pairs from ``answers`` and measure the judge on them. + + With *store* (``JudgeContextStore.from_artifact(data)``) each answer is + judged with its arm's stored treatment material, as the arms run judged it. + """ + if store is not None: + require_context_scorer(scorer) noop: Dict[str, int] = {} variants = list(variants or DEGRADATIONS) - pairs = _build(answers, noop, n_bases=n_bases, seed=seed, variants=variants, library=library) + by_arm: Dict[str, Dict[str, int]] = {} + built = _build(answers, noop, n_bases=n_bases, seed=seed, variants=variants, library=library, + store=store, by_arm=by_arm) + pairs = built.pairs + if store is not None: + refuse_unpaired_arms(pairs, by_arm, "--bases", variants) report = evaluate_pairs( scorer, pairs, judge=judge, min_accuracy=min_accuracy, seed=seed, progress=progress, concurrency=concurrency, pair_scorer=pair_scorer, ) report.n_noop = noop report.variants = variants + report.context = "stored" if store is not None else "none" + report.n_without_context = built.n_dropped + report.by_arm = by_arm return report -def _build(answers, noop, *, n_bases, seed, variants, library) -> List[Dict[str, Any]]: +def attach_stored_context(pairs: List[Dict[str, Any]], store: Any) -> List[Dict[str, Any]]: + """Give each pair its arm's stored material. + + ``build_pairs`` has already dropped every base the store has no entry for, + so each pair here resolves. An arm whose stored + material is empty (``baseline``) is judged with the no-context marker, which + is what the arms run showed its judge for it. + """ + from dokimi.eval.judge_context import NO_CONTEXT + + kept = [] + for pair in pairs: + text = store.get(pair["base_id"], pair.get("arm", "")) + kept.append({**pair, "context": text or NO_CONTEXT}) + return kept + + +@dataclass(frozen=True) +class Built: + """Output of :func:`_build`: the pairs, and how many answers had no stored material.""" + + pairs: List[Dict[str, Any]] + n_dropped: int = 0 + + +def _build(answers, noop, *, n_bases, seed, variants, library, store=None, + by_arm=None) -> Built: + dropped_from_pairs: Dict[str, int] = {} + arms: Dict[str, Dict[str, int]] = by_arm if by_arm is not None else {} pairs = build_pairs( answers, noop=noop, @@ -472,12 +592,29 @@ def _build(answers, noop, *, n_bases, seed, variants, library) -> List[Dict[str, seed=seed, variants=variants or tuple(DEGRADATIONS), library=library, + store=store, + dropped=dropped_from_pairs, + by_arm=arms, ) - if not pairs and noop: - raise ValueError(f"Every degradation left the answer unchanged (no-op pairs: {noop})") if not pairs: - raise ValueError( + if noop: + raise NoUsablePairs(f"Every degradation left the answer unchanged (no-op pairs: {noop})") + if store is not None and dropped_from_pairs.get("total", 0): + raise NoUsablePairs( + "No answer has verified stored judge material (judge_materials) — the file was " + "written before arms runs kept it, or the material does not match its digests" + ) + raise NoUsablePairs( "No usable answers found — need answers longer than the minimum " "length with a question_id and question text" ) - return pairs + if store is not None: + lost = sorted(a for a, c in arms.items() if c["dropped"] and not c["kept"]) + if lost: + raise NoUsablePairs( + f"Arm(s) {lost} have answers but no verified judge material, so the gate would " + "not cover them: repair the artifact's judge_materials, or remove those arms from the answers file" + ) + pairs = attach_stored_context(pairs, store) + return Built(pairs, dropped_from_pairs.get("total", 0)) + return Built(pairs) diff --git a/dokimi/dokimi/eval/validity_scoring.py b/dokimi/dokimi/eval/validity_scoring.py index c1213c964..bdcabac8e 100644 --- a/dokimi/dokimi/eval/validity_scoring.py +++ b/dokimi/dokimi/eval/validity_scoring.py @@ -105,6 +105,13 @@ def to_dict(self) -> Dict[str, Any]: } +def _call(scorer: Scorer, pair: Dict[str, Any], answer: str) -> float: + """Score one member of a pair, with the arm's stored material when the pair carries it.""" + if "context" in pair: + return float(scorer(pair["library"], pair["question"], answer, context=pair["context"])) # type: ignore[call-arg] + return float(scorer(pair["library"], pair["question"], answer)) + + def _score_pairs( scorer: Scorer, pairs: Sequence[Dict[str, Any]], @@ -129,7 +136,7 @@ def intact(pair: Dict[str, Any]) -> Tuple[tuple, Any]: failure: Exception = RuntimeError("intact answer was not scored") for _ in range(INTACT_ATTEMPTS): try: - return key, float(scorer(pair["library"], pair["question"], pair["intact"])) + return key, _call(scorer, pair, pair["intact"]) except Exception as exc: # one bad pair must not lose the whole run failure = exc return key, failure @@ -153,9 +160,7 @@ def degraded(pair: Dict[str, Any]) -> ValidityRow: if isinstance(base, Exception): raise base row.intact_score = base - row.degraded_score = float( - scorer(pair["library"], pair["question"], pair["degraded"]) - ) + row.degraded_score = _call(scorer, pair, pair["degraded"]) except Exception as exc: # one bad pair must not lose the whole run row.error = f"{type(exc).__name__}: {exc}"[:200] return row diff --git a/dokimi/dokimi/report/model_compare.py b/dokimi/dokimi/report/model_compare.py index f6fd0b382..b88cf773a 100644 --- a/dokimi/dokimi/report/model_compare.py +++ b/dokimi/dokimi/report/model_compare.py @@ -307,8 +307,9 @@ def _check_judge_context(runs: list[tuple[str, dict]], group_name: str) -> None: f"Error [{group_name}]: runs were judged under different judge_context " f"protocols ({detail}). The judge saw different evidence for the same " "answers, so the scores cannot be pooled. Re-run answering and judging " - "together for the older runs: re-judging an answers file only sees the " - "200-char report snippets, which is the unstamped protocol again." + "together for the older runs: re-judging an answers file written before " + "judge_materials only sees the 200-char report snippets, which is the " + "unstamped protocol again." ) diff --git a/dokimi/tests/test_judge_reliability.py b/dokimi/tests/test_judge_reliability.py index 11c27fc7d..f65e79039 100644 --- a/dokimi/tests/test_judge_reliability.py +++ b/dokimi/tests/test_judge_reliability.py @@ -322,7 +322,10 @@ def test_a_failed_gate_also_silences_that_judges_grounded_measure(): report.apply_validity([{"judge": "p/j", "passed": False}]) assert (absolute.gate, grounded.gate) == ("FAIL", "FAIL") report.apply_validity([{"judge": "p/j", "passed": True}]) - assert absolute.gate == "PASS" and grounded.gate == "FAIL" # a PASS does not validate grounded scoring + assert (absolute.gate, grounded.gate) == ("FAIL", "FAIL") # a later PASS does not lift a FAIL + fresh = cj.Measure(name="p/j", kind="grounded") + cj.CrossJudgeReport(baseline_arm="baseline", measures=[fresh]).apply_validity([{"judge": "p/j", "passed": True}]) + assert fresh.gate != "PASS" # a PASS does not validate grounded scoring @pytest.mark.parametrize("rows,expected", [ @@ -478,9 +481,18 @@ def test_stored_measure_is_named_for_its_judge(): report = cj.CrossJudgeReport(baseline_arm="baseline", measures=[stored, rescored]) # Both measures are Haiku: agreement between them is not agreement across judges. assert report.summary()["arms"]["skill"]["label"] == "CONSISTENT + (one judge)" - # A failed gate for that judge reaches the stored scores, so they stop voting. - assert report.apply_validity([{"judge": "amazon-bedrock/haiku", "passed": False}]) == [] - assert stored.gate == "FAIL" + assert stored.context == "stored" and rescored.context == "none" + # A PASS gates only the measure of its own context. + assert report.apply_validity([{"judge": "amazon-bedrock/haiku", "context": "stored", "passed": True}]) == [] + assert (stored.gate, rescored.gate) == ("PASS", None) + stored.gate = None + assert report.apply_validity([{"judge": "amazon-bedrock/haiku", "passed": True}]) == [] # no key: none + assert (stored.gate, rescored.gate) == (None, "PASS") + # A FAIL in either context reaches every measure of the judge. + for ctx in ("stored", "none"): + stored.gate = rescored.gate = None + assert report.apply_validity([{"judge": "amazon-bedrock/haiku", "context": ctx, "passed": False}]) == [] + assert (stored.gate, rescored.gate) == ("FAIL", "FAIL") assert cj.stored_measure({"evaluations": data["evaluations"]}).name == "stored judge" @@ -505,6 +517,38 @@ def test_an_arm_whose_material_the_file_keeps_is_still_unreproducible_for_pairwi assert cj.unreproducible_arms(data) == ["skill"] +@pytest.mark.parametrize("source", ["snippet_fallback", None]) +def test_a_stored_measure_judged_on_snippets_is_none_context_and_gated_by_a_none_report(source): + data = _stamped_run(source) if source else _stamped_run() + if source is None: # a file from before judge_context: no source at all + for ev in data["evaluations"]: + for s in ev["scores"].values(): + s.pop("judge_context_source", None) + stored = cj.stored_measure(data) + assert stored.context == "none" + report = cj.CrossJudgeReport(baseline_arm="baseline", measures=[stored]) + assert report.apply_validity([{"judge": "amazon-bedrock/haiku", "passed": True}]) == [] + assert stored.gate == "PASS" + + +def test_a_stored_fail_also_silences_the_judges_coverage_and_pairwise_measures(): + measures = [cj.Measure(name="p/j", kind=k, context=c) for k, c in + (("coverage", "none"), ("pairwise", "none"), ("grounded", "stored"), ("aggregate", "stored"))] + report = cj.CrossJudgeReport(baseline_arm="baseline", measures=measures) + assert report.apply_validity([{"judge": "p/j", "context": "stored", "passed": False}]) == [] + assert [m.gate for m in measures] == ["FAIL"] * 4 + + +@pytest.mark.parametrize("order", [(0, 1), (1, 0)]) +def test_a_fail_wins_over_a_pass_in_either_report_order(order): + reports = [{"judge": "p/j", "context": "stored", "passed": False}, + {"judge": "p/j", "context": "none", "passed": True}] + measures = [cj.Measure(name="p/j", kind="aggregate", context=c) for c in ("none", "stored")] + report = cj.CrossJudgeReport(baseline_arm="baseline", measures=measures) + assert report.apply_validity([reports[i] for i in order]) == [] + assert [m.gate for m in measures] == ["FAIL", "FAIL"] + + def test_score_measure_hands_the_stored_material_to_the_rescored_judge(): from dokimi.eval.judge_context import JudgeContextStore @@ -1483,3 +1527,149 @@ def boom(self, *_, **__): with pytest.raises(RuntimeError, match="render failed"): args.func(args) assert json.loads(out.read_text(encoding="utf-8"))["summary"]["n_questions"] == 1 + + +def test_score_measure_marks_a_measure_stored_only_if_an_arm_saw_material(): + from dokimi.eval.judge_context import JudgeContextStore + + records = [{"question_id": "q1", "question_text": "q?", + "arms": {"baseline": {"answer": "a0"}, "skill": {"answer": "a1"}}}] + + def scorer(library, question, answer, ground_truth, context): + return 50.0 + + real = JudgeContextStore() + real.put("q1", "skill", "real material") + # the baseline answer falls back to snippets, so the judge saw both kinds: mixed, not stored + assert cj.score_measure(records, scorer, name="p/m", library="lib", store=real).context == "mixed" + real.put("q1", "baseline", "other material") + assert cj.score_measure(records, scorer, name="p/m", library="lib", store=real).context == "stored" + # An empty stored baseline is still "stored" (the arm was judged with the no-context + # marker), and the skill answer falls back to snippets: a mix, which validates nothing. + empty = JudgeContextStore() + empty.put("q1", "baseline", "") + assert cj.score_measure(records, scorer, name="p/m", library="lib", store=empty).context == "mixed" + assert cj.score_measure(records, scorer, name="p/m", library="lib").context == "none" + + +def test_score_measure_with_no_scored_answer_is_context_none_even_with_a_store(): + from dokimi.eval.judge_context import JudgeContextStore + + # Every answer is blank, so no job is sent to a judge. Nothing was judged in any + # context, so the measure must not claim "stored" and let a stored PASS gate it. + records = [{"question_id": "q1", "question_text": "q?", + "arms": {"baseline": {"answer": ""}, "skill": {"answer": ""}}}] + store = JudgeContextStore() + store.put("q1", "skill", "real material") + + def scorer(library, question, answer, ground_truth, context): + raise AssertionError("a blank answer must not reach the judge") + + m = cj.score_measure(records, scorer, name="p/m", library="lib", store=store) + assert m.scores == {} + assert m.context == "none" + + +def _material_arms_run(material="FULL SKILL BODY"): + """An arms file that keeps its judge material, with digests on answers and stored scores.""" + from dokimi.utils import digest_text + + digest = digest_text(material) + return { + "library_name": "L", "judge_provider": "amazon-bedrock", "judge_model": "m", + "judge_context": "treatment_material.v1", + "judge_materials": {digest: material}, + "answers": [{"question_id": f"Q{i}", "question_text": "q", "arms": { + "baseline": {"answer": f"base {i}", "metadata": {"arm_kind": "baseline"}}, + "skill": {"answer": f"skill {i}", "metadata": {"arm_kind": "skill"}, + "judge_context_sha256": digest}, + }} for i in range(8)], + "evaluations": [{"question_id": f"Q{i}", "scores": { + "baseline": {"aggregate": 70, "judge_context_source": "stored"}, + "skill": {"aggregate": 90, "judge_context_source": "stored", + "judge_context_sha256": digest}}} for i in range(8)], + } + + +def _cross_judge_args(tmp_path, data, *extra): + import json + + from dokimi.cli import build_parser + + answers = tmp_path / "run.json" + answers.write_text(json.dumps(data), encoding="utf-8") + return build_parser().parse_args(["eval", "cross-judge", "--answers", str(answers), + "--output", str(tmp_path / "cj.json"), *extra]) + + +def _recording_judge(monkeypatch, seen): + from dokimi.eval import judge as judge_mod + + class FakeJudge: + def __init__(self, **_): + pass + + def score_answer(self, *, answer, context="", **_): + seen[answer] = context + return {"aggregate": 50.0} + + monkeypatch.setattr(judge_mod, "Judge", FakeJudge) + + +def test_cross_judge_judges_hands_the_arms_material_to_the_rescored_judge(tmp_path, monkeypatch): + seen = {} + _recording_judge(monkeypatch, seen) + args = _cross_judge_args(tmp_path, _material_arms_run(), "--judges", "amazon-bedrock/m") + args.func(args) + assert seen["skill 0"] == "FULL SKILL BODY" + + +def test_cross_judge_judges_refuses_edited_material(tmp_path, monkeypatch, capsys): + seen = {} + _recording_judge(monkeypatch, seen) + data = _material_arms_run() + digest = next(iter(data["judge_materials"])) + data["judge_materials"][digest] = "EDITED" + args = _cross_judge_args(tmp_path, data, "--judges", "amazon-bedrock/m") + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 and "skill" in capsys.readouterr().err + assert not seen + + +def test_cross_judge_judges_with_pairwise_refuses_stored_scores(tmp_path, monkeypatch, capsys): + seen = {} + _recording_judge(monkeypatch, seen) + args = _cross_judge_args(tmp_path, _material_arms_run(), "--judges", "amazon-bedrock/m", + "--pairwise", str(tmp_path / "pw.json")) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 and "skill" in capsys.readouterr().err + + + +def test_a_stored_measure_mixing_stored_and_snippet_scores_is_mixed_and_no_pass_validates_it(): + data = _stamped_run() + for ev in data["evaluations"][:4]: + for s in ev["scores"].values(): + s["judge_context_source"] = "snippet_fallback" + assert cj.stored_measure(data).context == "mixed" + assert cj.stored_measure(_stamped_run()).context == "stored" + for context in ("stored", "none"): + m = cj.stored_measure(data) + report = cj.CrossJudgeReport(baseline_arm="baseline", measures=[m]) + report.apply_validity([{"judge": "amazon-bedrock/haiku", "passed": True, "context": context}]) + assert m.gate is None + m = cj.stored_measure(data) + report = cj.CrossJudgeReport(baseline_arm="baseline", measures=[m]) + report.apply_validity([{"judge": "amazon-bedrock/haiku", "passed": False, "context": "stored"}]) + assert m.gate == "FAIL" + + +def test_a_pass_in_one_context_never_names_the_context_of_a_grounded_measure(): + # A PASS validates only aggregate/pairwise measures: a grounded measure in another + # context must not turn the PASS into an "its measures are ..." note. + grounded = cj.Measure(name="p/j", kind="grounded", context="none") + report = cj.CrossJudgeReport(baseline_arm="baseline", measures=[grounded]) + assert report.apply_validity([{"judge": "p/j", "context": "stored", "passed": True}]) == ["p/j"] + assert grounded.gate is None diff --git a/dokimi/tests/test_judge_validity.py b/dokimi/tests/test_judge_validity.py index 3f0bf5ee1..996219644 100644 --- a/dokimi/tests/test_judge_validity.py +++ b/dokimi/tests/test_judge_validity.py @@ -33,6 +33,7 @@ ) from dokimi.eval.judge_validity import ( DEFAULT_MIN_ACCURACY, + NoUsablePairs, ValidityReport, ValidityRow, evaluate_pairs, @@ -1421,3 +1422,814 @@ def test_wrong_numbers_survives_a_digit_run_too_long_for_a_float(tail): huge = "9" * 400 out = wrong_numbers(f"Digest {huge}{tail}") assert out.startswith(f"Digest {huge}") and out != f"Digest {huge}{tail}" + + +# ── --context stored: the gate in the setting arms runs score in ───────────── + +def _with_materials(skill_text="FULL SKILL BODY"): + """`_answers()` as an arms artifact that keeps its judge materials.""" + from dokimi.utils import digest_text + + data = {"answers": _answers(), "judge_materials": {}} + for rec in data["answers"]: + for arm_name, arm in rec["arms"].items(): + text = skill_text if arm_name == "skill" else "" + digest = digest_text(text) + arm["judge_context_sha256"] = digest + data["judge_materials"][digest] = text + return data + + +def test_stored_context_reaches_the_judge_for_every_member_of_a_pair(): + from dokimi.eval.judge_context import NO_CONTEXT, JudgeContextStore + + data = _with_materials() + seen = [] + + def scorer(lib, q, a, context=None): + seen.append(context) + return float(len(a)) + + report = run_validity(data["answers"], scorer, n_bases=3, variants=["truncate_20"], + store=JudgeContextStore.from_artifact(data)) + assert report.context == "stored" and report.summary()["context"] == "stored" + assert set(seen) == {"FULL SKILL BODY", NO_CONTEXT} # baseline's empty material + assert None not in seen + assert "stored treatment material" in report.format_summary() + + +def test_bases_with_no_stored_material_are_dropped_and_counted(): + from dokimi.eval.judge_context import JudgeContextStore + + import copy + + data = _with_materials() + third = copy.deepcopy(data["answers"][0]) + third["question_id"] = "Q003" + data["answers"].append(third) # skill stays covered through Q003 + del data["answers"][0]["arms"]["skill"]["judge_context_sha256"] + report = run_validity(data["answers"], lambda l, q, a, context=None: 1.0, n_bases=9, + variants=["truncate_20"], store=JudgeContextStore.from_artifact(data)) + assert report.n_without_context == 1 + assert report.by_arm == {"baseline": {"kept": 3, "dropped": 0, "sampled": 3}, + "skill": {"kept": 1, "dropped": 1, "sampled": 1}} + assert report.summary()["by_arm"] == report.by_arm + text = report.format_summary() + assert "1 answers without verified material (counted before sampling)" in text + assert "skill: 1 kept, 1 dropped, 1 bases sampled" in text + + +def test_build_returns_pairs_and_dropped_count_without_an_out_param(): + import dataclasses + import copy + + from dokimi.eval.judge_context import JudgeContextStore + from dokimi.eval.judge_validity import Built, _build + + data = _with_materials() + third = copy.deepcopy(data["answers"][0]) + third["question_id"] = "Q003" + data["answers"].append(third) + del data["answers"][0]["arms"]["skill"]["judge_context_sha256"] + built = _build(data["answers"], {}, n_bases=9, seed=0, variants=["truncate_20"], library="lib", + store=JudgeContextStore.from_artifact(data), by_arm={}) + assert isinstance(built, Built) + assert built.n_dropped == 1 + assert built.pairs and all("context" in p for p in built.pairs) + with pytest.raises(dataclasses.FrozenInstanceError): + built.n_dropped = 5 + + no_store = _build(_answers(), {}, n_bases=3, seed=0, variants=["truncate_20"], library="lib") + assert no_store.n_dropped == 0 and no_store.pairs + + +def test_no_stored_material_at_all_is_an_error_not_a_no_context_run(): + from dokimi.eval.judge_context import JudgeContextStore + + store = JudgeContextStore() # nothing stored + with pytest.raises(NoUsablePairs, match="judge_materials"): + run_validity(_answers(), lambda l, q, a, context=None: 1.0, n_bases=3, + variants=["truncate_20"], store=store) + + +def test_a_whole_arm_without_verified_material_is_refused(): + from dokimi.eval.judge_context import JudgeContextStore + + data = _with_materials() + del data["answers"][0]["arms"]["skill"]["judge_context_sha256"] # skill's only record + with pytest.raises(NoUsablePairs, match=r"\['skill'\]"): + run_validity(data["answers"], lambda l, q, a, context=None: 1.0, n_bases=3, + variants=["truncate_20"], store=JudgeContextStore.from_artifact(data)) + + +def test_all_noop_with_a_store_reports_noop_not_missing_material(): + from unittest import mock + + from dokimi.eval.judge_context import JudgeContextStore + + data = _with_materials() + with mock.patch.dict("dokimi.eval.degrade.DEGRADATIONS", {"truncate_20": lambda text, **_: text}): + with pytest.raises(NoUsablePairs, match="unchanged"): + run_validity(data["answers"], lambda l, q, a, context=None: 1.0, n_bases=3, + variants=["truncate_20"], store=JudgeContextStore.from_artifact(data)) + + +def test_without_a_store_the_scorer_is_called_as_before(): + calls = [] + run_validity(_answers(), lambda l, q, a: calls.append(1) or 1.0, n_bases=3, + variants=["truncate_20"]) + assert calls + + +def test_judge_scorer_forwards_the_context_only_when_given(): + from dokimi.eval.judge_validity import judge_scorer + + got = [] + + class J: + def score_answer(self, **kw): + got.append(kw) + return {"aggregate": 1.0} + + s = judge_scorer(J()) + s("lib", "q", "a") + s("lib", "q", "a", context="material") + s_with_draw = judge_scorer(J(), draw=2) + s_with_draw("lib", "q", "a") + assert "context" not in got[0] and got[1]["context"] == "material" + assert "draw" not in got[0] + assert got[2]["draw"] == 2 and "context" not in got[2] + + +def test_sequential_mode_takes_the_store_too(): + from dokimi.eval.judge_context import JudgeContextStore + from dokimi.eval.judge_sequential import run_sequential + + data = _with_materials() + seen = set() + + def scorer(lib, q, a, context=None): + seen.add(context) + return float(len(a)) + + report = run_sequential(data["answers"], scorer, max_bases=3, variants=["truncate_20"], + store=JudgeContextStore.from_artifact(data)) + assert report.context == "stored" and "FULL SKILL BODY" in seen + + +@pytest.mark.parametrize("extra, message", [ + ([], "judge_materials"), # file without materials + (["--scorer", "typed:laya"], "takes context"), # typed scorer has no context slot + (["--mode", "pairwise"], "takes context"), +]) +def test_cli_refuses_stored_context_it_cannot_honour(tmp_path, capsys, extra, message): + import json + + from dokimi.cli import build_parser + + answers = tmp_path / "a.json" + answers.write_text(json.dumps({"answers": _answers()})) + args = build_parser().parse_args(["eval", "judge-validity", "--answers", str(answers), + "--context", "stored", *extra]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 and message in capsys.readouterr().err + + +def test_cli_judges_each_answer_with_its_arms_stored_material(tmp_path, monkeypatch): + import json + + from dokimi.cli import build_parser + from dokimi.eval import judge as judge_mod + + seen = [] + + class FakeJudge: + def __init__(self, **_): + pass + + def score_answer(self, library_name, question, answer, context="(none passed)", **_): + seen.append(context) + return {"aggregate": float(len(answer) % 97)} + + monkeypatch.setattr(judge_mod, "Judge", FakeJudge) + answers = tmp_path / "arms.json" + answers.write_text(json.dumps(_with_materials())) + out = tmp_path / "v.json" + args = build_parser().parse_args(["eval", "judge-validity", "--answers", str(answers), + "--context", "stored", "--bases", "3", + "--variants", "truncate_20", "--output", str(out)]) + code = None + try: + args.func(args) + except SystemExit as exc: # the gate may fail on a toy judge; that is exit 1, not 2 + code = exc.code + assert code in (None, 1) + assert "FULL SKILL BODY" in seen and "(none passed)" not in seen + assert json.loads(out.read_text())["summary"]["context"] == "stored" + + +def test_a_lone_surrogate_in_material_is_hashed_not_fatal(): + from dokimi.eval.judge_context import JudgeContextStore + + text = "chunk ends mid-pair " + chr(0xD83D) + store = JudgeContextStore() + store.put("q1", "docs", text) + digest = store.digest_of("q1", "docs") + assert digest.startswith("sha256:") and store.materials() == {digest: text} + data = {"answers": [{"question_id": "q1", "arms": {"docs": {"judge_context_sha256": digest}}}], + "judge_materials": {digest: text}} + assert JudgeContextStore.from_artifact(data).get("q1", "docs") == text + + +def test_the_sampled_bases_come_from_those_with_material(): + """--bases N is drawn after the bases without material are dropped (Thread 1).""" + from dokimi.eval.judge_context import JudgeContextStore + + answers = [{"question_id": f"Q{i}", "question_text": "q", + "arms": {a: {"answer": ANSWER + f" {a}{i}"} for a in ("baseline", "skill", "docs")}} + for i in range(40)] + store = JudgeContextStore() + for rec in answers: + store.put(rec["question_id"], "baseline", "") + store.put(rec["question_id"], "skill", "SKILL") # docs has no verified material + # Without the docs arm in the store the arm is dropped entirely; keep one docs record. + store.put("Q0", "docs", "DOCS") + report = run_validity(answers, lambda l, q, a, context=None: 1.0, n_bases=30, + variants=["truncate_20"], store=store) + assert len({(r.base_id, r.arm) for r in report.rows}) == 30 + + +def _stored_cli(tmp_path, monkeypatch, *argv, data=None, score=lambda a: float(len(a) % 97)): + import json + + from dokimi.cli import build_parser + from dokimi.eval import judge as judge_mod + + class FakeJudge: + def __init__(self, **_): + pass + + def score_answer(self, library_name, question, answer, context="(none passed)", **_): + return {"aggregate": score(answer)} + + monkeypatch.setattr(judge_mod, "Judge", FakeJudge) + answers = tmp_path / "arms.json" + answers.write_text(json.dumps(data if data is not None else _with_materials())) + out = tmp_path / "v.json" + args = build_parser().parse_args(["eval", "judge-validity", "--answers", str(answers), + "--judge-provider", "amazon-bedrock", "--judge-model", "m", + "--variants", "truncate_20", "--output", str(out), *argv]) + code = None + try: + args.func(args) + except SystemExit as exc: + code = exc.code + assert code in (None, 1) # the gate failing is 1; 2 is a refusal + return json.loads(out.read_text())["summary"] + + +@pytest.mark.parametrize("mode", [["--bases", "3"], ["--sequential", "--max-bases", "3"]]) +def test_cli_stored_context_reports_stored_with_an_unsuffixed_label(tmp_path, monkeypatch, mode): + summary = _stored_cli(tmp_path, monkeypatch, "--context", "stored", *mode) + assert summary["context"] == "stored" + assert summary["judge"] == "amazon-bedrock/m" + assert summary["mode"] == ("sequential" if "--sequential" in mode else "fixed") + assert set(summary["by_arm"]) == {"baseline", "skill"} + + +def test_cli_judge_validity_exits_2_when_no_answer_is_long_enough(tmp_path, monkeypatch, capsys): + import json + + from dokimi.cli import build_parser + + answers = tmp_path / "a.json" + answers.write_text(json.dumps({"answers": [{"question_id": "Q1", "question_text": "q", + "arms": {"baseline": {"answer": "short"}}}]})) + args = build_parser().parse_args(["eval", "judge-validity", "--answers", str(answers), + "--judge-provider", "amazon-bedrock"]) + with pytest.raises(SystemExit) as exc: + args.func(args) + assert exc.value.code == 2 and "Error:" in capsys.readouterr().err + + +def test_a_bug_valueerror_in_a_degradation_is_not_turned_into_a_user_error(tmp_path, monkeypatch): + from unittest import mock + + def boom(text, **_): + raise ValueError("internal bug in truncate_20") + + with mock.patch.dict("dokimi.eval.degrade.DEGRADATIONS", {"truncate_20": boom}): + with pytest.raises(ValueError, match="internal bug"): + _stored_cli(tmp_path, monkeypatch, "--bases", "3") + + +def test_cli_stored_context_on_a_baseline_only_artifact_exits_2(tmp_path, monkeypatch, capsys): + data = _with_materials() + for rec in data["answers"]: + rec["arms"].pop("skill", None) + data["judge_materials"] = {k: v for k, v in data["judge_materials"].items() if v == ""} + with pytest.raises(AssertionError): # _stored_cli asserts exit code 1; here it is 2 + _stored_cli(tmp_path, monkeypatch, "--context", "stored", data=data) + assert "judge_materials" in capsys.readouterr().err + + +def test_a_stored_validity_report_gates_the_rescored_judge_in_cross_judge(tmp_path, monkeypatch, capsys): + """judge-validity --context stored, then cross-judge --validity on its report.""" + import json + + from dokimi.cli import build_parser + from dokimi.utils import digest_text + + # A judge that ranks the longer answer lower fails the gate (intact must win). + failing = _stored_cli(tmp_path, monkeypatch, "--context", "stored", "--bases", "3", + score=lambda a: -float(len(a))) + assert failing["passed"] is False and failing["judge"] == "amazon-bedrock/m" + stored_report = tmp_path / "stored.json" + stored_report.write_text(json.dumps({"summary": failing})) + none_report = tmp_path / "none.json" + none_report.write_text(json.dumps({"summary": {**failing, "context": "none"}})) + none_pass = tmp_path / "none_pass.json" + none_pass.write_text(json.dumps({"summary": {**failing, "context": "none", "passed": True}})) + stored_pass = tmp_path / "stored_pass.json" + stored_pass.write_text(json.dumps({"summary": {**failing, "passed": True}})) + + digest = digest_text("FULL SKILL BODY") + empty = digest_text("") # the baseline arm's stored material is empty, and still stored + run = {"library_name": "L", "judge_provider": "amazon-bedrock", "judge_model": "m", + "judge_context": "treatment_material.v1", + "judge_materials": {digest: "FULL SKILL BODY", empty: ""}, + "answers": [{"question_id": f"Q{i}", "question_text": "q", "arms": { + "baseline": {"answer": f"b{i}", "judge_context_sha256": empty}, + "skill": {"answer": f"s{i}", "judge_context_sha256": digest}}} + for i in range(8)]} + + def gates(report_path): + run_path = tmp_path / "run.json" + run_path.write_text(json.dumps(run)) + out = tmp_path / "cj.json" + args = build_parser().parse_args(["eval", "cross-judge", "--answers", str(run_path), "--no-stored", + "--judges", "amazon-bedrock/m", "--validity", str(report_path), + "--output", str(out)]) + args.func(args) + rows = json.loads(out.read_text())["summary"]["arms"]["skill"]["rows"] + return {r["measure"]: r["gate"] for r in rows} + + assert gates(stored_report) == {"amazon-bedrock/m": "FAIL"} + # A FAIL reaches the judge in any context; a PASS only in the context it ran in. + assert gates(none_report) == {"amazon-bedrock/m": "FAIL"} + assert gates(stored_pass) == {"amazon-bedrock/m": "PASS"} + capsys.readouterr() + assert gates(none_pass) == {"amazon-bedrock/m": None} + assert "report context none; its measures are stored" in capsys.readouterr().err + + +def test_a_missing_material_warning_does_not_promise_a_snippet_fallback_on_the_gate(caplog): + from dokimi.eval.judge_context import JudgeContextStore + + with caplog.at_level("WARNING", logger="dokimi.eval.judge_context"): + assert JudgeContextStore().get("Q1", "skill") is None + text = caplog.text + assert "No stored judge context for Q1/skill" in text and "drops such bases" in text + + +def test_sequential_stored_context_reports_pool_and_dropped_answers(): + """Only answers with verified material count toward the pool, so an exhausted pool is 'pool'.""" + from dokimi.eval.judge_context import JudgeContextStore + from dokimi.eval.judge_sequential import run_sequential + from dokimi.utils import digest_text + + digest = digest_text("FULL SKILL BODY") + answers = [{"question_id": f"Q{i:03d}", "question_text": f"q{i}?", + "arms": {"skill": {"answer": ANSWER + f" Detail {i}. ", + **({"judge_context_sha256": digest} if i < 4 else {})}}} + for i in range(20)] + data = {"answers": answers, "judge_materials": {digest: "FULL SKILL BODY"}} + report = run_sequential(data["answers"], lambda l, q, a, context=None: float(len(a)), + max_bases=10, variants=["truncate_20"], + store=JudgeContextStore.from_artifact(data)) + assert report.n_without_context == 16 + assert report.by_arm == {"skill": {"kept": 4, "dropped": 16, "sampled": 4, "scored": 4}} + assert "16 answers without verified material" in report.format_summary() + d = report.sequential["truncate_20"] + assert d["decision"] == "continue" and d["stopped_by"] == "pool" + + + +def _starvable(n=30, with_material=3): + """n baseline answers and n skill answers, of which only ``with_material`` hold material.""" + from dokimi.utils import digest_text + + empty, body = digest_text(""), digest_text("FULL SKILL BODY") + answers = [{"question_id": f"Q{i:03d}", "question_text": f"q{i}?", "arms": { + "baseline": {"answer": ANSWER + f" Base {i}. ", "judge_context_sha256": empty}, + "skill": {"answer": ANSWER + f" Skill {i}. ", + **({"judge_context_sha256": body} if i < with_material else {})}}} + for i in range(n)] + return {"answers": answers, "judge_materials": {empty: "", body: "FULL SKILL BODY"}} + + +def test_an_arm_with_material_but_no_sampled_base_is_refused(): + """A --context stored PASS must not rest on the other arm's bases alone.""" + from dokimi.eval.judge_context import JudgeContextStore + from dokimi.eval.judge_validity import NoUsablePairs + + data = _starvable() + refused = covered = 0 + for seed in range(20): + try: + report = run_validity(data["answers"], lambda l, q, a, context=None: 1.0, n_bases=4, + seed=seed, variants=["truncate_20"], + store=JudgeContextStore.from_artifact(data)) + except NoUsablePairs as exc: + message = str(exc) + refused += 1 + assert "['skill']" in message and "no base was scored" in message + else: + assert report.by_arm["skill"]["sampled"] > 0 + # Fixed mode doesn't track "scored", just verify rows exist for skill + assert any(r.arm == "skill" for r in report.rows) + covered += 1 + assert refused and covered + + +def test_a_scorer_without_a_context_argument_is_refused_before_pairs_are_built(): + from dokimi.eval.judge_context import JudgeContextStore + from dokimi.eval.judge_validity import NoUsablePairs + from dokimi.utils import digest_text + + digest = digest_text("FULL SKILL BODY") + data = {"answers": [{"question_id": "Q0", "question_text": "q?", "arms": { + "skill": {"answer": ANSWER, "judge_context_sha256": digest}}}], + "judge_materials": {digest: "FULL SKILL BODY"}} + with pytest.raises(NoUsablePairs, match="does not accept a `context=` argument"): + run_validity(data["answers"], lambda lib, q, a: 1.0, n_bases=5, variants=["truncate_20"], + store=JudgeContextStore.from_artifact(data)) + + +def test_a_noop_variant_leaving_an_arm_unpaired_is_refused_before_scoring(): + """strip_code is a no-op on the skill answers (no fence), so the skill arm, which has + verified material, gets no pair. The refusal names it and no judge call is made.""" + from dokimi.eval.judge_context import JudgeContextStore + from dokimi.eval.judge_validity import NoUsablePairs + from dokimi.utils import digest_text + + digest = digest_text("FULL SKILL BODY") + empty = digest_text("") + plain = "Plain skill prose without any code fence, long enough to be a usable answer. " * 25 + answers = [{"question_id": f"Q{i:03d}", "question_text": f"q{i}?", "arms": { + "baseline": {"answer": ANSWER + f" Base {i}. ", "judge_context_sha256": empty}}} + for i in range(10)] + for i in range(3): + answers[i]["arms"]["skill"] = {"answer": plain + f" {i}", "judge_context_sha256": digest} + data = {"answers": answers, "judge_materials": {empty: "", digest: "FULL SKILL BODY"}} + calls = [] + + def scorer(lib, q, a, context=None): + calls.append(a) + return 1.0 + + with pytest.raises(NoUsablePairs, match=r"\['skill'\]"): + run_validity(data["answers"], scorer, n_bases=50, seed=0, variants=["strip_code"], + store=JudgeContextStore.from_artifact(data)) + assert calls == [] + + +def test_fixed_mode_arm_with_material_and_only_noop_bases_is_refused(): + """A fixed-mode run where one arm gets no scored base refuses despite material.""" + from dokimi.eval.judge_context import JudgeContextStore + from dokimi.eval.judge_validity import NoUsablePairs + from dokimi.utils import digest_text + + digest = digest_text("FULL SKILL BODY") + empty = digest_text("") + # 100 baseline answers, 1 skill answer with material. + # With n_bases=50, skill arm has 1 kept but typically gets no sampled base due to + # low odds in pooled shuffle. When no base is sampled and later no rows scored, + # the gate would not cover the arm. + answers = [{"question_id": f"Q{i:03d}", "question_text": f"q{i}?", "arms": { + "baseline": {"answer": ANSWER + f" Base {i}. ", "judge_context_sha256": empty}}} + for i in range(100)] + answers[0]["arms"]["skill"] = {"answer": ANSWER + " Skill 0. ", "judge_context_sha256": digest} + data = {"answers": answers, "judge_materials": {empty: "", digest: "FULL SKILL BODY"}} + + # With only 1 skill answer out of 100 and n_bases=50, skill has low odds of being sampled. + refused = covered = 0 + for seed in range(10): + try: + run_validity(data["answers"], lambda l, q, a, context=None: 1.0, n_bases=50, + seed=seed, variants=["truncate_20"], + store=JudgeContextStore.from_artifact(data)) + covered += 1 + except NoUsablePairs as exc: + message = str(exc) + refused += 1 + assert "['skill']" in message and "no base was scored" in message + # We expect some to be refused when skill doesn't get sampled + assert refused > 0, "Expected at least one seed to refuse due to skill arm not being scored" + + +def test_sequential_stopped_by_counts_kept_answers_not_sampled_ones(): + """kept > max_bases: a higher --max-bases would add bases, so an undecided variant is 'cap'.""" + from dokimi.eval.judge_context import JudgeContextStore + from dokimi.eval.judge_sequential import run_sequential + from dokimi.utils import digest_text + + digest = digest_text("FULL SKILL BODY") + plain = "Plain prose answer without any code block, long enough to count. " * 30 + answers = [{"question_id": f"Q{i:03d}", "question_text": f"q{i}?", + "arms": {"skill": {"answer": (ANSWER if i < 3 else plain) + f" Detail {i}. ", + "judge_context_sha256": digest}}} + for i in range(60)] + data = {"answers": answers, "judge_materials": {digest: "FULL SKILL BODY"}} + report = run_sequential(data["answers"], lambda l, q, a, context=None: 50.0, max_bases=50, + variants=["strip_code"], store=JudgeContextStore.from_artifact(data)) + assert report.by_arm["skill"]["kept"] == 60 and report.by_arm["skill"]["sampled"] == 50 + d = report.sequential["strip_code"] + assert d["decision"] == "continue" and d["stopped_by"] == "cap" + + +@pytest.mark.parametrize("extra, message, wrong", [ + (["--scorer", "bogus"], "unknown --scorer 'bogus'", "takes context"), + (["--scorer", "typed:laya"], "the typed scorer does not", "pairwise"), + (["--mode", "pairwise"], "--mode pairwise does not", "typed"), +]) +def test_cli_stored_context_refusal_names_the_real_cause(tmp_path, capsys, extra, message, wrong): + import json + + from dokimi.cli import build_parser + + answers = tmp_path / "a.json" + answers.write_text(json.dumps(_with_materials())) + args = build_parser().parse_args(["eval", "judge-validity", "--answers", str(answers), + "--context", "stored", *extra]) + with pytest.raises(SystemExit) as exc: + args.func(args) + err = capsys.readouterr().err + assert exc.value.code == 2 and message in err and wrong not in err + + +def test_sequential_reports_the_bases_scored_not_the_pool_built(): + """An early SPRT stop scores fewer bases than --max-bases built; the summary says which.""" + from dokimi.eval.judge_context import JudgeContextStore + from dokimi.eval.judge_sequential import run_sequential + from dokimi.utils import digest_text + + digest = digest_text("FULL SKILL BODY") + answers = [{"question_id": f"Q{i:03d}", "question_text": f"q{i}?", + "arms": {"skill": {"answer": ANSWER + f" Detail {i}. ", + "judge_context_sha256": digest}}} + for i in range(100)] + data = {"answers": answers, "judge_materials": {digest: "FULL SKILL BODY"}} + report = run_sequential(data["answers"], lambda l, q, a, context=None: float(len(a)), + max_bases=50, variants=["truncate_20"], + store=JudgeContextStore.from_artifact(data)) + assert report.sequential["truncate_20"]["decision"] == "PASS" + counts = report.by_arm["skill"] + scored = len({r.base_id for r in report.rows}) + assert counts["sampled"] == 50 and counts["scored"] == scored < 50 + text = report.format_summary() + assert f"{scored} bases scored" in text and "bases sampled" not in text + + +def test_sequential_per_arm_scored_counts_differ(): + """Sequential mode tracks bases scored per arm, pinning the per-arm filter.""" + from dokimi.eval.judge_context import JudgeContextStore + from dokimi.eval.judge_sequential import run_sequential + from dokimi.utils import digest_text + + # Just verify that the "scored" field is populated per-arm in sequential mode. + # The mutation is that the per-arm filter is removed, so all arms print the same total. + digest = digest_text("FULL SKILL BODY") + answers = [{"question_id": f"Q{i:03d}", "question_text": f"q{i}?", + "arms": {"skill": {"answer": ANSWER + f" Detail {i}. ", + "judge_context_sha256": digest}}} + for i in range(100)] + data = {"answers": answers, "judge_materials": {digest: "FULL SKILL BODY"}} + report = run_sequential(data["answers"], lambda l, q, a, context=None: float(len(a)), + max_bases=50, variants=["truncate_20"], + store=JudgeContextStore.from_artifact(data)) + # Verify that "scored" is tracked + assert "scored" in report.by_arm["skill"], "scored field should be in by_arm" + assert report.by_arm["skill"]["scored"] > 0, "scored count should be > 0" + + +def test_sequential_refuses_an_arm_with_material_left_out_of_the_pool(): + """--max-bases can leave the only skill base out of the pool: refuse before any call.""" + from dokimi.eval.judge_context import JudgeContextStore + from dokimi.eval.judge_sequential import run_sequential + from dokimi.eval.judge_validity import NoUsablePairs + + data = _starvable(n=100, with_material=1) + for rec in data["answers"][1:]: + del rec["arms"]["skill"] + calls = [] + + def scorer(lib, q, a, context=None): + calls.append(a) + return float(len(a)) + + store = JudgeContextStore.from_artifact(data) + with pytest.raises(NoUsablePairs, match=r"\['skill'\].*raise --max-bases"): + run_sequential(data["answers"], scorer, max_bases=20, seed=1, variants=["truncate_20"], store=store) + assert calls == [] + + +def test_refusal_does_not_say_raise_the_sample_when_every_kept_base_was_sampled(): + from dokimi.eval.judge_validity import NoUsablePairs, refuse_unpaired_arms + + by_arm = {"skill": {"kept": 2, "dropped": 0, "sampled": 2}} + with pytest.raises(NoUsablePairs) as exc: + refuse_unpaired_arms([], by_arm, "--bases") + message = str(exc.value) + assert "cannot help" in message + assert "raise --bases" not in message + + +def test_refusal_still_says_raise_the_sample_when_a_kept_base_was_not_drawn(): + from dokimi.eval.judge_validity import NoUsablePairs, refuse_unpaired_arms + + by_arm = {"skill": {"kept": 3, "dropped": 0, "sampled": 1}} + with pytest.raises(NoUsablePairs, match=r"\['skill'\].*raise --bases"): + refuse_unpaired_arms([], by_arm, "--bases") + + +def _baseline_heavy_arms(): + """100 baseline + 3 skill answers, each citing its verified material.""" + from dokimi.utils import digest_text + + data = {"answers": [], "judge_materials": {}} + for i in range(103): + arm_name = "baseline" if i < 100 else "skill" + text = "" if arm_name == "baseline" else "SKILL" + digest = digest_text(text) + data["judge_materials"][digest] = text + data["answers"].append({ + "question_id": f"Q{i}", "question_text": f"Question number {i}?", + "arms": {arm_name: {"answer": ANSWER + f"Variant {i}. ", "judge_context_sha256": digest}}, + }) + return data + + +def test_sequential_refuses_an_arm_the_loop_never_scored(): + """The SPRT can decide on baseline bases alone and stop, leaving the skill + arm with 0 scored bases.""" + from dokimi.eval.judge_context import JudgeContextStore + from dokimi.eval.judge_sequential import run_sequential + + data = _baseline_heavy_arms() + with pytest.raises(NoUsablePairs, match=r"\['skill'\].*no base was scored"): + run_sequential(data["answers"], lambda l, q, a, context=None: float(len(a)), + max_bases=50, variants=["truncate_20"], + store=JudgeContextStore.from_artifact(data)) + + +def test_sequential_unscored_arm_refusal_names_a_remedy_that_covers_the_arm(): + """Past the pool size a higher --max-bases stops at the same place; the + fixed mode the message names does score the arm.""" + from dokimi.eval.judge_context import JudgeContextStore + from dokimi.eval.judge_sequential import run_sequential + + data = _baseline_heavy_arms() + store = JudgeContextStore.from_artifact(data) + + def scorer(lib, question, answer, context=None): + return float(len(answer)) + + for max_bases in (200, 400): + with pytest.raises(NoUsablePairs, match=r"\['skill'\]") as exc: + run_sequential(data["answers"], scorer, max_bases=max_bases, variants=["truncate_20"], store=store) + assert "raise --max-bases" not in str(exc.value) + assert "without --sequential" in str(exc.value) + report = run_validity(data["answers"], scorer, n_bases=103, variants=["truncate_20"], store=store) + assert any(r.arm == "skill" and not r.neutral for r in report.rows) + + +@pytest.mark.parametrize("seed", range(4)) +def test_sequential_reports_a_fail_even_when_an_arm_was_never_scored(seed): + """A judge that cannot tell a degraded answer apart fails on the bases it + was scored on; that FAIL is the result, not a refusal over the other arm.""" + from dokimi.eval.judge_context import JudgeContextStore + from dokimi.eval.judge_sequential import run_sequential + + data = _baseline_heavy_arms() + report = run_sequential(data["answers"], lambda l, q, a, context=None: 5.0, max_bases=50, + seed=seed, variants=["truncate_20"], store=JudgeContextStore.from_artifact(data)) + assert report.sequential["truncate_20"]["decision"] == "FAIL" + assert report.summary()["passed"] is False + assert report.by_arm["skill"]["scored"] == 0 + + +def _fenced_baseline_arms(n_skill): + """100 baseline answers with a code fence, ``n_skill`` skill answers without one.""" + from dokimi.utils import digest_text + + data = {"answers": [], "judge_materials": {}} + for i in range(100 + n_skill): + arm_name = "baseline" if i < 100 else "skill" + text = "" if arm_name == "baseline" else "SKILL" + digest = digest_text(text) + data["judge_materials"][digest] = text + body = ANSWER + f"Variant {i}. " + if arm_name == "skill": + body = body.replace("```", "") + data["answers"].append({ + "question_id": f"Q{i}", "question_text": f"Question number {i}?", + "arms": {arm_name: {"answer": body, "judge_context_sha256": digest}}, + }) + return data + + +@pytest.mark.parametrize("n_skill", [3, 1]) +def test_sequential_padding_alone_does_not_cover_an_arm(n_skill): + """A skill base scored only by the neutral ``padding`` variant has no graded + row, so the gate may not PASS on it, whatever the seed.""" + from dokimi.eval.judge_context import JudgeContextStore + from dokimi.eval.judge_sequential import run_sequential + + data = _fenced_baseline_arms(n_skill) + store = JudgeContextStore.from_artifact(data) + for seed in range(10): + with pytest.raises(NoUsablePairs, match=r"\['skill'\]"): + run_sequential(data["answers"], lambda l, q, a, context=None: float(len(a)), + max_bases=120, seed=seed, variants=["strip_code", "padding"], store=store) + + +def test_sequential_unscored_arm_refusal_names_every_arm_sorted(): + from dokimi.eval.judge_context import JudgeContextStore + from dokimi.eval.judge_sequential import run_sequential + from dokimi.utils import digest_text + + data = {"answers": [], "judge_materials": {}} + for i in range(106): + arm_name = "baseline" if i < 100 else ("zeta" if i < 103 else "alpha") + text = "" if arm_name == "baseline" else arm_name + digest = digest_text(text) + data["judge_materials"][digest] = text + body = ANSWER + f"Variant {i}. " + if arm_name != "baseline": + body = body.replace("```", "") + data["answers"].append({ + "question_id": f"Q{i}", "question_text": f"Question number {i}?", + "arms": {arm_name: {"answer": body, "judge_context_sha256": digest}}, + }) + store = JudgeContextStore.from_artifact(data) + messages = set() + for seed in range(10): + with pytest.raises(NoUsablePairs) as exc: + run_sequential(data["answers"], lambda l, q, a, context=None: float(len(a)), max_bases=120, + seed=seed, variants=["strip_code", "padding"], store=store) + messages.add(str(exc.value)) + assert any("['alpha', 'zeta']" in m for m in messages) + + +@pytest.mark.parametrize("n_skill", [3, 1]) +def test_fixed_mode_refuses_an_arm_whose_only_pairs_are_padding(n_skill): + """The remedy the sequential refusal names: with every base sampled, a + skill arm that only padding changes has no graded row, so it is refused + before any call instead of passing on baseline rows alone.""" + from dokimi.eval.judge_context import JudgeContextStore + from dokimi.eval.judge_sequential import run_sequential + + data = _fenced_baseline_arms(n_skill) + store = JudgeContextStore.from_artifact(data) + calls = [] + + def scorer(lib, question, answer, context=None): + calls.append(1) + return float(len(answer)) + + n = len(data["answers"]) + with pytest.raises(NoUsablePairs, match=r"\['skill'\].*variant \(strip_code\) left") as exc: + run_validity(data["answers"], scorer, n_bases=n, variants=["strip_code", "padding"], store=store) + assert "raise --bases" not in str(exc.value) + with pytest.raises(NoUsablePairs, match=r"\['skill'\].*use other degradations"): + run_sequential(data["answers"], scorer, max_bases=n, variants=["strip_code", "padding"], store=store) + assert calls == [] + # A run that requests no graded variant is not refused here; it fails as unmeasured. + report = run_validity(data["answers"], scorer, n_bases=n, variants=["padding"], store=store) + assert report.summary()["passed"] is False + + +@pytest.mark.parametrize("seed", [0, 1, 2, 5]) +def test_sequential_padding_after_a_graded_variant_closed_does_not_cover_an_arm(seed): + """The skill arm has truncate_20 pairs, but truncate_20 is decided before its + bases come up; strip_code (a no-op on it) stays open, so only its padding + pair is scored. That row is not a graded one: the PASS is refused.""" + import re + + from dokimi.eval.judge_context import JudgeContextStore + from dokimi.eval.judge_sequential import run_sequential + + data = _fenced_baseline_arms(3) + + def scorer(lib, question, answer, context=None): + # truncate_20 cuts the "Variant i." tail; such a text is scored by length. + m = re.search(r"Variant (\d+)\. ", answer) + i = int(m.group(1)) if m else -1 + return 1e6 if "```" not in answer and i % 6 == 0 and i < 100 else float(len(answer)) + + with pytest.raises(NoUsablePairs, match=r"\['skill'\].*before the sequential test stopped"): + run_sequential(data["answers"], scorer, max_bases=120, seed=seed, + variants=["truncate_20", "strip_code", "padding"], + store=JudgeContextStore.from_artifact(data))