diff --git a/src/vectrify/score/ensemble.py b/src/vectrify/score/ensemble.py index 780a910..e8790a4 100644 --- a/src/vectrify/score/ensemble.py +++ b/src/vectrify/score/ensemble.py @@ -45,6 +45,7 @@ import io import logging +import statistics from dataclasses import dataclass from typing import Any @@ -75,6 +76,13 @@ class PanelReference: image: Image.Image tiles: list[Any] + # Each member's distance from the target to a blank canvas, measured once. + # It is what makes a member's distance mean something on its own: raw + # cosine distances come from three different embedding spaces and span + # different widths, so an uncalibrated average is decided by whichever + # member happens to spread widest -- one model steering the run, which is + # what a panel exists to prevent. + blank: list[float] def _tiles(image: Image.Image) -> list[Image.Image]: @@ -106,12 +114,18 @@ def validate_environment(self) -> None: def prepare_reference(self, original_rgb: Image.Image) -> PanelReference: cells = _tiles(original_rgb) - return PanelReference( + reference = PanelReference( image=original_rgb, tiles=[m.embed_images(cells) for m in self._members], + blank=[], ) + empty = Image.new("RGB", original_rgb.size, (255, 255, 255)) + reference.blank.extend( + max(d[0], 1e-6) for d in self._raw_distances(reference, [empty]) + ) + return reference - def _distances(self, reference: PanelReference, images: list[Image.Image]): + def _raw_distances(self, reference: PanelReference, images: list[Image.Image]): """Each member's distance to every image, cell by cell then averaged.""" per_member = [] for member, reference_tiles in zip(self._members, reference.tiles, strict=True): @@ -124,40 +138,70 @@ def _distances(self, reference: PanelReference, images: list[Image.Image]): ) return per_member - def score(self, reference: PanelReference, candidate_png: bytes) -> float: - """Mean distance across the panel. + def _distances(self, reference: PanelReference, images: list[Image.Image]): + """Per-member distances, each as a fraction of that member's distance + from the target to a blank canvas. 0 is the target itself and about 1 + is as wrong as an empty drawing, on every member and every target.""" + raw = self._raw_distances(reference, images) + return [ + [value / scale for value in member] + for member, scale in zip(raw, reference.blank, strict=True) + ] - A single candidate has nothing to be compared against, so there is no - vote to take. This exists for callers that need a scalar per candidate; - the panel's actual judgement is ``rank``, which is what decides - direction. + def score(self, reference: PanelReference, candidate_png: bytes) -> float: + """The panel's verdict on one candidate: the median calibrated distance. + + The median rather than the mean, and that is the whole panel argument + in absolute form. With three members the median is the majority + position: for any standard you might hold a candidate to, "the panel + says it meets this" is true exactly when the median says so, and a + member that is idiosyncratic about this particular drawing cannot move + it. The pairwise vote said the same thing about pairs; this says it + about candidates, which is what lets two scores be compared at all. + + Absolute, so it means the same thing in every call and in every run -- + the property `rank` could not have, since counting rivals beaten only + describes the field a candidate was ranked against. """ try: image = Image.open(io.BytesIO(candidate_png)).convert("RGB") except Exception: return MAX_SCORE values = [d[0] for d in self._distances(reference, [image])] - return sum(values) / len(values) if values else MAX_SCORE + return statistics.median(values) if values else MAX_SCORE def rank( self, reference: PanelReference, candidate_pngs: list[bytes] ) -> list[float]: - """Score candidates by how many rivals the panel puts them ahead of. - - Every pair is put to the panel and the majority wins, then candidates - are ordered by wins less losses. That ordering step is not decoration: - a majority relation is a tournament and cycles, so it cannot be sorted - by pairwise comparison -- three candidates can each beat the next. - Counting wins ranks a tournament without needing it to be transitive. - - Returns one value per candidate, lower being better, so it drops into - a caller that expects a distance. Values span [0, 1] up to the width of - the tie-break term, which can carry the extremes a little past either - end; nothing compares them against an absolute threshold. + """Score every candidate, lower being better. + + One absolute number each, so the caller may compare them with anything + else the panel has scored -- a candidate from an earlier check, from an + earlier epoch, or from another run entirely. + + This used to put every pair to the panel and count rivals beaten, which + ranked a field correctly and said nothing outside it: the same drawing + scored differently depending on who it was ranked against, so two calls + could not be compared and nothing could be cached between them. """ if not candidate_pngs: return [] + images = [] + for png in candidate_pngs: + try: + images.append(Image.open(io.BytesIO(png)).convert("RGB")) + except Exception: + # A candidate that will not open is scored worst rather than + # failing the field it was ranked with. + images.append(Image.new("RGB", reference.image.size, (255, 255, 255))) + + per_member = self._distances(reference, images) + return [ + statistics.median([member[i] for member in per_member]) + for i in range(len(images)) + ] + images = [] for png in candidate_pngs: try: diff --git a/src/vectrify/vector/runner.py b/src/vectrify/vector/runner.py index 38cb334..e72ad01 100644 --- a/src/vectrify/vector/runner.py +++ b/src/vectrify/vector/runner.py @@ -1,6 +1,7 @@ import io import logging import os +from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor from typing import TYPE_CHECKING, Any @@ -99,6 +100,73 @@ def initial_seed_tasks(epoch_seeds: int, initial_nodes: list[SearchNode]) -> int return max(0, epoch_seeds - seeded) +def evaluate_front( + nodes: list[SearchNode], + *, + front_scorer: Callable[[], tuple[Any, Any]], + format_plugin: Any, + out_w: int, + out_h: int, +) -> list[SearchNode]: + """Order *nodes* by the evaluator, best first, scoring only what is new. + + *front_scorer* is called for (scorer, reference) and only when there is + something to score, so a call the cache answers in full never builds a + model. + + Re-rasterises rather than reading a node's stored render, which is only + kept when --write-lineage or --save-raster is on. + """ + renders: list[tuple[bytes, SearchNode]] = [] + for node in nodes: + # Already judged, and the judgement travels: the panel's score is a + # calibrated distance to the target, so it means the same thing in + # every call. Re-rasterising and re-embedding a node the evaluator has + # already seen would buy an identical number at full price -- and a run + # asks about the same pool members repeatedly. + if FRONT_SCORE in node.metrics: + continue + content = getattr(node.state.payload, "content", None) + if not content: + continue + try: + renders.append( + (format_plugin.rasterize(content, out_w=out_w, out_h=out_h), node) + ) + except Exception as exc: + log.debug(f"Front evaluation skipped node {node.id}: {exc}") + + if renders: + scorer, ref = front_scorer() + pngs = [png for png, _ in renders] + try: + values = scorer.rank(ref, pngs) + except AttributeError: + values = [scorer.score(ref, png) for png in pngs] + except Exception as exc: + log.warning(f"Front evaluation failed, keeping rank order: {exc}") + return nodes + + for value, (_png, node) in zip(values, renders, strict=True): + node.metrics[FRONT_SCORE] = value + + # Every node the panel has ever scored, freshly measured or recalled. + scored = [ + (node.metrics[FRONT_SCORE], node) + for node in nodes + if FRONT_SCORE in node.metrics + ] + if not scored: + return nodes + scored.sort(key=lambda pair: pair[0]) + log.info( + f"Front evaluated: {len(scored)} candidate(s) " + f"({len(renders)} newly scored), " + f"best {scored[0][0]:.6f}, worst {scored[-1][0]:.6f}" + ) + return [node for _value, node in scored] + + def run_vector_search( image_path: str, storage: StorageAdapter, @@ -256,56 +324,13 @@ def _front_scorer() -> tuple[Any, Any]: return _front[0], _front[1] def rank_front(nodes: list[SearchNode]) -> list[SearchNode]: - """Order a converged front by the run's real objective. - - Re-rasterises rather than reading the node's stored render, which is - only kept when --write-lineage or --save-raster is on. - """ - scorer, ref = _front_scorer() - renders: list[tuple[bytes, SearchNode]] = [] - for node in nodes: - content = getattr(node.state.payload, "content", None) - if not content: - continue - try: - renders.append( - ( - format_plugin.rasterize( - content, out_w=original_w, out_h=original_h - ), - node, - ) - ) - except Exception as exc: - log.debug(f"Front evaluation skipped node {node.id}: {exc}") - - if not renders: - return nodes - - pngs = [png for png, _ in renders] - try: - # A panel ranks the field as a whole, because a majority vote needs - # candidates to compare; a single scorer just scores each one. - values = scorer.rank(ref, pngs) - except AttributeError: - values = [scorer.score(ref, png) for png in pngs] - except Exception as exc: - log.warning(f"Front evaluation failed, keeping round order: {exc}") - return nodes - - scored: list[tuple[float, SearchNode]] = [] - for value, (_png, node) in zip(values, renders, strict=True): - node.metrics[FRONT_SCORE] = value - scored.append((value, node)) - - if not scored: - return nodes - scored.sort(key=lambda pair: pair[0]) - log.info( - f"Front evaluated: {len(scored)} candidate(s), " - f"best {scored[0][0]:.6f}, worst {scored[-1][0]:.6f}" + return evaluate_front( + nodes, + front_scorer=_front_scorer, + format_plugin=format_plugin, + out_w=original_w, + out_h=original_h, ) - return [node for _value, node in scored] engine = MultiprocessSearchEngine( workers=workers, diff --git a/tests/score/test_ensemble.py b/tests/score/test_ensemble.py index f619e3d..bb4edd8 100644 --- a/tests/score/test_ensemble.py +++ b/tests/score/test_ensemble.py @@ -1,20 +1,24 @@ +import statistics + from PIL import Image from vectrify.score.ensemble import EnsembleScorer, PanelReference def _panel(*rows: list[float]) -> tuple[EnsembleScorer, PanelReference]: - """A panel whose members report the distances the test dictates. + """A panel whose members report the calibrated distances the test dictates. Stubbed at the per-member distances rather than at the encoders: the panel cuts every picture into tiles and compares the embeddings itself, so faking an encoder would mean faking image decoding and tiling too, none of which - is what these tests are about. What is under test is the vote. + is what these tests are about. """ scorer = EnsembleScorer.__new__(EnsembleScorer) scorer._members = [None] * len(rows) # type: ignore[assignment] scorer._names = tuple(f"m{i}" for i in range(len(rows))) - reference = PanelReference(image=Image.new("RGB", (12, 12)), tiles=[]) + reference = PanelReference( + image=Image.new("RGB", (12, 12)), tiles=[], blank=[1.0] * len(rows) + ) def distances(_reference, images): return [list(row[: len(images)]) for row in rows] @@ -26,87 +30,60 @@ def distances(_reference, images): CANDIDATES = [b"0", b"1", b"2"] -def test_the_majority_outvotes_a_dissenting_member(): - """The property a single scorer cannot have: one member being idiosyncratic - about a particular pair does not decide the pair.""" +def test_the_majority_decides_and_a_dissenting_member_does_not(): + """The property a single scorer cannot have. With three members the median + is the majority position: one member being idiosyncratic about a drawing + cannot move the verdict, whatever value it reports.""" agree = [0.1, 0.2, 0.3] - dissent = [0.9, 0.2, 0.1] - scorer, reference = _panel(agree, agree, agree, dissent, dissent) + dissent = [0.9, 0.9, 0.9] + scorer, reference = _panel(agree, agree, dissent) ranked = scorer.rank(reference, CANDIDATES) - assert ranked[0] < ranked[1] < ranked[2] + assert ranked == agree -def test_a_voting_cycle_leaves_every_candidate_ranked(): - """Rock-paper-scissors: each candidate beats the next by a majority. The - relation cannot be sorted, and no candidate may be dropped or left without - a position.""" - scorer, reference = _panel( - [0.1, 0.2, 0.3], - [0.1, 0.2, 0.3], - [0.3, 0.1, 0.2], - [0.3, 0.1, 0.2], - [0.2, 0.3, 0.1], - ) - - ranked = scorer.rank(reference, CANDIDATES) +def test_a_score_does_not_depend_on_the_field_it_was_scored_with(): + """Why this replaced counting rivals beaten. The same drawing has to come + out the same however it is grouped, or two checks cannot be compared and + nothing can be cached from one to the next. + """ + rows = [[0.1, 0.5, 0.9], [0.2, 0.5, 0.8], [0.3, 0.5, 0.7]] + scorer, reference = _panel(*rows) - assert len(ranked) == 3 - assert all(0.0 <= value <= 1.0 for value in ranked) + alone = scorer.rank(reference, CANDIDATES[:1]) + with_others = scorer.rank(reference, CANDIDATES) + assert alone[0] == with_others[0] -def test_ranking_is_not_decided_by_one_member_s_scale(): - """A member reporting distances an order of magnitude larger than the rest - would dominate any averaging scheme. It gets one vote here.""" - small = [0.01, 0.02, 0.03] - huge = [90.0, 60.0, 30.0] - scorer, reference = _panel(small, small, small, huge, huge) - ranked = scorer.rank(reference, CANDIDATES) +def test_scores_are_the_median_of_the_calibrated_members(): + rows = [[0.1, 0.4], [0.2, 0.5], [0.9, 0.6]] + scorer, reference = _panel(*rows) - assert ranked[0] < ranked[2], "the loud member overruled the majority" + ranked = scorer.rank(reference, CANDIDATES[:2]) + assert ranked[0] == statistics.median([0.1, 0.2, 0.9]) + assert ranked[1] == statistics.median([0.4, 0.5, 0.6]) -def test_a_single_candidate_falls_back_to_the_mean_distance(): - """Nothing to compare against, so there is no vote to take.""" - scorer, reference = _panel([0.2, 0.4, 0.6], [0.4, 0.4, 0.4]) - assert scorer.rank(reference, [b"0"]) == [(0.2 + 0.4) / 2] - assert scorer.rank(reference, []) == [] - - -def test_a_tie_on_votes_is_settled_by_mean_rank(): - """Wins are integers, so a front of tens ties often, and the caller keeps - only the best few as parents. Leaving ties to fall through to pool order - would decide the next epoch arbitrarily.""" - # Every member ranks 0 and 1 adjacently but 2 last, so 0 and 1 tie on wins - # against each other while every member places 0 ahead of 1. - scorer, reference = _panel( - [0.10, 0.11, 0.90], - [0.10, 0.11, 0.90], - [0.10, 0.11, 0.90], - ) +def test_ranking_is_not_decided_by_one_member_s_scale(): + """Calibration is what earns this. Raw cosine distances come from three + embedding spaces of different widths, and an uncalibrated combination is + decided by whichever member spreads widest -- one model steering the run, + which is the thing a panel exists to prevent.""" + narrow = [0.10, 0.11] + also_narrow = [0.10, 0.11] + wide_but_opposed = [0.90, 0.10] + scorer, reference = _panel(narrow, also_narrow, wide_but_opposed) - ranked = scorer.rank(reference, CANDIDATES) + ranked = scorer.rank(reference, CANDIDATES[:2]) - assert ranked[0] < ranked[1] < ranked[2] - assert len(set(ranked)) == 3, "candidates were left tied" - - -def test_the_tie_break_never_overrides_a_vote(): - """A candidate the majority puts ahead must stay ahead however lopsided the - mean ranks are, or the tie-break has quietly become the ranking.""" - # 0 wins the majority (three members of five), while the two dissenting - # members rank it dead last by a wide margin. - scorer, reference = _panel( - [0.10, 0.20, 0.30], - [0.10, 0.20, 0.30], - [0.10, 0.20, 0.30], - [0.99, 0.01, 0.02], - [0.99, 0.01, 0.02], - ) + # The wide member prefers the second candidate by a distance that dwarfs + # the others; the majority still decides. + assert ranked[0] < ranked[1] - ranked = scorer.rank(reference, CANDIDATES) - assert ranked[0] < ranked[1], "the tie-break reordered across a majority" +def test_an_empty_field_ranks_to_nothing(): + scorer, reference = _panel([0.1], [0.2], [0.3]) + assert scorer.rank(reference, []) == [] diff --git a/tests/search/test_engine.py b/tests/search/test_engine.py index 2f70963..4bc207f 100644 --- a/tests/search/test_engine.py +++ b/tests/search/test_engine.py @@ -1081,6 +1081,7 @@ def score_fn(results): assert scored, "results scored before the failure should have survived it" + def test_the_epoch_budget_ends_an_epoch_that_has_not_gone_stale(): """Staleness measures whether the pool has stopped producing; the budget measures how long the proxy has run without the evaluator seeing anything. diff --git a/tests/vector/test_evaluate_front.py b/tests/vector/test_evaluate_front.py new file mode 100644 index 0000000..98f4f69 --- /dev/null +++ b/tests/vector/test_evaluate_front.py @@ -0,0 +1,166 @@ +"""The evaluator's cache: what it recalls, what it re-measures, what it costs. + +A panel call is the expensive thing in a run, and the pool it is asked about +changes slowly, so most of any call is a repeat of the last one. These pin that +the repeat is free -- and that the cheap paths stay cheap when the evaluator is +absent, broken, or has nothing new to look at. +""" + +from vectrify.formats.models import VectorStatePayload +from vectrify.score.metrics import FRONT_SCORE +from vectrify.search import ChainState, SearchNode +from vectrify.vector.runner import evaluate_front + + +class FakePlugin: + def rasterize(self, content, out_w, out_h): + _ = (out_w, out_h) + return content.encode() + + +class CountingScorer: + """Records every field it is asked to score.""" + + def __init__(self, values: list[float] | None = None): + self.calls: list[int] = [] + self._values = values + + def rank(self, _ref, pngs): + self.calls.append(len(pngs)) + if self._values is not None: + return self._values[: len(pngs)] + return [0.1 * (i + 1) for i in range(len(pngs))] + + +def _node(node_id: int, content: str = "") -> SearchNode: + return SearchNode( + score=0.0, + id=node_id, + parent_id=0, + state=ChainState( + score=0.0, + payload=VectorStatePayload( + content=content, + raster_data_url=None, + raster_preview_data_url=None, + origin=None, + ), + ), + ) + + +def _evaluate(nodes, scorer, built: list | None = None): + def front_scorer(): + if built is not None: + built.append(1) + return scorer, object() + + return evaluate_front( + nodes, + front_scorer=front_scorer, + format_plugin=FakePlugin(), + out_w=8, + out_h=8, + ) + + +def test_every_node_is_scored_the_first_time(): + scorer = CountingScorer() + nodes = [_node(i, f"") for i in range(1, 4)] + + ranked = _evaluate(nodes, scorer) + + assert scorer.calls == [3] + assert all(FRONT_SCORE in n.metrics for n in ranked) + + +def test_a_second_look_at_the_same_nodes_costs_nothing(): + scorer = CountingScorer() + nodes = [_node(i, f"") for i in range(1, 4)] + + _evaluate(nodes, scorer) + _evaluate(nodes, scorer) + + assert scorer.calls == [3] + + +def test_a_fully_cached_call_does_not_even_build_the_scorer(): + """The model is the expensive part, and a call the cache answers in full + has no reason to load one.""" + scorer = CountingScorer() + nodes = [_node(i, f"") for i in range(1, 4)] + _evaluate(nodes, scorer) + + built: list = [] + _evaluate(nodes, scorer, built=built) + + assert built == [] + + +def test_only_the_unseen_nodes_are_scored(): + scorer = CountingScorer() + old = [_node(i, f"") for i in range(1, 4)] + _evaluate(old, scorer) + + fresh = [_node(9, "")] + _evaluate([*old, *fresh], scorer) + + assert scorer.calls == [3, 1] + + +def test_a_recalled_score_orders_against_a_fresh_one(): + """The point of an absolute score. A value measured in an earlier call has + to be comparable with one measured now, or the cache would order the field + by when each candidate happened to be seen.""" + scorer = CountingScorer(values=[0.9]) + stale = _node(1, "") + _evaluate([stale], scorer) + + better = _node(2, "") + scorer._values = [0.1] + ranked = _evaluate([stale, better], scorer) + + assert [n.id for n in ranked] == [2, 1] + + +def test_nodes_without_content_are_left_out_rather_than_failing_the_field(): + scorer = CountingScorer() + nodes = [_node(1, ""), _node(2, "")] + + ranked = _evaluate(nodes, scorer) + + assert scorer.calls == [1] + assert [n.id for n in ranked] == [1] + + +def test_a_failing_evaluator_returns_the_nodes_it_was_given(): + class Exploding: + def rank(self, _ref, _pngs): + raise RuntimeError("no") + + nodes = [_node(1, ""), _node(2, "")] + + ranked = _evaluate(nodes, Exploding()) + + assert ranked == nodes + assert all(FRONT_SCORE not in n.metrics for n in ranked) + + +def test_a_scorer_without_rank_is_asked_one_candidate_at_a_time(): + """--scorer simple has no panel to put a field to, only a score per + candidate.""" + + class SingleOnly: + def __init__(self): + self.seen = 0 + + def score(self, _ref, _png): + self.seen += 1 + return 0.5 + + scorer = SingleOnly() + nodes = [_node(i, f"") for i in range(1, 4)] + + _evaluate(nodes, scorer) + + assert scorer.seen == 3