diff --git a/src/vectrify/score/ensemble.py b/src/vectrify/score/ensemble.py
index 780a910..e8790a4 100644
--- a/src/vectrify/score/ensemble.py
+++ b/src/vectrify/score/ensemble.py
@@ -45,6 +45,7 @@
import io
import logging
+import statistics
from dataclasses import dataclass
from typing import Any
@@ -75,6 +76,13 @@
class PanelReference:
image: Image.Image
tiles: list[Any]
+ # Each member's distance from the target to a blank canvas, measured once.
+ # It is what makes a member's distance mean something on its own: raw
+ # cosine distances come from three different embedding spaces and span
+ # different widths, so an uncalibrated average is decided by whichever
+ # member happens to spread widest -- one model steering the run, which is
+ # what a panel exists to prevent.
+ blank: list[float]
def _tiles(image: Image.Image) -> list[Image.Image]:
@@ -106,12 +114,18 @@ def validate_environment(self) -> None:
def prepare_reference(self, original_rgb: Image.Image) -> PanelReference:
cells = _tiles(original_rgb)
- return PanelReference(
+ reference = PanelReference(
image=original_rgb,
tiles=[m.embed_images(cells) for m in self._members],
+ blank=[],
)
+ empty = Image.new("RGB", original_rgb.size, (255, 255, 255))
+ reference.blank.extend(
+ max(d[0], 1e-6) for d in self._raw_distances(reference, [empty])
+ )
+ return reference
- def _distances(self, reference: PanelReference, images: list[Image.Image]):
+ def _raw_distances(self, reference: PanelReference, images: list[Image.Image]):
"""Each member's distance to every image, cell by cell then averaged."""
per_member = []
for member, reference_tiles in zip(self._members, reference.tiles, strict=True):
@@ -124,40 +138,70 @@ def _distances(self, reference: PanelReference, images: list[Image.Image]):
)
return per_member
- def score(self, reference: PanelReference, candidate_png: bytes) -> float:
- """Mean distance across the panel.
+ def _distances(self, reference: PanelReference, images: list[Image.Image]):
+ """Per-member distances, each as a fraction of that member's distance
+ from the target to a blank canvas. 0 is the target itself and about 1
+ is as wrong as an empty drawing, on every member and every target."""
+ raw = self._raw_distances(reference, images)
+ return [
+ [value / scale for value in member]
+ for member, scale in zip(raw, reference.blank, strict=True)
+ ]
- A single candidate has nothing to be compared against, so there is no
- vote to take. This exists for callers that need a scalar per candidate;
- the panel's actual judgement is ``rank``, which is what decides
- direction.
+ def score(self, reference: PanelReference, candidate_png: bytes) -> float:
+ """The panel's verdict on one candidate: the median calibrated distance.
+
+ The median rather than the mean, and that is the whole panel argument
+ in absolute form. With three members the median is the majority
+ position: for any standard you might hold a candidate to, "the panel
+ says it meets this" is true exactly when the median says so, and a
+ member that is idiosyncratic about this particular drawing cannot move
+ it. The pairwise vote said the same thing about pairs; this says it
+ about candidates, which is what lets two scores be compared at all.
+
+ Absolute, so it means the same thing in every call and in every run --
+ the property `rank` could not have, since counting rivals beaten only
+ describes the field a candidate was ranked against.
"""
try:
image = Image.open(io.BytesIO(candidate_png)).convert("RGB")
except Exception:
return MAX_SCORE
values = [d[0] for d in self._distances(reference, [image])]
- return sum(values) / len(values) if values else MAX_SCORE
+ return statistics.median(values) if values else MAX_SCORE
def rank(
self, reference: PanelReference, candidate_pngs: list[bytes]
) -> list[float]:
- """Score candidates by how many rivals the panel puts them ahead of.
-
- Every pair is put to the panel and the majority wins, then candidates
- are ordered by wins less losses. That ordering step is not decoration:
- a majority relation is a tournament and cycles, so it cannot be sorted
- by pairwise comparison -- three candidates can each beat the next.
- Counting wins ranks a tournament without needing it to be transitive.
-
- Returns one value per candidate, lower being better, so it drops into
- a caller that expects a distance. Values span [0, 1] up to the width of
- the tie-break term, which can carry the extremes a little past either
- end; nothing compares them against an absolute threshold.
+ """Score every candidate, lower being better.
+
+ One absolute number each, so the caller may compare them with anything
+ else the panel has scored -- a candidate from an earlier check, from an
+ earlier epoch, or from another run entirely.
+
+ This used to put every pair to the panel and count rivals beaten, which
+ ranked a field correctly and said nothing outside it: the same drawing
+ scored differently depending on who it was ranked against, so two calls
+ could not be compared and nothing could be cached between them.
"""
if not candidate_pngs:
return []
+ images = []
+ for png in candidate_pngs:
+ try:
+ images.append(Image.open(io.BytesIO(png)).convert("RGB"))
+ except Exception:
+ # A candidate that will not open is scored worst rather than
+ # failing the field it was ranked with.
+ images.append(Image.new("RGB", reference.image.size, (255, 255, 255)))
+
+ per_member = self._distances(reference, images)
+ return [
+ statistics.median([member[i] for member in per_member])
+ for i in range(len(images))
+ ]
+
images = []
for png in candidate_pngs:
try:
diff --git a/src/vectrify/vector/runner.py b/src/vectrify/vector/runner.py
index 38cb334..e72ad01 100644
--- a/src/vectrify/vector/runner.py
+++ b/src/vectrify/vector/runner.py
@@ -1,6 +1,7 @@
import io
import logging
import os
+from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor
from typing import TYPE_CHECKING, Any
@@ -99,6 +100,73 @@ def initial_seed_tasks(epoch_seeds: int, initial_nodes: list[SearchNode]) -> int
return max(0, epoch_seeds - seeded)
+def evaluate_front(
+ nodes: list[SearchNode],
+ *,
+ front_scorer: Callable[[], tuple[Any, Any]],
+ format_plugin: Any,
+ out_w: int,
+ out_h: int,
+) -> list[SearchNode]:
+ """Order *nodes* by the evaluator, best first, scoring only what is new.
+
+ *front_scorer* is called for (scorer, reference) and only when there is
+ something to score, so a call the cache answers in full never builds a
+ model.
+
+ Re-rasterises rather than reading a node's stored render, which is only
+ kept when --write-lineage or --save-raster is on.
+ """
+ renders: list[tuple[bytes, SearchNode]] = []
+ for node in nodes:
+ # Already judged, and the judgement travels: the panel's score is a
+ # calibrated distance to the target, so it means the same thing in
+ # every call. Re-rasterising and re-embedding a node the evaluator has
+ # already seen would buy an identical number at full price -- and a run
+ # asks about the same pool members repeatedly.
+ if FRONT_SCORE in node.metrics:
+ continue
+ content = getattr(node.state.payload, "content", None)
+ if not content:
+ continue
+ try:
+ renders.append(
+ (format_plugin.rasterize(content, out_w=out_w, out_h=out_h), node)
+ )
+ except Exception as exc:
+ log.debug(f"Front evaluation skipped node {node.id}: {exc}")
+
+ if renders:
+ scorer, ref = front_scorer()
+ pngs = [png for png, _ in renders]
+ try:
+ values = scorer.rank(ref, pngs)
+ except AttributeError:
+ values = [scorer.score(ref, png) for png in pngs]
+ except Exception as exc:
+ log.warning(f"Front evaluation failed, keeping rank order: {exc}")
+ return nodes
+
+ for value, (_png, node) in zip(values, renders, strict=True):
+ node.metrics[FRONT_SCORE] = value
+
+ # Every node the panel has ever scored, freshly measured or recalled.
+ scored = [
+ (node.metrics[FRONT_SCORE], node)
+ for node in nodes
+ if FRONT_SCORE in node.metrics
+ ]
+ if not scored:
+ return nodes
+ scored.sort(key=lambda pair: pair[0])
+ log.info(
+ f"Front evaluated: {len(scored)} candidate(s) "
+ f"({len(renders)} newly scored), "
+ f"best {scored[0][0]:.6f}, worst {scored[-1][0]:.6f}"
+ )
+ return [node for _value, node in scored]
+
+
def run_vector_search(
image_path: str,
storage: StorageAdapter,
@@ -256,56 +324,13 @@ def _front_scorer() -> tuple[Any, Any]:
return _front[0], _front[1]
def rank_front(nodes: list[SearchNode]) -> list[SearchNode]:
- """Order a converged front by the run's real objective.
-
- Re-rasterises rather than reading the node's stored render, which is
- only kept when --write-lineage or --save-raster is on.
- """
- scorer, ref = _front_scorer()
- renders: list[tuple[bytes, SearchNode]] = []
- for node in nodes:
- content = getattr(node.state.payload, "content", None)
- if not content:
- continue
- try:
- renders.append(
- (
- format_plugin.rasterize(
- content, out_w=original_w, out_h=original_h
- ),
- node,
- )
- )
- except Exception as exc:
- log.debug(f"Front evaluation skipped node {node.id}: {exc}")
-
- if not renders:
- return nodes
-
- pngs = [png for png, _ in renders]
- try:
- # A panel ranks the field as a whole, because a majority vote needs
- # candidates to compare; a single scorer just scores each one.
- values = scorer.rank(ref, pngs)
- except AttributeError:
- values = [scorer.score(ref, png) for png in pngs]
- except Exception as exc:
- log.warning(f"Front evaluation failed, keeping round order: {exc}")
- return nodes
-
- scored: list[tuple[float, SearchNode]] = []
- for value, (_png, node) in zip(values, renders, strict=True):
- node.metrics[FRONT_SCORE] = value
- scored.append((value, node))
-
- if not scored:
- return nodes
- scored.sort(key=lambda pair: pair[0])
- log.info(
- f"Front evaluated: {len(scored)} candidate(s), "
- f"best {scored[0][0]:.6f}, worst {scored[-1][0]:.6f}"
+ return evaluate_front(
+ nodes,
+ front_scorer=_front_scorer,
+ format_plugin=format_plugin,
+ out_w=original_w,
+ out_h=original_h,
)
- return [node for _value, node in scored]
engine = MultiprocessSearchEngine(
workers=workers,
diff --git a/tests/score/test_ensemble.py b/tests/score/test_ensemble.py
index f619e3d..bb4edd8 100644
--- a/tests/score/test_ensemble.py
+++ b/tests/score/test_ensemble.py
@@ -1,20 +1,24 @@
+import statistics
+
from PIL import Image
from vectrify.score.ensemble import EnsembleScorer, PanelReference
def _panel(*rows: list[float]) -> tuple[EnsembleScorer, PanelReference]:
- """A panel whose members report the distances the test dictates.
+ """A panel whose members report the calibrated distances the test dictates.
Stubbed at the per-member distances rather than at the encoders: the panel
cuts every picture into tiles and compares the embeddings itself, so faking
an encoder would mean faking image decoding and tiling too, none of which
- is what these tests are about. What is under test is the vote.
+ is what these tests are about.
"""
scorer = EnsembleScorer.__new__(EnsembleScorer)
scorer._members = [None] * len(rows) # type: ignore[assignment]
scorer._names = tuple(f"m{i}" for i in range(len(rows)))
- reference = PanelReference(image=Image.new("RGB", (12, 12)), tiles=[])
+ reference = PanelReference(
+ image=Image.new("RGB", (12, 12)), tiles=[], blank=[1.0] * len(rows)
+ )
def distances(_reference, images):
return [list(row[: len(images)]) for row in rows]
@@ -26,87 +30,60 @@ def distances(_reference, images):
CANDIDATES = [b"0", b"1", b"2"]
-def test_the_majority_outvotes_a_dissenting_member():
- """The property a single scorer cannot have: one member being idiosyncratic
- about a particular pair does not decide the pair."""
+def test_the_majority_decides_and_a_dissenting_member_does_not():
+ """The property a single scorer cannot have. With three members the median
+ is the majority position: one member being idiosyncratic about a drawing
+ cannot move the verdict, whatever value it reports."""
agree = [0.1, 0.2, 0.3]
- dissent = [0.9, 0.2, 0.1]
- scorer, reference = _panel(agree, agree, agree, dissent, dissent)
+ dissent = [0.9, 0.9, 0.9]
+ scorer, reference = _panel(agree, agree, dissent)
ranked = scorer.rank(reference, CANDIDATES)
- assert ranked[0] < ranked[1] < ranked[2]
+ assert ranked == agree
-def test_a_voting_cycle_leaves_every_candidate_ranked():
- """Rock-paper-scissors: each candidate beats the next by a majority. The
- relation cannot be sorted, and no candidate may be dropped or left without
- a position."""
- scorer, reference = _panel(
- [0.1, 0.2, 0.3],
- [0.1, 0.2, 0.3],
- [0.3, 0.1, 0.2],
- [0.3, 0.1, 0.2],
- [0.2, 0.3, 0.1],
- )
-
- ranked = scorer.rank(reference, CANDIDATES)
+def test_a_score_does_not_depend_on_the_field_it_was_scored_with():
+ """Why this replaced counting rivals beaten. The same drawing has to come
+ out the same however it is grouped, or two checks cannot be compared and
+ nothing can be cached from one to the next.
+ """
+ rows = [[0.1, 0.5, 0.9], [0.2, 0.5, 0.8], [0.3, 0.5, 0.7]]
+ scorer, reference = _panel(*rows)
- assert len(ranked) == 3
- assert all(0.0 <= value <= 1.0 for value in ranked)
+ alone = scorer.rank(reference, CANDIDATES[:1])
+ with_others = scorer.rank(reference, CANDIDATES)
+ assert alone[0] == with_others[0]
-def test_ranking_is_not_decided_by_one_member_s_scale():
- """A member reporting distances an order of magnitude larger than the rest
- would dominate any averaging scheme. It gets one vote here."""
- small = [0.01, 0.02, 0.03]
- huge = [90.0, 60.0, 30.0]
- scorer, reference = _panel(small, small, small, huge, huge)
- ranked = scorer.rank(reference, CANDIDATES)
+def test_scores_are_the_median_of_the_calibrated_members():
+ rows = [[0.1, 0.4], [0.2, 0.5], [0.9, 0.6]]
+ scorer, reference = _panel(*rows)
- assert ranked[0] < ranked[2], "the loud member overruled the majority"
+ ranked = scorer.rank(reference, CANDIDATES[:2])
+ assert ranked[0] == statistics.median([0.1, 0.2, 0.9])
+ assert ranked[1] == statistics.median([0.4, 0.5, 0.6])
-def test_a_single_candidate_falls_back_to_the_mean_distance():
- """Nothing to compare against, so there is no vote to take."""
- scorer, reference = _panel([0.2, 0.4, 0.6], [0.4, 0.4, 0.4])
- assert scorer.rank(reference, [b"0"]) == [(0.2 + 0.4) / 2]
- assert scorer.rank(reference, []) == []
-
-
-def test_a_tie_on_votes_is_settled_by_mean_rank():
- """Wins are integers, so a front of tens ties often, and the caller keeps
- only the best few as parents. Leaving ties to fall through to pool order
- would decide the next epoch arbitrarily."""
- # Every member ranks 0 and 1 adjacently but 2 last, so 0 and 1 tie on wins
- # against each other while every member places 0 ahead of 1.
- scorer, reference = _panel(
- [0.10, 0.11, 0.90],
- [0.10, 0.11, 0.90],
- [0.10, 0.11, 0.90],
- )
+def test_ranking_is_not_decided_by_one_member_s_scale():
+ """Calibration is what earns this. Raw cosine distances come from three
+ embedding spaces of different widths, and an uncalibrated combination is
+ decided by whichever member spreads widest -- one model steering the run,
+ which is the thing a panel exists to prevent."""
+ narrow = [0.10, 0.11]
+ also_narrow = [0.10, 0.11]
+ wide_but_opposed = [0.90, 0.10]
+ scorer, reference = _panel(narrow, also_narrow, wide_but_opposed)
- ranked = scorer.rank(reference, CANDIDATES)
+ ranked = scorer.rank(reference, CANDIDATES[:2])
- assert ranked[0] < ranked[1] < ranked[2]
- assert len(set(ranked)) == 3, "candidates were left tied"
-
-
-def test_the_tie_break_never_overrides_a_vote():
- """A candidate the majority puts ahead must stay ahead however lopsided the
- mean ranks are, or the tie-break has quietly become the ranking."""
- # 0 wins the majority (three members of five), while the two dissenting
- # members rank it dead last by a wide margin.
- scorer, reference = _panel(
- [0.10, 0.20, 0.30],
- [0.10, 0.20, 0.30],
- [0.10, 0.20, 0.30],
- [0.99, 0.01, 0.02],
- [0.99, 0.01, 0.02],
- )
+ # The wide member prefers the second candidate by a distance that dwarfs
+ # the others; the majority still decides.
+ assert ranked[0] < ranked[1]
- ranked = scorer.rank(reference, CANDIDATES)
- assert ranked[0] < ranked[1], "the tie-break reordered across a majority"
+def test_an_empty_field_ranks_to_nothing():
+ scorer, reference = _panel([0.1], [0.2], [0.3])
+ assert scorer.rank(reference, []) == []
diff --git a/tests/search/test_engine.py b/tests/search/test_engine.py
index 2f70963..4bc207f 100644
--- a/tests/search/test_engine.py
+++ b/tests/search/test_engine.py
@@ -1081,6 +1081,7 @@ def score_fn(results):
assert scored, "results scored before the failure should have survived it"
+
def test_the_epoch_budget_ends_an_epoch_that_has_not_gone_stale():
"""Staleness measures whether the pool has stopped producing; the budget
measures how long the proxy has run without the evaluator seeing anything.
diff --git a/tests/vector/test_evaluate_front.py b/tests/vector/test_evaluate_front.py
new file mode 100644
index 0000000..98f4f69
--- /dev/null
+++ b/tests/vector/test_evaluate_front.py
@@ -0,0 +1,166 @@
+"""The evaluator's cache: what it recalls, what it re-measures, what it costs.
+
+A panel call is the expensive thing in a run, and the pool it is asked about
+changes slowly, so most of any call is a repeat of the last one. These pin that
+the repeat is free -- and that the cheap paths stay cheap when the evaluator is
+absent, broken, or has nothing new to look at.
+"""
+
+from vectrify.formats.models import VectorStatePayload
+from vectrify.score.metrics import FRONT_SCORE
+from vectrify.search import ChainState, SearchNode
+from vectrify.vector.runner import evaluate_front
+
+
+class FakePlugin:
+ def rasterize(self, content, out_w, out_h):
+ _ = (out_w, out_h)
+ return content.encode()
+
+
+class CountingScorer:
+ """Records every field it is asked to score."""
+
+ def __init__(self, values: list[float] | None = None):
+ self.calls: list[int] = []
+ self._values = values
+
+ def rank(self, _ref, pngs):
+ self.calls.append(len(pngs))
+ if self._values is not None:
+ return self._values[: len(pngs)]
+ return [0.1 * (i + 1) for i in range(len(pngs))]
+
+
+def _node(node_id: int, content: str = "") -> SearchNode:
+ return SearchNode(
+ score=0.0,
+ id=node_id,
+ parent_id=0,
+ state=ChainState(
+ score=0.0,
+ payload=VectorStatePayload(
+ content=content,
+ raster_data_url=None,
+ raster_preview_data_url=None,
+ origin=None,
+ ),
+ ),
+ )
+
+
+def _evaluate(nodes, scorer, built: list | None = None):
+ def front_scorer():
+ if built is not None:
+ built.append(1)
+ return scorer, object()
+
+ return evaluate_front(
+ nodes,
+ front_scorer=front_scorer,
+ format_plugin=FakePlugin(),
+ out_w=8,
+ out_h=8,
+ )
+
+
+def test_every_node_is_scored_the_first_time():
+ scorer = CountingScorer()
+ nodes = [_node(i, f"") for i in range(1, 4)]
+
+ ranked = _evaluate(nodes, scorer)
+
+ assert scorer.calls == [3]
+ assert all(FRONT_SCORE in n.metrics for n in ranked)
+
+
+def test_a_second_look_at_the_same_nodes_costs_nothing():
+ scorer = CountingScorer()
+ nodes = [_node(i, f"") for i in range(1, 4)]
+
+ _evaluate(nodes, scorer)
+ _evaluate(nodes, scorer)
+
+ assert scorer.calls == [3]
+
+
+def test_a_fully_cached_call_does_not_even_build_the_scorer():
+ """The model is the expensive part, and a call the cache answers in full
+ has no reason to load one."""
+ scorer = CountingScorer()
+ nodes = [_node(i, f"") for i in range(1, 4)]
+ _evaluate(nodes, scorer)
+
+ built: list = []
+ _evaluate(nodes, scorer, built=built)
+
+ assert built == []
+
+
+def test_only_the_unseen_nodes_are_scored():
+ scorer = CountingScorer()
+ old = [_node(i, f"") for i in range(1, 4)]
+ _evaluate(old, scorer)
+
+ fresh = [_node(9, "")]
+ _evaluate([*old, *fresh], scorer)
+
+ assert scorer.calls == [3, 1]
+
+
+def test_a_recalled_score_orders_against_a_fresh_one():
+ """The point of an absolute score. A value measured in an earlier call has
+ to be comparable with one measured now, or the cache would order the field
+ by when each candidate happened to be seen."""
+ scorer = CountingScorer(values=[0.9])
+ stale = _node(1, "")
+ _evaluate([stale], scorer)
+
+ better = _node(2, "")
+ scorer._values = [0.1]
+ ranked = _evaluate([stale, better], scorer)
+
+ assert [n.id for n in ranked] == [2, 1]
+
+
+def test_nodes_without_content_are_left_out_rather_than_failing_the_field():
+ scorer = CountingScorer()
+ nodes = [_node(1, ""), _node(2, "")]
+
+ ranked = _evaluate(nodes, scorer)
+
+ assert scorer.calls == [1]
+ assert [n.id for n in ranked] == [1]
+
+
+def test_a_failing_evaluator_returns_the_nodes_it_was_given():
+ class Exploding:
+ def rank(self, _ref, _pngs):
+ raise RuntimeError("no")
+
+ nodes = [_node(1, ""), _node(2, "")]
+
+ ranked = _evaluate(nodes, Exploding())
+
+ assert ranked == nodes
+ assert all(FRONT_SCORE not in n.metrics for n in ranked)
+
+
+def test_a_scorer_without_rank_is_asked_one_candidate_at_a_time():
+ """--scorer simple has no panel to put a field to, only a score per
+ candidate."""
+
+ class SingleOnly:
+ def __init__(self):
+ self.seen = 0
+
+ def score(self, _ref, _png):
+ self.seen += 1
+ return 0.5
+
+ scorer = SingleOnly()
+ nodes = [_node(i, f"") for i in range(1, 4)]
+
+ _evaluate(nodes, scorer)
+
+ assert scorer.seen == 3