diff --git a/scripts/clean_runs.py b/scripts/clean_runs.py
index b72d924..35386f5 100755
--- a/scripts/clean_runs.py
+++ b/scripts/clean_runs.py
@@ -28,29 +28,45 @@
def collect_node_files(nodes_dir: Path) -> list[dict]:
"""
Read node files (any known output extension) from a nodes directory.
- Supports two filename formats:
- New: {score}_{id}.{ext} e.g. 0.069113_2.svg
- Old: score{score}_node{id}_... e.g. score00000.069113_node00002_parent00000.svg
+ Supports the shapes storage has written:
+ Current: {id}.{ext} e.g. 2.svg
+ Current: eval{score}_{id}.{ext} e.g. eval0.004392_2.svg
+ Legacy: {round_score}_{id}.{ext} e.g. 0.069113_2.svg
+ Legacy: score{score}_node{id}_... e.g. score00000.069113_node00002_...
+
+ Only the eval prefix carries a score that means anything: it is the
+ evaluator's, the run's only score. The legacy leading number was a blended
+ proxy that nothing ranked on, so it is parsed for the id and ignored.
"""
ext_pattern = "|".join(re.escape(e) for e in OUTPUT_EXTENSIONS)
# New format: plain score_id.ext
# `inf` must be its own alternative: storage writes f"{score:.6f}", which
# yields a bare "inf" for INVALID_SCORE, so requiring digits first made the
# optional (?:inf)? branch dead and left inf_*.svg files unmatched entirely.
- _new = re.compile(rf"^(inf|[0-9.]+)_(\d+)(?:{ext_pattern})$")
+ _plain = re.compile(rf"^(\d+)(?:{ext_pattern})$")
+ _eval = re.compile(rf"^eval(-?[0-9.]+)_(\d+)(?:{ext_pattern})$")
+ _legacy = re.compile(rf"^(inf|[0-9.]+)_(\d+)(?:{ext_pattern})$")
# Old format: score00000.069113_node00002_parent00000.svg
_old = re.compile(rf"^score([0-9.]+)_node(\d+)_parent\d+(?:{ext_pattern})$")
nodes = []
for node_path in sorted(nodes_dir.iterdir()):
- m = _new.match(node_path.name) or _old.match(node_path.name)
- if not m:
- continue
- try:
- score = float(m.group(1))
- except ValueError:
- score = float("inf")
- node_id = int(m.group(2))
+ bare = _plain.match(node_path.name)
+ if bare:
+ score, node_id = float("inf"), int(bare.group(1))
+ else:
+ m = (
+ _eval.match(node_path.name)
+ or _legacy.match(node_path.name)
+ or _old.match(node_path.name)
+ )
+ if not m:
+ continue
+ try:
+ score = float(m.group(1))
+ except ValueError:
+ score = float("inf")
+ node_id = int(m.group(2))
nodes.append(
{
"id": node_id,
diff --git a/scripts/plot_run.py b/scripts/plot_run.py
index 1aeb8d3..d23e321 100755
--- a/scripts/plot_run.py
+++ b/scripts/plot_run.py
@@ -96,7 +96,7 @@ def _float(key, default=0.0):
"seeds_target": _float("seeds_target"),
"epochs_completed": _float("epoch"),
"epoch_patience_config": _float("epoch_patience"),
- "epoch_diversity_config": _float("epoch_diversity"),
+ "epoch_max_tasks_config": _float("epoch_max_tasks"),
"pool_diversity_final": _float("pool_diversity"),
"pool_score_std_final": _float("pool_score_std"),
}
diff --git a/src/vectrify/cli.py b/src/vectrify/cli.py
index de7c3f4..8f86a22 100644
--- a/src/vectrify/cli.py
+++ b/src/vectrify/cli.py
@@ -33,7 +33,13 @@
# before going stale at task 5200. Given floors of 0.10 and 0.05, all four
# epochs of a real run ended on a pool measure rather than on staleness and the
# run stopped after 3821 of its 12000 tasks.
-DEFAULT_EPOCH_DIVERSITY = 0.0
+# A ceiling on how long one epoch may run before the evaluator gets to steer
+# again. Off by default: the right value is a judgement about how much drift on
+# the cheap measures is acceptable between evaluations, and nothing measured so
+# far pins it. Measured on one run, staleness at 500 first fired 150,800 tasks
+# into an epoch -- all of it with no evaluator in the loop, which is where the
+# proxy runs away from what a viewer would call better.
+DEFAULT_EPOCH_MAX_TASKS = None
# Tasks without improvement before an epoch is called converged. Measured over
# eleven runs and 145 improvements, the gap between one improvement and the
# next is 25 tasks at the median, 142 at the 95th percentile and 497 at the
@@ -221,28 +227,26 @@ def parse_args(args: list[str] | None = None) -> argparse.Namespace:
"so this and --seeds together fix the run's entire LLM spend. "
f"Default: {DEFAULT_EPOCHS}",
)
+ g_epoch.add_argument(
+ "--epoch-max-tasks",
+ type=int,
+ default=DEFAULT_EPOCH_MAX_TASKS,
+ dest="epoch_max_tasks",
+ metavar="N",
+ help="End the epoch after this many local tasks whether or not it has "
+ "gone stale, so the evaluator ranks the front and the model re-seeds "
+ "from its choice at least this often. Unset by default.",
+ )
g_epoch.add_argument(
"--epoch-patience",
type=int,
default=DEFAULT_EPOCH_PATIENCE,
dest="epoch_patience",
metavar="N",
- help="End the epoch and re-seed if the best score does not improve by "
- "reaching the best-ranked tier over this many consecutive local tasks. "
- "0 disables. "
+ help="End the epoch and re-seed if no candidate reaches the "
+ "best-ranked tier over this many consecutive local tasks. 0 disables. "
f"Default: {DEFAULT_EPOCH_PATIENCE}",
)
- g_epoch.add_argument(
- "--epoch-diversity",
- type=float,
- default=DEFAULT_EPOCH_DIVERSITY,
- dest="epoch_diversity",
- metavar="THR",
- help="End an epoch once pool diversity has fallen to this fraction "
- "of what it was when the epoch opened, e.g. 0.3. A fraction rather "
- "than a fixed level, because how varied a pool starts out depends on "
- "the drawing. 0 disables.",
- )
g_search.add_argument(
"--tournament-size",
type=int,
diff --git a/src/vectrify/dashboard.py b/src/vectrify/dashboard.py
index ce13af0..0579861 100644
--- a/src/vectrify/dashboard.py
+++ b/src/vectrify/dashboard.py
@@ -20,19 +20,6 @@ def _bar(fraction: float, width: int = 12) -> str:
return "█" * filled + "░" * (width - filled)
-def _diversity_color(pool_diversity: float, epoch_diversity: float) -> str:
- """Diversity keeps its own ladder — red once it has reached the stop
- threshold, yellow while still within 2x of it — rather than the
- fraction-based one the other criteria use."""
- if epoch_diversity <= 0:
- return "cyan"
- if pool_diversity < epoch_diversity:
- return "red"
- if pool_diversity < epoch_diversity * 2:
- return "yellow"
- return "green"
-
-
def _threshold_color(fraction: float) -> str:
"""Green while there is headroom, red as a stop threshold is approached."""
if fraction > 0.8:
@@ -100,10 +87,9 @@ def _build_renderable(stats: SearchStats) -> Panel:
)
# Pool stats: single line with diversity + variance values
- div_color = _diversity_color(s.pool_diversity, s.epoch_diversity)
pool_line = (
- f" diversity [{div_color}]{s.pool_diversity:.3f}[/{div_color}]"
+ f" diversity [dim]{s.pool_diversity:.3f}[/dim]"
f" spread [dim]{s.pool_score_std:.4f}[/dim]"
f" stale [dim]{s.epoch_no_improve:,}[/dim]"
)
@@ -111,17 +97,6 @@ def _build_renderable(stats: SearchStats) -> Panel:
# Stop criteria rows (only when enabled)
stop_rows: list[tuple[str, str]] = []
- if s.epoch_diversity > 0:
- # Diversity's bar tracks the raw value, and its color its own ladder.
- stop_rows.append(
- (
- "div stop",
- f" [{div_color}]{_bar(s.pool_diversity, width=20)}[/{div_color}]"
- f" {s.pool_diversity:.3f}"
- f" [dim]epoch at < {s.epoch_diversity:.3f}[/dim]",
- )
- )
-
if s.phase == "seed" and s.seeds_target > 0:
# Not a stop criterion, but the epoch is waiting on it all the same.
stop_rows.append(
diff --git a/src/vectrify/main.py b/src/vectrify/main.py
index 75640e8..8f50eb5 100755
--- a/src/vectrify/main.py
+++ b/src/vectrify/main.py
@@ -136,7 +136,7 @@ def main():
epoch_patience=args.epoch_patience or None,
pool_size=args.pool_size,
seeds=args.seeds,
- epoch_diversity=args.epoch_diversity,
+ epoch_max_tasks=args.epoch_max_tasks,
tournament_size=args.tournament_size,
adaptive_operators=args.adaptive_operators,
epochs=args.epochs,
diff --git a/src/vectrify/score/metrics.py b/src/vectrify/score/metrics.py
index d4071c5..d39ca6b 100644
--- a/src/vectrify/score/metrics.py
+++ b/src/vectrify/score/metrics.py
@@ -7,8 +7,6 @@
from collections.abc import Mapping
-from vectrify.score.moments import MOMENT_WEIGHT
-
EDGE = "edge"
COLOUR = "colour"
SHAPE = "shape"
@@ -21,64 +19,10 @@
# at different things.
SCORER_METRICS: tuple[str, ...] = (EDGE, COLOUR, SHAPE, DETAIL)
-# What selection ranks candidates by: the chromatic and structural distances,
-# blended. No embedding: the round score no longer runs a model at all.
-#
-# It used to, weighted half and half with colour, on the strength of a
-# measurement that ranked each candidate rule by how often it agreed with the
-# evaluator panel. That is only as sound as the panel, and the panel then read
-# whole images, which is the thing it was worst at. Against the distortion
-# screen instead -- damage of a known severity in a known order, so nothing has
-# to be trusted as a reference -- the ordering is different. As a share of
-# level pairs ordered correctly on vector damage:
-#
-# 0.25 colour + 0.50 edge 96.7% no forward pass
-# edge alone 96.2% no forward pass
-# colour alone 95.8% no forward pass
-# 0.50 embedding + 0.50 colour 95.5% one forward pass, what shipped
-# embedding at three cells 95.4% nine forward passes
-# embedding whole 92.8% one forward pass
-#
-# Adding an embedding back to the winning pair moves it by a tenth of a point
-# for four to nine forward passes per candidate, which is the whole of the
-# round's model cost for nothing.
-#
-# Edge overlap was dropped from this blend once for scoring 41.5% against the
-# panel. It is the strongest single ingredient there is when measured against
-# damage that is known rather than judged.
-#
-# Nothing counts elements or bytes. No operator adds an element, so a measure
-# built on how many there are says nothing the score does not already say.
-OBJECTIVE_NAMES: tuple[str, ...] = SCORER_METRICS
-
-# Weights within the blend, from the same sweep, which searched a grid of
-# 0, 0.25 and 0.5 and picked colour at 0.25 against edge at 0.50. What it chose
-# is the one-to-two ratio; the pair is written normalised so the round score is
-# on the scale it appears to be on, since it is recorded per node and read back
-# as an absolute number. Ranking is unaffected either way -- both this and
-# build_objectives are linear in the weights, so a common factor cancels.
-#
-# The optimum is broad: holding edge at two thirds, colour anywhere from a
-# quarter to a half of it lands within a tenth of a point.
-# Colour and edge keep their one-to-two ratio; the shape term takes its weight
-# from what is left. See score.moments for why it earns a place and why the
-# place is a small one.
-COLOUR_WEIGHT = (1.0 - MOMENT_WEIGHT) / 3.0
-EDGE_WEIGHT = 2.0 * (1.0 - MOMENT_WEIGHT) / 3.0
-SHAPE_WEIGHT = MOMENT_WEIGHT
-
-
-def round_score(colour: float, edge: float, shape: float = 0.0) -> float:
- """What a candidate is ranked by, before the population is known.
-
- build_objectives scales each part by its population maximum, which needs a
- population; a candidate arriving on its own needs an absolute number for
- the run's best-so-far and the lineage. Both measures already sit near
- [0, 1], so the same weights apply unscaled.
- """
- return COLOUR_WEIGHT * colour + EDGE_WEIGHT * edge + SHAPE_WEIGHT * shape
-
-
+# No weights, and nothing to blend. Selection ranks by dominance over the
+# vector of measures, which compares them component by component -- so no
+# measure is privileged and a weight between them would not change a single
+# verdict. The only score in the run is the evaluator's, below.
# The evaluator's verdict on a converged front member. Recorded so a run can be
# read back, and deliberately NOT an objective: it exists on a handful of nodes
# per epoch, and a metric present on only part of the population reads as 0.0
diff --git a/src/vectrify/search/__init__.py b/src/vectrify/search/__init__.py
index 2f5cdaf..a905c33 100644
--- a/src/vectrify/search/__init__.py
+++ b/src/vectrify/search/__init__.py
@@ -1,11 +1,19 @@
from vectrify.search.base import SearchStrategy, StorageAdapter
from vectrify.search.collector import StatCollector
from vectrify.search.engine import MultiprocessSearchEngine
-from vectrify.search.models import INVALID_SCORE, ChainState, Result, SearchNode, Task
+from vectrify.search.models import (
+ INVALID_SCORE,
+ VALID_SCORE,
+ ChainState,
+ Result,
+ SearchNode,
+ Task,
+)
from vectrify.search.nsga import NsgaStrategy
__all__ = [
"INVALID_SCORE",
+ "VALID_SCORE",
"ChainState",
"MultiprocessSearchEngine",
"NsgaStrategy",
diff --git a/src/vectrify/search/base.py b/src/vectrify/search/base.py
index 9507b1e..ec48df6 100644
--- a/src/vectrify/search/base.py
+++ b/src/vectrify/search/base.py
@@ -16,13 +16,6 @@ def top_tier_ids(self, pool: list[SearchNode]) -> set[int]:
progress, so this replaces comparing a blended score."""
...
- def should_diversify(self, pool: list[SearchNode]) -> tuple[bool, float]:
- """Return (trigger_epoch, diversity).
-
- diversity is the mean normalised Hamming distance across sampled node pairs.
- """
- ...
-
def select_survivors(
self, nodes: list[SearchNode[TState]], max_keep: int
) -> list[SearchNode[TState]]:
@@ -50,7 +43,15 @@ class StorageAdapter(Protocol[TState]):
def initialize(self) -> None: ...
- def save_node(self, node: SearchNode[TState], tasks_completed: int = 0) -> None: ...
+ def save_node(
+ self,
+ node: SearchNode[TState],
+ tasks_completed: int = 0,
+ keep_content: bool = True,
+ ) -> None:
+ """Record *node*. *keep_content* asks for the drawing itself as well as
+ the lineage row, which is how a run stays a readable directory."""
+ ...
def save_best(self, node: SearchNode[TState]) -> None:
"""Write the best final candidate to the top-level output path."""
diff --git a/src/vectrify/search/collector.py b/src/vectrify/search/collector.py
index 6fbfe75..7a3a9cc 100644
--- a/src/vectrify/search/collector.py
+++ b/src/vectrify/search/collector.py
@@ -46,7 +46,7 @@ def _rounded(field: str, digits: int) -> Callable[["SearchStats"], float]:
"pool_diversity": _rounded("pool_diversity", 4),
"pool_score_std": _rounded("pool_score_std", 6),
"epoch_patience": lambda s: s.epoch_patience,
- "epoch_diversity": _rounded("epoch_diversity", 4),
+ "epoch_max_tasks": lambda s: s.epoch_max_tasks,
}
STATS_COLUMNS = list(STATS_FIELDS)
@@ -72,10 +72,10 @@ def __init__(self, stats: "SearchStats", run_dir: Path | None = None) -> None:
def configure_run(
self,
*,
- epoch_diversity: float,
+ epoch_max_tasks: int | None,
) -> None:
s = self._stats
- s.epoch_diversity = epoch_diversity
+ s.epoch_max_tasks = epoch_max_tasks or 0
def seed_initial_score(self, best_score: float) -> None:
s = self._stats
diff --git a/src/vectrify/search/engine.py b/src/vectrify/search/engine.py
index 4fbc686..a5cd9b1 100644
--- a/src/vectrify/search/engine.py
+++ b/src/vectrify/search/engine.py
@@ -7,8 +7,10 @@
from collections.abc import Callable
from typing import Any, Generic, TypeVar
+from vectrify.score.metrics import FRONT_SCORE
from vectrify.search.base import SearchStrategy, StorageAdapter
from vectrify.search.collector import StatCollector
+from vectrify.search.diversity import pool_diversity
from vectrify.search.models import (
INVALID_SCORE,
ChainState,
@@ -117,7 +119,7 @@ def run(
epoch_seeds: int = 0,
initial_seeds: int | None = None,
epochs: int | None = None,
- epoch_diversity: float = 0.0,
+ epoch_max_tasks: int | None = None,
operator_policy: OperatorPolicy | None = None,
collector: StatCollector | None = None,
) -> None:
@@ -205,9 +207,13 @@ def _scorer_worker():
seed_archive: dict[int, SearchNode[TState]] = {}
seed_archive_cap = max(1, active_pool_size // SEED_ARCHIVE_POOL_SHARE)
- sorted_initial = sorted(initial_nodes, key=lambda n: n.score)
- active_pool: list[SearchNode[TState]] = sorted_initial[:active_pool_size]
- best_node = sorted_initial[0] if sorted_initial else None
+ # No ordering to apply: the measures are traded off by dominance and
+ # nothing ranks a candidate on its own. The pool is a set, and the cap
+ # takes whatever arrived.
+ active_pool: list[SearchNode[TState]] = list(initial_nodes)[:active_pool_size]
+ # Set by the evaluator, the run's only score, at each epoch boundary and
+ # at the end. There is deliberately no best between those points.
+ best_node: SearchNode[TState] | None = None
# Children are held back and merged as a generation, NSGA-II's mu+lambda
# replacement: the truncation is a whole-population sort, so paying it
@@ -220,9 +226,9 @@ def _scorer_worker():
epoch = 0
epoch_no_improve = 0
+ epoch_started_at = 0
# Reset at every transition, so each epoch is judged against the
# pool it opened with rather than against the first one.
- epoch_baseline: float | None = None
pool_refilling = False # True until a fresh epoch's pool reaches capacity
# Children accumulate outside active_pool: they replace it wholesale
@@ -249,9 +255,7 @@ def _scorer_worker():
self.storage.max_node_id, max((n.id for n in initial_nodes), default=0)
)
- log.info(
- f"Search started. Initial best: {best_node.score if best_node else 'N/A'}"
- )
+ log.info(f"Search started with {len(active_pool)} candidate(s) in the pool.")
if phase == SEED_PHASE:
log.info(
f"Epoch 0: seeding with {seeds_target} LLM call(s) "
@@ -269,7 +273,8 @@ def _begin_seed_phase() -> None:
seeds_target, \
seeds_dispatched, \
seeds_completed, \
- seed_task_ids
+ seed_task_ids, \
+ best_node
# The remembered seeds enter the ranking as candidates rather than
# being handed a reserved slot: a seed the pool has genuinely
@@ -279,6 +284,11 @@ def _begin_seed_phase() -> None:
# local search has wandered away from it -- from there the same
# comparison that ranks everything else can decide.
pool_ids = {n.id for n in active_pool}
+ # The standing best joins the comparison so the evaluator can keep
+ # it: without that, a boundary could replace a candidate it already
+ # judged better.
+ if best_node is not None:
+ pool_ids.add(best_node.id)
candidates = active_pool + [
n for n in seed_archive.values() if n.id not in pool_ids
]
@@ -288,8 +298,19 @@ def _begin_seed_phase() -> None:
if parents and self.rank_front is not None:
try:
parents = self.rank_front(parents)
+ # The evaluator has just spoken, which is the only occasion
+ # anything in the run is called best. Its top pick stands
+ # until the next boundary, unless it already prefers the
+ # standing one -- best_node is in the ranked set, so if it
+ # comes out ahead it stays.
+ if parents:
+ best_node = parents[0]
+ log.info(
+ f"Best so far: node={best_node.id} "
+ f"evaluator={best_node.metrics.get(FRONT_SCORE, 0.0):.6f}"
+ )
except Exception as exc:
- log.warning(f"Front evaluation failed, keeping L1 order: {exc}")
+ log.warning(f"Front evaluation failed, keeping rank order: {exc}")
parents = parents[:epoch_seeds]
if not parents:
parents = list(active_pool)
@@ -442,31 +463,28 @@ def _make_node(res: Result, *, new_lineage: bool = False) -> SearchNode[TState]:
operator=res.operator,
)
- def _note_best(new_node: SearchNode[TState], res: Result) -> bool:
- nonlocal best_node
-
- is_new_best = best_node is None or new_node.score < best_node.score
+ def _outranks(a: SearchNode[TState], b: SearchNode[TState]) -> bool:
+ """Whether *a* beats *b* under the strategy's own relation."""
+ if b.score >= INVALID_SCORE:
+ return a.score < INVALID_SCORE
+ if a.score >= INVALID_SCORE:
+ return False
+ return a.id in self.strategy.top_tier_ids([a, b])
+
+ def _note_accepted(new_node: SearchNode[TState], res: Result) -> None:
+ """Record an accepted candidate. Nothing here decides it is best:
+ that is the evaluator's call and it happens at epoch boundaries."""
if collector is not None:
collector.on_accepted(
new_node,
- is_new_best=is_new_best,
+ is_new_best=False,
elapsed=time.monotonic() - start_time,
llm_type=res.llm_type,
)
- if is_new_best:
- best_node = new_node
- log.info(
- f"[{res.llm_type.upper() if res.llm_type else 'NEW BEST'}] "
- f"node={new_node.id} score={new_node.score:.6f}"
- )
- elif res.llm_type:
- log.info(
- f"[{res.llm_type.upper()} ACCEPTED] "
- f"node={new_node.id} score={new_node.score:.6f}"
- )
+ if res.llm_type:
+ log.info(f"[{res.llm_type.upper()} ACCEPTED] node={new_node.id}")
else:
- log.debug(f"[ACCEPTED] node={new_node.id} score={new_node.score:.6f}")
- return is_new_best
+ log.debug(f"[ACCEPTED] node={new_node.id}")
def _archive_seed(node: SearchNode[TState]) -> None:
"""Keep an LLM seed available to the fronts of later epochs.
@@ -477,11 +495,18 @@ def _archive_seed(node: SearchNode[TState]) -> None:
the archive holding the seeds most likely to still be worth editing.
"""
current = seed_archive.get(node.root_id)
- if current is not None and current.score <= node.score:
+ if current is not None and not _outranks(node, current):
return
seed_archive[node.root_id] = node
if len(seed_archive) > seed_archive_cap:
- worst = max(seed_archive.values(), key=lambda n: n.score)
+ # The entry the rest of the archive beats most often. Dominance
+ # rather than a score, so no measure is privileged here either.
+ entries = list(seed_archive.values())
+ losses = {
+ n.root_id: sum(1 for m in entries if m is not n and _outranks(m, n))
+ for n in entries
+ }
+ worst = max(entries, key=lambda n: losses[n.root_id])
del seed_archive[worst.root_id]
def _process_seed_result(res: Result) -> None:
@@ -489,7 +514,7 @@ def _process_seed_result(res: Result) -> None:
seed_children.append(new_node)
_archive_seed(new_node)
node_states[new_node.id] = new_node.state
- _note_best(new_node, res)
+ _note_accepted(new_node, res)
self.storage.save_node(new_node, tasks_completed)
def _close_generation() -> None:
@@ -516,7 +541,8 @@ def _close_generation() -> None:
# can get to the front any more, whatever the numbers happen to be
# denominated in.
new_ids = {n.id for n in pending_children}
- if new_ids & self.strategy.top_tier_ids(combined):
+ top_tier = self.strategy.top_tier_ids(combined)
+ if new_ids & top_tier:
epoch_no_improve = 0
if collector is not None:
collector.on_no_improve_reset()
@@ -526,7 +552,14 @@ def _close_generation() -> None:
operator_policy.update(child.operator, child.id in kept)
if child.id in kept:
node_states[child.id] = child.state
- self.storage.save_node(child, tasks_completed)
+ # Content only for candidates that reached the best-ranked
+ # tier. A run admits most of what it produces -- one wrote
+ # 106,640 files -- and a node that never outranked anything
+ # is not worth reading back. The lineage row is written
+ # either way, so the record of what happened is complete.
+ self.storage.save_node(
+ child, tasks_completed, keep_content=child.id in top_tier
+ )
continue
# A child can be the run's best and still lose its generation on
# another objective. Save it anyway: save_best is about to write
@@ -554,7 +587,7 @@ def _close_generation() -> None:
def _process_local_result(res: Result) -> None:
new_node = _make_node(res)
pending_children.append(new_node)
- _note_best(new_node, res)
+ _note_accepted(new_node, res)
# Progress is decided when the generation closes, where the pool is
# ranked -- see _close_generation. A candidate cannot be known to
@@ -563,9 +596,9 @@ def _process_local_result(res: Result) -> None:
_close_generation()
def _do_epoch_transition(reason: str) -> None:
- nonlocal epoch, epoch_baseline
+ nonlocal epoch, epoch_started_at
- epoch_baseline = None
+ epoch_started_at = tasks_completed
# The next seed batch edits this pool's front, so the children that
# arrived since the last generation have to land in it first.
@@ -582,7 +615,7 @@ def _do_epoch_transition(reason: str) -> None:
_begin_seed_phase()
def _check_epoch_end():
- nonlocal pool_refilling, epoch_baseline
+ nonlocal pool_refilling
if pool_refilling:
if len(active_pool) < active_pool_size:
@@ -592,20 +625,30 @@ def _check_epoch_end():
staleness = (
epoch_patience is not None and epoch_no_improve >= epoch_patience
)
- _unused, pool_div = self.strategy.should_diversify(active_pool)
+ # Still reported, no longer a criterion: read against the epoch's
+ # opening value it was a ratio to a moment, and across real runs that
+ # moment was a trough as often as a peak -- epoch 0 opened on the
+ # "too little data" sentinel of 1.0 and later epochs ended above
+ # their own baseline, so the rule fired at once or never.
+ pool_div = pool_diversity(active_pool)
pool_std = score_std(valid_scores(active_pool))
if collector is not None:
collector.on_pool_state(diversity=pool_div, score_std=pool_std)
- # Diversity is read against where this epoch started rather than
- # against a fixed number: it sat anywhere between 0.05 and 0.33
- # across real runs depending only on how intricate the drawing is,
- # so no absolute threshold means the same thing twice.
- if epoch_baseline is None:
- epoch_baseline = max(pool_div, 1e-9)
- low_diversity = (
- epoch_diversity > 0 and pool_div / epoch_baseline < epoch_diversity
+ # A ceiling on how long one epoch may run. Staleness measures
+ # whether the pool has stopped producing; this measures how long the
+ # proxy has been left unsupervised, which is a different thing.
+ # Measured on one run, the gap between top-tier entries has a median
+ # of 68 tasks and a 99th percentile of 285, so staleness at 500 only
+ # arrives at the far tail -- 150,800 tasks into the epoch, all of it
+ # without the evaluator seeing anything. The epoch boundary is where
+ # the evaluator ranks the front and the model re-seeds from its
+ # choice, so capping the epoch caps the drift.
+ over_budget = (
+ epoch_max_tasks is not None
+ and epoch_max_tasks > 0
+ and tasks_completed - epoch_started_at >= epoch_max_tasks
)
# Any one of these ending the epoch, rather than all of them
@@ -627,29 +670,36 @@ def _check_epoch_end():
f"staleness ({epoch_no_improve} >="
f" {epoch_patience} tasks without improvement)"
)
- elif low_diversity:
+ elif over_budget:
reason = (
- f"diversity fell to {pool_div / epoch_baseline:.2f}"
- " of its opening value"
+ f"epoch budget ({tasks_completed - epoch_started_at} >="
+ f" {epoch_max_tasks} tasks)"
)
else:
return
_do_epoch_transition(reason)
+ def _any_top_tier() -> SearchNode[TState] | None:
+ """A member of the best-ranked tier, for when the evaluator never
+ ran or failed. Nothing else can name a best: with the measures
+ traded off there is no scalar to sort by, so any unbeaten candidate
+ is as defensible as another -- and writing one of those beats
+ losing the run's artifact to a scorer error at shutdown.
+ """
+ valid = [n for n in active_pool if n.score < INVALID_SCORE]
+ if not valid:
+ return None
+ top = self.strategy.top_tier_ids(valid)
+ return next((n for n in valid if n.id in top), valid[0])
+
def _final_artifact() -> SearchNode[TState] | None:
"""The candidate to write out, chosen by the evaluator.
- best_node is the winner on the round's score, which is a cheap
- stand-in for the real objective and ranks candidates at about rho
- 0.83 against it. Within one front the evaluator finds roughly a 2x
- spread, so trusting the proxy here is close to picking arbitrarily
- among the good candidates -- and the run has already paid for every
- one of them.
-
- best_node is included in the comparison rather than replaced, so
- this cannot do worse by the evaluator's own judgement than the
- score-based pick did.
+ best_node is whatever the evaluator chose at the last epoch
+ boundary. It is included in the comparison rather than replaced, so
+ this cannot come out worse by the evaluator's own judgement than its
+ previous pick.
The whole pool is evaluated, not the capped front an epoch boundary
gets: FRONT_EVAL_CAP exists because a boundary pays that cost every
@@ -658,20 +708,21 @@ def _final_artifact() -> SearchNode[TState] | None:
case the pool held a candidate the evaluator scored 8x better than
the one the round score picked.
"""
+ fallback = best_node or _any_top_tier()
if self.rank_front is None or not active_pool:
- return best_node
+ return fallback
finalists = [n for n in active_pool if n.score < INVALID_SCORE]
if best_node is not None and all(n.id != best_node.id for n in finalists):
finalists.append(best_node)
if not finalists:
- return best_node
+ return fallback
try:
return self.rank_front(finalists)[0]
except Exception as exc:
- log.warning(f"Final evaluation failed, keeping the best score: {exc}")
- return best_node
+ log.warning(f"Final evaluation failed, keeping a top-tier node: {exc}")
+ return fallback
try:
while True:
diff --git a/src/vectrify/search/models.py b/src/vectrify/search/models.py
index 456852b..ff4b217 100644
--- a/src/vectrify/search/models.py
+++ b/src/vectrify/search/models.py
@@ -3,6 +3,13 @@
INVALID_SCORE = float("inf")
+# A candidate that was measured. `score` is a validity marker and nothing more:
+# the measures are traded off by dominance, so no single number orders
+# candidates mid-run, and inventing one is what the blended round score was.
+# The run's only score is the evaluator's, recorded as metrics[FRONT_SCORE] on
+# the nodes it has actually seen.
+VALID_SCORE = 0.0
+
TState = TypeVar("TState")
TResultPayload = TypeVar("TResultPayload")
diff --git a/src/vectrify/search/nsga.py b/src/vectrify/search/nsga.py
index 3b057ef..bbd02c5 100644
--- a/src/vectrify/search/nsga.py
+++ b/src/vectrify/search/nsga.py
@@ -4,7 +4,7 @@
from typing import Any, Generic, TypeVar
from vectrify.score.metrics import SCORER_METRICS
-from vectrify.search.diversity import hamming_distance, pool_diversity
+from vectrify.search.diversity import hamming_distance
from vectrify.search.models import INVALID_SCORE, SearchNode
log = logging.getLogger(__name__)
@@ -156,11 +156,11 @@ def build_objectives(nodes: list[SearchNode]) -> dict[int, Objectives]:
by component, so any positive rescaling of a component leaves every
dominance verdict unchanged, and a weight there would be inert.
- Trading the three off against each other is the reason each is measured
+ Trading them off against each other is the reason each is measured
separately: a measure that only ever contributes a fraction of a sum can
- never outvote the other two on the candidates it disagrees about, which is
- exactly where it earns its place. The weighted sum is what `round_score`
- reports as the run's headline number; it is not what ranks the pool.
+ never outvote the others on the candidates it disagrees about, which is
+ exactly where it earns its place. No measure is privileged and there is no
+ blend of them anywhere -- the run's only score is the evaluator's.
Callers must pass only valid nodes (score < INVALID_SCORE); an infinite
score would corrupt the normalization for every other node.
@@ -202,12 +202,10 @@ def __init__(
self,
pool_size: int = 20,
crossover_distance_threshold: int = 10,
- epoch_diversity: float = 0.0,
tournament_size: int = 2,
):
self.pool_size = pool_size
self.crossover_distance_threshold = crossover_distance_threshold
- self.epoch_diversity = epoch_diversity
# Selection intensity is a function of the tournament size alone -- the
# winner's expected quantile is ~1/(size+1) -- so this is an absolute
# count rather than a fraction of the pool, and stays meaningful when
@@ -372,7 +370,3 @@ def top_tier_ids(self, pool: list[SearchNode[TState]]) -> set[int]:
return set()
objectives = build_objectives(valid)
return {n.id for n in pareto_front(valid, lambda n: objectives[n.id])}
-
- def should_diversify(self, pool: list[SearchNode[TState]]) -> tuple[bool, float]:
- diversity = pool_diversity(pool)
- return self.epoch_diversity > 0 and diversity < self.epoch_diversity, diversity
diff --git a/src/vectrify/search/stats.py b/src/vectrify/search/stats.py
index 86616f2..4615cb9 100644
--- a/src/vectrify/search/stats.py
+++ b/src/vectrify/search/stats.py
@@ -65,7 +65,7 @@ class SearchStats:
seeds_completed: int = 0
seeds_target: int = 0
pool_diversity: float = 1.0
- epoch_diversity: float = 0.0
+ epoch_max_tasks: int = 0
tasks_completed: int = 0
accepted_count: int = 0
diff --git a/src/vectrify/vector/resume.py b/src/vectrify/vector/resume.py
index 199d12c..61b0ba0 100644
--- a/src/vectrify/vector/resume.py
+++ b/src/vectrify/vector/resume.py
@@ -10,10 +10,11 @@
from vectrify.score.compare import compare
from vectrify.score.complexity import detail_excess
from vectrify.score.edges import overlap_distance
-from vectrify.score.metrics import COLOUR, DETAIL, EDGE, SHAPE, round_score
+from vectrify.score.metrics import COLOUR, DETAIL, EDGE, SHAPE
from vectrify.score.simple import SimpleFallbackScorer
from vectrify.search import (
INVALID_SCORE,
+ VALID_SCORE,
ChainState,
SearchNode,
StorageAdapter,
@@ -156,7 +157,7 @@ def _prep(item: tuple) -> PreppedNode:
metrics[COLOUR] = float(comparison.colour.mean())
metrics[SHAPE] = comparison.shape
metrics[DETAIL] = detail_excess(reference_detail, item.png)
- new_score = round_score(metrics[COLOUR], metrics[EDGE], metrics[SHAPE])
+ new_score = VALID_SCORE
node = SearchNode(
score=new_score,
id=current_new_id,
diff --git a/src/vectrify/vector/runner.py b/src/vectrify/vector/runner.py
index 959d3c8..38cb334 100644
--- a/src/vectrify/vector/runner.py
+++ b/src/vectrify/vector/runner.py
@@ -12,7 +12,7 @@
from PIL import Image, UnidentifiedImageError
from vectrify.cli import (
- DEFAULT_EPOCH_DIVERSITY,
+ DEFAULT_EPOCH_MAX_TASKS,
DEFAULT_MAX_TOTAL_TASKS,
DEFAULT_POOL_SIZE,
DEFAULT_RESOLUTION_LLM,
@@ -36,12 +36,12 @@
EDGE,
FRONT_SCORE,
SHAPE,
- round_score,
)
from vectrify.score.utils import MAX_SCORE
from vectrify.score.vision import DEFAULT_VISION_MODEL
from vectrify.search import (
INVALID_SCORE,
+ VALID_SCORE,
ChainState,
MultiprocessSearchEngine,
NsgaStrategy,
@@ -120,7 +120,7 @@ def run_vector_search(
epoch_patience: int | None = None,
pool_size: int = DEFAULT_POOL_SIZE,
seeds: int | None = None,
- epoch_diversity: float = DEFAULT_EPOCH_DIVERSITY,
+ epoch_max_tasks: int | None = DEFAULT_EPOCH_MAX_TASKS,
tournament_size: int = DEFAULT_TOURNAMENT_SIZE,
adaptive_operators: bool = True,
epochs: int | None = None,
@@ -232,7 +232,7 @@ def run_vector_search(
)
if collector is not None:
collector.configure_run(
- epoch_diversity=epoch_diversity,
+ epoch_max_tasks=epoch_max_tasks,
)
valid = [n for n in initial_nodes if n.score < INVALID_SCORE]
if valid:
@@ -311,7 +311,6 @@ def rank_front(nodes: list[SearchNode]) -> list[SearchNode]:
workers=workers,
strategy=NsgaStrategy[VectorStatePayload](
pool_size=pool_size,
- epoch_diversity=epoch_diversity,
tournament_size=tournament_size,
),
storage=storage,
@@ -358,9 +357,10 @@ def _pixel_objectives(res) -> None:
res.metrics[COLOUR] = float(comparison.colour.mean())
res.metrics[SHAPE] = comparison.shape
res.metrics[DETAIL] = detail_excess(reference_detail, png)
- res.score = round_score(
- res.metrics[COLOUR], res.metrics[EDGE], res.metrics[SHAPE]
- )
+ # Measured, so valid. `score` carries no magnitude any more: the
+ # measures are ranked by dominance and the only score in the run is
+ # the evaluator's, recorded as FRONT_SCORE on the nodes it sees.
+ res.score = VALID_SCORE
except Exception as exc:
log.debug(f"Pixel objectives skipped: {exc}")
@@ -410,7 +410,7 @@ def score_fn(results):
epoch_seeds=epoch_seeds,
initial_seeds=first_batch,
epochs=epochs,
- epoch_diversity=epoch_diversity,
+ epoch_max_tasks=epoch_max_tasks,
operator_policy=operator_policy,
collector=collector,
)
diff --git a/src/vectrify/vector/storage.py b/src/vectrify/vector/storage.py
index a9742aa..4d24415 100644
--- a/src/vectrify/vector/storage.py
+++ b/src/vectrify/vector/storage.py
@@ -124,26 +124,37 @@ def load_resume_nodes(self) -> list[tuple[int, str]]:
log.info(f"Loading nodes to resume from latest run: {latest_run.name}")
ext = re.escape(self.file_extension)
- # `inf` is its own alternative because save_node writes f"{score:.6f}",
- # which renders INVALID_SCORE as a bare "inf".
- file_pattern = re.compile(rf"^(inf|[0-9.]+)_(\d+){ext}$")
- parsed_files: list[tuple[int, Path, float]] = []
+ # Two shapes, both written by _node_basename: a bare id, and an
+ # evaluator score followed by an id. Older runs used a blended score in
+ # that leading position, which parses here as a score that no longer
+ # means anything -- so --resume-top only trusts an `eval` prefix and
+ # otherwise takes the newest ids, which is the honest ordering when
+ # nothing in the directory has been evaluated.
+ scored = re.compile(rf"^eval(-?[0-9.]+)_(\d+){ext}$")
+ plain = re.compile(rf"^(\d+){ext}$")
+ parsed_files: list[tuple[int, Path, float | None]] = []
glob_pattern = f"*{self.file_extension}"
for file_path in target_nodes_dir.glob(glob_pattern):
- match = file_pattern.match(file_path.name)
- node_id = int(match.group(2)) if match else self._max_id + 1
- # Score comes from the match, not from re-splitting the stem: a
- # user-dropped file with a non-numeric prefix would otherwise raise
- # an unhandled ValueError while sorting for --resume-top.
- score = float(match.group(1)) if match else float("inf")
+ score: float | None = None
+ match = scored.match(file_path.name)
+ if match:
+ node_id, score = int(match.group(2)), float(match.group(1))
+ else:
+ bare = plain.match(file_path.name)
+ node_id = int(bare.group(1)) if bare else self._max_id + 1
self._max_id = max(self._max_id, node_id)
parsed_files.append((node_id, file_path, score))
if self.resume_top is not None:
- parsed_files.sort(key=lambda item: item[2])
- parsed_files = parsed_files[: self.resume_top]
+ evaluated = [item for item in parsed_files if item[2] is not None]
+ if evaluated:
+ evaluated.sort(key=lambda item: item[2])
+ parsed_files = evaluated[: self.resume_top]
+ else:
+ parsed_files.sort(key=lambda item: item[0], reverse=True)
+ parsed_files = parsed_files[: self.resume_top]
resumed_data = []
for node_id, file_path, _score in parsed_files:
@@ -157,8 +168,17 @@ def load_resume_nodes(self) -> list[tuple[int, str]]:
return sorted(resumed_data, key=lambda x: x[0])
def save_node(
- self, node: SearchNode[VectorStatePayload], tasks_completed: int = 0
+ self,
+ node: SearchNode[VectorStatePayload],
+ tasks_completed: int = 0,
+ keep_content: bool = True,
) -> None:
+ """Record *node* in lineage.csv, and write its content when asked.
+
+ *keep_content* is how a run stays a readable directory rather than a
+ hundred thousand files: the lineage row is cheap and always written,
+ the drawing itself only for candidates worth reading back.
+ """
if self.nodes_dir is None or self.lineage_csv is None:
return
@@ -167,16 +187,21 @@ def save_node(
# --no-write-lineage suppresses the per-node files and lineage.csv, but
# the raster/heatmap sidecars stay under their own flags.
if not self.write_lineage:
- self._save_sidecars(node)
+ if keep_content:
+ self._save_sidecars(node)
return
- base_fn = f"{node.score:.6f}_{node.id}"
-
- if node.state.payload.content:
- content_path = self.nodes_dir / f"{base_fn}{self.file_extension}"
- content_path.write_text(node.state.payload.content, encoding="utf-8")
-
- self._save_sidecars(node)
+ # Named by id alone. The name used to lead with a blended score, which
+ # meant a directory listing sorted by a number nothing in the run ranks
+ # on: on one run the best-named file was 0.033325 while the artifact the
+ # evaluator actually chose read 0.052259 by that same number, so anyone
+ # reading the directory would pick the wrong file.
+ if keep_content:
+ base_fn = self._node_basename(node)
+ if node.state.payload.content:
+ content_path = self.nodes_dir / f"{base_fn}{self.file_extension}"
+ content_path.write_text(node.state.payload.content, encoding="utf-8")
+ self._save_sidecars(node)
content_md5 = (
hashlib.md5(node.state.payload.content.encode()).hexdigest()
@@ -206,10 +231,23 @@ def save_node(
}
)
+ @staticmethod
+ def _node_basename(node: SearchNode[VectorStatePayload]) -> str:
+ """Id, prefixed by the evaluator's verdict where there is one.
+
+ Only nodes the evaluator has seen carry a score at all, so only those
+ can be usefully sorted by name; the rest are named by id and ordered by
+ arrival, which is the truth about them.
+ """
+ panel = node.metrics.get(FRONT_SCORE)
+ if panel is not None:
+ return f"eval{panel:.6f}_{node.id}"
+ return str(node.id)
+
def _save_sidecars(self, node: SearchNode[VectorStatePayload]) -> None:
"""Write the optional .png / .heatmap.png next to a node."""
assert self.nodes_dir is not None
- base_fn = f"{node.score:.6f}_{node.id}"
+ base_fn = self._node_basename(node)
if self.save_raster and node.state.payload.raster_data_url:
_, b64 = split_data_url(node.state.payload.raster_data_url)
diff --git a/tests/score/test_metrics.py b/tests/score/test_metrics.py
index 0c500d3..037e605 100644
--- a/tests/score/test_metrics.py
+++ b/tests/score/test_metrics.py
@@ -2,7 +2,6 @@
from vectrify.score.metrics import (
METRIC_NAMES,
- OBJECTIVE_NAMES,
SCORER_METRICS,
read_metrics,
row_has_metrics,
@@ -19,7 +18,7 @@ def test_the_evaluator_verdict_is_recorded_but_never_an_objective():
reads as 0.0 -- best possible for a minimised objective -- which would let
every unevaluated candidate dominate every evaluated one."""
assert "front_score" in METRIC_NAMES
- assert "front_score" not in OBJECTIVE_NAMES
+ assert "front_score" not in SCORER_METRICS
def test_read_metrics_defaults_missing_columns_to_zero():
diff --git a/tests/search/test_collector.py b/tests/search/test_collector.py
index 742ded9..3a96286 100644
--- a/tests/search/test_collector.py
+++ b/tests/search/test_collector.py
@@ -44,8 +44,8 @@ def _collector(tmp_path: Path) -> tuple[StatCollector, SearchStats]:
def test_configure_run_records_the_epoch_thresholds(tmp_path):
collector, stats = _collector(tmp_path)
- collector.configure_run(epoch_diversity=0.3)
- assert stats.epoch_diversity == 0.3
+ collector.configure_run(epoch_max_tasks=300)
+ assert stats.epoch_max_tasks == 300
def test_seed_initial_score_anchors_the_history_at_zero():
diff --git a/tests/search/test_engine.py b/tests/search/test_engine.py
index ec93997..2f70963 100644
--- a/tests/search/test_engine.py
+++ b/tests/search/test_engine.py
@@ -27,10 +27,6 @@ def select_parent(
_ = nodes
return 1, None
- def should_diversify(self, pool: list[SearchNode]) -> tuple[bool, float]:
- _ = pool
- return False, 1.0
-
def select_survivors(
self, nodes: list[SearchNode], max_keep: int
) -> list[SearchNode]:
@@ -56,8 +52,13 @@ def load_resume_nodes(self, max_nodes: int = 20) -> list:
_ = max_nodes
return []
- def save_node(self, node: SearchNode, tasks_completed: int = 0) -> None:
- _ = (node, tasks_completed)
+ def save_node(
+ self,
+ node: SearchNode,
+ tasks_completed: int = 0,
+ keep_content: bool = True,
+ ) -> None:
+ _ = (node, tasks_completed, keep_content)
self.save_called = True
def save_best(self, node: SearchNode) -> None:
@@ -995,9 +996,10 @@ def test_the_final_artifact_is_chosen_by_the_evaluator():
assert store.best_saved.score == 0.5
-def test_a_failing_evaluator_falls_back_to_the_best_score():
+def test_a_failing_evaluator_still_writes_a_top_tier_candidate():
"""Losing the run's single most important artifact to a scorer error at
- shutdown would be the worst possible time for it."""
+ shutdown would be the worst possible time for it. With no blended score to
+ fall back on, any unbeaten candidate is written instead."""
def explode(_nodes):
raise RuntimeError("no")
@@ -1006,7 +1008,7 @@ def explode(_nodes):
_run_two_children(_engine_with_evaluator(store, explode))
assert store.best_saved is not None
- assert store.best_saved.score == 0.1
+ assert store.best_saved.score < INVALID_SCORE
def test_scorer_thread_scores_queued_results_together():
@@ -1079,23 +1081,15 @@ def score_fn(results):
assert scored, "results scored before the failure should have survived it"
-
-def test_a_collapsed_pool_ends_the_epoch_without_waiting_for_staleness():
- """Each criterion is set tight enough that reaching it is reason enough on
- its own: a pool that has become clones of one drawing is finished whatever
- the score is still doing, and requiring every criterion to agree would let
- a rarely-reached one block the transition and spend the run as a single
- local search."""
+def test_the_epoch_budget_ends_an_epoch_that_has_not_gone_stale():
+ """Staleness measures whether the pool has stopped producing; the budget
+ measures how long the proxy has run without the evaluator seeing anything.
+ Here nothing goes stale, and the epoch ends anyway."""
from unittest.mock import MagicMock
- class Collapsed(FakeStrategy):
- def should_diversify(self, pool: list[SearchNode]) -> tuple[bool, float]:
- _ = pool
- return False, 0.0
-
collector = MagicMock()
engine = MultiprocessSearchEngine(
- workers=1, strategy=Collapsed(), storage=FakeStorage(), max_total_tasks=6
+ workers=1, strategy=FakeStrategy(), storage=FakeStorage(), max_total_tasks=6
)
for task_id in range(1, 7):
engine.unscored_q.put(
@@ -1110,9 +1104,9 @@ def should_diversify(self, pool: list[SearchNode]) -> tuple[bool, float]:
max_wall_seconds=None,
active_pool_size=2,
epochs=4,
- # Far beyond the task budget, so staleness cannot be what ends it.
+ # Far beyond the run, so staleness cannot be what ends the epoch.
epoch_patience=10_000,
- epoch_diversity=0.5,
+ epoch_max_tasks=2,
collector=collector,
)
diff --git a/tests/search/test_nsga.py b/tests/search/test_nsga.py
index e73f0a1..9629142 100644
--- a/tests/search/test_nsga.py
+++ b/tests/search/test_nsga.py
@@ -3,12 +3,7 @@
import pytest
-from vectrify.score.metrics import (
- COLOUR_WEIGHT,
- EDGE_WEIGHT,
- SCORER_METRICS,
- SHAPE_WEIGHT,
-)
+from vectrify.score.metrics import SCORER_METRICS
from vectrify.search import ChainState, SearchNode, nsga
from vectrify.search.diversity import simhash
from vectrify.search.models import INVALID_SCORE
@@ -340,43 +335,6 @@ def test_pool_size_one_always_returns_same_node():
assert selected == {1}
-def test_should_diversify_small_pool_needs_boost():
- strategy = NsgaStrategy(epoch_diversity=0.5)
- nodes = [make_node(i, 0.1, content="") for i in range(1, 5)]
- triggered, diversity = strategy.should_diversify(nodes)
- assert triggered is True
- assert 0.0 <= diversity <= 1.0
-
-
-def test_should_diversify_large_pool_needs_boost():
- strategy = NsgaStrategy(epoch_diversity=0.5)
- nodes = [make_node(i, 0.1, content="") for i in range(1, 21)]
- triggered, diversity = strategy.should_diversify(nodes)
- assert triggered is True
- assert 0.0 <= diversity <= 1.0
-
-
-def test_should_not_diversify_diverse_pool():
- strategy = NsgaStrategy(epoch_diversity=0.01)
- nodes = [
- make_node(
- i, 0.1, content=f""
- )
- for i in range(1, 5)
- ]
- triggered, diversity = strategy.should_diversify(nodes)
- assert triggered is False
- assert 0.0 <= diversity <= 1.0
-
-
-def test_should_not_diversify_too_few_nodes():
- strategy = NsgaStrategy(epoch_diversity=0.99)
- nodes = [make_node(i, 0.1) for i in range(1, 4)]
- triggered, diversity = strategy.should_diversify(nodes)
- assert triggered is False
- assert diversity == 1.0
-
-
def test_epoch_parents_returns_pareto_front():
strategy = NsgaStrategy(pool_size=10)
nodes = [
@@ -760,14 +718,13 @@ def test_the_third_measure_decides_when_the_other_two_disagree():
# The blend the pool used to be ranked by prefers b, because edge carries
# nearly four times shape's weight and wins by a full unit here.
- # Positions come from the registry; the vector is in its order, not the
- # order the weights happen to be written in.
- ic, ie, ish = (SCORER_METRICS.index(n) for n in ("colour", "edge", "shape"))
-
- def blend(v):
- return COLOUR_WEIGHT * v[ic] + EDGE_WEIGHT * v[ie] + SHAPE_WEIGHT * v[ish]
-
- assert blend(vb) < blend(va)
+ # And it decides on which candidate is better, not by how much: b wins its
+ # one axis by a full unit and still loses, which no weighted sum could
+ # reproduce -- the smallest weight is always buried by a large margin
+ # elsewhere. That sum no longer exists anywhere in the run.
+ ie = SCORER_METRICS.index("edge")
+ assert vb[ie] < va[ie]
+ assert va[ie] - vb[ie] > 0.8
def test_a_candidate_winning_on_one_measure_alone_is_still_dominated():
diff --git a/tests/test_cli.py b/tests/test_cli.py
index 7fe19cb..d146f50 100644
--- a/tests/test_cli.py
+++ b/tests/test_cli.py
@@ -70,7 +70,7 @@ def test_defaults_pinned():
assert args.pool_size == 100
# Off: a pool collapses into agreement long before it stops improving, so
# a threshold that looks safe ends search while it is still working.
- assert args.epoch_diversity == 0.0
+ assert args.epoch_max_tasks is None
# Unset: it was binding before the limits that describe the search.
assert args.max_total_tasks is None
# Patience counts local tasks only; a seed batch is not a hill-climb and
diff --git a/tests/vector/test_runner.py b/tests/vector/test_runner.py
index d4d14d2..41b89cc 100644
--- a/tests/vector/test_runner.py
+++ b/tests/vector/test_runner.py
@@ -17,7 +17,7 @@ def test_runner_defaults_match_cli_defaults():
for p in inspect.signature(run_vector_search).parameters.values()
}
assert defaults["pool_size"] == cli.DEFAULT_POOL_SIZE
- assert defaults["epoch_diversity"] == cli.DEFAULT_EPOCH_DIVERSITY
+ assert defaults["epoch_max_tasks"] == cli.DEFAULT_EPOCH_MAX_TASKS
assert defaults["vision_model"] == cli.DEFAULT_VISION_MODEL
# seeds has no static default; the runner derives it from pool_size so the
# per-epoch LLM batch scales with the pool it has to fill.
diff --git a/tests/vector/test_storage.py b/tests/vector/test_storage.py
index e5d7eba..50dad12 100644
--- a/tests/vector/test_storage.py
+++ b/tests/vector/test_storage.py
@@ -86,7 +86,7 @@ def test_save_node_and_lineage(tmp_path, dummy_node):
adapter.save_node(dummy_node)
assert adapter.nodes_dir is not None
- svg_path = adapter.nodes_dir / "0.123456_42.svg"
+ svg_path = adapter.nodes_dir / "42.svg"
assert svg_path.is_file()
assert adapter.max_node_id == 42
@@ -137,7 +137,7 @@ def test_load_resume_nodes(tmp_path):
prev_run_nodes = adapter.runs_dir / "2020-01-01_00-00-00" / "nodes"
prev_run_nodes.mkdir(parents=True)
- with (prev_run_nodes / "0.555000_15.svg").open("w") as f:
+ with (prev_run_nodes / "15.svg").open("w") as f:
f.write(valid_svg)
nodes = adapter.load_resume_nodes()
@@ -182,7 +182,7 @@ def test_save_raster_writes_png(tmp_path):
raster_data_url=png_bytes_to_data_url(_make_png())
)
adapter.save_node(node)
- assert (adapter.nodes_dir / "0.500000_1.png").is_file()
+ assert (adapter.nodes_dir / "1.png").is_file()
def test_save_raster_false_does_not_write_png(tmp_path):
@@ -193,7 +193,7 @@ def test_save_raster_false_does_not_write_png(tmp_path):
raster_data_url=png_bytes_to_data_url(_make_png())
)
adapter.save_node(node)
- assert not (adapter.nodes_dir / "0.500000_1.png").is_file()
+ assert not (adapter.nodes_dir / "1.png").is_file()
def test_save_heatmap_false_does_not_write_heatmap_png(tmp_path):
@@ -204,7 +204,7 @@ def test_save_heatmap_false_does_not_write_heatmap_png(tmp_path):
heatmap_data_url=png_bytes_to_data_url(_make_png("blue"))
)
adapter.save_node(node)
- assert not (adapter.nodes_dir / "0.500000_1.heatmap.png").is_file()
+ assert not (adapter.nodes_dir / "1.heatmap.png").is_file()
def test_save_node_content_none_does_not_write_content_file(tmp_path):
@@ -238,7 +238,7 @@ def test_save_heatmap_content_is_valid_png(tmp_path):
heatmap_data_url=png_bytes_to_data_url(original_png)
)
adapter.save_node(node)
- written = (adapter.nodes_dir / "0.500000_1.heatmap.png").read_bytes()
+ written = (adapter.nodes_dir / "1.heatmap.png").read_bytes()
assert written == original_png
@@ -264,7 +264,7 @@ def test_write_lineage_true_still_writes(tmp_path, dummy_node):
assert adapter.lineage_csv is not None
assert adapter.lineage_csv.exists()
assert adapter.nodes_dir is not None
- assert [p.name for p in adapter.nodes_dir.iterdir()] == ["0.123456_42.svg"]
+ assert [p.name for p in adapter.nodes_dir.iterdir()] == ["42.svg"]
def test_extensionless_output_does_not_collide_with_the_project_dir(
@@ -298,11 +298,12 @@ def test_runs_started_in_the_same_second_get_distinct_directories(tmp_path):
assert len(dirs) == 5
-def test_resume_parses_inf_scored_node_files(tmp_path):
+def test_resume_reads_both_name_shapes(tmp_path):
+ """A bare id, and an evaluator score followed by an id."""
nodes = tmp_path / "out" / "runs" / "2026-01-01_00-00-00" / "nodes"
nodes.mkdir(parents=True)
- (nodes / "0.200000_2.svg").write_text("", encoding="utf-8")
- (nodes / "inf_7.svg").write_text("", encoding="utf-8")
+ (nodes / "2.svg").write_text("", encoding="utf-8")
+ (nodes / "eval0.004392_7.svg").write_text("", encoding="utf-8")
adapter = FileStorageAdapter(str(tmp_path / "out.svg"), resume=True)
resumed = adapter.load_resume_nodes()
@@ -317,13 +318,32 @@ def test_resume_top_tolerates_a_non_numeric_filename(tmp_path):
"""
nodes = tmp_path / "out" / "runs" / "2026-01-01_00-00-00" / "nodes"
nodes.mkdir(parents=True)
- (nodes / "0.100000_1.svg").write_text("", encoding="utf-8")
- (nodes / "0.900000_2.svg").write_text("", encoding="utf-8")
+ (nodes / "eval0.100000_1.svg").write_text("", encoding="utf-8")
+ (nodes / "eval0.900000_2.svg").write_text("", encoding="utf-8")
(nodes / "handwritten.svg").write_text("", encoding="utf-8")
adapter = FileStorageAdapter(str(tmp_path / "out.svg"), resume=True, resume_top=2)
resumed = adapter.load_resume_nodes() # must not raise
- # The best-scoring real node is kept; the unparseable one sorts last (inf).
- assert 1 in {node_id for node_id, _ in resumed}
- assert len(resumed) == 2
+ # Only evaluated nodes carry a score, so --resume-top ranks among those and
+ # the unparseable file is simply not one of them.
+ ids = {node_id for node_id, _ in resumed}
+ assert ids == {1, 2}
+
+
+def test_lineage_is_written_even_when_the_drawing_is_not(tmp_path, dummy_node):
+ """A run admits most of what it produces, and writing every drawing left
+ one run with 106,640 files. The lineage row is cheap and keeps the record
+ complete; the drawing is only worth writing for candidates worth reading
+ back."""
+ adapter = FileStorageAdapter(str(tmp_path / "out.svg"))
+ adapter.initialize()
+
+ adapter.save_node(dummy_node, tasks_completed=7, keep_content=False)
+
+ assert adapter.nodes_dir is not None
+ assert list(adapter.nodes_dir.iterdir()) == []
+ assert adapter.lineage_csv is not None
+ rows = list(csv.DictReader(adapter.lineage_csv.open(encoding="utf-8")))
+ assert rows[0]["id"] == "42"
+ assert rows[0]["task"] == "7"