diff --git a/src/vectrify/cli.py b/src/vectrify/cli.py index d3fd5f8..de7c3f4 100644 --- a/src/vectrify/cli.py +++ b/src/vectrify/cli.py @@ -43,11 +43,6 @@ # found, which argues for clearing the observed maximum rather than sitting # just above the percentile. DEFAULT_EPOCH_PATIENCE = 500 -# Unset: any improvement at all resets patience. A fixed delta is denominated -# in whatever the round objective is, so one value cannot mean the same thing -# on two images or survive a change to the objective. Unset, --epoch-patience -# reads only "is this one better", which carries across images unchanged. -DEFAULT_EPOCH_MIN_DELTA = None DEFAULT_TOURNAMENT_SIZE = 2 DEFAULT_ADAPTIVE_OPERATORS = True # Unset: the run is bounded by --epochs and --max-wall-seconds, which are the @@ -233,17 +228,10 @@ def parse_args(args: list[str] | None = None) -> argparse.Namespace: dest="epoch_patience", metavar="N", help="End the epoch and re-seed if the best score does not improve by " - "--epoch-min-delta over this many consecutive local tasks. 0 disables. " + "reaching the best-ranked tier over this many consecutive local tasks. " + "0 disables. " f"Default: {DEFAULT_EPOCH_PATIENCE}", ) - g_epoch.add_argument( - "--epoch-min-delta", - type=float, - default=DEFAULT_EPOCH_MIN_DELTA, - metavar="DELTA", - help="Minimum score improvement that resets --epoch-patience. " - "Unset by default, so any improvement counts.", - ) g_epoch.add_argument( "--epoch-diversity", type=float, diff --git a/src/vectrify/main.py b/src/vectrify/main.py index d131b0c..75640e8 100755 --- a/src/vectrify/main.py +++ b/src/vectrify/main.py @@ -134,7 +134,6 @@ def main(): write_lineage=args.write_lineage, save_raster=args.save_raster, epoch_patience=args.epoch_patience or None, - epoch_min_delta=args.epoch_min_delta, pool_size=args.pool_size, seeds=args.seeds, epoch_diversity=args.epoch_diversity, diff --git a/src/vectrify/search/base.py b/src/vectrify/search/base.py index 0cb4429..9507b1e 100644 --- a/src/vectrify/search/base.py +++ b/src/vectrify/search/base.py @@ -11,6 +11,11 @@ def select_parent( self, nodes: list[SearchNode[TState]] ) -> tuple[int, int | None]: ... + def top_tier_ids(self, pool: list[SearchNode]) -> set[int]: + """Ids of the best-ranked tier. Entry into it is what counts as + progress, so this replaces comparing a blended score.""" + ... + def should_diversify(self, pool: list[SearchNode]) -> tuple[bool, float]: """Return (trigger_epoch, diversity). diff --git a/src/vectrify/search/engine.py b/src/vectrify/search/engine.py index d4d400e..4fbc686 100644 --- a/src/vectrify/search/engine.py +++ b/src/vectrify/search/engine.py @@ -111,7 +111,6 @@ def run( initial_nodes: list[SearchNode[TState]], max_wall_seconds: float | None = None, epoch_patience: int | None = None, - epoch_min_delta: float | None = None, active_pool_size: int = 20, generation_size: int | None = None, score_fn: Callable[[list[Result]], None] | None = None, @@ -224,7 +223,6 @@ def _scorer_worker(): # Reset at every transition, so each epoch is judged against the # pool it opened with rather than against the first one. epoch_baseline: float | None = None - epoch_patience_best = best_node.score if best_node else INVALID_SCORE pool_refilling = False # True until a fresh epoch's pool reaches capacity # Children accumulate outside active_pool: they replace it wholesale @@ -318,13 +316,7 @@ def _begin_seed_phase() -> None: def _finish_seed_phase() -> None: """Install the LLM children as the epoch's pool and start refining.""" - nonlocal \ - phase, \ - active_pool, \ - node_states, \ - epoch_no_improve, \ - epoch_patience_best, \ - pool_refilling + nonlocal phase, active_pool, node_states, epoch_no_improve, pool_refilling valid_children = [c for c in seed_children if c.score < INVALID_SCORE] previous_ids = {n.id for n in active_pool} @@ -357,8 +349,6 @@ def _finish_seed_phase() -> None: phase = LOCAL_PHASE epoch_no_improve = 0 - scores = valid_scores(active_pool) - epoch_patience_best = min(scores) if scores else INVALID_SCORE pool_refilling = True log.info( f"Epoch {epoch}: refining {len(active_pool)} candidate(s) locally." @@ -511,7 +501,7 @@ def _close_generation() -> None: full sort per result, which at pool size 20 costs about as much as producing the candidate did. """ - nonlocal active_pool + nonlocal active_pool, epoch_no_improve if not pending_children: return @@ -520,6 +510,17 @@ def _close_generation() -> None: survivors = self.strategy.select_survivors(combined, active_pool_size) kept = {n.id for n in survivors} + # Progress is a new candidate reaching the best-ranked tier. Read + # off the dominance relation, so it needs no blended score and no + # threshold on a magnitude -- an epoch goes stale when nothing new + # can get to the front any more, whatever the numbers happen to be + # denominated in. + new_ids = {n.id for n in pending_children} + if new_ids & self.strategy.top_tier_ids(combined): + epoch_no_improve = 0 + if collector is not None: + collector.on_no_improve_reset() + for child in pending_children: if operator_policy is not None: operator_policy.update(child.operator, child.id in kept) @@ -551,18 +552,13 @@ def _close_generation() -> None: pending_children.clear() def _process_local_result(res: Result) -> None: - nonlocal epoch_patience_best, epoch_no_improve - new_node = _make_node(res) pending_children.append(new_node) _note_best(new_node, res) - if new_node.score <= epoch_patience_best - (epoch_min_delta or 0.0): - epoch_patience_best = new_node.score - epoch_no_improve = 0 - if collector is not None: - collector.on_no_improve_reset() - + # Progress is decided when the generation closes, where the pool is + # ranked -- see _close_generation. A candidate cannot be known to + # have reached the top tier before it has been ranked against one. if len(pending_children) >= lambda_size: _close_generation() diff --git a/src/vectrify/search/nsga.py b/src/vectrify/search/nsga.py index 239e149..431bea6 100644 --- a/src/vectrify/search/nsga.py +++ b/src/vectrify/search/nsga.py @@ -365,6 +365,21 @@ def epoch_parents( parents = pareto_nodes[:max_parents] return parents or valid[:max_parents] + def top_tier_ids(self, pool: list[SearchNode[TState]]) -> set[int]: + """Ids of the best-ranked tier under the majority relation. + + The tier rather than a single winner: candidates can be mutually + unbeaten, and where the relation cycles the whole cycle lands in the top + tier together. Callers take entry into it as the definition of progress, + which needs no blended score and no magnitude -- only whether one + candidate beats another. + """ + valid = [n for n in pool if n.score < INVALID_SCORE] + if not valid: + return set() + objectives = build_objectives(valid) + return {n.id for n in pareto_front(valid, lambda n: objectives[n.id])} + def should_diversify(self, pool: list[SearchNode[TState]]) -> tuple[bool, float]: diversity = pool_diversity(pool) return self.epoch_diversity > 0 and diversity < self.epoch_diversity, diversity diff --git a/src/vectrify/vector/runner.py b/src/vectrify/vector/runner.py index 67589df..600e4b7 100644 --- a/src/vectrify/vector/runner.py +++ b/src/vectrify/vector/runner.py @@ -110,7 +110,6 @@ def run_vector_search( write_lineage: bool = True, save_raster: bool = False, epoch_patience: int | None = None, - epoch_min_delta: float | None = None, pool_size: int = DEFAULT_POOL_SIZE, seeds: int | None = None, epoch_diversity: float = DEFAULT_EPOCH_DIVERSITY, @@ -387,7 +386,6 @@ def score_fn(results): initial_nodes, max_wall_seconds=max_wall_seconds, epoch_patience=epoch_patience, - epoch_min_delta=epoch_min_delta, active_pool_size=pool_size, score_fn=score_fn, epoch_seeds=epoch_seeds, diff --git a/tests/search/test_engine.py b/tests/search/test_engine.py index bf34793..ec93997 100644 --- a/tests/search/test_engine.py +++ b/tests/search/test_engine.py @@ -7,7 +7,19 @@ from vectrify.search.engine import MultiprocessSearchEngine -class FakeStrategy: +class _TierMixin: + """The best-ranked tier the engine asks every strategy for. These fakes + score on one number, so the tier is whatever ties for lowest.""" + + def top_tier_ids(self, pool) -> set[int]: + valid = [n for n in pool if n.score < INVALID_SCORE] + if not valid: + return set() + best = min(n.score for n in valid) + return {n.id for n in valid if n.score == best} + + +class FakeStrategy(_TierMixin): def select_parent( self, nodes: list[SearchNode], @@ -163,13 +175,16 @@ def epoch_parents(self, pool, max_parents): initial_nodes=[initial_node], max_wall_seconds=None, epoch_patience=3, - epoch_min_delta=0.1, ) assert strat.epoch_parents_calls >= 1 assert store.save_called -def test_engine_epoch_patience_resets_on_improvement(): +def test_epoch_patience_resets_when_a_child_reaches_the_top_tier(): + """Progress is entry into the best-ranked tier, not a margin on a blended + score. Each of these children is the best yet, so each resets patience and + no transition may fire.""" + class TrackingStrategy(FakeStrategy): def __init__(self): self.epoch_parents_calls = 0 @@ -184,18 +199,9 @@ def epoch_parents(self, pool, max_parents): workers=1, strategy=strat, storage=store, max_total_tasks=3 ) - # Each result improves on the previous best by more than epoch_min_delta, - # so the patience counter resets every time and no transition may fire. for score in (0.35, 0.2, 0.05): engine.unscored_q.put( - Result( - task_id=1, - parent_id=1, - valid=True, - score=score, - payload="p", - llm_type="llm-generate", - ) + Result(task_id=1, parent_id=1, valid=True, score=score, payload="p") ) initial_node = SearchNode( @@ -205,7 +211,8 @@ def epoch_parents(self, pool, max_parents): initial_nodes=[initial_node], max_wall_seconds=None, epoch_patience=2, - epoch_min_delta=0.1, + active_pool_size=4, + generation_size=1, ) assert strat.epoch_parents_calls == 0 assert store.save_called @@ -426,7 +433,6 @@ def epoch_parents(self, pool, max_parents): max_wall_seconds=None, epoch_seeds=3, epoch_patience=1, - epoch_min_delta=0.1, ) assert strat.epoch_parents_calls == 0 @@ -497,7 +503,6 @@ def test_local_results_that_outlive_their_epoch_do_not_count_as_seeds(caplog): max_wall_seconds=None, epoch_seeds=1, epoch_patience=1, - epoch_min_delta=0.1, active_pool_size=2, epochs=5, ) @@ -588,7 +593,6 @@ def rank_front(nodes): max_wall_seconds=None, epoch_seeds=1, epoch_patience=1, - epoch_min_delta=0.1, active_pool_size=2, epochs=5, ) @@ -643,7 +647,6 @@ def test_a_new_epoch_can_be_seeded_from_the_llm_seed_local_search_replaced(): max_wall_seconds=None, epoch_seeds=1, epoch_patience=1, - epoch_min_delta=0.1, active_pool_size=1, generation_size=1, epochs=2, @@ -683,7 +686,6 @@ def test_remembered_seeds_stay_within_their_share_of_the_front(): max_wall_seconds=None, epoch_seeds=2, epoch_patience=1, - epoch_min_delta=0.1, active_pool_size=4, generation_size=1, epochs=3, @@ -717,7 +719,6 @@ def test_a_resumed_run_treats_no_restored_node_as_an_llm_seed(): epoch_seeds=1, initial_seeds=0, epoch_patience=1, - epoch_min_delta=0.1, active_pool_size=1, generation_size=1, epochs=2, @@ -747,7 +748,6 @@ def test_a_run_without_llm_seeds_offers_the_epoch_only_the_evolved_pool(): max_wall_seconds=None, epoch_seeds=0, epoch_patience=1, - epoch_min_delta=0.1, active_pool_size=1, generation_size=1, epochs=2, @@ -779,7 +779,6 @@ def test_a_failing_evaluator_does_not_stop_the_run(): max_wall_seconds=None, epoch_seeds=1, epoch_patience=1, - epoch_min_delta=0.1, active_pool_size=2, epochs=4, ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 084de8b..7fe19cb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -71,8 +71,7 @@ def test_defaults_pinned(): # Off: a pool collapses into agreement long before it stops improving, so # a threshold that looks safe ends search while it is still working. assert args.epoch_diversity == 0.0 - # Unset: both were binding before the limits that describe the search. - assert args.epoch_min_delta is None + # Unset: it was binding before the limits that describe the search. assert args.max_total_tasks is None # Patience counts local tasks only; a seed batch is not a hill-climb and # cannot go stale. Raised from 200 once the gap between improvements was