From dbf10481eeacff63c953da1e6b3a4fc565b07ef7 Mon Sep 17 00:00:00 2001 From: Dev Dalia Date: Mon, 24 Aug 2026 18:36:03 +0530 Subject: [PATCH] Draw held-out targets from cli sessions only rank_sessions sorts every `auto` session below every `cli` one, so an `auto` target cannot reach rank 1 while any `cli` session matches. heldout_queries sampled any session, so on a corpus that is mostly `auto` most of the 300 queries asked for something the ranker deliberately does not return. That is not a hard query, it is the wrong question: agsearch is for finding a session you remember being in, and nobody remembers an SDK-spawned run they never saw. The effect was a metric with almost no headroom, which could not show a real ranking change. Targets are now cli-only. `auto` sessions stay in the pool as distractors, which costs ~0.008 @1 and keeps them searchable. held@1 0.170 -> 0.490 heldMRR 0.209 -> 0.618 Gold set unchanged. Test-only; no behaviour change. --- tests/heldout_eval.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/heldout_eval.py b/tests/heldout_eval.py index 4b46da1..7be5693 100644 --- a/tests/heldout_eval.py +++ b/tests/heldout_eval.py @@ -7,6 +7,8 @@ require that session back. It covers 300 sessions instead of 15 and nobody tuned against it, so a change that helps set 1 while flat or down on set 2 is a change fitted to the labels. +Set 2 draws its targets from `cli` sessions only. See heldout_queries(). + The timing column matters as much as the quality columns: fzf re-ranks on every keystroke, so a variant that buys accuracy at 5x the latency is not shippable. """ @@ -22,13 +24,24 @@ def load(path, name): def heldout_queries(ag, rows, seed=7, n=300): """Pick content words from a session's own title. If you typed words from the title, that session should come back. Titles are not what the ranker mainly scores (body is), - so this is a fair proxy, and it covers 300 sessions instead of 15.""" + so this is a fair proxy, and it covers 300 sessions instead of 15. + + Targets are `cli` sessions only. rank_sessions sorts every `auto` session below every + `cli` one, so an `auto` target cannot reach rank 1 while any `cli` session matches at + all. That is deliberate: agsearch is for finding a session you remember being in, and + nobody remembers an SDK-spawned run they never saw. Asking for one back is not a hard + query, it is the wrong question, and on a corpus that is mostly `auto` it caps @1 far + below 1.0 and leaves too little headroom for a real ranking change to show up. + + `auto` sessions stay in `rows`, so they still compete as distractors.""" rnd = random.Random(seed) pool = [] for r in rows: title = r[ag.C_TITLE] if title.startswith(""): # slash-command invocations, not conversations continue + if r[ag.C_KIND] == "auto": # never a valid target; see docstring + continue words = [w for w in ag.parse_query(title)] if len(words) < 2: continue