Make the baseline harness measure what the pipeline actually does

bin/retrieval_baseline

@@ -39,7 +39,12 @@ from langchain_core.retrievers import BaseRetriever
 from nltk.tokenize import word_tokenize
 
 from agent.models import get_embedding, get_llm
-from retrievers.csv_chroma import chroma_settings, list_chroma_subdirectories
+from retrievers.csv_chroma import (
+    VECTOR_OVERFETCH,
+    chroma_settings,
+    dedupe_by_entity,
+    list_chroma_subdirectories,
+)
 from retrievers.reactome.metadata_info import (
     reactome_descriptions_info,
     reactome_field_info,
@@ -83,18 +88,24 @@ def build_retrievers(
         client_settings=chroma_settings,
     )
 
+    # Both vector-side retrievers are asked for more than k and collapsed to k
+    # distinct entities, exactly as HybridRetriever does. Measuring a raw k here
+    # would compare against a configuration the pipeline no longer uses -- stale
+    # numbers that look fresh.
+    overfetch = k * VECTOR_OVERFETCH
+
     retrievers: dict[str, BaseRetriever] = {
         "bm25": bm25,
         # The plain semantic retriever: similarity search with no LLM in the loop.
-        "vector": vectordb.as_retriever(search_kwargs={"k": k}),
+        "vector": vectordb.as_retriever(search_kwargs={"k": overfetch}),
     }
     if with_selfquery:
         retrievers["selfquery"] = SelfQueryRetriever.from_llm(
             llm=llm,
             vectorstore=vectordb,
             document_contents=reactome_descriptions_info[collection],
             metadata_field_info=reactome_field_info[collection],
-            search_kwargs={"k": k},
+            search_kwargs={"k": overfetch},
         )
     return retrievers
 
@@ -129,6 +140,10 @@ def capture(args: argparse.Namespace) -> None:
             per_retriever: dict[str, list[str]] = {}
             for name, retriever in retrievers.items():
                 docs = retriever.invoke(question)
+                if name != "bm25":
+                    # BM25 returns distinct rows already; the vector side needs
+                    # the same collapse the pipeline applies.
+                    docs = dedupe_by_entity(docs, args.k)
                 per_retriever[name] = [doc_id(d) for d in docs]
             result[question][collection] = per_retriever
 
@@ -189,6 +204,10 @@ def overlap(args: argparse.Namespace) -> None:
             sq, vec = retrievers["selfquery"], retrievers["vector"]
             sets.setdefault(collection, []).append(_jaccard(sq, vec))
             ranks.setdefault(collection, []).append(_rank_agreement(sq, vec))
+            # Measured AFTER de-duplication, so this is 1.0 by construction and
+            # says nothing about duplication in the raw result. Kept only as a
+            # guard: anything below 1.0 means dedupe_by_entity failed to fill k,
+            # i.e. VECTOR_OVERFETCH is too low (see #177).
             distinct.setdefault(collection, []).append(
                 len(set(vec)) / len(vec) if vec else 1.0
             )
@@ -212,8 +231,9 @@ def overlap(args: argparse.Namespace) -> None:
     )
     print(
         "\n  set overlap ignores duplicates and order; rank agree counts a "
-        "document\n  only if it is in the same position; distinct/k is the "
-        "share of the vector\n  result that is not a repeat (see issue #169)."
+        "document\n  only if it is in the same position. distinct/k is measured "
+        "after\n  de-duplication, so 1.00 is expected; below 1.00 means the "
+        "over-fetch\n  is too low to fill k (see #177)."
     )
 
 

specs/001-retriever-rewrite/spec.md

@@ -133,19 +133,34 @@ with no LLM in the loop.
 - **It removes the component most likely to break on LangChain 1.x.** SelfQuery
   depends on `lark` and on structured-output behaviour that has moved between
   versions.
-- **Retrieval results change materially.** #171 measured SelfQuery as *stable run
-  to run*, agreeing with plain vector on 0.48 of documents and 0.19 of positions.
-  This is a real change in what reaches the model, not the removal of noise.
+- **Retrieval results change materially.** Recaptured 2026-09-08 against current
+  `main`, with both retrievers at `k = 10 x VECTOR_OVERFETCH` collapsed to 10
+  distinct entities -- the configuration the pipeline actually uses:
+
+  | collection | set overlap | rank agreement |
+  |---|---|---|
+  | complexes | 0.49 | 0.18 |
+  | ewas | 0.54 | 0.17 |
+  | reactions | 0.58 | 0.14 |
+  | summations | 0.57 | 0.24 |
+  | **overall** | **0.55** | **0.18** |
+
+  SelfQuery and plain semantic search return **about half the same documents, in
+  almost entirely different order**. SelfQuery is stable run to run, so this is a
+  real change in what reaches the model, not the removal of noise.
 - **Metadata filtering is lost as a capability**, not merely as code. SelfQuery
   translated a question into a Chroma metadata filter; nothing replaces that.
 - **339 lines across the `metadata_info.py` files become dead in the retrieval
   path**, still referenced only by `src/evaluation/evaluator.py` and
   `bin/retrieval_baseline`, which construct SelfQuery for comparison. They should
   not be deleted while those still need them.
 
-**Verification**: recapture #171's numbers against current `main` first. They
-predate the #169 and #170 fixes, which changed what the vector side returns, so
-the 0.48/0.19 figures describe a retriever that no longer exists.
+**Verification**: done. The figures above supersede the 0.48/0.19 in #171, which
+predated the #169 and #170 fixes and described a retriever that no longer exists.
+Set overlap moved 0.48 -> 0.55 once both sides were measured at the pipeline's
+real configuration; rank agreement was unchanged at ~0.18. The conclusion is
+unmoved: this is a substantial change to retrieval, and it should be judged on
+answer quality rather than on overlap, which cannot say which set is better.
 
 ### D2 — The budget is per collection — DECIDED