diff --git a/bin/retrieval_baseline b/bin/retrieval_baseline index d8c1125..124dbd1 100755 --- a/bin/retrieval_baseline +++ b/bin/retrieval_baseline @@ -39,7 +39,12 @@ from langchain_core.retrievers import BaseRetriever from nltk.tokenize import word_tokenize from agent.models import get_embedding, get_llm -from retrievers.csv_chroma import chroma_settings, list_chroma_subdirectories +from retrievers.csv_chroma import ( + VECTOR_OVERFETCH, + chroma_settings, + dedupe_by_entity, + list_chroma_subdirectories, +) from retrievers.reactome.metadata_info import ( reactome_descriptions_info, reactome_field_info, @@ -83,10 +88,16 @@ def build_retrievers( client_settings=chroma_settings, ) + # Both vector-side retrievers are asked for more than k and collapsed to k + # distinct entities, exactly as HybridRetriever does. Measuring a raw k here + # would compare against a configuration the pipeline no longer uses -- stale + # numbers that look fresh. + overfetch = k * VECTOR_OVERFETCH + retrievers: dict[str, BaseRetriever] = { "bm25": bm25, # The plain semantic retriever: similarity search with no LLM in the loop. - "vector": vectordb.as_retriever(search_kwargs={"k": k}), + "vector": vectordb.as_retriever(search_kwargs={"k": overfetch}), } if with_selfquery: retrievers["selfquery"] = SelfQueryRetriever.from_llm( @@ -94,7 +105,7 @@ def build_retrievers( vectorstore=vectordb, document_contents=reactome_descriptions_info[collection], metadata_field_info=reactome_field_info[collection], - search_kwargs={"k": k}, + search_kwargs={"k": overfetch}, ) return retrievers @@ -129,6 +140,10 @@ def capture(args: argparse.Namespace) -> None: per_retriever: dict[str, list[str]] = {} for name, retriever in retrievers.items(): docs = retriever.invoke(question) + if name != "bm25": + # BM25 returns distinct rows already; the vector side needs + # the same collapse the pipeline applies. + docs = dedupe_by_entity(docs, args.k) per_retriever[name] = [doc_id(d) for d in docs] result[question][collection] = per_retriever @@ -189,6 +204,10 @@ def overlap(args: argparse.Namespace) -> None: sq, vec = retrievers["selfquery"], retrievers["vector"] sets.setdefault(collection, []).append(_jaccard(sq, vec)) ranks.setdefault(collection, []).append(_rank_agreement(sq, vec)) + # Measured AFTER de-duplication, so this is 1.0 by construction and + # says nothing about duplication in the raw result. Kept only as a + # guard: anything below 1.0 means dedupe_by_entity failed to fill k, + # i.e. VECTOR_OVERFETCH is too low (see #177). distinct.setdefault(collection, []).append( len(set(vec)) / len(vec) if vec else 1.0 ) @@ -212,8 +231,9 @@ def overlap(args: argparse.Namespace) -> None: ) print( "\n set overlap ignores duplicates and order; rank agree counts a " - "document\n only if it is in the same position; distinct/k is the " - "share of the vector\n result that is not a repeat (see issue #169)." + "document\n only if it is in the same position. distinct/k is measured " + "after\n de-duplication, so 1.00 is expected; below 1.00 means the " + "over-fetch\n is too low to fill k (see #177)." ) diff --git a/specs/001-retriever-rewrite/spec.md b/specs/001-retriever-rewrite/spec.md index ba958dd..0bba52f 100644 --- a/specs/001-retriever-rewrite/spec.md +++ b/specs/001-retriever-rewrite/spec.md @@ -133,9 +133,21 @@ with no LLM in the loop. - **It removes the component most likely to break on LangChain 1.x.** SelfQuery depends on `lark` and on structured-output behaviour that has moved between versions. -- **Retrieval results change materially.** #171 measured SelfQuery as *stable run - to run*, agreeing with plain vector on 0.48 of documents and 0.19 of positions. - This is a real change in what reaches the model, not the removal of noise. +- **Retrieval results change materially.** Recaptured 2026-09-08 against current + `main`, with both retrievers at `k = 10 x VECTOR_OVERFETCH` collapsed to 10 + distinct entities -- the configuration the pipeline actually uses: + + | collection | set overlap | rank agreement | + |---|---|---| + | complexes | 0.49 | 0.18 | + | ewas | 0.54 | 0.17 | + | reactions | 0.58 | 0.14 | + | summations | 0.57 | 0.24 | + | **overall** | **0.55** | **0.18** | + + SelfQuery and plain semantic search return **about half the same documents, in + almost entirely different order**. SelfQuery is stable run to run, so this is a + real change in what reaches the model, not the removal of noise. - **Metadata filtering is lost as a capability**, not merely as code. SelfQuery translated a question into a Chroma metadata filter; nothing replaces that. - **339 lines across the `metadata_info.py` files become dead in the retrieval @@ -143,9 +155,12 @@ with no LLM in the loop. `bin/retrieval_baseline`, which construct SelfQuery for comparison. They should not be deleted while those still need them. -**Verification**: recapture #171's numbers against current `main` first. They -predate the #169 and #170 fixes, which changed what the vector side returns, so -the 0.48/0.19 figures describe a retriever that no longer exists. +**Verification**: done. The figures above supersede the 0.48/0.19 in #171, which +predated the #169 and #170 fixes and described a retriever that no longer exists. +Set overlap moved 0.48 -> 0.55 once both sides were measured at the pipeline's +real configuration; rank agreement was unchanged at ~0.18. The conclusion is +unmoved: this is a substantial change to retrieval, and it should be judged on +answer quality rather than on overlap, which cannot say which set is better. ### D2 — The budget is per collection — DECIDED