Hello
x bold drop")
+ assert result == "Hello\n `x` **bold** drop"
+
+
+def test_split_by_headers_splits_on_markdown_headers():
+ p = make_processor()
+ text = "# Title\ncontent one\n## Subtitle\ncontent two"
+ sections = p._split_by_headers(text)
+ assert sections == ["# Title\ncontent one", "## Subtitle\ncontent two"]
+
+
+def test_split_oversized_chunks_splits_on_newlines_when_over_target():
+ p = make_processor()
+ chunk = "a" * 5 + "\n" + "b" * 5 + "\n" + "c" * 5
+ result = p._split_oversized_chunks([chunk], target_length=8)
+ assert result == ["aaaaa", "bbbbb", "ccccc"]
+
+
+def test_accumulate_chunks_merges_up_to_target_length():
+ p = make_processor()
+ splits = ["a" * 5, "b" * 5, "c" * 5]
+ result = p._accumulate_chunks(splits, target_length=12, overlap=1, min_length=8)
+ assert result == ["aaaaabbbbb", "aaaaabbbbbccccc"]
+
+
+def test_chunk_adaptor_docs_respects_custom_target_and_min_length():
+ p = make_processor(target_length=20, min_length=15, overlap=1)
+ json_data = [{"name": "doc-a.md", "docs": "# Header\n" + ("word " * 10).strip()}]
+
+ chunks, metadata_dict = p._chunk_adaptor_docs(json_data)
+
+ assert all(c["name"] == "doc-a.md" for c in chunks)
+ assert all(c["docs_type"] == "general_docs" for c in chunks)
+ assert "doc-a.md" in metadata_dict
+
+
+def test_chunk_adaptor_docs_skips_ignored_docs():
+ p = DocsiteProcessor(docs_type="general_docs", docs_to_ignore=["skip-me.md"])
+ json_data = [{"name": "skip-me.md", "docs": "content"}]
+
+ chunks, metadata_dict = p._chunk_adaptor_docs(json_data)
+
+ assert chunks == []
+ assert metadata_dict == {}
+
+
+def test_constructor_defaults_match_previous_hardcoded_values():
+ p = DocsiteProcessor(docs_type="general_docs")
+ assert p.target_length == 1000
+ assert p.min_length == 700
+ assert p.overlap == 1
diff --git a/services/embed_docsite/tests/unit/test_embed_docsite.py b/services/embed_docsite/tests/unit/test_embed_docsite.py
new file mode 100644
index 00000000..9651ff90
--- /dev/null
+++ b/services/embed_docsite/tests/unit/test_embed_docsite.py
@@ -0,0 +1,213 @@
+"""Unit tests for embed_docsite's orchestration. DocsiteProcessor/DocsiteIndexer
+and get_db_connection are all mocked."""
+
+from unittest.mock import MagicMock, patch
+
+import embed_docsite.embed_docsite as m
+
+
+def test_main_orchestrates_full_batch_lifecycle_and_returns_summary():
+ fake_conn = MagicMock()
+ fake_indexer = MagicMock()
+ fake_indexer.start_batch.return_value = 7
+ fake_indexer.insert_documents.return_value = 10
+ fake_indexer.copy_forward_missing_docs_types.return_value = 3
+ fake_indexer.prune_old_batches.return_value = [4]
+
+ fake_processor = MagicMock()
+ fake_processor.get_preprocessed_docs.return_value = ([{"name": "a.md", "docs_type": "general_docs", "doc_chunk": "x"}], {"a.md": {}})
+
+ with patch.object(m, "get_db_connection", return_value=fake_conn), \
+ patch.object(m, "register_vector_type"), \
+ patch.object(m, "DocsiteProcessor", return_value=fake_processor), \
+ patch.object(m, "DocsiteIndexer", return_value=fake_indexer), \
+ patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test"}):
+ result = m.main({"docs_to_upload": ["general_docs"], "target": "postgres"})
+
+ fake_indexer.start_batch.assert_called_once_with(fake_conn, ["general_docs"])
+ fake_indexer.insert_documents.assert_called_once()
+ fake_indexer.copy_forward_missing_docs_types.assert_called_once_with(fake_conn, 7, ["general_docs"])
+ fake_indexer.build_index.assert_called_once_with(fake_conn, 7)
+ fake_indexer.promote_batch.assert_called_once_with(fake_conn, 7, 13) # 10 inserted + 3 copied forward
+ fake_indexer.prune_old_batches.assert_called_once_with(fake_conn)
+ fake_conn.close.assert_called_once()
+
+ assert result == {
+ "target": "postgres",
+ "batch_id": 7,
+ "docs_types": ["general_docs"],
+ "chunk_count": 10,
+ "copied_forward": 3,
+ "pruned_batches": [4],
+ "promoted": True,
+ }
+
+def test_main_defaults_docs_to_upload_to_all_types():
+ fake_conn = MagicMock()
+ fake_indexer = MagicMock()
+ fake_indexer.start_batch.return_value = 1
+ fake_indexer.insert_documents.return_value = 0
+ fake_indexer.copy_forward_missing_docs_types.return_value = 0
+ fake_indexer.prune_old_batches.return_value = []
+ fake_processor = MagicMock()
+ fake_processor.get_preprocessed_docs.return_value = ([], {})
+
+ with patch.object(m, "get_db_connection", return_value=fake_conn), \
+ patch.object(m, "register_vector_type"), \
+ patch.object(m, "DocsiteProcessor", return_value=fake_processor) as mock_processor_cls, \
+ patch.object(m, "DocsiteIndexer", return_value=fake_indexer), \
+ patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test"}):
+ m.main({"target": "postgres"})
+
+ called_docs_types = [call.kwargs["docs_type"] for call in mock_processor_cls.call_args_list]
+ assert called_docs_types == m.ALL_DOCS_TYPES
+
+
+def test_main_defaults_to_pinecone_target():
+ """Default must match main's behavior: write to Pinecone, never open Postgres."""
+ fake_indexer = MagicMock()
+ fake_processor = MagicMock()
+ fake_processor.get_preprocessed_docs.return_value = ([], {})
+
+ with patch.object(m, "get_db_connection") as mock_get_conn, \
+ patch.object(m, "DocsiteProcessor", return_value=fake_processor), \
+ patch.object(m, "LegacyPineconeDocsiteIndexer", return_value=fake_indexer) as mock_legacy_cls, \
+ patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test", "PINECONE_API_KEY": "pc-test"}):
+ m.main({"docs_to_upload": ["general_docs"]})
+
+ mock_legacy_cls.assert_called_once()
+ mock_get_conn.assert_not_called()
+
+
+def test_main_pinecone_target_does_not_require_postgres_url():
+ """With target=pinecone the service must not touch Postgres at all, so
+ POSTGRES_URL need not be set — matching main's dependency surface."""
+ fake_processor = MagicMock()
+ fake_processor.get_preprocessed_docs.return_value = ([], {})
+
+ with patch.object(m, "get_db_connection") as mock_get_conn, \
+ patch.object(m, "DocsiteProcessor", return_value=fake_processor), \
+ patch.object(m, "LegacyPineconeDocsiteIndexer"), \
+ patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test", "PINECONE_API_KEY": "pc-test"}, clear=True):
+ m.main({})
+
+ mock_get_conn.assert_not_called()
+
+
+def test_main_rejects_unknown_target():
+ import pytest
+ from util import ApolloError
+
+ with patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test"}):
+ with pytest.raises(ApolloError) as exc:
+ m.main({"target": "elasticsearch"})
+
+ assert exc.value.code == 400
+
+
+def test_postgres_upload_runs_migrations_before_registering_vector_type():
+ """Order is load-bearing, not incidental: register_vector runs
+ to_regtype('vector') and raises unless CREATE EXTENSION has already run."""
+ calls = []
+ fake_conn = MagicMock()
+ fake_indexer = MagicMock()
+ fake_indexer.start_batch.return_value = 7
+ fake_indexer.insert_documents.return_value = 1
+ fake_indexer.copy_forward_missing_docs_types.return_value = 0
+ fake_indexer.prune_old_batches.return_value = []
+ fake_processor = MagicMock()
+ fake_processor.get_preprocessed_docs.return_value = ([], {})
+
+ with patch.object(m, "get_db_connection", return_value=fake_conn), \
+ patch.object(m, "run_migrations", side_effect=lambda _conn: calls.append("migrate")), \
+ patch.object(m, "register_vector_type", side_effect=lambda _conn: calls.append("register")), \
+ patch.object(m, "DocsiteProcessor", return_value=fake_processor), \
+ patch.object(m, "DocsiteIndexer", return_value=fake_indexer), \
+ patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test"}):
+ m.main({"docs_to_upload": ["general_docs"], "target": "postgres"})
+
+ assert calls == ["migrate", "register"]
+
+
+def test_postgres_upload_marks_batch_failed_and_reraises():
+ """Schema allows status='failed' and nothing ever set it, so an interrupted
+ index run left a 'building' row that no later run could interpret."""
+ import pytest
+
+ fake_conn = MagicMock()
+ fake_indexer = MagicMock()
+ fake_indexer.start_batch.return_value = 7
+ fake_indexer.insert_documents.side_effect = RuntimeError("embedding API down")
+ fake_processor = MagicMock()
+ fake_processor.get_preprocessed_docs.return_value = ([], {})
+
+ with patch.object(m, "get_db_connection", return_value=fake_conn), \
+ patch.object(m, "run_migrations"), \
+ patch.object(m, "register_vector_type"), \
+ patch.object(m, "DocsiteProcessor", return_value=fake_processor), \
+ patch.object(m, "DocsiteIndexer", return_value=fake_indexer), \
+ patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test"}):
+ with pytest.raises(RuntimeError, match="embedding API down"):
+ m.main({"docs_to_upload": ["general_docs"], "target": "postgres"})
+
+ fake_indexer.fail_batch.assert_called_once_with(fake_conn, 7)
+ fake_conn.close.assert_called_once()
+
+
+def test_prune_failure_after_promote_does_not_mark_batch_failed():
+ """Pruning cleans up OLDER, unrelated batches. If it throws after the new
+ batch was already promoted, the new batch must stay 'complete' — it must
+ not be retroactively marked 'failed', and the call must still succeed."""
+ fake_conn = MagicMock()
+ fake_indexer = MagicMock()
+ fake_indexer.start_batch.return_value = 7
+ fake_indexer.insert_documents.return_value = 10
+ fake_indexer.copy_forward_missing_docs_types.return_value = 3
+ fake_indexer.prune_old_batches.side_effect = RuntimeError("lock conflict on DROP INDEX CONCURRENTLY")
+ fake_processor = MagicMock()
+ fake_processor.get_preprocessed_docs.return_value = ([{"name": "a.md", "docs_type": "general_docs", "doc_chunk": "x"}], {"a.md": {}})
+
+ with patch.object(m, "get_db_connection", return_value=fake_conn), \
+ patch.object(m, "run_migrations"), \
+ patch.object(m, "register_vector_type"), \
+ patch.object(m, "DocsiteProcessor", return_value=fake_processor), \
+ patch.object(m, "DocsiteIndexer", return_value=fake_indexer), \
+ patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test"}):
+ result = m.main({"docs_to_upload": ["general_docs"], "target": "postgres"})
+
+ fake_indexer.promote_batch.assert_called_once_with(fake_conn, 7, 13)
+ fake_indexer.fail_batch.assert_not_called()
+ fake_conn.close.assert_called_once()
+
+ assert result == {
+ "target": "postgres",
+ "batch_id": 7,
+ "docs_types": ["general_docs"],
+ "chunk_count": 10,
+ "copied_forward": 3,
+ "pruned_batches": [],
+ "promoted": True,
+ }
+
+
+def test_failed_marking_never_masks_the_original_error():
+ """Bookkeeping must not become the reported failure — the operator needs to
+ see what actually broke."""
+ import pytest
+
+ fake_conn = MagicMock()
+ fake_indexer = MagicMock()
+ fake_indexer.start_batch.return_value = 7
+ fake_indexer.insert_documents.side_effect = RuntimeError("embedding API down")
+ fake_indexer.fail_batch.side_effect = RuntimeError("connection already gone")
+ fake_processor = MagicMock()
+ fake_processor.get_preprocessed_docs.return_value = ([], {})
+
+ with patch.object(m, "get_db_connection", return_value=fake_conn), \
+ patch.object(m, "run_migrations"), \
+ patch.object(m, "register_vector_type"), \
+ patch.object(m, "DocsiteProcessor", return_value=fake_processor), \
+ patch.object(m, "DocsiteIndexer", return_value=fake_indexer), \
+ patch.dict("os.environ", {"OPENAI_API_KEY": "sk-test"}):
+ with pytest.raises(RuntimeError, match="embedding API down"):
+ m.main({"docs_to_upload": ["general_docs"], "target": "postgres"})
diff --git a/services/job_chat/retrieve_docs.py b/services/job_chat/retrieve_docs.py
index 5c0e644f..33d35a89 100644
--- a/services/job_chat/retrieve_docs.py
+++ b/services/job_chat/retrieve_docs.py
@@ -1,23 +1,24 @@
-import os
import json
+import os
+
import anthropic
+import sentry_sdk
from anthropic import (
APIConnectionError,
- BadRequestError,
AuthenticationError,
- PermissionDeniedError,
+ BadRequestError,
+ InternalServerError,
NotFoundError,
- UnprocessableEntityError,
+ PermissionDeniedError,
RateLimitError,
- InternalServerError,
+ UnprocessableEntityError,
)
-import sentry_sdk
from langfuse import observe
-from util import ApolloError, create_logger
from models import resolve_model
-from search_docsite.search_docsite import DocsiteSearch
+from search_docsite.search_docsite import resolve_backend
+from util import ApolloError, create_logger
+
from .rag_config_loader import ConfigLoader
-from streaming_util import StreamManager
logger = create_logger("job_chat.retrieve_docs")
@@ -82,7 +83,7 @@ def retrieve_knowledge(content, history, code="", adaptor="", api_key=None, stre
search_results = list(set(search_results))
search_results_sections = list(set(result.metadata["doc_title"] for result in search_results))
except Exception as e:
- logger.error(f"Pinecone search failed: {e}")
+ logger.error(f"Docsite search failed: {e}")
sentry_sdk.capture_exception(e)
# Continue with empty results - chat can still work without docs
search_results = []
@@ -163,26 +164,33 @@ def generate_queries(content, client, user_context=""):
"Failed to generate search queries - invalid response from AI service",
type="INVALID_LLM_RESPONSE",
details={"response_preview": text[:200]}
- )
+ ) from e
if len(answer_parsed) >= 4:
answer_parsed = answer_parsed[:4]
return (answer_parsed, usage)
-def search_docs(search_queries, top_k, threshold):
- """Search the docsite vector store using search queries."""
- docsite_search = DocsiteSearch()
+def search_docs(search_queries, top_k, threshold=None):
+ """Search the docsite store. Both backends run semantic search, so the
+ threshold applies identically.
+
+ Set DOCSITE_SEARCH_BACKEND=postgres to use Postgres.
+
+ :param threshold: Cosine-similarity cutoff
+ """
+ searcher = resolve_backend()()
search_results = []
for q in search_queries:
- query_search_result = docsite_search.search(
- q.get("query"),
- top_k=top_k,
- threshold=threshold,
+ query_search_result = searcher.search(
+ q.get("query"),
+ top_k=top_k,
+ threshold=threshold,
+ strategy="semantic",
docs_type="general_docs"
)
search_results.extend(query_search_result)
-
+
return search_results
def format_context(adaptor, code, history):
@@ -247,10 +255,10 @@ def call_llm(model, temperature, system_prompt, user_prompt, client, output_sche
"Unable to reach the AI service for documentation search",
type="CONNECTION_ERROR",
details=details,
- )
+ ) from e
except AuthenticationError as e:
logger.error(f"Authentication error during knowledge retrieval: {e}")
- raise ApolloError(401, "Authentication failed with AI service", type="AUTH_ERROR")
+ raise ApolloError(401, "Authentication failed with AI service", type="AUTH_ERROR") from e
except RateLimitError as e:
logger.error(f"Rate limit error during knowledge retrieval: {e}")
retry_after = int(e.response.headers.get('retry-after', 60)) if hasattr(e, 'response') else 60
@@ -259,22 +267,22 @@ def call_llm(model, temperature, system_prompt, user_prompt, client, output_sche
"Rate limit exceeded for documentation search, please try again later",
type="RATE_LIMIT",
details={"retry_after": retry_after}
- )
+ ) from e
except BadRequestError as e:
logger.error(f"Bad request error during knowledge retrieval: {e}")
- raise ApolloError(400, f"Invalid request to AI service: {str(e)}", type="BAD_REQUEST")
+ raise ApolloError(400, f"Invalid request to AI service: {str(e)}", type="BAD_REQUEST") from e
except PermissionDeniedError as e:
logger.error(f"Permission denied error during knowledge retrieval: {e}")
- raise ApolloError(403, "Not authorized to perform this action", type="FORBIDDEN")
+ raise ApolloError(403, "Not authorized to perform this action", type="FORBIDDEN") from e
except NotFoundError as e:
logger.error(f"Not found error during knowledge retrieval: {e}")
- raise ApolloError(404, "Resource not found", type="NOT_FOUND")
+ raise ApolloError(404, "Resource not found", type="NOT_FOUND") from e
except UnprocessableEntityError as e:
logger.error(f"Unprocessable entity error during knowledge retrieval: {e}")
- raise ApolloError(422, str(e), type="INVALID_REQUEST")
+ raise ApolloError(422, str(e), type="INVALID_REQUEST") from e
except InternalServerError as e:
logger.error(f"Internal server error from AI service during knowledge retrieval: {e}")
- raise ApolloError(500, "The AI service encountered an error", type="PROVIDER_ERROR")
+ raise ApolloError(500, "The AI service encountered an error", type="PROVIDER_ERROR") from e
except Exception as e:
logger.error(f"Unexpected error during LLM call for knowledge retrieval: {str(e)}")
- raise ApolloError(500, f"Unexpected error during documentation search: {str(e)}", type="UNKNOWN_ERROR")
\ No newline at end of file
+ raise ApolloError(500, f"Unexpected error during documentation search: {str(e)}", type="UNKNOWN_ERROR") from e
\ No newline at end of file
diff --git a/services/job_chat/tests/integration/test_adaptor_docs_pipeline.py b/services/job_chat/tests/integration/test_adaptor_docs_pipeline.py
index 78ec9979..f926316e 100644
--- a/services/job_chat/tests/integration/test_adaptor_docs_pipeline.py
+++ b/services/job_chat/tests/integration/test_adaptor_docs_pipeline.py
@@ -3,7 +3,6 @@
import psycopg2
import pytest
from dotenv import load_dotenv
-
from job_chat.prompt import generate_system_message
load_dotenv()
@@ -147,7 +146,7 @@ def test_generate_queries_returns_valid_structure():
print("==================TEST==================")
print("Description: Testing generate_queries returns valid JSON structure")
- from job_chat.retrieve_docs import generate_queries, get_client, format_context
+ from job_chat.retrieve_docs import format_context, generate_queries, get_client
# Step 1: Prepare test inputs
print("\n1. Preparing test inputs...")
@@ -159,7 +158,7 @@ def test_generate_queries_returns_valid_structure():
# Step 2: Call generate_queries
print("\n2. Calling generate_queries...")
client = get_client()
- queries, usage = generate_queries(question, client, user_context)
+ queries, _usage = generate_queries(question, client, user_context)
print(f" Generated {len(queries)} queries")
# Step 3: Validate structure
diff --git a/services/job_chat/tests/unit/conftest.py b/services/job_chat/tests/unit/conftest.py
index 1816dd69..ce4637da 100644
--- a/services/job_chat/tests/unit/conftest.py
+++ b/services/job_chat/tests/unit/conftest.py
@@ -4,8 +4,7 @@
module-level `OpenAIEmbeddings()` default arg validates credentials at construction
(openai 2.x / langchain-openai 1.x). A key must therefore exist at import time.
-Dummy placeholders only — unit tests mock every network seam, and the repo-root
-conftest blocks real client construction. `setdefault` lets a real key win.
+Dummy placeholders only.
"""
import os
diff --git a/services/job_chat/tests/unit/test_retrieve_docs.py b/services/job_chat/tests/unit/test_retrieve_docs.py
index 54fb3558..45b57ddf 100644
--- a/services/job_chat/tests/unit/test_retrieve_docs.py
+++ b/services/job_chat/tests/unit/test_retrieve_docs.py
@@ -14,11 +14,11 @@
from unittest.mock import MagicMock, patch
import pytest
-
+from embeddings.embeddings import SearchResult
from job_chat import retrieve_docs as rd
+from job_chat.retrieve_docs import search_docs
from util import ApolloError
-
# --- generate_queries ----------------------------------------------------------
def test_generate_queries_truncates_to_four():
@@ -76,3 +76,54 @@ def test_call_llm_wraps_unexpected_error_as_apollo_error():
assert exc.value.code == 500
assert exc.value.type == "UNKNOWN_ERROR"
+
+
+# --- search_docs ----------------------------------------------------------------
+
+
+def _fake_result(title):
+ return SearchResult(f"text for {title}", {"doc_title": title, "docs_type": "general_docs"}, 0.9)
+
+
+def test_search_docs_forwards_query_args_to_resolved_backend(monkeypatch):
+ monkeypatch.delenv("DOCSITE_SEARCH_BACKEND", raising=False)
+
+ with patch.object(rd, "resolve_backend") as mock_resolve:
+ backend = mock_resolve.return_value.return_value
+ backend.search.return_value = [_fake_result("A")]
+ results = search_docs([{"query": "q"}], top_k=5)
+
+ assert [r.metadata["doc_title"] for r in results] == ["A"]
+ backend.search.assert_called_once_with(
+ "q", top_k=5, threshold=None, strategy="semantic", docs_type="general_docs"
+ )
+
+
+def test_search_docs_passes_threshold_through(monkeypatch):
+ """Threshold is a cosine-similarity cutoff that must reach the backend — this
+ regressed once already when the backend flag was introduced, and rag.yaml's
+ threshold silently stopped applying."""
+ monkeypatch.delenv("DOCSITE_SEARCH_BACKEND", raising=False)
+
+ with patch.object(rd, "resolve_backend") as mock_resolve:
+ backend = mock_resolve.return_value.return_value
+ backend.search.return_value = [_fake_result("A")]
+ search_docs([{"query": "q"}], top_k=5, threshold=0.8)
+
+ backend.search.assert_called_once_with(
+ "q", top_k=5, threshold=0.8, strategy="semantic", docs_type="general_docs"
+ )
+
+
+def test_search_docs_accumulates_results_across_queries(monkeypatch):
+ monkeypatch.delenv("DOCSITE_SEARCH_BACKEND", raising=False)
+
+ with patch.object(rd, "resolve_backend") as mock_resolve:
+ backend = mock_resolve.return_value.return_value
+ backend.search.side_effect = [[_fake_result("A")], [_fake_result("B")]]
+ results = search_docs([{"query": "q1"}, {"query": "q2"}], top_k=5)
+
+ assert [r.metadata["doc_title"] for r in results] == ["A", "B"]
+ assert backend.search.call_count == 2
+
+
diff --git a/services/migrations/20260728000000_docsite_batches_and_chunks.sql b/services/migrations/20260728000000_docsite_batches_and_chunks.sql
new file mode 100644
index 00000000..0b8dfa53
--- /dev/null
+++ b/services/migrations/20260728000000_docsite_batches_and_chunks.sql
@@ -0,0 +1,34 @@
+-- Docsite chunk storage (Postgres + pgvector).
+-- Applied by services/db_migrations.py; recorded in _migrations_docs.
+
+CREATE EXTENSION IF NOT EXISTS vector;
+
+CREATE TABLE IF NOT EXISTS docsite_batches (
+ id BIGSERIAL PRIMARY KEY,
+ status VARCHAR(20) NOT NULL DEFAULT 'building'
+ CHECK (status IN ('building', 'complete', 'failed')),
+ docs_types TEXT[] NOT NULL,
+ chunk_target_length INT NOT NULL,
+ chunk_min_length INT NOT NULL,
+ embedding_model VARCHAR(100) NOT NULL,
+ chunk_count INT,
+ started_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ completed_at TIMESTAMPTZ
+);
+
+CREATE TABLE IF NOT EXISTS docsite_chunks (
+ id BIGSERIAL PRIMARY KEY,
+ batch_id BIGINT NOT NULL REFERENCES docsite_batches(id) ON DELETE CASCADE,
+ doc_title VARCHAR(500) NOT NULL,
+ docs_type VARCHAR(50) NOT NULL,
+ chunk_index INT NOT NULL,
+ text TEXT NOT NULL,
+ embedding vector(1536) NOT NULL,
+ text_search tsvector GENERATED ALWAYS AS (to_tsvector('english', text)) STORED,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT now()
+);
+
+CREATE INDEX IF NOT EXISTS idx_docsite_chunks_batch ON docsite_chunks(batch_id);
+CREATE INDEX IF NOT EXISTS idx_docsite_chunks_doc_title ON docsite_chunks(batch_id, doc_title);
+CREATE INDEX IF NOT EXISTS idx_docsite_chunks_docs_type ON docsite_chunks(batch_id, docs_type);
+CREATE INDEX IF NOT EXISTS idx_docsite_chunks_fts ON docsite_chunks USING gin(text_search);
diff --git a/services/search_docsite/README.md b/services/search_docsite/README.md
index ecbe0146..c7380344 100644
--- a/services/search_docsite/README.md
+++ b/services/search_docsite/README.md
@@ -26,16 +26,24 @@ bun py search_docsite tmp/payload.json -O
## Implementation
The service uses the DocsiteSearch class to query the database (Pinecone). It embeds semantic search queries using OpenAI.
+To compare backends on the same query, run the service twice with different
+`backend` values and diff the results. This replaces the shadow-mode comparison
+that was considered for the Postgres migration.
+
## Payload Reference
The input payload is a JSON object with the following structure:
```js
{
- "query": "What is Asana", // Input query
- "collection_name": "Docsite-20250225", // Name of the collection in the vector database
- "docs_type": "adaptor_docs", // Filter for document type adaptor_docs, adaptor_functions, general_docs (optional)
- "doc_title": "Asana", // Filter for document title (optional)
- "top_k": 5 // Adjust the number of search results (optional)
+ "query": "What is Asana", // Input query (required)
+ "backend": "pinecone", // 'pinecone' | 'postgres'. Defaults to DOCSITE_SEARCH_BACKEND, itself defaulting to pinecone.
+ "docs_type": "adaptor_docs", // Filter for adaptor_docs | adaptor_functions | general_docs (optional)
+ "doc_title": "Asana", // Filter for document title (optional)
+ "top_k": 5, // Number of search results (optional)
+ "threshold": 0.8, // Cosine cutoff. Only valid with strategy 'semantic'. (optional)
+ "strategy": "semantic", // Postgres backend only: 'semantic' | 'keyword' | 'hybrid'
+ "batch_id": 12, // Postgres backend only: pin a specific batch (optional)
+ "collection_name": "docsite-..." // Pinecone backend only: pin a namespace (optional)
}
```
diff --git a/services/search_docsite/pinecone_legacy_search.py b/services/search_docsite/pinecone_legacy_search.py
new file mode 100644
index 00000000..384f7ef3
--- /dev/null
+++ b/services/search_docsite/pinecone_legacy_search.py
@@ -0,0 +1,111 @@
+import os
+
+from embeddings.embeddings import SearchResult
+from langchain_openai import OpenAIEmbeddings
+from langchain_pinecone import PineconeVectorStore
+from pinecone import Pinecone
+from util import ApolloError, create_logger
+
+logger = create_logger("LegacyPineconeDocsiteSearch")
+
+
+class LegacyPineconeDocsiteSearch:
+ """
+ Legacy Pinecone-backed docsite search, still the default backend and the
+ rollback path for the Postgres migration. Selected via resolve_backend()
+ in services/search_docsite/search_docsite.py.
+
+ :param collection_name: Vectorstore collection name (namespace) to store documents
+ :param index_name: Vectorstore index name (default: docsite)
+ :param default_top_k: Default number of results to return (default: 5)
+ :param embeddings: LangChain embedding type (default: OpenAIEmbeddings())
+ """
+ def __init__(self, collection_name=None, index_name="docsite", default_top_k=5, embeddings=None):
+ self.index_client = index_name
+ self.default_top_k = default_top_k
+ self.embeddings = embeddings if embeddings is not None else OpenAIEmbeddings()
+
+ if collection_name is None:
+ logger.info("Collection name not provided; retrieving the most recent collection name.")
+ collection_name = self._get_most_recent_namespace()
+
+ self.collection_name = collection_name
+ self.vectorstore = PineconeVectorStore(index_name=index_name, namespace=collection_name, embedding=self.embeddings)
+
+ def search(self, query, top_k=None, threshold=None, strategy='semantic', doc_title=None, docs_type=None):
+ filters = self._build_filter(doc_title=doc_title, docs_type=docs_type)
+ logger.info("Metadata filters built")
+
+ if strategy != 'semantic':
+ raise ApolloError(
+ 400,
+ f"The Pinecone backend only supports strategy='semantic', got '{strategy}'",
+ type="BAD_REQUEST",
+ )
+
+ return self._semantic_search(query=query, top_k=top_k, threshold=threshold, filters=filters)
+
+ def _semantic_search(self, query, top_k=None, threshold=None, filters=None):
+ if top_k is None and threshold is None:
+ top_k = self.default_top_k
+
+ max_k = top_k or 50
+
+ scored_docs = self.vectorstore.similarity_search_with_score(
+ query=query,
+ k=max_k,
+ filter=filters
+ )
+
+ logger.info(f"Similar documents retrieved: {len(scored_docs)}")
+
+ results = []
+ for doc, score in scored_docs:
+ if threshold is not None and score < threshold:
+ continue
+
+ if top_k is not None and len(results) >= top_k and threshold is None:
+ break
+
+ results.append(SearchResult(doc.page_content, doc.metadata, score))
+
+ logger.info(f"Filtered to {len(results)} results")
+ return results
+
+ def _build_filter(self, **kwargs):
+ conditions = []
+
+ if kwargs.get('doc_title'):
+ conditions.append({"doc_title": {"$eq": kwargs['doc_title']}})
+
+ if kwargs.get('docs_type'):
+ conditions.append({"docs_type": {"$eq": kwargs['docs_type']}})
+
+ if not conditions:
+ return None
+
+ if len(conditions) == 1:
+ return conditions[0]
+
+ return {"$and": conditions}
+
+ def _get_most_recent_namespace(self):
+ pc = Pinecone(api_key=os.environ.get("PINECONE_API_KEY"))
+ index = pc.Index("docsite")
+ index_stats = index.describe_index_stats()
+ namespaces = index_stats.get('namespaces', {}).keys()
+
+ # The indexer names namespaces docsite-%Y%m%d%H%M (20 chars); namespaces
+ # created by hand use docsite-%Y%m%d (16). Accept both, so a namespace the
+ # indexer just wrote is discoverable.
+ valid_namespaces = sorted(
+ (ns for ns in namespaces if ns.startswith("docsite-") and ns[8:].isdigit() and len(ns) in (16, 20)),
+ reverse=True
+ )
+
+ if not valid_namespaces:
+ raise ApolloError(404, "No valid namespaces found in the index", type="NOT_FOUND")
+
+ most_recent_namespace = valid_namespaces[0]
+ logger.info(f"Most recent docsite collection name found: {most_recent_namespace}")
+ return most_recent_namespace
diff --git a/services/search_docsite/search_docsite.py b/services/search_docsite/search_docsite.py
index f7355ea7..7638a55a 100644
--- a/services/search_docsite/search_docsite.py
+++ b/services/search_docsite/search_docsite.py
@@ -1,174 +1,265 @@
import os
+
+import psycopg2
from dotenv import load_dotenv
-from pinecone import Pinecone
-from langchain_pinecone import PineconeVectorStore
-from langchain_openai import OpenAIEmbeddings
-from util import create_logger, ApolloError
from embeddings.embeddings import SearchResult
+from langchain_openai import OpenAIEmbeddings
+from pgvector import Vector
+from pgvector.psycopg2 import register_vector
+from search_docsite.pinecone_legacy_search import LegacyPineconeDocsiteSearch
+from util import ApolloError, create_logger, get_db_connection
+
logger = create_logger("DocsiteSearch")
+SCHEMA_MISSING_MESSAGE = (
+ "Docsite schema not initialised — run embed_docsite with target=postgres"
+)
+
+
+def register_vector_type(conn):
+ """Register the pgvector adapter on this connection."""
+ register_vector(conn)
+
class DocsiteSearch:
"""
- Initialize the docsite vectorstore and search it with optional metadata filters.
-
- :param collection_name: Vectorstore collection name (namespace) to store documents
- :param index_name: Vectorstore index name (default: docsite)
+ Search embedded docsite chunks in Postgres using semantic (pgvector cosine),
+ keyword (Postgres full-text search), or hybrid (Reciprocal Rank Fusion) strategies.
+
+ :param batch_id: Explicit batch to search. If None, resolves to the newest 'complete' batch.
:param default_top_k: Default number of results to return (default: 5)
- :param embeddings: LangChain embedding type (default: OpenAIEmbeddings())
"""
- def __init__(self, collection_name=None, index_name="docsite", default_top_k=5, embeddings=OpenAIEmbeddings()):
- self.index_client = index_name
+
+ def __init__(self, batch_id=None, default_top_k=5):
self.default_top_k = default_top_k
+ self._explicit_batch_id = batch_id
+ self._embeddings = None
+
+ @property
+ def embeddings(self):
+ if self._embeddings is None:
+ self._embeddings = OpenAIEmbeddings()
+ return self._embeddings
- if collection_name is None:
- logger.info("Collection name not provided; retrieving the most recent collection name.")
- collection_name = self._get_most_recent_namespace()
+ def _connect(self):
+ """Open a connection with pgvector registered.
+ """
+ conn = get_db_connection()
+ try:
+ register_vector_type(conn)
+ except psycopg2.ProgrammingError as exc:
+ conn.close()
+ raise ApolloError(503, SCHEMA_MISSING_MESSAGE, type="DATABASE_ERROR") from exc
+ return conn
- self.collection_name = collection_name
- self.vectorstore = PineconeVectorStore(index_name=index_name, namespace=collection_name, embedding=embeddings)
-
def search(self, query, top_k=None, threshold=None, strategy='semantic', doc_title=None, docs_type=None):
"""
- Search database with optional filters.
+ Search docsite_chunks with optional filters.
:param query: Search query string
:param top_k: Number of results to return
- :param threshold: Score threshold for semantic search
- :param strategy: Search strategy (default: 'semantic')
+ :param threshold: Cosine-similarity cutoff. Valid only for
+ strategy='semantic'; raises for other strategies.
+ :param strategy: 'semantic' | 'keyword' | 'hybrid' (default: 'semantic')
:param doc_title: Filter by document title
:param docs_type: Filter by document type
:return: List of SearchResult objects
"""
- filters = self._build_filter(doc_title=doc_title, docs_type=docs_type)
- logger.info("Metadata filters built")
+ if threshold is not None and strategy != 'semantic':
+ raise ApolloError(
+ 400,
+ f"threshold is only supported for strategy='semantic', got '{strategy}'",
+ type="BAD_REQUEST",
+ )
+
+ conn = self._connect()
+ try:
+ batch_id = self._explicit_batch_id or self._resolve_current_batch(conn)
- if strategy == 'semantic':
- return self._semantic_search(query=query, top_k=top_k, threshold=threshold, filters=filters)
+ if strategy == 'semantic':
+ return self._semantic_search(conn, batch_id, query, top_k, threshold, doc_title, docs_type)
+ if strategy == 'keyword':
+ return self._keyword_search(conn, batch_id, query, top_k, doc_title, docs_type)
+ if strategy == 'hybrid':
+ return self._hybrid_search(conn, batch_id, query, top_k, doc_title, docs_type)
- def _semantic_search(self, query, top_k=None, threshold=None, filters=None):
- """Search the vectorstore using semantic search."""
+ raise ApolloError(400, f"Unknown search strategy: {strategy}", type="BAD_REQUEST")
+ finally:
+ conn.close()
+
+ def _resolve_current_batch(self, conn):
+ """Find the newest complete batch id."""
+ try:
+ with conn.cursor() as cur:
+ cur.execute("SELECT id FROM docsite_batches WHERE status = 'complete' ORDER BY id DESC LIMIT 1")
+ row = cur.fetchone()
+ except psycopg2.errors.UndefinedTable as exc:
+ raise ApolloError(503, SCHEMA_MISSING_MESSAGE, type="DATABASE_ERROR") from exc
+ if row is None:
+ raise ApolloError(404, "No complete docsite batch found", type="NOT_FOUND")
+ return row[0]
+
+ def _semantic_search(self, conn, batch_id, query, top_k, threshold, doc_title, docs_type):
if top_k is None and threshold is None:
top_k = self.default_top_k
-
max_k = top_k or 50
-
- scored_docs = self.vectorstore.similarity_search_with_score(
- query=query,
- k=max_k,
- filter=filters
- )
-
- logger.info(f"Similar documents retrieved: {len(scored_docs)}")
-
+
+ query_embedding = Vector(self.embeddings.embed_query(query))
+
+ sql = """
+ SELECT text, doc_title, docs_type, 1 - (embedding <=> %(query_embedding)s) AS score
+ FROM docsite_chunks
+ WHERE batch_id = %(batch_id)s
+ AND (%(doc_title)s IS NULL OR doc_title = %(doc_title)s)
+ AND (%(docs_type)s IS NULL OR docs_type = %(docs_type)s)
+ ORDER BY embedding <=> %(query_embedding)s
+ LIMIT %(max_k)s
+ """
+ params = {
+ "query_embedding": query_embedding, "batch_id": batch_id,
+ "doc_title": doc_title, "docs_type": docs_type, "max_k": max_k,
+ }
+ with conn.cursor() as cur:
+ cur.execute(sql, params)
+ rows = cur.fetchall()
+
results = []
- for doc, score in scored_docs:
+ for text, title, dtype, score in rows:
if threshold is not None and score < threshold:
continue
-
- # If we've reached top_k docs and no threshold is set, stop
if top_k is not None and len(results) >= top_k and threshold is None:
break
-
- results.append(SearchResult(doc.page_content, doc.metadata, score))
-
- logger.info(f"Filtered to {len(results)} results")
+ results.append(SearchResult(text, {"doc_title": title, "docs_type": dtype}, score))
+
+ logger.info(f"Semantic search returned {len(results)} results")
return results
-
- def _build_filter(self, **kwargs):
- """Build filter conditions to search the vectorstore."""
- conditions = []
-
- # Add exact match conditions
- if kwargs.get('doc_title'):
- conditions.append({"doc_title": {"$eq": kwargs['doc_title']}})
-
-
- if kwargs.get('docs_type'):
- conditions.append({"docs_type": {"$eq": kwargs['docs_type']}})
-
- # If no conditions were added, return None
- if not conditions:
- return None
-
- # If only one condition, return it directly
- if len(conditions) == 1:
- return conditions[0]
-
- # If multiple conditions, combine them with $and
- return {"$and": conditions}
-
- def _get_most_recent_namespace(self):
- """Retrieve the most recent docsite upload by collection name from Pinecone."""
-
- pc = Pinecone(api_key=os.environ.get("PINECONE_API_KEY"))
- index = pc.Index("docsite")
- index_stats = index.describe_index_stats()
- namespaces = index_stats.get('namespaces', {}).keys()
-
- valid_namespaces = sorted(
- (ns for ns in namespaces if ns.startswith("docsite-") and ns[8:].isdigit() and len(ns) == 16),
- reverse=True
- )
- if not valid_namespaces:
- raise ApolloError(404, "No valid namespaces found in the index.", type="NOT_FOUND")
+ def _keyword_search(self, conn, batch_id, query, top_k, doc_title, docs_type):
+ max_k = top_k or self.default_top_k
- most_recent_namespace = valid_namespaces[0]
- logger.info(f"Most recent docsite collection name found: {most_recent_namespace}")
- return most_recent_namespace
+ sql = """
+ SELECT text, doc_title, docs_type,
+ ts_rank_cd(text_search, plainto_tsquery('english', %(query)s)) AS score
+ FROM docsite_chunks
+ WHERE batch_id = %(batch_id)s
+ AND text_search @@ plainto_tsquery('english', %(query)s)
+ AND (%(doc_title)s IS NULL OR doc_title = %(doc_title)s)
+ AND (%(docs_type)s IS NULL OR docs_type = %(docs_type)s)
+ ORDER BY score DESC
+ LIMIT %(max_k)s
+ """
+ params = {"query": query, "batch_id": batch_id, "doc_title": doc_title, "docs_type": docs_type, "max_k": max_k}
+ with conn.cursor() as cur:
+ cur.execute(sql, params)
+ rows = cur.fetchall()
+
+ results = [SearchResult(text, {"doc_title": title, "docs_type": dtype}, score) for text, title, dtype, score in rows]
+ logger.info(f"Keyword search returned {len(results)} results")
+ return results
+
+ def _hybrid_search(self, conn, batch_id, query, top_k, doc_title, docs_type):
+ max_k = top_k or self.default_top_k
+ candidate_k = 50
+
+ query_embedding = Vector(self.embeddings.embed_query(query))
+
+ sql = """
+ WITH semantic AS (
+ SELECT id, text, doc_title, docs_type,
+ ROW_NUMBER() OVER (ORDER BY embedding <=> %(query_embedding)s) AS rnk
+ FROM docsite_chunks
+ WHERE batch_id = %(batch_id)s
+ AND (%(doc_title)s IS NULL OR doc_title = %(doc_title)s)
+ AND (%(docs_type)s IS NULL OR docs_type = %(docs_type)s)
+ ORDER BY embedding <=> %(query_embedding)s
+ LIMIT %(candidate_k)s
+ ),
+ keyword AS (
+ SELECT id, text, doc_title, docs_type,
+ ROW_NUMBER() OVER (ORDER BY ts_rank_cd(text_search, plainto_tsquery('english', %(query)s)) DESC) AS rnk
+ FROM docsite_chunks
+ WHERE batch_id = %(batch_id)s
+ AND text_search @@ plainto_tsquery('english', %(query)s)
+ AND (%(doc_title)s IS NULL OR doc_title = %(doc_title)s)
+ AND (%(docs_type)s IS NULL OR docs_type = %(docs_type)s)
+ ORDER BY rnk
+ LIMIT %(candidate_k)s
+ )
+ SELECT COALESCE(s.text, k.text) AS text,
+ COALESCE(s.doc_title, k.doc_title) AS doc_title,
+ COALESCE(s.docs_type, k.docs_type) AS docs_type,
+ COALESCE(1.0::float8 / (60 + s.rnk), 0) + COALESCE(1.0::float8 / (60 + k.rnk), 0) AS score
+ FROM semantic s FULL OUTER JOIN keyword k ON s.id = k.id
+ ORDER BY score DESC
+ LIMIT %(max_k)s
+ """
+ params = {
+ "query_embedding": query_embedding, "query": query, "batch_id": batch_id,
+ "doc_title": doc_title, "docs_type": docs_type, "candidate_k": candidate_k, "max_k": max_k,
+ }
+ with conn.cursor() as cur:
+ cur.execute(sql, params)
+ rows = cur.fetchall()
+
+ results = [
+ SearchResult(text, {"doc_title": title, "docs_type": dtype}, float(score))
+ for text, title, dtype, score in rows
+ ]
+ logger.info(f"Hybrid search returned {len(results)} results")
+ return results
+
+
+BACKEND_INDEX_PARAMS = {
+ "postgres": ["batch_id", "default_top_k"],
+ "pinecone": ["collection_name", "index_name", "default_top_k", "embeddings"],
+}
+
+
+def resolve_backend(override=None):
+ """Return the search class for the configured backend.
+
+ :param override: Backend name that is prioritised over DOCSITE_SEARCH_BACKEND
+ """
+ name = override or os.environ.get("DOCSITE_SEARCH_BACKEND", "pinecone")
+ if name not in BACKEND_INDEX_PARAMS:
+ raise ApolloError(400, f"Unknown backend '{name}'. Expected 'pinecone' or 'postgres'", type="BAD_REQUEST")
+ return DocsiteSearch if name == "postgres" else LegacyPineconeDocsiteSearch
def main(data):
logger.info("Starting...")
required_fields = ["query"]
-
missing = [field for field in required_fields if field not in data]
-
if missing:
logger.error(f"Missing required fields in data: {', '.join(missing)}")
- return
+ return None
- index_params = {}
- search_params = {"query": data["query"]}
+ backend = data.get("backend") or os.environ.get("DOCSITE_SEARCH_BACKEND", "pinecone")
+ search_cls = resolve_backend(backend)
- # Add optional parameters
+ search_params = {"query": data["query"]}
optional_search_params = ["docs_type", "doc_title", "top_k", "threshold", "strategy"]
- optional_index_params = ["collection_name", "index_name", "default_top_k", "embeddings"]
-
for key in optional_search_params:
if key in data:
search_params[key] = data[key]
- for key in optional_index_params:
- if key in data:
- index_params[key] = data[key]
+ index_params = {key: data[key] for key in BACKEND_INDEX_PARAMS[backend] if key in data}
- # Set API keys
load_dotenv(override=True)
- OPENAI_API_KEY = os.environ.get('OPENAI_API_KEY')
- PINECONE_API_KEY = os.environ.get('PINECONE_API_KEY')
-
- # Check for missing keys
- missing_keys = []
-
- if not OPENAI_API_KEY:
- missing_keys.append("OPENAI_API_KEY")
- if not PINECONE_API_KEY:
- missing_keys.append("PINECONE_API_KEY")
-
- if missing_keys:
- msg = f"Missing API keys: {', '.join(missing_keys)}"
+ openai_api_key = os.environ.get('OPENAI_API_KEY')
+ if not openai_api_key:
+ msg = "Missing API key: OPENAI_API_KEY"
logger.error(msg)
- raise ApolloError(500, f"Missing API keys: {', '.join(missing_keys)}", type="BAD_REQUEST")
+ raise ApolloError(500, msg, type="BAD_REQUEST")
- # Initialize search engine
- docsite_search = DocsiteSearch(**index_params)
- logger.info("Docsite database initialised")
+ logger.info(f"Searching docsite via the {backend} backend")
+
+ docsite_search = search_cls(**index_params)
results = docsite_search.search(**search_params)
-
+
return [result.to_json() for result in results]
+
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main({})
diff --git a/services/search_docsite/tests/conftest.py b/services/search_docsite/tests/conftest.py
index d066197b..eb256c39 100644
--- a/services/search_docsite/tests/conftest.py
+++ b/services/search_docsite/tests/conftest.py
@@ -5,9 +5,7 @@
credentials at construction time. That happens when the test module is imported,
before any test runs — so a key must exist in the environment or import fails.
-These are dummy placeholders only: unit tests inject mocks for every network
-seam and the repo-root conftest additionally blocks real client construction, so
-no real key is ever used. `setdefault` means a real key (from services/.env) wins.
+We only set dummy environment variables for unit tests.
"""
import os
diff --git a/services/search_docsite/tests/eval/__init__.py b/services/search_docsite/tests/eval/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/services/search_docsite/tests/eval/golden_queries.yaml b/services/search_docsite/tests/eval/golden_queries.yaml
new file mode 100644
index 00000000..9f4cbad1
--- /dev/null
+++ b/services/search_docsite/tests/eval/golden_queries.yaml
@@ -0,0 +1,109 @@
+# Representative job_chat/general-docs queries with expected doc titles.
+#
+# recall@k counts a query as a hit when ANY expected title appears in the top k,
+# so each entry lists every doc that would be a reasonable answer, not only the
+# single best one. Titles are docsite filenames without .md, matching
+# docsite_chunks.doc_title.
+#
+# category is 'conceptual' (natural-language questions) or 'keyword' (exact-term
+# lookups). run_eval reports recall per category, because hybrid search is
+# expected to help on keyword lookups and tie on conceptual ones — a blended
+# figure would hide exactly that effect.
+#
+# Labels were derived from the general_docs corpus via full-text search on each
+# query's key terms, deliberately not from vector-search output: labelling from
+# the thing under test would be circular.
+queries:
+ # --- conceptual -------------------------------------------------------------
+ - query: "how do I configure a webhook trigger"
+ category: conceptual
+ expected_doc_titles: ["triggers", "webhook-auth"]
+ - query: "how do I set up a cron trigger"
+ category: conceptual
+ expected_doc_titles: ["triggers"]
+ - query: "what is a run in OpenFn"
+ category: conceptual
+ expected_doc_titles: ["terminology", "glossary"]
+ - query: "how do I use collections to store state between runs"
+ category: conceptual
+ expected_doc_titles: ["collections", "state", "cli-collections"]
+ - query: "how do I configure a credential for an adaptor"
+ category: conceptual
+ expected_doc_titles: ["credentials", "manage-credentials"]
+ - query: "what is the difference between a job and a workflow"
+ category: conceptual
+ expected_doc_titles: ["terminology", "glossary", "workflows"]
+ - query: "how do I deploy a project using the CLI"
+ category: conceptual
+ expected_doc_titles: ["cli-sync", "portability"]
+ - query: "how do I debug a failed run"
+ category: conceptual
+ expected_doc_titles: ["troubleshooting", "rerunning-workflow", "inspect-runs"]
+ - query: "how do I write a data transform function"
+ category: conceptual
+ expected_doc_titles: ["data-transformation", "job-writing-guide", "operations"]
+ - query: "how do I control who can access a project"
+ category: conceptual
+ expected_doc_titles: ["collaboration", "user-roles-permissions"]
+ - query: "how do I set up a sandbox environment"
+ category: conceptual
+ expected_doc_titles: ["sandboxes"]
+ - query: "how long is my run data kept"
+ category: conceptual
+ expected_doc_titles: ["retention-periods", "io-data-storage", "security-for-devs"]
+ - query: "how do I get notified when a workflow fails"
+ category: conceptual
+ expected_doc_titles: ["notifications"]
+ - query: "how do I version control my project with GitHub"
+ category: conceptual
+ expected_doc_titles: ["link-to-gh", "cli-sync"]
+ - query: "what security measures does OpenFn have"
+ category: conceptual
+ expected_doc_titles: ["security", "security-compliance"]
+
+ # --- keyword ----------------------------------------------------------------
+ - query: "state.cursor"
+ category: keyword
+ expected_doc_titles: ["using-cursors"]
+ - query: "--force flag on project push"
+ category: keyword
+ expected_doc_titles: ["cli-sync"]
+ - query: "webhook auth method"
+ category: keyword
+ expected_doc_titles: ["webhook-auth"]
+ - query: "workflow snapshots"
+ category: keyword
+ expected_doc_titles: ["workflow-snapshots"]
+ - query: "lazy state operator"
+ category: keyword
+ expected_doc_titles: ["lazy-state-operator"]
+ - query: "dataValue"
+ category: keyword
+ expected_doc_titles: ["job-snippets"]
+ - query: "each operation"
+ category: keyword
+ expected_doc_titles: ["operations"]
+ - query: "rerun a workflow"
+ category: keyword
+ expected_doc_titles: ["rerunning-workflow"]
+ - query: "collections CLI commands"
+ category: keyword
+ expected_doc_titles: ["cli-collections"]
+ - query: "activity history"
+ category: keyword
+ expected_doc_titles: ["activity-history"]
+ - query: "project.yaml"
+ category: keyword
+ expected_doc_titles: ["portability-v3"]
+ - query: "step editor"
+ category: keyword
+ expected_doc_titles: ["step-editor"]
+ - query: "git branch"
+ category: keyword
+ expected_doc_titles: ["working-with-branches", "cli-sync"]
+ - query: "sandbox"
+ category: keyword
+ expected_doc_titles: ["sandboxes"]
+ - query: "security compliance"
+ category: keyword
+ expected_doc_titles: ["security-compliance"]
diff --git a/services/search_docsite/tests/eval/run_eval.py b/services/search_docsite/tests/eval/run_eval.py
new file mode 100644
index 00000000..f20b6c10
--- /dev/null
+++ b/services/search_docsite/tests/eval/run_eval.py
@@ -0,0 +1,223 @@
+"""Offline recall@k + latency comparison between docsite search backends.
+
+Usage: poetry run python -m search_docsite.tests.eval.run_eval
+
+Compares Postgres semantic and hybrid search at each indexed chunk size against
+the legacy Pinecone baseline, over the golden query set. Recall is reported per
+query category, because hybrid is expected to help on keyword lookups and tie on
+conceptual ones — a blended figure would hide that.
+
+Requires a Postgres batch per chunk size in CHUNK_SIZES; configurations without
+one are skipped. Index them with embed_docsite, passing chunk_target_length,
+chunk_min_length, and keep_batches >= 3 so earlier batches are not pruned.
+
+Golden queries live in services/search_docsite/tests/eval/golden_queries.yaml
+— edit that file to add queries or curate expected_doc_titles.
+"""
+
+import time
+from pathlib import Path
+
+import yaml
+from dotenv import load_dotenv
+
+# Needed as LegacyPineconeDocsiteSearch evaluates OpenAIEmbeddings() as a
+# default argument.
+load_dotenv()
+
+from search_docsite.pinecone_legacy_search import LegacyPineconeDocsiteSearch
+from search_docsite.search_docsite import DocsiteSearch
+from util import get_db_connection
+
+GOLDEN_QUERIES_PATH = Path(__file__).parent / "golden_queries.yaml"
+
+# Chunk sizes to evaluate, matching docsite_batches.chunk_target_length.
+CHUNK_SIZES = [1000, 1800, 2500]
+STRATEGIES = ["semantic", "hybrid"]
+
+# The migration-fidelity pairing: same strategy and chunk size, different store.
+AGREEMENT_PAIR = ("Pinecone semantic", "Postgres semantic (1000)")
+
+
+def resolve_batch_id(chunk_target_length):
+ """Newest complete batch indexed at the given chunk size, or None if there is none.
+
+ Batch ids are environment-specific, so the eval looks them up by the indexing
+ configuration recorded on each batch rather than hardcoding them.
+ """
+ conn = get_db_connection()
+ try:
+ with conn.cursor() as cur:
+ cur.execute(
+ "SELECT id FROM docsite_batches WHERE status = 'complete' "
+ "AND chunk_target_length = %s ORDER BY id DESC LIMIT 1",
+ (chunk_target_length,),
+ )
+ row = cur.fetchone()
+ finally:
+ conn.close()
+
+ return row[0] if row else None
+
+
+def compute_recall_at_k(retrieved_titles, expected_titles):
+ """True if any expected title appears among the retrieved titles."""
+ return bool(set(retrieved_titles) & set(expected_titles))
+
+
+def compute_agreement(report_a, report_b):
+ """Doc-title overlap between two backends' result sets, per query and averaged.
+
+ Needs no ground truth, so it gives a usable comparison signal before
+ golden_queries.yaml is curated.
+ """
+ per_query = []
+ for a, b in zip(report_a["per_query"], report_b["per_query"], strict=True):
+ titles_a = {t for t in a["retrieved_titles"] if t is not None}
+ titles_b = {t for t in b["retrieved_titles"] if t is not None}
+ union = titles_a | titles_b
+ overlap = titles_a & titles_b
+ per_query.append({
+ "query": a["query"],
+ "overlap": len(overlap),
+ "union": len(union),
+ "jaccard": (len(overlap) / len(union)) if union else 0.0,
+ })
+
+ mean_jaccard = (sum(q["jaccard"] for q in per_query) / len(per_query)) if per_query else 0.0
+ return {"per_query": per_query, "mean_jaccard": mean_jaccard}
+
+
+def run_eval(golden_queries, make_backend, strategy, top_k=5):
+ """Run every golden query against one backend/strategy and return a report dict."""
+ backend = make_backend()
+ per_query = []
+ latencies = []
+ scored = 0
+ skipped = 0
+ hits = 0
+ by_category = {}
+
+ for item in golden_queries:
+ query = item["query"]
+ expected_titles = item.get("expected_doc_titles", [])
+ category = item.get("category")
+
+ start = time.time()
+ results = backend.search(query, top_k=top_k, strategy=strategy, docs_type="general_docs")
+ elapsed = time.time() - start
+ latencies.append(elapsed)
+
+ retrieved_titles = [r.metadata.get("doc_title") for r in results]
+
+ if expected_titles:
+ hit = compute_recall_at_k(retrieved_titles, expected_titles)
+ scored += 1
+ hits += int(hit)
+ if category:
+ counts = by_category.setdefault(category, {"scored": 0, "hits": 0})
+ counts["scored"] += 1
+ counts["hits"] += int(hit)
+ else:
+ hit = None
+ skipped += 1
+
+ per_query.append({
+ "query": query,
+ "category": category,
+ "retrieved_titles": retrieved_titles,
+ "expected_titles": expected_titles,
+ "hit": hit,
+ "latency_s": elapsed,
+ })
+
+ latencies_sorted = sorted(latencies)
+ p50 = latencies_sorted[len(latencies_sorted) // 2] if latencies_sorted else 0.0
+ p95_index = min(len(latencies_sorted) - 1, int(len(latencies_sorted) * 0.95)) if latencies_sorted else 0
+ p95 = latencies_sorted[p95_index] if latencies_sorted else 0.0
+
+ return {
+ "recall_at_k": (hits / scored) if scored else None,
+ "recall_by_category": {
+ name: {"recall": c["hits"] / c["scored"], "scored": c["scored"]}
+ for name, c in sorted(by_category.items())
+ },
+ "queries_scored": scored,
+ "queries_skipped": skipped,
+ "p50_latency_s": p50,
+ "p95_latency_s": p95,
+ "per_query": per_query,
+ }
+
+
+def print_per_query_results(label_a, report_a, label_b, report_b):
+ """Print each golden query's expected titles and both backends' hit/miss + retrieved titles."""
+ print("\nPer-query results:")
+ for a, b in zip(report_a["per_query"], report_b["per_query"], strict=True):
+ expected = ", ".join(a["expected_titles"]) or "(unlabelled — excluded from recall)"
+ print(f" [{a['category'] or 'uncategorised'}] {a['query']}")
+ print(f" expected: {expected}")
+ for label, item in ((label_a, a), (label_b, b)):
+ status = {True: "HIT ", False: "MISS", None: "-- "}[item["hit"]]
+ titles = ", ".join(t for t in item["retrieved_titles"] if t is not None)
+ print(f" {label:26} {status} {item['latency_s']:.3f}s {titles}")
+
+
+def build_configs():
+ """Every (label, backend factory, strategy) to evaluate.
+
+ Postgres configurations for a chunk size with no complete batch are skipped
+ with a notice, so the eval still runs when only some batches are indexed.
+ """
+ configs = [("Pinecone semantic", LegacyPineconeDocsiteSearch, "semantic")]
+
+ for chunk_size in CHUNK_SIZES:
+ batch_id = resolve_batch_id(chunk_size)
+ if batch_id is None:
+ print(f"Skipping Postgres configs for chunk size {chunk_size} — no complete batch")
+ continue
+ for strategy in STRATEGIES:
+ label = f"Postgres {strategy} ({chunk_size})"
+ configs.append((label, lambda b=batch_id: DocsiteSearch(batch_id=b), strategy))
+
+ return configs
+
+
+def _fmt(recall):
+ """Render a recall figure, or a dash when the category had no scored queries."""
+ return "-" if recall is None else f"{recall:.2f}"
+
+
+def main():
+ with open(GOLDEN_QUERIES_PATH) as f:
+ golden_queries = yaml.safe_load(f)["queries"]
+
+ reports = {}
+ for label, make_backend, strategy in build_configs():
+ reports[label] = run_eval(golden_queries, make_backend, strategy)
+
+ print(f"\n{'configuration':<28} {'conceptual':>11} {'keyword':>9} {'overall':>9} {'p50':>8} {'p95':>8}")
+ for label, report in reports.items():
+ by_cat = report["recall_by_category"]
+ conceptual = by_cat.get("conceptual", {}).get("recall")
+ keyword = by_cat.get("keyword", {}).get("recall")
+ print(f"{label:<28} "
+ f"{_fmt(conceptual):>11} {_fmt(keyword):>9} {_fmt(report['recall_at_k']):>9} "
+ f"{report['p50_latency_s']:>7.3f}s {report['p95_latency_s']:>7.3f}s")
+
+ scored = next(iter(reports.values()))
+ print(f"\nScored {scored['queries_scored']} queries, skipped {scored['queries_skipped']}")
+
+ label_a, label_b = AGREEMENT_PAIR
+ if label_a in reports and label_b in reports:
+ agreement = compute_agreement(reports[label_a], reports[label_b])
+ print(f"\nMigration fidelity ({label_a} vs {label_b}):")
+ print(f" mean doc-title Jaccard={agreement['mean_jaccard']:.3f} "
+ f"across {len(agreement['per_query'])} queries")
+ for q in agreement["per_query"]:
+ print(f" {q['jaccard']:.2f} overlap={q['overlap']}/{q['union']} {q['query']}")
+ print_per_query_results(label_a, reports[label_a], label_b, reports[label_b])
+
+
+if __name__ == "__main__":
+ main()
diff --git a/services/search_docsite/tests/unit/test_docsite_search.py b/services/search_docsite/tests/unit/test_docsite_search.py
index 37630f90..c283481f 100644
--- a/services/search_docsite/tests/unit/test_docsite_search.py
+++ b/services/search_docsite/tests/unit/test_docsite_search.py
@@ -1,140 +1,253 @@
-"""Unit tests for DocsiteSearch — the Pinecone + OpenAI seam used by job_chat.
+"""Unit tests for the Postgres-backed DocsiteSearch (semantic/keyword/hybrid).
-These pin the contracts most exposed by the dependency bump (langchain-pinecone
-0.2.2→0.2.13, langchain-openai →1.x, pinecone 5→7):
-
- - the langchain `similarity_search_with_score(query=, k=, filter=)` signature
- and its `[(Document, score), ...]` return shape, consumed by `_semantic_search`
- - the pinecone `describe_index_stats().get("namespaces")` shape, consumed by
- `_get_most_recent_namespace`
- - that the module's dependency symbols still import under the new versions
-
-Every external boundary is mocked, so no network/credentials are touched (the
-repo-root conftest also blocks real anthropic/openai client construction here).
+get_db_connection and register_vector_type are mocked throughout, no real
+Postgres connection is made. The OpenAI embeddings client is mocked via the
+`_embeddings` attribute, matching the DocsiteIndexer test pattern.
"""
+import json
+from decimal import Decimal
from unittest.mock import MagicMock, patch
+import psycopg2
import pytest
-
import search_docsite.search_docsite as m
+from pgvector import Vector
from util import ApolloError
-class FakeDoc:
- """Stand-in for a langchain Document (page_content + metadata)."""
+def make_conn():
+ conn = MagicMock()
+ cur = MagicMock()
+ conn.cursor.return_value.__enter__.return_value = cur
+ return conn, cur
- def __init__(self, text, metadata=None):
- self.page_content = text
- self.metadata = metadata or {}
+def make_search(**kwargs):
+ ds = m.DocsiteSearch(**kwargs)
+ ds._embeddings = MagicMock()
+ ds._embeddings.embed_query.return_value = [0.1, 0.2, 0.3]
+ return ds
-def make_search(default_top_k=5):
- """Construct DocsiteSearch offline: collection_name given (skips the
- namespace lookup) and PineconeVectorStore patched (no real client)."""
- with patch.object(m, "PineconeVectorStore", return_value=MagicMock()):
- return m.DocsiteSearch(
- collection_name="docsite-20240101",
- default_top_k=default_top_k,
- embeddings=MagicMock(),
- )
+def patched(conn):
+ return patch.object(m, "get_db_connection", return_value=conn), patch.object(m, "register_vector_type")
-# --- _build_filter (pure logic) ------------------------------------------------
-@pytest.mark.parametrize(
- "kwargs, expected",
- [
- ({"doc_title": "Adaptor X"}, {"doc_title": {"$eq": "Adaptor X"}}),
- ({"docs_type": "general_docs"}, {"docs_type": {"$eq": "general_docs"}}),
- ],
-)
-def test_build_filter_single_key(kwargs, expected):
- ds = make_search()
- assert ds._build_filter(**kwargs) == expected
+# --- strategy dispatch -----------------------------------------------------
+
+def test_search_dispatches_to_semantic_strategy():
+ conn, _ = make_conn()
+ ds = make_search(batch_id=1)
+ with patched(conn)[0], patched(conn)[1], patch.object(ds, "_semantic_search", return_value=["r"]) as mock_sem:
+ result = ds.search("query", strategy="semantic")
+ assert result == ["r"]
+ mock_sem.assert_called_once()
+
+
+def test_search_raises_on_unknown_strategy():
+ conn, _ = make_conn()
+ ds = make_search(batch_id=1)
+ with patched(conn)[0], patched(conn)[1]:
+ with pytest.raises(ApolloError) as exc:
+ ds.search("query", strategy="nonsense")
+ assert exc.value.code == 400
+
+
+@pytest.mark.parametrize("strategy", ["hybrid", "keyword"])
+def test_search_rejects_threshold_for_non_semantic_strategies(strategy):
+ """RRF/FTS scores are not comparable to a cosine cutoff. Silently ignoring a
+ threshold here is a landmine: 0.8 against a 0.033-max score would drop
+ every result with no error."""
+ conn, _ = make_conn()
+ ds = make_search(batch_id=1)
+ with patched(conn)[0], patched(conn)[1]:
+ with pytest.raises(ApolloError) as exc:
+ ds.search("query", threshold=0.8, strategy=strategy)
+ assert exc.value.code == 400
+
+
+@pytest.mark.parametrize("strategy", ["hybrid", "keyword"])
+def test_search_allows_none_threshold_for_non_semantic_strategies(strategy):
+ conn, _ = make_conn()
+ ds = make_search(batch_id=1)
+ with patched(conn)[0], patched(conn)[1], \
+ patch.object(ds, "_keyword_search", return_value=["r"]), \
+ patch.object(ds, "_hybrid_search", return_value=["r"]):
+ assert ds.search("query", threshold=None, strategy=strategy) == ["r"]
-def test_build_filter_both_combines_with_and():
+# --- _resolve_current_batch --------------------------------------------------
+
+def test_resolve_current_batch_returns_newest_complete_batch_id():
+ conn, cur = make_conn()
+ cur.fetchone.return_value = (9,)
ds = make_search()
- assert ds._build_filter(doc_title="X", docs_type="general_docs") == {
- "$and": [{"doc_title": {"$eq": "X"}}, {"docs_type": {"$eq": "general_docs"}}]
- }
+ assert ds._resolve_current_batch(conn) == 9
-def test_build_filter_none_returns_none():
+def test_resolve_current_batch_raises_when_none_complete():
+ conn, cur = make_conn()
+ cur.fetchone.return_value = None
ds = make_search()
- assert ds._build_filter() is None
+ with pytest.raises(ApolloError) as exc:
+ ds._resolve_current_batch(conn)
+ assert exc.value.code == 404
-# --- _semantic_search (langchain return-shape contract) ------------------------
+# --- _semantic_search: (top_k, threshold) fallback semantics, ported from Pinecone tests ---
-def test_semantic_search_applies_threshold_and_passes_signature():
+def test_semantic_search_applies_threshold_and_falls_back_to_k_50():
+ conn, cur = make_conn()
+ cur.fetchall.return_value = [("a", "Doc A", "general_docs", 0.9), ("b", "Doc B", "general_docs", 0.4)]
ds = make_search()
- ds.vectorstore.similarity_search_with_score.return_value = [
- (FakeDoc("a"), 0.9),
- (FakeDoc("b"), 0.6),
- (FakeDoc("c"), 0.4), # below threshold, dropped
- ]
- results = ds._semantic_search(query="q", threshold=0.5)
+ results = ds._semantic_search(conn, batch_id=1, query="q", top_k=None, threshold=0.5, doc_title=None, docs_type=None)
- assert [r.score for r in results] == [0.9, 0.6]
- assert [r.text for r in results] == ["a", "b"]
- # Pin the langchain-pinecone call signature; threshold-only => k falls back to 50.
- ds.vectorstore.similarity_search_with_score.assert_called_once_with(
- query="q", k=50, filter=None
- )
+ assert [r.text for r in results] == ["a"]
+ params = cur.execute.call_args[0][1]
+ assert params["max_k"] == 50
def test_semantic_search_truncates_to_top_k_when_no_threshold():
+ conn, cur = make_conn()
+ cur.fetchall.return_value = [("a", "A", "t", 0.9), ("b", "B", "t", 0.8), ("c", "C", "t", 0.7)]
ds = make_search()
- ds.vectorstore.similarity_search_with_score.return_value = [
- (FakeDoc(t), s) for t, s in [("a", 0.9), ("b", 0.8), ("c", 0.7), ("d", 0.6)]
- ]
- results = ds._semantic_search(query="q", top_k=2)
+ results = ds._semantic_search(conn, batch_id=1, query="q", top_k=2, threshold=None, doc_title=None, docs_type=None)
assert [r.text for r in results] == ["a", "b"]
- ds.vectorstore.similarity_search_with_score.assert_called_once_with(
- query="q", k=2, filter=None
- )
def test_semantic_search_defaults_to_default_top_k():
+ conn, cur = make_conn()
+ cur.fetchall.return_value = [(str(i), str(i), "t", 0.9) for i in range(7)]
ds = make_search(default_top_k=5)
- ds.vectorstore.similarity_search_with_score.return_value = [
- (FakeDoc(str(i)), 0.9) for i in range(7)
- ]
- # Neither top_k nor threshold given => default_top_k (5) applies.
- results = ds._semantic_search(query="q")
+ results = ds._semantic_search(conn, batch_id=1, query="q", top_k=None, threshold=None, doc_title=None, docs_type=None)
assert len(results) == 5
- ds.vectorstore.similarity_search_with_score.assert_called_once_with(
- query="q", k=5, filter=None
- )
-# --- _get_most_recent_namespace (pinecone describe_index_stats shape) ----------
+# --- _keyword_search ---------------------------------------------------------
+
+def test_keyword_search_uses_ts_rank_and_returns_results():
+ conn, cur = make_conn()
+ cur.fetchall.return_value = [("a", "Doc A", "general_docs", 0.5)]
+ ds = make_search()
+
+ results = ds._keyword_search(conn, batch_id=1, query="webhook", top_k=None, doc_title=None, docs_type="general_docs")
+
+ assert len(results) == 1
+ assert results[0].text == "a"
+ sql = cur.execute.call_args[0][0]
+ assert "ts_rank_cd" in sql
+ assert "plainto_tsquery" in sql
+
+
+# --- _hybrid_search ------------------------------------------------------------
+
+def test_hybrid_search_runs_rrf_query_and_returns_results():
+ conn, cur = make_conn()
+ cur.fetchall.return_value = [("a", "Doc A", "general_docs", 0.032)]
+ ds = make_search()
+
+ results = ds._hybrid_search(conn, batch_id=1, query="webhook", top_k=5, doc_title=None, docs_type="general_docs")
+
+ assert len(results) == 1
+ sql = cur.execute.call_args[0][0]
+ assert "FULL OUTER JOIN" in sql
+ params = cur.execute.call_args[0][1]
+ assert params["candidate_k"] == 50
+ assert params["max_k"] == 5
-def _patch_pinecone(namespaces):
- index = MagicMock()
- index.describe_index_stats.return_value = {"namespaces": {ns: {} for ns in namespaces}}
- client = MagicMock()
- client.Index.return_value = index
- return patch.object(m, "Pinecone", return_value=client)
+
+def test_hybrid_search_score_is_json_serializable_float():
+ """Postgres returns RRF as `numeric`, which psycopg2 hands back as Decimal.
+ Decimal is not JSON-serializable, and entry.py's json.dump sits outside its
+ try/except — so this would kill the process, not return a 500."""
+ conn, cur = make_conn()
+ cur.fetchall.return_value = [("a", "Doc A", "general_docs", Decimal("0.032"))]
+ ds = make_search()
+
+ results = ds._hybrid_search(conn, batch_id=1, query="webhook", top_k=5, doc_title=None, docs_type="general_docs")
+
+ assert isinstance(results[0].score, float)
+ json.dumps(results[0].to_json()) # must not raise
-def test_get_most_recent_namespace_picks_latest_valid():
+def test_hybrid_search_casts_rrf_to_float8_in_sql():
+ """Belt and braces: the SQL itself must not produce numeric in the first place."""
+ conn, cur = make_conn()
+ cur.fetchall.return_value = []
ds = make_search()
- namespaces = ["docsite-20231231", "docsite-20240101", "other", "docsite-bad"]
- with _patch_pinecone(namespaces):
- assert ds._get_most_recent_namespace() == "docsite-20240101"
+ ds._hybrid_search(conn, batch_id=1, query="webhook", top_k=5, doc_title=None, docs_type=None)
+
+ sql = cur.execute.call_args[0][0]
+ assert "float8" in sql
+
+
+# --- missing schema ----------------------------------------------------------
-def test_get_most_recent_namespace_raises_when_none_valid():
+def test_search_maps_missing_pgvector_extension_to_503():
+ """register_vector raises ProgrammingError('vector type not found in the
+ database') when CREATE EXTENSION has not run. Since migrations moved to the
+ indexer, that is the first thing a reader hits on an un-indexed database."""
+ conn, _ = make_conn()
+ ds = make_search(batch_id=1)
+
+ with patch.object(m, "get_db_connection", return_value=conn), \
+ patch.object(m, "register_vector_type",
+ side_effect=psycopg2.ProgrammingError("vector type not found in the database")):
+ with pytest.raises(ApolloError) as exc:
+ ds.search("query")
+
+ assert exc.value.code == 503
+ assert "embed_docsite" in exc.value.message
+ conn.close.assert_called_once()
+
+
+def test_search_maps_missing_docsite_tables_to_503():
+ """Extension present, tables absent — the second way a database can be
+ un-indexed."""
+ conn, cur = make_conn()
+ cur.execute.side_effect = psycopg2.errors.UndefinedTable(
+ 'relation "docsite_batches" does not exist',
+ )
ds = make_search()
- with _patch_pinecone(["other", "docsite-bad", "docsite-2024"]):
+
+ with patch.object(m, "get_db_connection", return_value=conn), \
+ patch.object(m, "register_vector_type"):
with pytest.raises(ApolloError) as exc:
- ds._get_most_recent_namespace()
- assert exc.value.code == 404
+ ds.search("query")
+
+ assert exc.value.code == 503
+ assert "embed_docsite" in exc.value.message
+
+
+# --- vector binding ----------------------------------------------------------
+
+def test_semantic_search_binds_the_embedding_as_a_vector():
+ """psycopg2 has no adapter for list, so a list is sent as numeric[] and
+ `vector <=> numeric[]` resolves to no operator. Only Vector and ndarray are
+ registered by pgvector."""
+ conn, cur = make_conn()
+ cur.fetchall.return_value = []
+ ds = make_search()
+
+ ds._semantic_search(conn, batch_id=1, query="q", top_k=5, threshold=None, doc_title=None, docs_type=None)
+
+ params = cur.execute.call_args[0][1]
+ assert params["query_embedding"] == Vector([0.1, 0.2, 0.3])
+
+
+def test_hybrid_search_binds_the_embedding_as_a_vector():
+ conn, cur = make_conn()
+ cur.fetchall.return_value = []
+ ds = make_search()
+
+ ds._hybrid_search(conn, batch_id=1, query="q", top_k=5, doc_title=None, docs_type=None)
+
+ params = cur.execute.call_args[0][1]
+ assert params["query_embedding"] == Vector([0.1, 0.2, 0.3])
diff --git a/services/search_docsite/tests/unit/test_pinecone_legacy_search.py b/services/search_docsite/tests/unit/test_pinecone_legacy_search.py
new file mode 100644
index 00000000..ed67f6e6
--- /dev/null
+++ b/services/search_docsite/tests/unit/test_pinecone_legacy_search.py
@@ -0,0 +1,161 @@
+"""Unit tests for DocsiteSearch — the Pinecone + OpenAI seam used by job_chat.
+
+These pin the contracts most exposed by the dependency bump (langchain-pinecone
+0.2.2→0.2.13, langchain-openai →1.x, pinecone 5→7):
+
+ - the langchain `similarity_search_with_score(query=, k=, filter=)` signature
+ and its `[(Document, score), ...]` return shape, consumed by `_semantic_search`
+ - the pinecone `describe_index_stats().get("namespaces")` shape, consumed by
+ `_get_most_recent_namespace`
+ - that the module's dependency symbols still import under the new versions
+
+Every external boundary is mocked, so no network/credentials are touched (the
+repo-root conftest also blocks real anthropic/openai client construction here).
+"""
+
+from unittest.mock import MagicMock, patch
+
+import pytest
+import search_docsite.pinecone_legacy_search as m
+from util import ApolloError
+
+
+class FakeDoc:
+ """Stand-in for a langchain Document (page_content + metadata)."""
+
+ def __init__(self, text, metadata=None):
+ self.page_content = text
+ self.metadata = metadata or {}
+
+
+def make_search(default_top_k=5):
+ """Construct DocsiteSearch offline: collection_name given (skips the
+ namespace lookup) and PineconeVectorStore patched (no real client)."""
+ with patch.object(m, "PineconeVectorStore", return_value=MagicMock()):
+ return m.LegacyPineconeDocsiteSearch(
+ collection_name="docsite-20240101",
+ default_top_k=default_top_k,
+ embeddings=MagicMock(),
+ )
+
+
+# --- _build_filter (pure logic) ------------------------------------------------
+
+@pytest.mark.parametrize(
+ "kwargs, expected",
+ [
+ ({"doc_title": "Adaptor X"}, {"doc_title": {"$eq": "Adaptor X"}}),
+ ({"docs_type": "general_docs"}, {"docs_type": {"$eq": "general_docs"}}),
+ ],
+)
+def test_build_filter_single_key(kwargs, expected):
+ ds = make_search()
+ assert ds._build_filter(**kwargs) == expected
+
+
+def test_build_filter_both_combines_with_and():
+ ds = make_search()
+ assert ds._build_filter(doc_title="X", docs_type="general_docs") == {
+ "$and": [{"doc_title": {"$eq": "X"}}, {"docs_type": {"$eq": "general_docs"}}]
+ }
+
+
+def test_build_filter_none_returns_none():
+ ds = make_search()
+ assert ds._build_filter() is None
+
+
+# --- _semantic_search (langchain return-shape contract) ------------------------
+
+def test_semantic_search_applies_threshold_and_passes_signature():
+ ds = make_search()
+ ds.vectorstore.similarity_search_with_score.return_value = [
+ (FakeDoc("a"), 0.9),
+ (FakeDoc("b"), 0.6),
+ (FakeDoc("c"), 0.4), # below threshold, dropped
+ ]
+
+ results = ds._semantic_search(query="q", threshold=0.5)
+
+ assert [r.score for r in results] == [0.9, 0.6]
+ assert [r.text for r in results] == ["a", "b"]
+ # Pin the langchain-pinecone call signature; threshold-only => k falls back to 50.
+ ds.vectorstore.similarity_search_with_score.assert_called_once_with(
+ query="q", k=50, filter=None
+ )
+
+
+def test_semantic_search_truncates_to_top_k_when_no_threshold():
+ ds = make_search()
+ ds.vectorstore.similarity_search_with_score.return_value = [
+ (FakeDoc(t), s) for t, s in [("a", 0.9), ("b", 0.8), ("c", 0.7), ("d", 0.6)]
+ ]
+
+ results = ds._semantic_search(query="q", top_k=2)
+
+ assert [r.text for r in results] == ["a", "b"]
+ ds.vectorstore.similarity_search_with_score.assert_called_once_with(
+ query="q", k=2, filter=None
+ )
+
+
+def test_semantic_search_defaults_to_default_top_k():
+ ds = make_search(default_top_k=5)
+ ds.vectorstore.similarity_search_with_score.return_value = [
+ (FakeDoc(str(i)), 0.9) for i in range(7)
+ ]
+
+ # Neither top_k nor threshold given => default_top_k (5) applies.
+ results = ds._semantic_search(query="q")
+
+ assert len(results) == 5
+ ds.vectorstore.similarity_search_with_score.assert_called_once_with(
+ query="q", k=5, filter=None
+ )
+
+
+# --- _get_most_recent_namespace (pinecone describe_index_stats shape) ----------
+
+def _patch_pinecone(namespaces):
+ index = MagicMock()
+ index.describe_index_stats.return_value = {"namespaces": {ns: {} for ns in namespaces}}
+ client = MagicMock()
+ client.Index.return_value = index
+ return patch.object(m, "Pinecone", return_value=client)
+
+
+def test_get_most_recent_namespace_picks_latest_valid():
+ ds = make_search()
+ namespaces = ["docsite-20231231", "docsite-20240101", "other", "docsite-bad"]
+ with _patch_pinecone(namespaces):
+ assert ds._get_most_recent_namespace() == "docsite-20240101"
+
+
+def test_get_most_recent_namespace_raises_when_none_valid():
+ ds = make_search()
+ with _patch_pinecone(["other", "docsite-bad", "docsite-2024"]):
+ with pytest.raises(ApolloError) as exc:
+ ds._get_most_recent_namespace()
+ assert exc.value.code == 404
+
+
+# --- strategy guard -------------------------------------------------------------
+
+@pytest.mark.parametrize("strategy", ["hybrid", "keyword", "nonsense"])
+def test_legacy_search_raises_on_unsupported_strategy(strategy):
+ """Previously fell off the end of the method and returned None implicitly,
+ which surfaces downstream as a confusing TypeError. Reachable now that the
+ backend is selectable per request."""
+ ds = make_search()
+ with pytest.raises(ApolloError) as exc:
+ ds.search("query", strategy=strategy)
+ assert exc.value.code == 400
+
+
+def test_legacy_search_still_dispatches_semantic():
+ ds = make_search()
+ ds.vectorstore.similarity_search_with_score.return_value = [(FakeDoc("a"), 0.9)]
+
+ results = ds.search("query", strategy="semantic")
+
+ assert [r.text for r in results] == ["a"]
diff --git a/services/search_docsite/tests/unit/test_run_eval.py b/services/search_docsite/tests/unit/test_run_eval.py
new file mode 100644
index 00000000..02f62428
--- /dev/null
+++ b/services/search_docsite/tests/unit/test_run_eval.py
@@ -0,0 +1,189 @@
+"""Unit tests for the offline golden-query eval's scoring logic.
+Backends are fully faked — no real search/DB/network involved."""
+
+from unittest.mock import MagicMock
+
+import pytest
+from search_docsite.tests.eval.run_eval import compute_agreement, compute_recall_at_k, run_eval
+
+
+def test_compute_recall_at_k_true_when_any_expected_title_present():
+ assert compute_recall_at_k(["Doc A", "Doc B"], ["Doc B", "Doc C"]) is True
+
+
+def test_compute_recall_at_k_false_when_no_overlap():
+ assert compute_recall_at_k(["Doc A"], ["Doc Z"]) is False
+
+
+def make_fake_backend(titles_by_query):
+ """Fake backend whose .search() returns SearchResults with the given doc_titles per query."""
+ from embeddings.embeddings import SearchResult
+
+ def fake_search(query, top_k=None, strategy=None, docs_type=None):
+ return [SearchResult(f"text-{t}", {"doc_title": t, "docs_type": "general_docs"}, 0.9) for t in titles_by_query.get(query, [])]
+
+ backend = MagicMock()
+ backend.search.side_effect = fake_search
+ backend_cls = MagicMock(return_value=backend)
+ return backend_cls
+
+
+def test_run_eval_scores_labeled_queries_and_skips_unlabeled():
+ golden_queries = [
+ {"query": "how do I configure a webhook", "expected_doc_titles": ["Webhooks"]},
+ {"query": "what is a run", "expected_doc_titles": []}, # unlabeled — skipped from recall aggregate
+ ]
+ backend_cls = make_fake_backend({"how do I configure a webhook": ["Webhooks", "Other Doc"], "what is a run": ["Runs"]})
+
+ report = run_eval(golden_queries, backend_cls, strategy="hybrid", top_k=5)
+
+ assert report["queries_scored"] == 1
+ assert report["queries_skipped"] == 1
+ assert report["recall_at_k"] == 1.0
+ assert len(report["per_query"]) == 2
+
+
+def test_run_eval_computes_recall_across_multiple_labeled_queries():
+ golden_queries = [
+ {"query": "q1", "expected_doc_titles": ["A"]},
+ {"query": "q2", "expected_doc_titles": ["Z"]}, # backend won't return Z -> miss
+ ]
+ backend_cls = make_fake_backend({"q1": ["A"], "q2": ["B"]})
+
+ report = run_eval(golden_queries, backend_cls, strategy="semantic", top_k=5)
+
+ assert report["queries_scored"] == 2
+ assert report["recall_at_k"] == 0.5
+
+
+def test_run_eval_reports_latency_percentiles():
+ golden_queries = [{"query": "q1", "expected_doc_titles": ["A"]}]
+ backend_cls = make_fake_backend({"q1": ["A"]})
+
+ report = run_eval(golden_queries, backend_cls, strategy="hybrid", top_k=5)
+
+ assert "p50_latency_s" in report
+ assert "p95_latency_s" in report
+ assert report["p50_latency_s"] >= 0
+
+
+def test_compute_agreement_reports_perfect_overlap():
+ report_a = {"per_query": [{"query": "q", "retrieved_titles": ["A", "B"]}]}
+ report_b = {"per_query": [{"query": "q", "retrieved_titles": ["B", "A"]}]}
+
+ agreement = compute_agreement(report_a, report_b)
+
+ assert agreement["per_query"][0]["overlap"] == 2
+ assert agreement["per_query"][0]["jaccard"] == 1.0
+ assert agreement["mean_jaccard"] == 1.0
+
+
+def test_compute_agreement_reports_partial_overlap():
+ report_a = {"per_query": [{"query": "q", "retrieved_titles": ["A", "B"]}]}
+ report_b = {"per_query": [{"query": "q", "retrieved_titles": ["B", "C"]}]}
+
+ agreement = compute_agreement(report_a, report_b)
+
+ assert agreement["per_query"][0]["overlap"] == 1
+ assert agreement["per_query"][0]["union"] == 3
+ assert agreement["per_query"][0]["jaccard"] == pytest.approx(1 / 3)
+
+
+def test_compute_agreement_handles_both_backends_returning_nothing():
+ report_a = {"per_query": [{"query": "q", "retrieved_titles": []}]}
+ report_b = {"per_query": [{"query": "q", "retrieved_titles": []}]}
+
+ agreement = compute_agreement(report_a, report_b)
+
+ assert agreement["per_query"][0]["jaccard"] == 0.0
+ assert agreement["mean_jaccard"] == 0.0
+
+
+def test_compute_agreement_means_across_queries():
+ report_a = {"per_query": [
+ {"query": "q1", "retrieved_titles": ["A"]},
+ {"query": "q2", "retrieved_titles": ["X"]},
+ ]}
+ report_b = {"per_query": [
+ {"query": "q1", "retrieved_titles": ["A"]},
+ {"query": "q2", "retrieved_titles": ["Y"]},
+ ]}
+
+ agreement = compute_agreement(report_a, report_b)
+
+ assert agreement["mean_jaccard"] == pytest.approx(0.5)
+
+
+def test_run_eval_reports_recall_per_category():
+ golden_queries = [
+ {"query": "c1", "category": "conceptual", "expected_doc_titles": ["A"]},
+ {"query": "c2", "category": "conceptual", "expected_doc_titles": ["B"]},
+ {"query": "k1", "category": "keyword", "expected_doc_titles": ["C"]},
+ {"query": "k2", "category": "keyword", "expected_doc_titles": ["MISSING"]},
+ ]
+ backend_cls = make_fake_backend({"c1": ["A"], "c2": ["B"], "k1": ["C"], "k2": ["Z"]})
+
+ report = run_eval(golden_queries, backend_cls, strategy="semantic", top_k=5)
+
+ assert report["recall_by_category"]["conceptual"] == {"recall": 1.0, "scored": 2}
+ assert report["recall_by_category"]["keyword"] == {"recall": 0.5, "scored": 2}
+ assert report["recall_at_k"] == 0.75
+
+
+def test_run_eval_tolerates_queries_without_a_category():
+ golden_queries = [
+ {"query": "q1", "expected_doc_titles": ["A"]},
+ {"query": "q2", "category": "keyword", "expected_doc_titles": ["B"]},
+ ]
+ backend_cls = make_fake_backend({"q1": ["A"], "q2": ["B"]})
+
+ report = run_eval(golden_queries, backend_cls, strategy="semantic", top_k=5)
+
+ assert report["recall_at_k"] == 1.0
+ assert report["recall_by_category"] == {"keyword": {"recall": 1.0, "scored": 1}}
+
+
+def test_run_eval_excludes_unlabelled_queries_from_category_recall():
+ golden_queries = [
+ {"query": "k1", "category": "keyword", "expected_doc_titles": ["A"]},
+ {"query": "k2", "category": "keyword", "expected_doc_titles": []},
+ ]
+ backend_cls = make_fake_backend({"k1": ["A"], "k2": ["B"]})
+
+ report = run_eval(golden_queries, backend_cls, strategy="semantic", top_k=5)
+
+ assert report["recall_by_category"]["keyword"] == {"recall": 1.0, "scored": 1}
+ assert report["queries_skipped"] == 1
+
+
+def test_resolve_batch_id_returns_newest_matching_batch():
+ from unittest.mock import patch
+
+ import search_docsite.tests.eval.run_eval as m
+
+ conn = MagicMock()
+ cur = MagicMock()
+ conn.cursor.return_value.__enter__.return_value = cur
+ cur.fetchone.return_value = (11,)
+
+ with patch.object(m, "get_db_connection", return_value=conn):
+ assert m.resolve_batch_id(2500) == 11
+
+ assert cur.execute.call_args[0][1] == (2500,)
+ conn.close.assert_called_once()
+
+
+def test_resolve_batch_id_returns_none_when_no_batch_matches():
+ from unittest.mock import patch
+
+ import search_docsite.tests.eval.run_eval as m
+
+ conn = MagicMock()
+ cur = MagicMock()
+ conn.cursor.return_value.__enter__.return_value = cur
+ cur.fetchone.return_value = None
+
+ with patch.object(m, "get_db_connection", return_value=conn):
+ assert m.resolve_batch_id(1800) is None
+
+ conn.close.assert_called_once()
diff --git a/services/search_docsite/tests/unit/test_search_docsite_main.py b/services/search_docsite/tests/unit/test_search_docsite_main.py
new file mode 100644
index 00000000..c6724b84
--- /dev/null
+++ b/services/search_docsite/tests/unit/test_search_docsite_main.py
@@ -0,0 +1,98 @@
+"""Unit tests for search_docsite.main's backend selection and resolve_backend.
+
+The `backend` payload field lets the same query be run against either backend
+on demand for manual comparison.
+"""
+
+from unittest.mock import patch
+
+import pytest
+import search_docsite.search_docsite as m
+from util import ApolloError
+
+
+def test_main_defaults_to_pinecone_backend(monkeypatch):
+ monkeypatch.delenv("DOCSITE_SEARCH_BACKEND", raising=False)
+ monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
+
+ with patch.object(m, "LegacyPineconeDocsiteSearch") as mock_legacy_cls:
+ mock_legacy_cls.return_value.search.return_value = []
+ m.main({"query": "webhooks"})
+
+ mock_legacy_cls.assert_called_once_with()
+ mock_legacy_cls.return_value.search.assert_called_once_with(query="webhooks")
+
+
+def test_main_payload_backend_overrides_env(monkeypatch):
+ monkeypatch.setenv("DOCSITE_SEARCH_BACKEND", "pinecone")
+ monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
+
+ with patch.object(m, "DocsiteSearch") as mock_pg_cls:
+ mock_pg_cls.return_value.search.return_value = []
+ m.main({"query": "webhooks", "backend": "postgres"})
+
+ mock_pg_cls.return_value.search.assert_called_once_with(query="webhooks")
+
+
+def test_main_env_selects_postgres_when_no_payload_override(monkeypatch):
+ monkeypatch.setenv("DOCSITE_SEARCH_BACKEND", "postgres")
+ monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
+
+ with patch.object(m, "DocsiteSearch") as mock_pg_cls:
+ mock_pg_cls.return_value.search.return_value = []
+ m.main({"query": "webhooks"})
+
+ mock_pg_cls.return_value.search.assert_called_once_with(query="webhooks")
+
+
+def test_main_rejects_unknown_backend(monkeypatch):
+ monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
+
+ with pytest.raises(ApolloError) as exc:
+ m.main({"query": "webhooks", "backend": "sqlite"})
+
+ assert exc.value.code == 400
+
+
+def test_main_routes_index_params_per_backend(monkeypatch):
+ """The two classes take different constructor params. batch_id is meaningless
+ to Pinecone; collection_name is meaningless to Postgres."""
+ monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
+
+ with patch.object(m, "DocsiteSearch") as mock_pg_cls:
+ mock_pg_cls.return_value.search.return_value = []
+ m.main({"query": "q", "backend": "postgres", "batch_id": 3, "collection_name": "ignored"})
+
+ mock_pg_cls.assert_called_once_with(batch_id=3)
+
+
+def test_main_routes_collection_name_to_pinecone_only(monkeypatch):
+ monkeypatch.setenv("OPENAI_API_KEY", "sk-test")
+
+ with patch.object(m, "LegacyPineconeDocsiteSearch") as mock_legacy_cls:
+ mock_legacy_cls.return_value.search.return_value = []
+ m.main({"query": "q", "backend": "pinecone", "collection_name": "docsite-202501010000", "batch_id": 9})
+
+ mock_legacy_cls.assert_called_once_with(collection_name="docsite-202501010000")
+
+
+def test_resolve_backend_defaults_to_pinecone(monkeypatch):
+ monkeypatch.delenv("DOCSITE_SEARCH_BACKEND", raising=False)
+ assert m.resolve_backend() is m.LegacyPineconeDocsiteSearch
+
+
+def test_resolve_backend_reads_env(monkeypatch):
+ monkeypatch.setenv("DOCSITE_SEARCH_BACKEND", "postgres")
+ assert m.resolve_backend() is m.DocsiteSearch
+
+
+def test_resolve_backend_override_wins_over_env(monkeypatch):
+ monkeypatch.setenv("DOCSITE_SEARCH_BACKEND", "pinecone")
+ assert m.resolve_backend("postgres") is m.DocsiteSearch
+
+
+def test_resolve_backend_rejects_unknown_name(monkeypatch):
+ monkeypatch.delenv("DOCSITE_SEARCH_BACKEND", raising=False)
+ with pytest.raises(ApolloError) as exc:
+ m.resolve_backend("sqlite")
+ assert exc.value.code == 400
diff --git a/services/tools/search_documentation/search_documentation.py b/services/tools/search_documentation/search_documentation.py
index 609c5aa8..018cc172 100644
--- a/services/tools/search_documentation/search_documentation.py
+++ b/services/tools/search_documentation/search_documentation.py
@@ -5,17 +5,16 @@
1. As a standalone service via entry.py: bun py tools/search_documentation
2. As a tool by supervisor via search_documentation_tool()
"""
-import os
import sys
-from pathlib import Path
-from typing import Dict, List, Optional
from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict
# Import utilities from services directory
sys.path.append(str(Path(__file__).parent.parent.parent))
-from util import create_logger, ApolloError
-from search_docsite.search_docsite import DocsiteSearch
+from search_docsite.search_docsite import resolve_backend
+from util import ApolloError, create_logger
logger = create_logger(__name__)
@@ -46,8 +45,10 @@ def _search_implementation(query: str, num_results: int) -> Dict:
"""
logger.info(f"Searching documentation for: {query[:100]}...")
+ # Both backends use semantic search with the same cosine-similarity cutoff, so
+ # results are directly comparable.
# Initialize docsite search
- docsite_search = DocsiteSearch()
+ docsite_search = resolve_backend()()
# Search with threshold for quality results
search_results = docsite_search.search(
@@ -91,7 +92,7 @@ def main(data: Dict) -> Dict:
raise
except Exception as e:
logger.exception("Error in search_documentation service")
- raise ApolloError(500, f"Documentation search failed: {str(e)}")
+ raise ApolloError(500, f"Documentation search failed: {str(e)}") from e
def search_documentation_tool(tool_input: Dict) -> str:
@@ -140,4 +141,4 @@ def search_documentation_tool(tool_input: Dict) -> str:
except Exception as e:
logger.exception("Error in search_documentation tool")
- raise ApolloError(500, f"Documentation search failed: {str(e)}")
+ raise ApolloError(500, f"Documentation search failed: {str(e)}") from e
diff --git a/services/tools/search_documentation/tests/unit/conftest.py b/services/tools/search_documentation/tests/unit/conftest.py
new file mode 100644
index 00000000..32aac060
--- /dev/null
+++ b/services/tools/search_documentation/tests/unit/conftest.py
@@ -0,0 +1,14 @@
+"""Test config for search_documentation unit tests.
+
+Importing `search_documentation.search_documentation` pulls in
+`search_docsite.pinecone_legacy_search`, whose module-level `OpenAIEmbeddings()`
+default arg validates credentials at construction (openai 2.x / langchain-openai 1.x).
+A key must therefore exist at import time.
+
+Dummy placeholders only.
+"""
+
+import os
+
+os.environ.setdefault("OPENAI_API_KEY", "sk-test-dummy")
+os.environ.setdefault("PINECONE_API_KEY", "pc-test-dummy")
diff --git a/services/tools/search_documentation/tests/unit/test_search_documentation.py b/services/tools/search_documentation/tests/unit/test_search_documentation.py
new file mode 100644
index 00000000..569f8af9
--- /dev/null
+++ b/services/tools/search_documentation/tests/unit/test_search_documentation.py
@@ -0,0 +1,20 @@
+"""Unit tests for search_documentation's delegation to the resolved backend."""
+
+from unittest.mock import patch
+
+import tools.search_documentation.search_documentation as m
+
+
+def test_search_implementation_applies_quality_gate_to_resolved_backend(monkeypatch):
+ """The 0.7 threshold is a quality gate on live traffic — it was silently
+ dropped once already."""
+ monkeypatch.delenv("DOCSITE_SEARCH_BACKEND", raising=False)
+
+ with patch.object(m, "resolve_backend") as mock_resolve:
+ backend = mock_resolve.return_value.return_value
+ backend.search.return_value = []
+ m._search_implementation("how do I use webhooks", 5)
+
+ backend.search.assert_called_once_with(
+ query="how do I use webhooks", top_k=5, threshold=0.7, strategy="semantic"
+ )