Clarify docsite index setup in docs and fix dependencies

.env.example

@@ -23,9 +23,11 @@
 ANTHROPIC_API_KEY=sk-YOUR-API-KEY-HERE
 
 OPENAI_API_KEY=sk-YOUR-API-KEY-HERE
-PINECONE_KEY=YOUR-API-KEY-HERE
+PINECONE_API_KEY=YOUR-API-KEY-HERE
 POSTGRES_URL=postgresql://localhost:5432/apollo_dev
-SENTRY_DSN=YOUR-API-KEY-HERE
+# Leave empty to run without Sentry. A placeholder value is not ignored: any
+# non-URL DSN makes sentry_sdk.init raise BadDsn and kills every Python service.
+SENTRY_DSN=
 GITHUB_TOKEN=KEY
 
 # Langfuse observability

README.md

@@ -65,6 +65,8 @@ bun dev
 
 To see an index of the available language services, head to `localhost:3000`.
 
+The chat services search a Pinecone index of the OpenFn documentation, which every environment populates itself. Run `bun py embed_docsite` to build yours — see [embed_docsite](services/embed_docsite/README.md).
+
 ## Python Setup
 
 This repo uses `poetry` to manage dependencies.

pyproject.toml

@@ -26,6 +26,7 @@ psycopg2-binary = "^2.9.10"
 langfuse = "^4.14.1"
 opentelemetry-instrumentation-anthropic = "^0.62.1"
 opentelemetry-instrumentation-threading = "0.65b0"
+pandas = "^2.2"
 
 [tool.poetry.group.dev]
 optional = false

services/embed_docsite/README.md

@@ -2,25 +2,37 @@
 
 This service embeds the OpenFn Documentation to a vector database. It downloads, chunks, processes metadata, embeds and uploads the documentation to a vector database (Pinecone). 
 
-## Usage - Embedding OpenFn Documentation
+## Setup
+
+Every environment maintains its own vector store, so there is no shared index to point at. Run this service to populate your own before using `search_docsite`.
 
-The vector database used here is Pinecone. To obtain the env variables follow these steps:
+1. Create an account on [Pinecone](https://www.pinecone.io/) and set up a free cluster.
+2. Add `PINECONE_API_KEY` and `OPENAI_API_KEY` to your `.env` file.
 
-1. Create an account on [Pinecone] and set up a free cluster.
-2. Obtain the URL and token for the cluster and add them to the `.env` file.
-3. You'll also need an OpenAI API key to generate embeddings.
+The service creates the `docsite` index if it does not already exist.
+
+## Usage - Embedding OpenFn Documentation
 
 ### With the CLI, returning to stdout:
 
 ```bash
 openfn apollo embed_docsite tmp/payload.json
 ```
-To run directly from this repo (note that the server must be started):
+
+### Directly from this repo:
+
+```bash
+bun py embed_docsite
+```
+
+The payload is optional. With no `--input`, the service indexes all documentation using the defaults below; to customise it, pass a payload file:
 
 ```bash
-bun py embed_docsite tmp/payload.json -O
+bun py embed_docsite --input tmp/payload.json
 ```
 
+A full run downloads the entire docs site and embeds several thousand chunks, so allow upwards of ten minutes.
+
 ## Implementation
 The service uses the DocsiteProcessor to download the documentation and chunk it into smaller parts. The DocsiteIndexer formats metadata, creates a new collection, embeds the chunked texts (OpenAI) and uploads them into the vector database (Pinecone).
 

services/embed_docsite/docsite_indexer.py

@@ -6,7 +6,7 @@
 from langchain_pinecone import PineconeVectorStore
 from langchain_openai import OpenAIEmbeddings
 from langchain_community.document_loaders import DataFrameLoader
-from util import create_logger, ApolloError
+from util import create_logger, ApolloError, is_docsite_collection
 
 logger = create_logger("DocsiteIndexer")
 
@@ -108,12 +108,13 @@ def delete_old_collections(self, max_total_collections):
             index_stats = index.describe_index_stats()
             namespaces = index_stats.get('namespaces', {}).keys()
             valid_namespaces = sorted(
-                (ns for ns in namespaces if ns.startswith("docsite-") and ns[8:].isdigit() and len(ns) == 16),
+                (ns for ns in namespaces if is_docsite_collection(ns)),
                 reverse=False
             )
-            if len(valid_namespaces) > max_total_collections:
+            excess = len(valid_namespaces) - max_total_collections
+            if excess > 0:
                 logger.info(f"Deleting outdated docsite collections")
-                for old_collection in valid_namespaces[:max_total_collections]:
+                for old_collection in valid_namespaces[:excess]:
                     self.index.delete(delete_all=True, namespace=old_collection)
                     logger.info(f"Deleted collection {old_collection}")
 

services/search_docsite/README.md

@@ -2,25 +2,27 @@
 
 This service searches the OpenFn Documentation vector database using a query and returns search matches. 
 
-The documenation is vectorized through the `embed_docsite` service.
+The documentation is vectorized through the `embed_docsite` service.
 
-## Usage - Searching OpenFn Documentation
+## Setup
+
+Searching requires a populated `docsite` index. Every environment maintains its own, so there is no shared index to point at: run `embed_docsite` to create and fill yours before searching.
 
-The vector database used here is Pinecone. To obtain the env variables follow these steps:
+1. Create an account on [Pinecone](https://www.pinecone.io/) and set up a free cluster.
+2. Add `PINECONE_API_KEY` and `OPENAI_API_KEY` to your `.env` file.
 
-1. Create an account on [Pinecone] and set up a free cluster.
-2. Obtain the URL and token for the cluster and add them to the `.env` file.
-3. You'll also need an OpenAI API key to generate embeddings for input queries.
+## Usage - Searching OpenFn Documentation
 
 ### With the CLI, returning to stdout:
 
 ```bash
 openfn apollo search_docsite tmp/payload.json
 ```
-To run directly from this repo (note that the server must be started):
+
+### Directly from this repo:
 
 ```bash
-bun py search_docsite tmp/payload.json -O
+bun py search_docsite --input tmp/payload.json
 ```
 
 ## Implementation

services/search_docsite/search_docsite.py

@@ -3,7 +3,7 @@
 from pinecone import Pinecone
 from langchain_pinecone import PineconeVectorStore
 from langchain_openai import OpenAIEmbeddings
-from util import create_logger, ApolloError
+from util import create_logger, ApolloError, is_docsite_collection
 from embeddings.embeddings import SearchResult
 logger = create_logger("DocsiteSearch")
 
@@ -113,7 +113,7 @@ def _get_most_recent_namespace(self):
             namespaces = index_stats.get('namespaces', {}).keys()
 
             valid_namespaces = sorted(
-                (ns for ns in namespaces if ns.startswith("docsite-") and ns[8:].isdigit() and len(ns) == 16),
+                (ns for ns in namespaces if is_docsite_collection(ns)),
                 reverse=True
             )
 

services/search_docsite/tests/unit/test_docsite_search.py

@@ -140,6 +140,27 @@ def test_get_most_recent_namespace_raises_when_none_valid():
     assert exc.value.code == 404
 
 
+def test_get_most_recent_namespace_accepts_date_time_format():
+    """DocsiteIndexer names collections docsite-YYYYMMDDHHMM; a fresh index
+    holds only that format, and rejecting it 404'd every search after a
+    successful embed."""
+    ds = make_search()
+    with _patch_pinecone(["docsite-202608131022"]):
+        assert ds._get_most_recent_namespace() == "docsite-202608131022"
+
+
+def test_get_most_recent_namespace_picks_latest_across_mixed_formats():
+    ds = make_search()
+    namespaces = [
+        "docsite-20250225",       # legacy date-only
+        "docsite-202608131022",   # current date+time
+        "docsite-20260813",       # date-only, same day as above
+        "docsite-2026081310",     # 10 digits: neither format
+    ]
+    with _patch_pinecone(namespaces):
+        assert ds._get_most_recent_namespace() == "docsite-202608131022"
+
+
 # --- lazy embeddings construction ----------------------------------------------
 
 def test_default_embeddings_built_on_construction_not_import():