From be3687ed4812e159f32d50a2b05c5fb1020858ea Mon Sep 17 00:00:00 2001 From: hanna-paasivirta Date: Wed, 12 Aug 2026 13:56:56 +0100 Subject: [PATCH 1/3] docs: clarify docsite index setup --- .env.example | 2 +- README.md | 2 ++ services/embed_docsite/README.md | 26 +++++++++++++++++++------- services/search_docsite/README.md | 18 ++++++++++-------- 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/.env.example b/.env.example index 67f1bd48..c1d573d5 100644 --- a/.env.example +++ b/.env.example @@ -23,7 +23,7 @@ ANTHROPIC_API_KEY=sk-YOUR-API-KEY-HERE OPENAI_API_KEY=sk-YOUR-API-KEY-HERE -PINECONE_KEY=YOUR-API-KEY-HERE +PINECONE_API_KEY=YOUR-API-KEY-HERE POSTGRES_URL=postgresql://localhost:5432/apollo_dev SENTRY_DSN=YOUR-API-KEY-HERE GITHUB_TOKEN=KEY diff --git a/README.md b/README.md index 2e4d7c3c..e3607e49 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,8 @@ bun dev To see an index of the available language services, head to `localhost:3000`. +The chat services search a Pinecone index of the OpenFn documentation, which every environment populates itself. Run `bun py embed_docsite` to build yours — see [embed_docsite](services/embed_docsite/README.md). + ## Python Setup This repo uses `poetry` to manage dependencies. diff --git a/services/embed_docsite/README.md b/services/embed_docsite/README.md index 6129f303..dee43547 100644 --- a/services/embed_docsite/README.md +++ b/services/embed_docsite/README.md @@ -2,25 +2,37 @@ This service embeds the OpenFn Documentation to a vector database. It downloads, chunks, processes metadata, embeds and uploads the documentation to a vector database (Pinecone). -## Usage - Embedding OpenFn Documentation +## Setup + +Every environment maintains its own vector store, so there is no shared index to point at. Run this service to populate your own before using `search_docsite`. -The vector database used here is Pinecone. To obtain the env variables follow these steps: +1. Create an account on [Pinecone](https://www.pinecone.io/) and set up a free cluster. +2. Add `PINECONE_API_KEY` and `OPENAI_API_KEY` to your `.env` file. -1. Create an account on [Pinecone] and set up a free cluster. -2. Obtain the URL and token for the cluster and add them to the `.env` file. -3. You'll also need an OpenAI API key to generate embeddings. +The service creates the `docsite` index if it does not already exist. + +## Usage - Embedding OpenFn Documentation ### With the CLI, returning to stdout: ```bash openfn apollo embed_docsite tmp/payload.json ``` -To run directly from this repo (note that the server must be started): + +### Directly from this repo: + +```bash +bun py embed_docsite +``` + +The payload is optional. With no `--input`, the service indexes all documentation using the defaults below; to customise it, pass a payload file: ```bash -bun py embed_docsite tmp/payload.json -O +bun py embed_docsite --input tmp/payload.json ``` +A full run downloads the entire docs site and embeds several thousand chunks, so allow upwards of ten minutes. + ## Implementation The service uses the DocsiteProcessor to download the documentation and chunk it into smaller parts. The DocsiteIndexer formats metadata, creates a new collection, embeds the chunked texts (OpenAI) and uploads them into the vector database (Pinecone). diff --git a/services/search_docsite/README.md b/services/search_docsite/README.md index ecbe0146..895c4a13 100644 --- a/services/search_docsite/README.md +++ b/services/search_docsite/README.md @@ -2,25 +2,27 @@ This service searches the OpenFn Documentation vector database using a query and returns search matches. -The documenation is vectorized through the `embed_docsite` service. +The documentation is vectorized through the `embed_docsite` service. -## Usage - Searching OpenFn Documentation +## Setup + +Searching requires a populated `docsite` index. Every environment maintains its own, so there is no shared index to point at: run `embed_docsite` to create and fill yours before searching. -The vector database used here is Pinecone. To obtain the env variables follow these steps: +1. Create an account on [Pinecone](https://www.pinecone.io/) and set up a free cluster. +2. Add `PINECONE_API_KEY` and `OPENAI_API_KEY` to your `.env` file. -1. Create an account on [Pinecone] and set up a free cluster. -2. Obtain the URL and token for the cluster and add them to the `.env` file. -3. You'll also need an OpenAI API key to generate embeddings for input queries. +## Usage - Searching OpenFn Documentation ### With the CLI, returning to stdout: ```bash openfn apollo search_docsite tmp/payload.json ``` -To run directly from this repo (note that the server must be started): + +### Directly from this repo: ```bash -bun py search_docsite tmp/payload.json -O +bun py search_docsite --input tmp/payload.json ``` ## Implementation From dfbf9534a8f869619ca7823a7eca8c9afcb55cc6 Mon Sep 17 00:00:00 2001 From: hanna-paasivirta Date: Thu, 13 Aug 2026 10:00:29 +0100 Subject: [PATCH 2/3] add pandas dep and empty sentry dsn example --- .env.example | 4 +- poetry.lock | 122 ++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 3 files changed, 125 insertions(+), 2 deletions(-) diff --git a/.env.example b/.env.example index c1d573d5..9d7f530a 100644 --- a/.env.example +++ b/.env.example @@ -25,7 +25,9 @@ ANTHROPIC_API_KEY=sk-YOUR-API-KEY-HERE OPENAI_API_KEY=sk-YOUR-API-KEY-HERE PINECONE_API_KEY=YOUR-API-KEY-HERE POSTGRES_URL=postgresql://localhost:5432/apollo_dev -SENTRY_DSN=YOUR-API-KEY-HERE +# Leave empty to run without Sentry. A placeholder value is not ignored: any +# non-URL DSN makes sentry_sdk.init raise BadDsn and kills every Python service. +SENTRY_DSN= GITHUB_TOKEN=KEY # Langfuse observability diff --git a/poetry.lock b/poetry.lock index c413ba5d..017cd3df 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1724,6 +1724,102 @@ files = [ {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, ] +[[package]] +name = "pandas" +version = "2.3.3" +description = "Powerful data structures for data analysis, time series, and statistics" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pandas-2.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:376c6446ae31770764215a6c937f72d917f214b43560603cd60da6408f183b6c"}, + {file = "pandas-2.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e19d192383eab2f4ceb30b412b22ea30690c9e618f78870357ae1d682912015a"}, + {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5caf26f64126b6c7aec964f74266f435afef1c1b13da3b0636c7518a1fa3e2b1"}, + {file = "pandas-2.3.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dd7478f1463441ae4ca7308a70e90b33470fa593429f9d4c578dd00d1fa78838"}, + {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:4793891684806ae50d1288c9bae9330293ab4e083ccd1c5e383c34549c6e4250"}, + {file = "pandas-2.3.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:28083c648d9a99a5dd035ec125d42439c6c1c525098c58af0fc38dd1a7a1b3d4"}, + {file = "pandas-2.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:503cf027cf9940d2ceaa1a93cfb5f8c8c7e6e90720a2850378f0b3f3b1e06826"}, + {file = "pandas-2.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:602b8615ebcc4a0c1751e71840428ddebeb142ec02c786e8ad6b1ce3c8dec523"}, + {file = "pandas-2.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8fe25fc7b623b0ef6b5009149627e34d2a4657e880948ec3c840e9402e5c1b45"}, + {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b468d3dad6ff947df92dcb32ede5b7bd41a9b3cceef0a30ed925f6d01fb8fa66"}, + {file = "pandas-2.3.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b98560e98cb334799c0b07ca7967ac361a47326e9b4e5a7dfb5ab2b1c9d35a1b"}, + {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37b5848ba49824e5c30bedb9c830ab9b7751fd049bc7914533e01c65f79791"}, + {file = "pandas-2.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:db4301b2d1f926ae677a751eb2bd0e8c5f5319c9cb3f88b0becbbb0b07b34151"}, + {file = "pandas-2.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:f086f6fe114e19d92014a1966f43a3e62285109afe874f067f5abbdcbb10e59c"}, + {file = "pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53"}, + {file = "pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35"}, + {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908"}, + {file = "pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89"}, + {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98"}, + {file = "pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084"}, + {file = "pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b"}, + {file = "pandas-2.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:56851a737e3470de7fa88e6131f41281ed440d29a9268dcbf0002da5ac366713"}, + {file = "pandas-2.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdcd9d1167f4885211e401b3036c0c8d9e274eee67ea8d0758a256d60704cfe8"}, + {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e32e7cc9af0f1cc15548288a51a3b681cc2a219faa838e995f7dc53dbab1062d"}, + {file = "pandas-2.3.3-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:318d77e0e42a628c04dc56bcef4b40de67918f7041c2b061af1da41dcff670ac"}, + {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4e0a175408804d566144e170d0476b15d78458795bb18f1304fb94160cabf40c"}, + {file = "pandas-2.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:93c2d9ab0fc11822b5eece72ec9587e172f63cff87c00b062f6e37448ced4493"}, + {file = "pandas-2.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:f8bfc0e12dc78f777f323f55c58649591b2cd0c43534e8355c51d3fede5f4dee"}, + {file = "pandas-2.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:75ea25f9529fdec2d2e93a42c523962261e567d250b0013b16210e1d40d7c2e5"}, + {file = "pandas-2.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:74ecdf1d301e812db96a465a525952f4dde225fdb6d8e5a521d47e1f42041e21"}, + {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6435cb949cb34ec11cc9860246ccb2fdc9ecd742c12d3304989017d53f039a78"}, + {file = "pandas-2.3.3-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:900f47d8f20860de523a1ac881c4c36d65efcb2eb850e6948140fa781736e110"}, + {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:a45c765238e2ed7d7c608fc5bc4a6f88b642f2f01e70c0c23d2224dd21829d86"}, + {file = "pandas-2.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:c4fc4c21971a1a9f4bdb4c73978c7f7256caa3e62b323f70d6cb80db583350bc"}, + {file = "pandas-2.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:ee15f284898e7b246df8087fc82b87b01686f98ee67d85a17b7ab44143a3a9a0"}, + {file = "pandas-2.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1611aedd912e1ff81ff41c745822980c49ce4a7907537be8692c8dbc31924593"}, + {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6d2cefc361461662ac48810cb14365a365ce864afe85ef1f447ff5a1e99ea81c"}, + {file = "pandas-2.3.3-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ee67acbbf05014ea6c763beb097e03cd629961c8a632075eeb34247120abcb4b"}, + {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:c46467899aaa4da076d5abc11084634e2d197e9460643dd455ac3db5856b24d6"}, + {file = "pandas-2.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6253c72c6a1d990a410bc7de641d34053364ef8bcd3126f7e7450125887dffe3"}, + {file = "pandas-2.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:1b07204a219b3b7350abaae088f451860223a52cfb8a6c53358e7948735158e5"}, + {file = "pandas-2.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:2462b1a365b6109d275250baaae7b760fd25c726aaca0054649286bcfbb3e8ec"}, + {file = "pandas-2.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0242fe9a49aa8b4d78a4fa03acb397a58833ef6199e9aa40a95f027bb3a1b6e7"}, + {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a21d830e78df0a515db2b3d2f5570610f5e6bd2e27749770e8bb7b524b89b450"}, + {file = "pandas-2.3.3-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2e3ebdb170b5ef78f19bfb71b0dc5dc58775032361fa188e814959b74d726dd5"}, + {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d051c0e065b94b7a3cea50eb1ec32e912cd96dba41647eb24104b6c6c14c5788"}, + {file = "pandas-2.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:3869faf4bd07b3b66a9f462417d0ca3a9df29a9f6abd5d0d0dbab15dac7abe87"}, + {file = "pandas-2.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:c503ba5216814e295f40711470446bc3fd00f0faea8a086cbc688808e26f92a2"}, + {file = "pandas-2.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a637c5cdfa04b6d6e2ecedcb81fc52ffb0fd78ce2ebccc9ea964df9f658de8c8"}, + {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:854d00d556406bffe66a4c0802f334c9ad5a96b4f1f868adf036a21b11ef13ff"}, + {file = "pandas-2.3.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf1f8a81d04ca90e32a0aceb819d34dbd378a98bf923b6398b9a3ec0bf44de29"}, + {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:23ebd657a4d38268c7dfbdf089fbc31ea709d82e4923c5ffd4fbd5747133ce73"}, + {file = "pandas-2.3.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:5554c929ccc317d41a5e3d1234f3be588248e61f08a74dd17c9eabb535777dc9"}, + {file = "pandas-2.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:d3e28b3e83862ccf4d85ff19cf8c20b2ae7e503881711ff2d534dc8f761131aa"}, + {file = "pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b"}, +] + +[package.dependencies] +numpy = {version = ">=1.23.2", markers = "python_version == \"3.11\""} +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.7" + +[package.extras] +all = ["PyQt5 (>=5.15.9)", "SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)", "beautifulsoup4 (>=4.11.2)", "bottleneck (>=1.3.6)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=2022.12.0)", "fsspec (>=2022.11.0)", "gcsfs (>=2022.11.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.9.2)", "matplotlib (>=3.6.3)", "numba (>=0.56.4)", "numexpr (>=2.8.4)", "odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "pandas-gbq (>=0.19.0)", "psycopg2 (>=2.9.6)", "pyarrow (>=10.0.1)", "pymysql (>=1.0.2)", "pyreadstat (>=1.2.0)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "qtpy (>=2.3.0)", "s3fs (>=2022.11.0)", "scipy (>=1.10.0)", "tables (>=3.8.0)", "tabulate (>=0.9.0)", "xarray (>=2022.12.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)", "zstandard (>=0.19.0)"] +aws = ["s3fs (>=2022.11.0)"] +clipboard = ["PyQt5 (>=5.15.9)", "qtpy (>=2.3.0)"] +compression = ["zstandard (>=0.19.0)"] +computation = ["scipy (>=1.10.0)", "xarray (>=2022.12.0)"] +consortium-standard = ["dataframe-api-compat (>=0.1.7)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.1.0)", "python-calamine (>=0.1.7)", "pyxlsb (>=1.0.10)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.5)"] +feather = ["pyarrow (>=10.0.1)"] +fss = ["fsspec (>=2022.11.0)"] +gcp = ["gcsfs (>=2022.11.0)", "pandas-gbq (>=0.19.0)"] +hdf5 = ["tables (>=3.8.0)"] +html = ["beautifulsoup4 (>=4.11.2)", "html5lib (>=1.1)", "lxml (>=4.9.2)"] +mysql = ["SQLAlchemy (>=2.0.0)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.9.0)"] +parquet = ["pyarrow (>=10.0.1)"] +performance = ["bottleneck (>=1.3.6)", "numba (>=0.56.4)", "numexpr (>=2.8.4)"] +plot = ["matplotlib (>=3.6.3)"] +postgresql = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "psycopg2 (>=2.9.6)"] +pyarrow = ["pyarrow (>=10.0.1)"] +spss = ["pyreadstat (>=1.2.0)"] +sql-other = ["SQLAlchemy (>=2.0.0)", "adbc-driver-postgresql (>=0.8.0)", "adbc-driver-sqlite (>=0.8.0)"] +test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.9.2)"] + [[package]] name = "pinecone" version = "7.3.0" @@ -2269,6 +2365,18 @@ files = [ [package.extras] cli = ["click (>=5.0)"] +[[package]] +name = "pytz" +version = "2026.3.post1" +description = "World timezone definitions, modern and historical" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "pytz-2026.3.post1-py2.py3-none-any.whl", hash = "sha256:dd95840dd199baea12d9cc096a1d452caa6596a1c1e4b5f3dbd1541855d5e815"}, + {file = "pytz-2026.3.post1.tar.gz", hash = "sha256:2211d3fcf9a797d3405cac96ac7f61d80e6a644f72a3309607282fe8a2010c5d"}, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -2958,6 +3066,18 @@ files = [ [package.dependencies] typing-extensions = ">=4.12.0" +[[package]] +name = "tzdata" +version = "2026.3" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +groups = ["main"] +files = [ + {file = "tzdata-2026.3-py2.py3-none-any.whl", hash = "sha256:dc096730c87af6cab1b171c9d532be840741ff5d459015e7f6947bd7d7e54931"}, + {file = "tzdata-2026.3.tar.gz", hash = "sha256:4a1518b8993086a7982523e071643f3c0e5f213e75b21318e78bcabfff9d1415"}, +] + [[package]] name = "urllib3" version = "2.7.0" @@ -3625,4 +3745,4 @@ cffi = ["cffi (>=1.17,<2.0) ; platform_python_implementation != \"PyPy\" and pyt [metadata] lock-version = "2.1" python-versions = "3.11.*" -content-hash = "6551e8fae076b04ea022e47a09446e6573992a90751c4ce633f9e2e0d73f035e" +content-hash = "3fe023990e7e8f97188f1bbfe61d195c894d12626b77ed9ff54801e64fcaa623" diff --git a/pyproject.toml b/pyproject.toml index f67ccac0..64c8bfc6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ psycopg2-binary = "^2.9.10" langfuse = "^4.14.1" opentelemetry-instrumentation-anthropic = "^0.62.1" opentelemetry-instrumentation-threading = "0.65b0" +pandas = "^2.2" [tool.poetry.group.dev] optional = false From b03bfc665e11226ab1c526a00800b77ae56c3d77 Mon Sep 17 00:00:00 2001 From: hanna-paasivirta Date: Thu, 13 Aug 2026 11:23:43 +0100 Subject: [PATCH 3/3] accept both docsite namespace formats in search and cleanup --- services/embed_docsite/docsite_indexer.py | 9 ++++---- services/search_docsite/search_docsite.py | 4 ++-- .../tests/unit/test_docsite_search.py | 21 +++++++++++++++++++ services/util.py | 15 +++++++++++++ 4 files changed, 43 insertions(+), 6 deletions(-) diff --git a/services/embed_docsite/docsite_indexer.py b/services/embed_docsite/docsite_indexer.py index 0e418948..ec34bffd 100644 --- a/services/embed_docsite/docsite_indexer.py +++ b/services/embed_docsite/docsite_indexer.py @@ -6,7 +6,7 @@ from langchain_pinecone import PineconeVectorStore from langchain_openai import OpenAIEmbeddings from langchain_community.document_loaders import DataFrameLoader -from util import create_logger, ApolloError +from util import create_logger, ApolloError, is_docsite_collection logger = create_logger("DocsiteIndexer") @@ -108,12 +108,13 @@ def delete_old_collections(self, max_total_collections): index_stats = index.describe_index_stats() namespaces = index_stats.get('namespaces', {}).keys() valid_namespaces = sorted( - (ns for ns in namespaces if ns.startswith("docsite-") and ns[8:].isdigit() and len(ns) == 16), + (ns for ns in namespaces if is_docsite_collection(ns)), reverse=False ) - if len(valid_namespaces) > max_total_collections: + excess = len(valid_namespaces) - max_total_collections + if excess > 0: logger.info(f"Deleting outdated docsite collections") - for old_collection in valid_namespaces[:max_total_collections]: + for old_collection in valid_namespaces[:excess]: self.index.delete(delete_all=True, namespace=old_collection) logger.info(f"Deleted collection {old_collection}") diff --git a/services/search_docsite/search_docsite.py b/services/search_docsite/search_docsite.py index 7939939b..ec4c309a 100644 --- a/services/search_docsite/search_docsite.py +++ b/services/search_docsite/search_docsite.py @@ -3,7 +3,7 @@ from pinecone import Pinecone from langchain_pinecone import PineconeVectorStore from langchain_openai import OpenAIEmbeddings -from util import create_logger, ApolloError +from util import create_logger, ApolloError, is_docsite_collection from embeddings.embeddings import SearchResult logger = create_logger("DocsiteSearch") @@ -113,7 +113,7 @@ def _get_most_recent_namespace(self): namespaces = index_stats.get('namespaces', {}).keys() valid_namespaces = sorted( - (ns for ns in namespaces if ns.startswith("docsite-") and ns[8:].isdigit() and len(ns) == 16), + (ns for ns in namespaces if is_docsite_collection(ns)), reverse=True ) diff --git a/services/search_docsite/tests/unit/test_docsite_search.py b/services/search_docsite/tests/unit/test_docsite_search.py index cb150138..b376e5d0 100644 --- a/services/search_docsite/tests/unit/test_docsite_search.py +++ b/services/search_docsite/tests/unit/test_docsite_search.py @@ -140,6 +140,27 @@ def test_get_most_recent_namespace_raises_when_none_valid(): assert exc.value.code == 404 +def test_get_most_recent_namespace_accepts_date_time_format(): + """DocsiteIndexer names collections docsite-YYYYMMDDHHMM; a fresh index + holds only that format, and rejecting it 404'd every search after a + successful embed.""" + ds = make_search() + with _patch_pinecone(["docsite-202608131022"]): + assert ds._get_most_recent_namespace() == "docsite-202608131022" + + +def test_get_most_recent_namespace_picks_latest_across_mixed_formats(): + ds = make_search() + namespaces = [ + "docsite-20250225", # legacy date-only + "docsite-202608131022", # current date+time + "docsite-20260813", # date-only, same day as above + "docsite-2026081310", # 10 digits: neither format + ] + with _patch_pinecone(namespaces): + assert ds._get_most_recent_namespace() == "docsite-202608131022" + + # --- lazy embeddings construction ---------------------------------------------- def test_default_embeddings_built_on_construction_not_import(): diff --git a/services/util.py b/services/util.py index 45d6d0f7..64c737a5 100644 --- a/services/util.py +++ b/services/util.py @@ -246,3 +246,18 @@ def add_page_prefix(content: str, page: dict | None) -> str: prefix = f"[pg:{'/'.join(prefix_parts)}]" return f"{prefix} {content}" + + +def is_docsite_collection(name: str) -> bool: + """ + True for docsite collection (namespace) names created by embed_docsite: + 'docsite-' followed by a date (YYYYMMDD) or date+time (YYYYMMDDHHMM). + + Both formats exist in real indexes, and both the searcher and the cleanup + must accept both — filtering on one format made freshly embedded + collections invisible to search. Lexicographic order on the full name + sorts these chronologically (the first 8 digits are the date; a + date+time name sorts after a date-only name for the same day). + """ + suffix = name.removeprefix("docsite-") + return name != suffix and suffix.isdigit() and len(suffix) in (8, 12)