diff --git a/.dockerignore b/.dockerignore index 79adb55..215e875 100644 --- a/.dockerignore +++ b/.dockerignore @@ -67,6 +67,7 @@ test_ocr_output*/ # Scripts (not needed at runtime inside the container) scripts/ +!scripts/migrate_sqlite_to_postgres.py # Claude / AI tooling .claude/ diff --git a/.env.example b/.env.example index 5f715e5..18544ca 100644 --- a/.env.example +++ b/.env.example @@ -193,32 +193,12 @@ VITE_KEYCLOAK_IDP_HINT=google KEYCLOAK_ADMIN_USERNAME=admin KEYCLOAK_ADMIN_PASSWORD= -# --- Master Catalog (Postgres) --- -# code/name/tool_name/prompt_snippet, synced on DEV ingest (status=dev) -# and PROD promote (status=live). No DB container is provisioned in docker-compose — -# point these at your existing Postgres instance (any host/server is fine; this just -# needs its own database + user on it, e.g. `CREATE DATABASE master_catalog`). -MASTER_CATALOG_PG_HOST= -MASTER_CATALOG_PG_PORT=5432 -MASTER_CATALOG_PG_DB=master_catalog -MASTER_CATALOG_PG_USER=master_catalog -MASTER_CATALOG_PG_PASSWORD= -MASTER_CATALOG_PG_SSLMODE=disable - -# --- AI layer Redis (bharat-oan-api) --- -# docs-pipeline pushes the catalog snapshot directly into this Redis instance -# after every Postgres sync. Leave AI_LAYER_REDIS_HOST empty to disable the -# push (Postgres sync still happens; only the Redis mirror is skipped). -AI_LAYER_REDIS_HOST= -AI_LAYER_REDIS_PORT=6379 -AI_LAYER_REDIS_DB=0 -AI_LAYER_REDIS_PASSWORD= -MASTER_CATALOG_REDIS_TTL_SECONDS=172800 - # --- Publish to Network (Discovery Service) --- -# After DEV ingest, the publishing_to_network stage POSTs a Beckn catalog/publish -# envelope announcing the kind of knowledge this provider serves. Required — there -# is no feature flag to disable the stage. Worker-only; the API and UI never read these. +# After a document is promoted to PROD, the publishing_to_network stage POSTs a +# Beckn catalog/publish envelope announcing the kind of knowledge this provider +# serves. Set ENABLE_PUBLISH_TO_NETWORK=true to enable publishing. +# DISCOVERY_SERVICE_ENDPOINT and NETWORK_SENDER_ID are required when enabled. +ENABLE_PUBLISH_TO_NETWORK=false DISCOVERY_SERVICE_ENDPOINT= NETWORK_SENDER_ID= # Publicly reachable base URL of this provider, sent as each catalog's bppUri. @@ -265,6 +245,14 @@ KC_SMTP_SSL=false KC_SMTP_USERNAME= KC_SMTP_PASSWORD= +# Shared secret for POST /webhook. Callers must send it in the `x-api-key` header. +# Required: if unset or empty, /webhook returns 503. Wrong or missing key returns 401. +# Generate a 32-char hex value with: openssl rand -hex 16 +# Never commit a real value. +WEBHOOK_API_KEY=your_32_character_hex_key_here + +# Max records accepted per POST /webhook request (more returns 413) +WEBHOOK_MAX_RECORDS=500 # --- Logs to ClickHouse over OTLP (ADR 0007). Empty endpoint = off. # Local: http://otel-collector:4318 with `docker compose --profile observability up -d` diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 6e01332..deaf75a 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -9,6 +9,22 @@ jobs: pipeline: name: pipeline (lint, test, coverage) runs-on: ubuntu-latest + services: + postgres: + image: postgres:15-alpine + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: knowledge_provider_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + TEST_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/knowledge_provider_test steps: - uses: actions/checkout@v4 diff --git a/AGENTS.md b/AGENTS.md index 406cb6e..a549ba8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,13 +4,13 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## What this is -A review-driven document ingestion pipeline (product name "Bharat Vistaar Docs Pipeline"): Temporal workflows + FastAPI + SQLite + MinIO + Qdrant. Documents move through explicit stages (OCR → review → translate → review → chunk → review → ingest DEV → Super Admin promote to PROD), with every major output persisted as an inspectable artifact rather than hidden. +A review-driven document ingestion pipeline (product name "Bharat Vistaar Docs Pipeline"): Temporal workflows + FastAPI + Postgres + MinIO + Qdrant. Documents move through explicit stages (OCR → review → translate → review → chunk → review → ingest DEV → Super Admin promote to PROD), with every major output persisted as an inspectable artifact rather than hidden. Full narrative docs (read these before deep architectural changes, don't re-derive): - `README.md` — what the system does, API surface, supported inputs, auth model - `docs/SYSTEM_DESIGN.md` — C4 diagrams, containers, DEV/PROD indexing, stage/activity table - `docs/architecture-pipeline-rbac.md` — login flow, roles, DEV vs Super-Admin-PROD gate -- `docs/MASTER_SCHEME_CATALOG_AND_AI_SYNC_DESIGN.md` — master catalog → Redis sync to AI layer (bharat-oan-api) +- `docs/MASTER_SCHEME_CATALOG_AND_AI_SYNC_DESIGN.md` — original scheme catalog design (historical: the Postgres master catalog and Redis push it describes were removed; the pull API `/catalog/v1/*` remains) - `ENV.md` — full annotated environment variable reference (`.env.example` for a working template) ## Commands @@ -69,7 +69,7 @@ Core modules (all under `pipeline/`): - `workflows.py` — Temporal workflow definitions (`DocumentPipelineWorkflow` is the main one; also `ReingestionWorkflow`, `PromoteToProdWorkflow`, `TranslationOnlyWorkflow`, `OcrOnlyWorkflow`, `ChunkingOnlyWorkflow`) - `activities.py` — Temporal activities invoked by the workflows (OCR, translate, chunk, ingest, state updates) - `worker.py` — Temporal worker process; registers workflows/activities on task queue `ocr-pipeline` -- `db.py` — SQLite access layer; SQLite is the **canonical** metadata/review-state store (documents, pages, chunks, jobs, artifacts, audit log), not just a cache +- `db.py` — Postgres access layer (psycopg3 sync connection pool); Postgres is the **canonical** metadata/review-state store (documents, workflows, pages, chunks, artifacts, audit log, settings), not just a cache. Documents and jobs come back as flat dicts keyed by `workflow_id` (the document's main Temporal workflow id); every function also accepts the document UUID. Schema: `migrations/` (applied at api/worker startup by `migrate.py`), described in `docs/POSTGRES_SCHEMA.md` - `document_repository.py` — `DocumentRepository`, a domain layer over `db.py` for document reads; `db.py` stays one-function-per-query with no domain knowledge - `network_constants.py` — fixed values sent on the network (catalog/resource ids, topics, languages, Beckn version). The file v2 edits when the AI layer starts deriving them - `catalog_builder.py` — pure builder mapping a document's knowledge kind (`advisory`/`scheme`) to the single `OnDemand` Beckn catalog announced for that kind; takes the resolved provider and prefixes catalog/resource ids for a Group; returns `None` for any other kind. No env, no I/O @@ -79,8 +79,8 @@ Core modules (all under `pipeline/`): - `models.py` — Pydantic models, including `DocumentStage` enum and `PIPELINE_STAGES` (the stepper-UI stage list) - `config.py` — `Config` dataclass reading env vars, with defaults - `observability.py` — bridges root-logger `INFO`+ lines to OTLP when `OTEL_EXPORTER_OTLP_ENDPOINT` is set (ClickHouse/HyperDX); no-op otherwise. See ADR 0007 -- `instances.py` — instance/tenant (state) code ↔ human name mapping; `bv` = Bharat Vistaar platform-wide, others are state codes matching Keycloak group paths (`/states/MH/...` → `mh`) -- `master_catalog_pg.py`, `scheme_catalog.py` — Postgres-backed master scheme catalog, synced to Redis for the AI layer (bharat-oan-api) to consume +- `instances.py` — tenant instance code ↔ human name mapping; `bv` = Bharat Vistaar platform-wide, others are state codes matching Keycloak group paths (`/states/MH/...` → `mh`). A tenant code is an `instances` row (name = code); a document's `instance` is derived through `documents.catalog_id → catalogs.instance_id`, there is no `documents.instance` column +- `scheme_catalog.py` — scheme catalog behind the pull-based `/catalog/v1/*` API, served from the computed `catalog_status` view (one row per document; `catalogs` rows are `(instance, type)`, `documents` carry `code`/`name`/`aliases`) - `ocr/`, `translation/`, `chunking/`, `vector_store/` — pluggable provider packages, each with a `base.py` protocol/ABC and a `service.py` (or equivalent) that dispatches to the configured provider - `auth/` — Keycloak JWT validation, group-based tenancy/permissions, email-OTP flow, admin user management @@ -106,7 +106,7 @@ registered → ocr_processing → ocr_review → translation_processing → tran ### Storage responsibilities (don't blur these) -- **SQLite** — canonical metadata/content/review state (documents, pages, chunks, jobs, artifacts, audit, search settings). Source of truth for anything editable. +- **Postgres** — canonical metadata/content/review state (documents, workflows, pages, chunks, artifacts, audit, search settings). Source of truth for anything editable. - **MinIO** — blob storage for original uploads, normalized files, and stage artifacts (OCR/translation/chunk exports, vector-index payload snapshots). - **Temporal** — orchestration/retries/review-gate signaling only; not a content store. - **Qdrant** — downstream search projections of approved chunk state; never the place to edit content. @@ -121,4 +121,4 @@ React 18 + Vite + Tailwind v4 + Radix UI operator console (not a plain upload fo ### Scripts (`scripts/`) -Operational tools, not one-off hacks — e.g. `list_failed_workflows.py`, `terminate_stuck_workflows.py`, `keycloak_bootstrap_docs_pipeline.py`, `drop_legacy_marqo_columns.py`, `backfill_provider_id.py` (sets `provider_id` on existing DEV/PROD chunks; dry run unless `--apply`). Check here before writing a new maintenance script — an equivalent likely exists. +Operational tools, not one-off hacks — e.g. `list_failed_workflows.py`, `terminate_stuck_workflows.py`, `keycloak_bootstrap_docs_pipeline.py`, `drop_legacy_marqo_columns.py`, `backfill_provider_id.py` (sets `provider_id` on existing DEV/PROD chunks; dry run unless `--apply`), `migrate_sqlite_to_postgres.py`. Check here before writing a new maintenance script — an equivalent likely exists. diff --git a/CONTEXT.md b/CONTEXT.md index 4038b18..504d8dd 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -12,16 +12,12 @@ _Avoid_: "publish" alone — ambiguous with Publish to Network. The pipeline stage that runs only after a document's PROD promotion succeeds, and POSTs a Beckn-shaped `publish` envelope to an external Discovery Service. Skipped entirely when PROD is disabled (`DISABLE_PROD_SETTING=true`) — a document never reaches the network without first promoting to PROD. The envelope it sends is kind-level (see Knowledge Kind), not document-level: it does not make any single document searchable, only re-announces that the document's provider (its Group Provider, or the Shared Provider) serves that kind at all. _Avoid_: "publish" alone — ambiguous with Publish to Dev. -**Master Catalog**: -The global, Postgres-backed registry of prompt/tool metadata (any content type: scheme, advisory, etc.), synced to Redis so the AI layer can pick up entries without a redeploy. -_Avoid_: "catalog" alone — ambiguous with Scheme Catalog and the Network Catalog Envelope. - **Scheme Catalog**: -The SQLite-backed, scheme-only registry that aggregates one or more documents sharing a `scheme_code`. Backs the pull-based `GET /catalog/v1/*` API that other BAPs poll. -_Avoid_: "catalog" alone — ambiguous with Master Catalog and the Network Catalog Envelope. +The scheme-only registry (`catalog_entries` rows plus the computed `catalog` view) that aggregates one or more documents sharing a `scheme_code`. Backs the pull-based `GET /catalog/v1/*` API that other BAPs poll. +_Avoid_: "catalog" alone — ambiguous with the Network Catalog Envelope. **Network Catalog Envelope**: -The Beckn wire-protocol `message.catalogs` object sent by Publish to Network to the Discovery Service. Structurally and semantically unrelated to the Master Catalog and Scheme Catalog tables — do not conflate the three. +The Beckn wire-protocol `message.catalogs` object sent by Publish to Network to the Discovery Service. Structurally and semantically unrelated to the Scheme Catalog tables — do not conflate the two. _Avoid_: "catalog" alone. **Group**: diff --git a/Dockerfile b/Dockerfile index dcf1998..dd993cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,8 +29,10 @@ RUN --mount=type=cache,target=/root/.cache/uv \ ENV PATH="/app/.venv/bin:$PATH" -# Copy pipeline code +# Copy pipeline code and the SQL migrations the api/worker apply at startup COPY pipeline/ ./pipeline/ +COPY migrations/ ./migrations/ +COPY scripts/migrate_sqlite_to_postgres.py ./scripts/migrate_sqlite_to_postgres.py # Catch imports that work locally but fail on the shipped Python 3.10 image. RUN python -c "import pipeline.api" diff --git a/ENV.md b/ENV.md index beb1e05..bae5e8e 100644 --- a/ENV.md +++ b/ENV.md @@ -64,7 +64,11 @@ Root `.env` (see `.env.example`). Required by FastAPI (`pipeline/api/`), Tempora | `LOG_LEVEL` | `INFO` | Worker log verbosity (`DEBUG`/`INFO`/`WARNING`/`ERROR`); the `temporalio` SDK logger stays at `INFO` regardless, to avoid its own noisy debug output | | `MINIO_ENDPOINT` | `localhost:9000` | MinIO API host:port | | `MINIO_BUCKET` | `documents` | Object storage bucket | -| `DOCUMENT_DB_PATH` | `/data/documents.db` | SQLite path (use `./data/documents.db` for local non-Docker) | +| `DATABASE_URL` | `postgresql://postgres:postgres@localhost:5432/knowledge_provider` | Application Postgres (documents, workflows, pages, chunks, audit, settings). The compose `postgres` service builds it from `DB_USERNAME` / `DB_PASSWORD` / `DB_NAME` (defaults `docs` / `docs` / `knowledge_provider`). Pending SQL migrations in `migrations/` are applied by the api and worker at startup | +| `DB_POOL_MIN_SIZE` | `1` | Connection pool minimum per process | +| `DB_POOL_MAX_SIZE` | `10` | Connection pool maximum per process (api and worker each hold their own pool) | +| `DB_POOL_TIMEOUT_SECONDS` | `30` | Seconds to wait for a free pooled connection | +| `DOCUMENT_DB_PATH` | `/data/documents.db` | Only read by `scripts/migrate_sqlite_to_postgres.py`: the legacy SQLite file to import | ### Logs to ClickHouse (OTLP) @@ -89,6 +93,18 @@ Off unless `OTEL_EXPORTER_OTLP_ENDPOINT` is set. When it is set, API and worker | `DOCS_PIPELINE_UI_URL` | `http://localhost:3000` | Public UI base for links | | `DEFAULT_INSTANCE` | `default` | Default tenant / instance id | +### Webhook (`POST /webhook`) + +| Variable | Default | Purpose | +|----------|---------|---------| +| `WEBHOOK_MAX_RECORDS` | `500` | Max records per `/webhook` request; more returns 413 | +| `WEBHOOK_API_KEY` | *(empty — required)* | 32-char hex secret. Callers send it in the `x-api-key` header. Missing/invalid header → 401. Unset/empty → 503 (fails closed, ignores `AUTH_DISABLED`). Generate with `openssl rand -hex 16`. | +| `ADVISORY_BACKFILL_ON_START` | `false` | When `true`, the worker queues an index workflow for every unindexed `webhook_records` row once at startup (backfill of existing records). Leave off normally; `POST /webhook/advisory/reindex` does the same on demand. | + +Each saved record whose `payload_hash` differs from its `indexed_hash` also starts one Temporal workflow `advisory-` that writes advisory chunks (`document_kind=advisory`) to the Qdrant collection `VECTOR_DB_COLLECTION_NAME`. Admin endpoints (Keycloak admin): `POST /webhook/advisory/reindex`, `POST /webhook/advisory/promote` (copies to `PROD_VECTOR_DB_COLLECTION_NAME` (on `PROD_VECTOR_DB_URL`)), `GET /webhook/advisory/status`. + +Body is a JSON array of objects (max `WEBHOOK_MAX_RECORDS`); stored in the `webhook_records` table, upserted on `(source, external_id)` where `external_id` is `contentID` (else `id`, `slug`, payload hash). + ### Auth / Keycloak (backend JWT validation) | Variable | Default | Purpose | @@ -237,24 +253,7 @@ Exposes `/catalog/v1/*` so **bharat-oan-api** and **bharat-provider-backend** ca | `PROD_SCHEME_VECTOR_DB_API_KEY` | `PROD_VECTOR_DB_API_KEY` | Scheme collection API key | | `PROD_SCHEME_VECTOR_DB_COLLECTION_NAME` | `schemes-index` | Must **not** equal documents collection | -**Endpoints:** `GET /catalog/v1/snapshot`, `/version`, `/schemes`, `/tool-prompt`; `POST /catalog/v1/rebuild`, `/bootstrap`; `PATCH /documents/{id}/scheme-metadata`. - -### Master Catalog — Postgres + Redis push (dev preview channel) - -Separate from the SQLite/Qdrant catalog above (which tracks PROD vector publication and is scheme-specific). This one is a generic Postgres table (`master_catalog`: code, content_type, name, tool_name, doc_id, prompt_snippet, status) — not scheme-only, since ingested documents can be schemes, advisories, or other kinds that later grow their own code/name/tool metadata. docs-pipeline writes to it on **DEV ingest complete** (`status=dev`) and **PROD promote complete** (`status=live`), then pushes directly into the AI layer's Redis so bharat-oan-api can test an entry's prompt/tool routing in the dev chatbot without an AI-layer redeploy. - -| Variable | Default | Purpose | -|----------|---------|---------| -| `MASTER_CATALOG_PG_HOST` | *(empty — required)* | Host of your existing Postgres instance. No DB container is provisioned by docker-compose for this — bring your own server and create the database on it (e.g. `CREATE DATABASE master_catalog`). Tables are created automatically on first sync. | -| `MASTER_CATALOG_PG_PORT` | `5432` | Postgres port | -| `MASTER_CATALOG_PG_DB` | `master_catalog` | Database name | -| `MASTER_CATALOG_PG_USER` / `MASTER_CATALOG_PG_PASSWORD` | `master_catalog` / *(empty)* | Credentials | -| `MASTER_CATALOG_PG_SSLMODE` | `disable` | Use `require` for managed prod Postgres | -| `AI_LAYER_REDIS_HOST` | *(empty)* | bharat-oan-api's Redis host. Empty = Postgres sync still happens, Redis push is skipped | -| `AI_LAYER_REDIS_PORT` / `AI_LAYER_REDIS_DB` / `AI_LAYER_REDIS_PASSWORD` | `6379` / `0` / *(empty)* | AI layer Redis connection | -| `MASTER_CATALOG_REDIS_TTL_SECONDS` | `172800` (48h) | Snapshot key TTL — a dead-man's switch, not a freshness mechanism (writes are push-driven) | - -**Redis keys:** `master-catalog:dev:snapshot` (dev + live entries — what the dev chatbot reads), `master-catalog:live:snapshot` (live only — what prod reads). Plain JSON via raw `redis-py`, not routed through bharat-oan-api's `aiocache` layer, since it's an external write contract rather than an internal cache value. +**Endpoints:** `GET /catalog/v1/snapshot`, `/version`, `/schemes`, `/tool-prompt`; `POST /catalog/v1/rebuild`; `PATCH /documents/{id}/scheme-metadata` (body: `kind`, `code`, `name`, `aliases`, `tool_routing`, `catalog_visible`, `network_visible`, `valid_from`, `valid_to`). The catalog is computed from documents (one row per document), so there is no bootstrap endpoint. `/upload`, `/documents` and `/documents/batch` take a required `kind` (`scheme` or `advisory`) and an optional `group` and `code`. ### Publish to Network @@ -285,7 +284,7 @@ Each instance has one provider for both its advisory and scheme catalogs: id and |---------|----|-----|--------| | `VITE_*` proxies & Keycloak browser login | ✅ | — | — | | JWT validation (`AUTH_*`, `KEYCLOAK_*`) | — | ✅ | — | -| Temporal / MinIO / SQLite | — | ✅ | ✅ | +| Temporal / MinIO / Postgres (`DATABASE_URL`) | — | ✅ | ✅ | | OCR / translation / chunking / domain tags | — | config / status | ✅ runs jobs | | Qdrant (`VECTOR_DB_*`) | — | ✅ | ✅ | | Publish to Network (`DISCOVERY_SERVICE_ENDPOINT`, `NETWORK_SENDER_ID`) | — | — | ✅ runs the publish activity | @@ -299,7 +298,7 @@ Each instance has one provider for both its advisory and scheme catalogs: id and ```bash set -a && source .env && set +a -export DOCUMENT_DB_PATH="$(pwd)/data/documents.db" +export DATABASE_URL="postgresql://docs:docs@localhost:5433/knowledge_provider" # the compose `postgres` service (`docker compose up -d postgres`) export ALLOWED_FILE_PATHS="$(pwd)/books,$(pwd)/data/documents" export CORS_ORIGINS="http://localhost:3000,http://localhost:3001" export PYTHONPATH="$(pwd)" diff --git a/README.md b/README.md index 3534caa..5c79f6d 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![codecov](https://codecov.io/gh/OpenAgriNet/knowledge-provider/graph/badge.svg?token=2TKM2CR2AC)](https://codecov.io/gh/OpenAgriNet/knowledge-provider) -This repository contains a review-driven document ingestion pipeline built around Temporal workflows, FastAPI, SQLite, MinIO, and Qdrant. It is designed for teams that need to normalize heterogeneous files, extract structured text, review and correct outputs, generate chunks, and publish searchable records into a vector index. +This repository contains a review-driven document ingestion pipeline built around Temporal workflows, FastAPI, Postgres, MinIO, and Qdrant. It is designed for teams that need to normalize heterogeneous files, extract structured text, review and correct outputs, generate chunks, and publish searchable records into a vector index. The system is intentionally operational, not just algorithmic. Documents move through explicit stages, every major output can be persisted as an artifact, and the operator UI is designed to inspect and manage the pipeline rather than hide it. @@ -63,7 +63,7 @@ The platform is composed of six main services: | | | | v v v v +--------+---------+ +--------+--+ +---+---------------+ +------------------+ -| MinIO | | SQLite | | Worker | | Lang Detect | +| MinIO | | Postgres | | Worker | | Lang Detect | | originals, | | canonical | | normalize, OCR, | | language hints | | normalized files, | | metadata, | | translate, chunk, | | before | | artifacts | | pages, | | ingest | | translation | @@ -90,7 +90,7 @@ The document has been accepted into the system and assigned a workflow identifie Typical outputs: -- document row in SQLite +- document row in Postgres - original file reference - initial job record @@ -191,20 +191,21 @@ Document-level view of what has been pushed to the vector index. ## Storage Responsibilities -### SQLite +### Postgres -SQLite is the canonical metadata and review-state store. +Postgres is the canonical metadata and review-state store. Its schema is in `migrations/` and described in `docs/POSTGRES_SCHEMA.md`; the api and worker apply pending migrations at startup. Moving an existing SQLite deployment over is a one-off run of `scripts/migrate_sqlite_to_postgres.py` (see the `migrate-sqlite` compose profile). It owns: - document rows +- workflow rows (one per processing run, shown as jobs) - page rows - chunk rows -- jobs - artifact metadata - audit logs - search settings - document/index status +- inbound webhook records (`webhook_records`, written by `POST /webhook`, `x-api-key` = `WEBHOOK_API_KEY`) ### MinIO @@ -319,7 +320,7 @@ Default local ports from `docker-compose.yml`: - Docker and Docker Compose - an OCR provider API key exposed as `MISTRAL_API_KEY` -- enough local disk for SQLite, MinIO artifacts, and Qdrant state +- enough local disk for Postgres, MinIO artifacts, and Qdrant state ### Start @@ -366,7 +367,7 @@ Important runtime variables include: - `MINIO_ACCESS_KEY` - `MINIO_SECRET_KEY` - `MINIO_BUCKET` -- `DOCUMENT_DB_PATH` +- `DATABASE_URL` - `TRANSLATION_PROVIDER` - `TRANSLATION_MODEL` - `TRANSLATION_PAGE_CONCURRENCY` diff --git a/docker-compose.yml b/docker-compose.yml index b649b0e..2853bf3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,6 +21,7 @@ # API 127.0.0.1:${API_HOST_PORT:-8011}:8001 # UI 127.0.0.1:${UI_HOST_PORT:-3011}:80 # Keycloak 127.0.0.1:${KEYCLOAK_HOST_PORT:-8081}:8080 +# Postgres 127.0.0.1:${DB_HOST_PORT:-5433}:5432 # Temporal 127.0.0.1:7233:7233 # MinIO API 127.0.0.1:9000:9000 # MinIO UI 127.0.0.1:9001:9001 @@ -123,6 +124,30 @@ services: MINIO_BUCKET: ${MINIO_BUCKET:-documents} restart: "no" + # --------------------------------------------------------------------------- + # Application database — documents, workflows, pages, chunks, audit, settings. + # Schema is applied by api/worker at startup from migrations/ (see + # docs/POSTGRES_SCHEMA.md). A deployment that wants its own Postgres can set + # DATABASE_URL in .env and drop this service's depends_on entries. + # --------------------------------------------------------------------------- + postgres: + image: postgres:15-alpine + platform: linux/amd64 + environment: + POSTGRES_USER: ${DB_USERNAME:-docs} + POSTGRES_PASSWORD: ${DB_PASSWORD:-docs} + POSTGRES_DB: ${DB_NAME:-knowledge_provider} + ports: + - "127.0.0.1:${DB_HOST_PORT:-5433}:5432" + volumes: + - app-db-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USERNAME:-docs} -d ${DB_NAME:-knowledge_provider}"] + interval: 5s + timeout: 5s + retries: 10 + restart: unless-stopped + # --------------------------------------------------------------------------- # Qdrant — DEV vector index (see docs/SYSTEM_DESIGN.md's DEV vs PROD dual # indexing). api/worker's VECTOR_DB_URL is always pointed at this bundled @@ -252,7 +277,7 @@ services: TEMPORAL_HOST: temporal:7233 MINIO_ENDPOINT: minio:9000 VECTOR_DB_URL: http://qdrant:6333 - DOCUMENT_DB_PATH: /data/documents.db + DATABASE_URL: postgresql://${DB_USERNAME:-docs}:${DB_PASSWORD:-docs}@postgres:5432/${DB_NAME:-knowledge_provider} ALLOWED_FILE_PATHS: /app/books,/data/documents # HF cache path propagated into the container HF_HOME: ${HF_HOME:-/opt/hf-cache} @@ -261,6 +286,8 @@ services: ports: - "127.0.0.1:${API_HOST_PORT:-8011}:8001" depends_on: + postgres: + condition: service_healthy temporal: condition: service_healthy minio: @@ -272,7 +299,6 @@ services: command: uvicorn pipeline.api:app --host 0.0.0.0 --port 8001 volumes: - ./books:/app/books:ro - - sqlite-data:/data - documents-data:/data/documents - ${HF_HOME:-/opt/hf-cache}:${HF_HOME:-/opt/hf-cache} extra_hosts: @@ -295,12 +321,14 @@ services: TEMPORAL_HOST: temporal:7233 MINIO_ENDPOINT: minio:9000 VECTOR_DB_URL: http://qdrant:6333 - DOCUMENT_DB_PATH: /data/documents.db + DATABASE_URL: postgresql://${DB_USERNAME:-docs}:${DB_PASSWORD:-docs}@postgres:5432/${DB_NAME:-knowledge_provider} ALLOWED_FILE_PATHS: /app/books,/data/documents HF_HOME: ${HF_HOME:-/opt/hf-cache} HF_HUB_CACHE: ${HF_HOME:-/opt/hf-cache} OTEL_SERVICE_NAME: knowledge-provider-worker depends_on: + postgres: + condition: service_healthy temporal: condition: service_healthy minio: @@ -312,13 +340,39 @@ services: command: python -m pipeline.worker volumes: - ./books:/app/books:ro - - sqlite-data:/data - documents-data:/data/documents - ${HF_HOME:-/opt/hf-cache}:${HF_HOME:-/opt/hf-cache} extra_hosts: - "host.docker.internal:host-gateway" restart: unless-stopped + # --------------------------------------------------------------------------- + # One-off SQLite -> Postgres data migration (maintenance window, api/worker + # stopped). The legacy volume is opened read-only by the script and is never + # modified, so rolling back is redeploying the previous images on it. + # docker compose --profile migrate run --rm migrate-sqlite # dry run + # MIGRATE_ARGS= docker compose --profile migrate run --rm migrate-sqlite # real run + # --------------------------------------------------------------------------- + migrate-sqlite: + image: ghcr.io/openagrinet/knowledge-provider:${IMAGE_TAG:-latest} + platform: linux/amd64 + profiles: + - migrate + build: + context: . + dockerfile: Dockerfile + environment: + DATABASE_URL: postgresql://${DB_USERNAME:-docs}:${DB_PASSWORD:-docs}@postgres:5432/${DB_NAME:-knowledge_provider} + DOCUMENT_DB_PATH: /legacy/documents.db + DEFAULT_INSTANCE: ${DEFAULT_INSTANCE:-default} + depends_on: + postgres: + condition: service_healthy + command: python scripts/migrate_sqlite_to_postgres.py ${MIGRATE_ARGS---dry-run} + volumes: + - sqlite-data:/legacy + restart: "no" + # --------------------------------------------------------------------------- # React UI # Dockerfile.prod builds a static nginx image; VITE_* args come from .env. @@ -387,6 +441,7 @@ volumes: keycloak-db-data: keycloak-data: minio-data: + app-db-data: sqlite-data: documents-data: clickhouse-data: diff --git a/docs/ADR/0006-instances-and-catalogs-model.md b/docs/ADR/0006-instances-and-catalogs-model.md new file mode 100644 index 0000000..91f4190 --- /dev/null +++ b/docs/ADR/0006-instances-and-catalogs-model.md @@ -0,0 +1,21 @@ +# Providers are instances, catalogs are (instance, type), documents are identified by (catalog, code) + +Context: the first Postgres schema modelled a tenant as a free-text `documents.instance`, a `catalog_entries` table keyed by `(type, code)` with one entry per scheme, and a free-form `document_kind` that had to match the entry's type through a composite foreign key. Review feedback was that the unit published on the network is a provider's catalog, that a provider is a Keycloak group, that each scheme is itself a document, and that database enums and demo flags do not belong in the schema. + +Decision: +- An **instance** owns provider catalogs. Use one `instances` row per Keycloak tenant node (`/states/MH`, `/global`). Role leaves are not mirrored; permissions still come from the token. `users.instance_id` is a single nullable FK for now (first tenant in the token, sorted, with a warning when there are several); a join table can replace it if multi-instance membership is needed. +- A **catalog** is `(instance, type)`: `catalogs.instance_id` plus `catalogs.type`, unique together. `type` holds the network schema name. The kind-to-schema map lives in `pipeline/network_constants.py` and is a placeholder: `scheme` is `openagrinet:AgricultureResource`, `advisory` is `openagrinet:KnowledgeAdvisory`, `video` is `openagrinet:VideoResource`. Code branches through that map, never on raw `openagrinet:*` strings. Catalog Beckn information (`publish_directives`, `visible_networks`, `criteria`, `network`) sits in `catalogs.metadata`. +- A **document** belongs to one catalog and is identified there by `code`: `documents.catalog_id` NOT NULL, `UNIQUE (catalog_id, code)`, with `name` and `aliases` on the document. The API fields `code`, `name` and `aliases` are those columns. `documents.instance`, `document_kind` and `is_demo` are gone; the API returns the derived `instance` (the instance name) and `kind` (from the type map). The `/catalog/v1/*` snapshot keys, the Qdrant payload keys and Temporal arguments keep their earlier names (`scheme_code`, `document_kind`, `instance`, ...) because other services and in-flight workflows depend on them. +- Re-uploading an existing code updates the same document (new workflow, `source` replaced, pages and chunks cleared, `reindex_required` set) rather than adding a second one. +- Catalogs are created on upload for the tenant and kind. The upload therefore takes the document type up front. +- Bootstrap schemes (the code-seeded entries with no documents) are dropped. `/catalog/v1/*` is served from the `catalog_status` view, one row per document, and a scheme appears once a document exists. +- Period, uploader and vector index state moved into `documents.metadata`; counts are derived from `pages` and `chunks`. +- Database enums are replaced by plain `TEXT`, validated by `pipeline/models.py` (`DocumentStage`, `WorkflowType`, `WorkflowStatus`, `AuditEntityType`). +- `audit_logs.instance` is replaced by `audit_logs.instance_id` (snapshot, `ON DELETE SET NULL`). +- No Postgres held the first schema, so `migrations/0001` was rewritten in place instead of adding an ALTER migration. `scripts/migrate_sqlite_to_postgres.py` produces v2 and skips `kind=document` and `is_demo` documents, reporting the counts. + +Alternatives considered: +- **Keep `documents.instance`.** Rejected: the catalog already carries the provider, and two sources of tenant truth drift. +- **Type as a short kind plus a separate `schema_type` column.** Rejected for now in favour of one `type` column holding the schema name; revisit if scheme and advisory end up sharing a schema (ADR 0003 types both as `KnowledgeAdvisory`). +- **`user_instances` join table now.** Deferred: the team wants one instance per user for now. +- **Nullable `catalog_id`/`code` until a reviewer classifies the document.** Rejected: a document without a catalog has no tenant once `instance` is gone. The upload form asks for the type instead. diff --git a/docs/POSTGRES_SCHEMA.md b/docs/POSTGRES_SCHEMA.md new file mode 100644 index 0000000..f9e260e --- /dev/null +++ b/docs/POSTGRES_SCHEMA.md @@ -0,0 +1,576 @@ +# Postgres Schema — knowledge-provider + +Postgres schema behind `pipeline/db.py`. A document is content and metadata. A workflow is one run of processing against a document. A document belongs to a catalog (a provider instance plus a resource type) and is identified there by its `code`. Source file and vector index state are stored as JSONB on the document. Translations are stored as JSONB on each page. + +The schema is created by `migrations/0001_initial_schema.sql` and applied by the api and worker at startup (`pipeline/migrate.py`). `pipeline/db.py` is the only code that reads or writes it. The previous SQLite store is described in `docs/SQLITE_SCHEMA.md` (legacy) and is moved over once by `scripts/migrate_sqlite_to_postgres.py`. + +## Conventions + +- Every table has a primary key `id`, a UUID v4 (`DEFAULT gen_random_uuid()`, built in from Postgres 13). +- A foreign key is named `_id` (for example `document_id`). A role prefix is added when the role matters (`uploaded_by_user_id`, `started_by_user_id`). +- Text identifiers other systems rely on (`documents.code`, `temporal_workflow_id`) are ordinary columns, not keys. +- `created_at` and `updated_at` are the last two columns of a table. `updated_at` is maintained by one shared trigger. Append-only or short-lived tables skip `updated_at`, as noted per table. +- Timestamps are `TIMESTAMPTZ` (UTC), booleans are `BOOLEAN`, JSON is `JSONB`, lists of strings are `TEXT[]`. +- JSONB columns have a `CHECK` that the value is an object (or array). Value rules inside them live in Pydantic models, not in the database. +- There are no database enums. Every value set (stages, workflow types and statuses, audit entity types) is plain `TEXT`, validated in code by `pipeline/models.py`; open sets (`artifact_type`, `action_type`) are `TEXT` too. +- Counts have `CHECK >= 0`. Slugs and identifiers have pattern `CHECK`s. + +### Shared trigger + +```sql +CREATE FUNCTION set_updated_at() RETURNS trigger AS $$ +BEGIN + NEW.updated_at := now(); + RETURN NEW; +END $$ LANGUAGE plpgsql; +-- one per table that has updated_at: +-- CREATE TRIGGER trg__updated_at BEFORE UPDATE ON
+-- FOR EACH ROW EXECUTE FUNCTION set_updated_at(); +``` + +### Value sets (plain `TEXT`, validated in code) + +| Column(s) | Python type (`pipeline/models.py`) | Values | +|---|---|---| +| `workflows.current_stage`, `workflow_artifacts.stage` | `DocumentStage` | `registered`, `ocr_processing`, `ocr_review`, `translation_processing`, `translation_review`, `chunking`, `chunk_review`, `ready_for_ingestion`, `ingesting`, `publishing_to_network`, `approval_for_prod`, `ingesting_prod`, `completed`, `failed` | +| `workflows.workflow_type` | `WorkflowType` | `pipeline`, `ocr_only`, `ocr_retry`, `translation_retry`, `chunking_retry`, `reingest`, `promote_prod` | +| `workflows.status` | `WorkflowStatus` | `running`, `waiting_review`, `completed`, `failed` | +| `audit_logs.entity_type` | `AuditEntityType` | `document`, `page`, `chunk`, `setting`, `catalog` | + +The database does not check these values, so adding one needs no migration. Raw SQL that writes them must use a value from the table. + +## Tables + +| # | Table | Purpose | +|---|---|---| +| 1 | `instances` | Provider tenant nodes (`/states/MH`, `/global`), mirrored from Keycloak | +| 2 | `users` | People who upload, review and act, mirrored from Keycloak | +| 3 | `catalogs` | One row per provider instance and resource type (scheme, advisory, video) | +| 4 | `documents` | One row per document. Content, metadata, source file, index state | +| 5 | `workflows` | One row per processing run of a document. Stage, status, config | +| 6 | `workflow_artifacts` | Files each workflow produces, stored in MinIO | +| 7 | `pages` | OCR layer and translations of each page | +| 8 | `chunks` | Chunked text and review state | +| 9 | `audit_logs` | Append-only action history | +| 10 | `search_settings` | Single-row search configuration | +| 11 | `email_otps` | Email OTP login codes | +| 12 | `webhook_records` | Records pushed to `POST /webhook` | +| — | `schema_migrations` | Applied migration files (created by the runner, not by a migration) | +| — | `catalog_status` (view) | One row per document with its catalog status. `/catalog/v1/*` serves the scheme type | +| — | `document_current_workflow` (view) | Latest workflow of each document | + +Relationships: + +``` +instances 1 ── * users (users.instance_id, nullable) +instances 1 ── * catalogs 1 ── * documents 1 ── * workflows 1 ── * workflow_artifacts + │ + ├── * pages + └── * chunks +instances 1 ── * audit_logs (audit_logs.instance_id, snapshot, SET NULL) +``` + +Dependency order for creation: `instances`, `users`, `catalogs`, `documents`, `workflows`, `workflow_artifacts`, `pages`, `chunks`, `audit_logs`, `search_settings`, `email_otps`, `webhook_records`, views. + +--- + +## 1. `instances` + +One row per tenant node (a state, the centre, or the platform), mirrored from Keycloak. An instance is a provider: it owns catalogs. Rows are created on demand from a tenant code (`get_or_create_instance`) or from a Keycloak group path (`upsert_instance`). + +Unique: `keycloak_group_id`, `path`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Internal instance ID. | +| keycloak_group_id | TEXT | UNIQUE | Keycloak group ID. NULL until a sync that sees the path fills it (rows made by the SQLite move or by an upload have none). | +| name | TEXT | NOT NULL | Tenant code, lowercase (`mh`, `bv`). This is what the API calls the document's `instance`. | +| path | TEXT | NOT NULL UNIQUE | Keycloak group path: `/states/MH`, or `/global` for `bv`. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Created. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). | + +Notes +- Only the tenant node is mirrored, not the role leaves (`/states/MH/admin`). Roles and permissions are read from the token, not from this table. +- Role leaves stay in Keycloak; `catalogs.instance_id` points at the tenant node. + +--- + +## 2. `users` + +Uploader, reviewer and actor identity, looked up once from the Keycloak token. + +Unique: `keycloak_user_id`; unique index on `lower(email)` where `email IS NOT NULL`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Internal user ID. | +| keycloak_user_id | TEXT | NOT NULL UNIQUE | Keycloak `sub` claim. `local-admin` when auth is disabled (not a UUID, so it cannot be the PK). | +| username | TEXT | | Current username. | +| email | TEXT | | Current email. | +| roles | TEXT[] | NOT NULL DEFAULT '{}' | Current roles. | +| instance_id | UUID | FK → `instances(id)` ON DELETE SET NULL | The user's instance (tenant). Single membership for now. | +| first_seen_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | First login or upload. | +| last_seen_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Most recent activity. | + +Notes +- Upserted on each authenticated upload or login. A `local-admin` row is seeded by the migration. +- A token can carry several tenants but `instance_id` holds one: `upsert_user(..., tenants=[...])` uses the first code in sorted order and logs a warning when there are several. Authorization still reads the token. +- If users later need several instances, add a `user_instances` join table and drop `instance_id`. +- Point-in-time identity is kept in `audit_logs.actor`. + +--- + +## 3. `catalogs` + +One row per provider instance and resource type. A catalog is what a provider publishes on the network: Mahavistar's `scheme` catalog and its `advisory` catalog are two rows. A new type adds rows, not tables. + +Unique: `(instance_id, type)`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Catalog ID. | +| instance_id | UUID | NOT NULL, FK → `instances(id)` ON DELETE RESTRICT | The provider that owns the catalog. | +| type | TEXT | NOT NULL, CHECK `^[a-z0-9]+:[A-Za-z0-9]+$` | Network schema/capability, e.g. `openagrinet:AgricultureResource`. | +| name | TEXT | NOT NULL | Catalog display name. Defaults to ` ` when created on upload. | +| metadata | JSONB | NOT NULL DEFAULT '{}', CHECK object | Beckn publish information. See keys below. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Created. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). | + +### `metadata` keys + +| Key | Type | Purpose | +|---|---|---| +| publish_directives | object | Beckn `publishDirectives`. | +| visible_networks | string[] | Networks the catalog is visible to. | +| criteria | object | Catalog criteria. | +| network | object `{catalog_id, last_published_at}` | Set once the catalog is published on the network. | + +Pydantic (`CatalogMetadata`) validates the keys; the database only checks for an object. + +Notes +- `type` stores the schema name, and code asks `pipeline/network_constants.py` what it means (`catalog_type_for_kind`, `kind_for_catalog_type`) instead of comparing `openagrinet:*` strings. The mapping is a placeholder: `scheme` is `openagrinet:AgricultureResource`, `advisory` is `openagrinet:KnowledgeAdvisory`, `video` is `openagrinet:VideoResource` (see ADR 0003 and ADR 0006). +- Every instance gets one catalog per supported type when the instance row is created (`upsert_instance` calls `ensure_instance_catalogs` in the same transaction). Startup (`init_db`) runs `sync_catalogs_for_all_instances`, so a type added to `KIND_TO_CATALOG_TYPE` reaches existing instances. Upload keeps `get_or_create_catalog` as a fallback for a catalog that is missing. Names are ` catalog`, for example `MH scheme catalog`. +- Computed catalog values (status, chunk count, network visibility, `promoted_at`) come from the `catalog_status` view, not stored here. + +--- + +## 4. `documents` + +One row per document: one resource (a scheme, an advisory) in one catalog, identified there by `code`. + +Unique: `(catalog_id, code)`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Document ID. | +| catalog_id | UUID | NOT NULL, FK → `catalogs(id)` ON DELETE RESTRICT | Catalog (provider instance + type) the document belongs to. | +| code | TEXT | NOT NULL, CHECK `^[a-z0-9]+(-[a-z0-9]+)*$` | Short code (`pm-kmy`) that catalog consumers see. Unique within the catalog. | +| name | TEXT | NOT NULL | Display name. | +| aliases | TEXT[] | NOT NULL DEFAULT '{}', CHECK no empty string | Ordered alternate names used to match queries. The first one is the prompt hint. | +| source | JSONB | NOT NULL, CHECK object, CHECK `source->>'fingerprint' ~ '^[0-9a-f]{32}$'` | The uploaded file. See shape below. | +| is_disabled | BOOLEAN | NOT NULL DEFAULT FALSE | Soft delete. Restore flips it back. | +| metadata | JSONB | NOT NULL, DEFAULT see below, CHECK object | Period, uploader, index state and flags. See keys below. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Row created. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). | + +Indexes: `(created_at DESC)`, `(catalog_id, (source->>'fingerprint'))`, plus the unique `(catalog_id, code)`. + +### `source` + +Facts about the uploaded file, not about processing. + +```json +{ + "fingerprint": "", + "original_filename": "pmkisan.pdf", + "storage_uri": "s3://...", + "source_type": "document", + "canonical_input_type": "pdf" +} +``` + +| Key | Type | Notes | +|---|---|---| +| fingerprint | string | MD5, 32 lowercase hex. Enforced by CHECK. Also the key the document has inside the vector index. | +| original_filename | string | Name as uploaded. | +| storage_uri | string | Location of the uploaded file. | +| source_type | string | `spreadsheet`, `document` or `unknown`. | +| canonical_input_type | string | `spreadsheet` or `pdf`. Normalized pipeline path. | + +There is no page count here: it is the number of `pages` rows. + +### `metadata` + +| Key | Type | Default | Purpose | +|---|---|---|---| +| valid_from | string `YYYY-MM-DD` | absent | Start of the period chunks are searchable. | +| valid_to | string `YYYY-MM-DD` | absent | End of the period. Absent means no end. Pydantic requires `valid_from <= valid_to`. | +| uploaded_by_user_id | string (users.id) | absent | Who uploaded. Not an FK, so a deleted user leaves a stale id; `audit_logs.actor_user_id` keeps real history. | +| index_status | object | `{}` | Vector index state per index. Keys `dev` and `prod`. See below. | +| catalog_visible | boolean | true | Counts toward the catalog. | +| network_visible | boolean | true | May be published to the Beckn network. | +| reindex_required | boolean | false | Approved content changed and the vector index is stale. | +| reindex_reason | string | absent | Why a reindex is needed. | +| chunking | object `{run_number, provider, model, config, created_at}` | absent | Settings of the run that produced the current chunks. | + +Column default: `'{"index_status": {}, "catalog_visible": true, "network_visible": true, "reindex_required": false}'`. + +`index_status` is one entry per vector index (DEV and PROD are separate Qdrant collections): + +```json +{ + "dev": {"status": "indexed", "chunk_count_indexed": 42, + "last_indexed_at": "2026-10-19T06:20:00Z", "last_verified_at": "2026-10-19T06:25:00Z", + "schema_version": "passage-v1", "details": {}}, + "prod": {"...": "..."} +} +``` + +- A missing key means the document was never ingested to that index. `status` is `unknown`, `indexed` or `missing`. +- Update one index atomically with `jsonb_set(metadata, '{index_status,dev}', $1::jsonb)`; the `index_status` key always exists, which `jsonb_set` needs. + +### Notes +- A document's stage is the `current_stage` of its latest workflow. List and filter queries join the `document_current_workflow` view. +- The API's `instance` is derived: `documents.catalog_id → catalogs.instance_id → instances.name`. There is no `documents.instance` column, and tenant filters join `instances`. +- The API's `document_kind` is derived from `catalogs.type` through the map in `network_constants.py`. `scheme_code`, `scheme_name` and `scheme_aliases` are `code`, `name` and `aliases`. +- `chunk_count` and `page_count` are counted from `chunks` and `pages`; they are not stored. +- A new document needs a `document_kind` and a `code`. Upload gets the catalog from the tenant and the kind (seeded with the instance; created when missing). A document's kind is fixed after upload, so there is no kind change. `DuplicateDocumentCode` is raised when the code is taken in that catalog. +- Re-uploading an existing code updates the same document: a new workflow starts, `source` is replaced, pages and chunks are cleared and `reindex_required` is set (`reset_document_for_reupload`). Earlier workflows and artifacts stay as history. +- Changing a document's kind moves it to the same instance's catalog of the new type (`update_document_fields(document_kind=...)`). +- SQL does not check types inside the JSONB columns. The insert path writes the default metadata keys, and Pydantic validates `source`, `metadata` and each `index_status` entry. +- Each document keeps its own `source`. There is no unique key on the fingerprint. +- Identifier lookup (`find_document_by_doc_identifier`) matches `documents.id`, `workflows.temporal_workflow_id`, `source->>'fingerprint'`, `source->>'original_filename'`, an index `doc_id` and the legacy `md5(source->>'fingerprint')`. +- No `deleted_at`: soft delete is `is_disabled`, hard delete is `purge_document`. + +--- + +## 5. `workflows` + +One row per processing run of a document: the main pipeline run, retries, reingest and PROD promotion. Holds all processing state. + +Indexes: `(document_id, started_at DESC)`, `(status, started_at DESC)`, `(temporal_workflow_id)`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Workflow ID. | +| document_id | UUID | NOT NULL, FK → `documents(id)` ON DELETE CASCADE | Document the workflow runs against. | +| workflow_type | TEXT | NOT NULL | Kind of run (`WorkflowType`). | +| temporal_workflow_id | TEXT | NOT NULL | Temporal workflow ID. `doc-` for the main run, suffixed for retries. Not unique (a restart reuses the ID). Used by the API, UI and Temporal. | +| status | TEXT | NOT NULL DEFAULT 'running' | `running`, `waiting_review`, `completed` or `failed` (`WorkflowStatus`). | +| current_stage | TEXT | NOT NULL DEFAULT 'registered' | Stage the workflow is in now (`DocumentStage`). This is the document's stage when the workflow is the latest. | +| stage_timestamps | JSONB | NOT NULL DEFAULT '{}', CHECK object with only keys `ocr`, `translation`, `chunks` | When each stage first completed in this run, ISO-8601 UTC. Set-once per key. | +| stop_after_ocr | BOOLEAN | NOT NULL DEFAULT FALSE | Halt after OCR (OCR-only batch runs). | +| error_message | TEXT | | Failure reason when status is `failed`. | +| config | JSONB | NOT NULL DEFAULT '{}', CHECK object | Settings the run used, plus `chunking_progress` while chunking. | +| started_by_user_id | UUID | FK → `users(id)` ON DELETE SET NULL | Who started it. For `promote_prod`, the person who requested promotion. NULL for system-started runs. | +| started_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | When the workflow started. Plays the role of `created_at`. | +| completed_at | TIMESTAMPTZ | | When it completed or failed. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). Progress updates touch the row often. | + +13 columns. + +Notes +- `stage_timestamps` is set-once: `stage_timestamps = jsonb_build_object('ocr', $1) || stage_timestamps` (right side wins, so existing keys are kept). +- Progress updates are atomic: `UPDATE ... SET config = config || jsonb_build_object('chunking_progress', ...)`. +- A PROD request inserts a `promote_prod` workflow in `waiting_review`, with `started_by_user_id` set to the requester and `started_at` to the request time. Approval moves it to `running`; when approval starts a `PromoteToProdWorkflow`, that run takes over the waiting row instead of adding a second one. +- The current workflow of a document is the latest by `started_at`. A retry started while an older run is still `running` takes over the stage display, and so does a retry on a `completed` document: its stage goes back to the retry's stage until the retry finishes. +- No CHECK ties `completed_at` to status: `reconcile_materialized_state` sets `waiting_review` without clearing `completed_at`. +- `cancelled` is not a status because nothing writes it. +- Do not copy the uploader snapshot into `config`; `started_by_user_id` and `audit_logs` cover it. + +### View: `document_current_workflow` + +Latest workflow per document. Stores nothing. + +```sql +SELECT DISTINCT ON (document_id) * +FROM workflows +ORDER BY document_id, started_at DESC, id DESC; +``` + +--- + +## 6. `workflow_artifacts` + +Ledger of every file a workflow produces or stores. Files live in MinIO; this table records where and what. Append-only. + +Indexes: `(workflow_id, created_at DESC)`, `(workflow_id, artifact_type, created_at DESC)`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Artifact ID, used in `/documents/{id}/artifacts/{artifact_id}`. | +| workflow_id | UUID | NOT NULL, FK → `workflows(id)` ON DELETE CASCADE | Workflow that produced it. | +| artifact_type | TEXT | NOT NULL, CHECK not blank | Kind of output (`original_upload`, `normalized_pdf`, `normalized_spreadsheet`, `ocr_pages_json`, `translation_pages_json`, `chunk_json_export`, `vector_payload_export`, network publish exchange). Open set, so not an enum. | +| stage | TEXT | | Stage that created it (`DocumentStage`). | +| storage_uri | TEXT | NOT NULL | MinIO location of the blob. | +| mime_type | TEXT | | MIME type. | +| filename | TEXT | | File name of the blob. | +| size_bytes | BIGINT | CHECK ≥ 0 | Blob size. | +| metadata | JSONB | NOT NULL DEFAULT '{}', CHECK object | Chunking config for chunk exports, request and response for network publishes, source path for uploads. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | When the artifact was recorded. | + +10 columns. + +Notes +- Every artifact belongs to a workflow. The upload path creates the first `pipeline` workflow before recording `original_upload`. +- The document is reached through `workflows.document_id`. Per-document listing: `SELECT a.* FROM workflow_artifacts a JOIN workflows w ON w.id = a.workflow_id WHERE w.document_id = $1 ORDER BY a.created_at DESC`. +- The latest original or normalized artifact uses the same join with `artifact_type = 'original_upload'` (or `normalized_%`). +- Deleting a workflow deletes its artifact rows. Blobs in MinIO are not removed by the database; cleanup stays in `purge_document`. +- No `updated_at`: rows are append-only. +- Chunking run history lives in `chunk_json_export` metadata. + +--- + +## 7. `pages` + +OCR layer and translations of a page. + +Unique: `(document_id, page_number)`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Page ID. | +| document_id | UUID | NOT NULL, FK → `documents(id)` ON DELETE CASCADE | Owning document. | +| page_number | INTEGER | NOT NULL, CHECK ≥ 1 | 1-based page position. | +| original_markdown | TEXT | | Raw OCR output, kept unchanged. | +| edited_markdown | TEXT | | Reviewer-corrected OCR text. Used instead of the original when present. | +| translations | JSONB | NOT NULL DEFAULT '{}', CHECK `jsonb_typeof(translations) = 'object'` | Translations of this page, keyed by target language. See shape below. | +| metadata | JSONB | NOT NULL DEFAULT '{"is_reviewed": false}', CHECK object | OCR review and language fields. See keys below. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | When the page row was first created. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). | + +### `metadata` keys + +| Key | Type | Default | Purpose | +|---|---|---|---| +| is_reviewed | boolean | false | OCR review done. | +| reviewer_notes | string | absent | Reviewer comments on the OCR text. | +| detected_language | string | absent | Language code detected. Non-`en` means the page needs translation. | + +SQL does not check these types; Pydantic does. The review stage gate counts reviewed pages with `(metadata->>'is_reviewed')::boolean`, and the translation step finds pages to translate with `metadata->>'detected_language'`. Add expression indexes on both if those queries are slow: `((metadata->>'is_reviewed')::boolean)` and `((metadata->>'detected_language'))`. + +### `translations` + +```json +{ + "gu": { + "translated_markdown": "...", + "edited_translation": null, + "is_reviewed": false, + "reviewer_notes": null, + "provider": "azure", + "model": "gpt-5.4-mini", + "translated_at": "2026-10-19T06:14:00Z" + } +} +``` + +- Key is the target language. Several languages per page are possible. +- Pages that need no translation (English) keep `'{}'`. An entry exists only for pages actually translated. +- `edited_translation` takes precedence over `translated_markdown` when present. +- Writes target one path, for example `SET translations = jsonb_set(translations, '{gu,edited_translation}', to_jsonb($1::text))`. +- Re-running OCR on a page resets its `translations` to `'{}'`, since the page text changed. +- Counting translated pages is `WHERE translations <> '{}'`. Finding unreviewed translations needs `jsonb_each` over the small per-page value. + +Notes +- `save_pages` replaces the whole row, so it must pass `translations` through unchanged unless the translation activity is writing it. Otherwise an OCR save would wipe translations. +- `save_pages` is `INSERT ... ON CONFLICT (document_id, page_number) DO UPDATE SET `. It is called during OCR segments, after OCR, after translation and at persist time, and each call replaces the whole row, so the caller passes every field. +- `save_pages` also passes `metadata` through whole. Update single keys with `metadata = metadata || jsonb_build_object('is_reviewed', true)` instead of rewriting the object. +- Page edits are audited in `audit_logs` (`page_edit`, `page_reset`), so no edit-history columns. +- Pages belong to a document, not to the source file: edits, review flags and OCR retries are per document. + +--- + +## 8. `chunks` + +Unique: `(document_id, chunk_number)`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Chunk ID. | +| document_id | UUID | NOT NULL, FK → `documents(id)` ON DELETE CASCADE | Owning document. | +| chunk_number | INTEGER | NOT NULL, CHECK ≥ 1 | Order within the document. | +| original_text | TEXT | NOT NULL | Chunk text as produced by the chunker. | +| edited_text | TEXT | | Reviewer-edited text. Indexed when present. | +| token_count | INTEGER | NOT NULL DEFAULT 0, CHECK ≥ 0 | Length in tokens. | +| page_start | INTEGER | NOT NULL DEFAULT 1, CHECK ≥ 1 | First page covered. Kept for display and provenance links. | +| page_end | INTEGER | NOT NULL DEFAULT 1, CHECK ≥ `page_start` | Last page covered. | +| source_spans | JSONB | NOT NULL DEFAULT '[]', CHECK array | List of `{page_number, start_char, end_char}` pointing back into the page text. | +| section_title | TEXT | | Heading of the section. | +| content_type | TEXT | | Kind of content (`heading`, `body`, ...). Only those two seen in the chunkers, so left as `TEXT`. | +| is_reference | BOOLEAN | NOT NULL DEFAULT FALSE | Reference material. Search can exclude it. | +| is_reviewed | BOOLEAN | NOT NULL DEFAULT FALSE | Reviewed by a human. | +| is_excluded | BOOLEAN | NOT NULL DEFAULT FALSE | Excluded from indexing. | +| reviewer_notes | TEXT | | Reviewer comments. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Created. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). | + +Notes +- Chunking provider, model, config and run number are stored once per document in `documents.metadata.chunking`, not per chunk. +- The distinct page numbers a chunk draws from are derived from `source_spans`. +- Re-chunk runs in one transaction: delete the document's chunks, insert the new ones, update `metadata.chunking` (with `run_number` incremented) (the chunk count is counted from `chunks`, not stored). Chunk UUIDs change on every re-chunk; nothing references `chunks.id` (audit uses chunk number). +- `search_chunks` does a substring match on `lower(coalesce(edited_text, original_text))`, served by a trigram GIN index on that expression (the migration creates the `pg_trgm` extension). + +> **Why `pages` and `chunks` stay separate tables:** a page is a physical unit from OCR and a chunk is a semantic unit from the chunker, so their boundaries do not align. One chunk can span several pages and one page can hold several chunks, which makes the relationship many-to-many. Storing page text inside each chunk would copy a page into every chunk it overlaps, so any page edit, OCR retry or translation change would have to be rewritten in all of those chunks. + + +--- + +## 9. `audit_logs` + +Append-only history of actions. Must outlive documents, so there is no cascade delete. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Audit entry ID. | +| document_id | UUID | FK → `documents(id)` ON DELETE SET NULL | Document the action applied to. NULL for system-level entries and after a purge. | +| workflow_id | TEXT | | Temporal workflow ID snapshot. Survives a purge. | +| instance_id | UUID | FK → `instances(id)` ON DELETE SET NULL | Provider instance snapshot, so tenant scoping works after the document is gone. | +| action_type | TEXT | NOT NULL, CHECK not blank | What happened (`page_edit`, `purge_document`, ...). 22 values and growing, so not an enum. | +| entity_type | TEXT | | `document`, `page`, `chunk`, `setting` or `catalog` (`AuditEntityType`). | +| entity_number | INTEGER | | Page number or chunk number the action applied to. | +| field_name | TEXT | | Field that changed. | +| old_value | TEXT | | Value before the change. | +| new_value | TEXT | | Value after the change. | +| metadata | JSONB | NOT NULL DEFAULT '{}', CHECK object | Extra context. | +| actor_user_id | UUID | FK → `users(id)` ON DELETE SET NULL | Who acted. NULL for system actions. | +| actor | JSONB | NOT NULL DEFAULT '{}', CHECK object | Identity at the time: `{username, email, roles[]}`. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | When it happened. | + +Table CHECK: `entity_type IN ('setting','catalog') OR workflow_id IS NOT NULL`. + +Indexes: `(document_id, created_at DESC)`, `(instance_id, created_at DESC)`, `(created_at DESC)`, `(action_type, created_at DESC)`, `(created_at DESC) WHERE entity_type = 'setting'`. + +Notes +- The FK to `documents` is `SET NULL`, not cascade. A cascade would erase the audit trail on purge, and a strict FK would reject setting and catalog entries that have no document. +- Setting and catalog entries have NULL `document_id` and `workflow_id`, with `entity_type` set to `setting` or `catalog`. +- `instance_id` is a snapshot of the document's provider instance, so a purged document's history stays visible only to its own tenant. +- `entity_number` holds a page or chunk number, not an ID (chunk UUIDs would dangle after a re-chunk). +- `get_settings_audit_logs` is `WHERE entity_type = 'setting'`. +- `log_audit` fills `instance_id`, resolves the actor to a `users` row and builds the `actor` JSON. +- No `updated_at`: entries are immutable. +- The table only grows. Monthly partitioning is possible later without changing this design. + +--- + +## 10. `search_settings` + +Single row of search configuration. + +Unique: `singleton`. + +| Column | Type | Constraints / Default | Keys inside | Purpose | +|---|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | | Row ID. | +| singleton | BOOLEAN | NOT NULL DEFAULT TRUE UNIQUE, CHECK `singleton` | | Guarantees only one row. | +| retrieval | JSONB | NOT NULL DEFAULT '{}', CHECK object | `searchMethod`, `alpha`, `rankingMethod`, `hybridRrfK` | How semantic and keyword matching are combined and ranked. | +| candidates | JSONB | NOT NULL DEFAULT '{}', CHECK object | `indexName`, `efSearch`, `candidateCap`, `candidateMultiplier` | Which vector index is searched and how many candidates are pulled. | +| results | JSONB | NOT NULL DEFAULT '{}', CHECK object | `limit`, `maxChunksPerDoc`, `excludeReference`, `showHighlights`, `rerankMode` | How the final result list is shaped. | +| query | JSONB | NOT NULL DEFAULT '{}', CHECK object | `useE5Prefix`, `queryExpansionProfile` | How the query is prepared before searching. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | | Created. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | | Last change (trigger). | + +15 settings in four groups (4 + 4 + 5 + 2). All are global, with no per-tenant scope. + +Notes +- The database enforces only "is a JSON object". Value rules live in Pydantic: `SearchSettings` has four nested sub-models (`alpha` 0 to 1, integers ≥ 1, `searchMethod` in `TENSOR|LEXICAL|HYBRID`, `rankingMethod` in `rrf|normalize_linear`, unknown keys rejected). +- The API stays flat. `db.py` keeps one dict mapping each flat key to its group; reads flatten the columns and writes route each key to its column. +- Defaults live only in the Pydantic models. Reads merge stored JSON over the defaults. The default for `alpha` is 0.6. +- Partial update is an atomic merge per group: `UPDATE search_settings SET retrieval = retrieval || $1::jsonb`. +- Audit: one `audit_logs` row per changed key, `entity_type = 'setting'`, `field_name` such as `retrieval.alpha`. +- `rerankMode` has at least `none` and `bm25lite`; the full set was not seen, so no enum. +- Adding a setting to an existing group needs no migration; a new group needs a column. + +--- + +## 11. `email_otps` + +Server-side email-OTP login codes. Short-lived. + +Unique: `email`; index on `(expires_at)`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Row ID. | +| email | TEXT | NOT NULL UNIQUE, CHECK `email = lower(email)` | Address the code was requested for. One live code per email. | +| code_hash | TEXT | NOT NULL | Hash of the one-time code. The plain code is never stored. Random for unknown accounts. | +| expires_at | TIMESTAMPTZ | NOT NULL | When the code stops being valid. | +| attempts | INTEGER | NOT NULL DEFAULT 0, CHECK ≥ 0 | Failed verifications. Reset to 0 on resend. | +| last_sent_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | When the code was last sent. Used for the resend cooldown. | + +Notes +- No `created_at` or `updated_at`: rows are overwritten on each send. +- `email` is deliberately not an FK to `users`. A row is written for every request, even for unknown addresses (anti-enumeration, `email_otp.py:276-310`). +- Periodic cleanup (`DELETE FROM email_otps WHERE expires_at < now() - interval '1 day'`) is not scheduled yet. The endpoint is unauthenticated, so unverified rows stay until it is. +- `verify_otp` increments attempts atomically: `UPDATE ... SET attempts = attempts + 1 WHERE email = $1 RETURNING attempts`, or `SELECT ... FOR UPDATE`. +- Code constants (length 6, TTL 300 s, resend cooldown 30 s, max 5 attempts) stay in code. +- Optional: make the table `UNLOGGED`. Losing it in a crash only forces users to request a new code. + +--- + +## View: `catalog_status` + +Computed from `documents`, `catalogs`, `instances`, `workflows` and `document_current_workflow`. Stores nothing. One row per document. The `/catalog/v1/*` snapshot API reads it for the scheme type (`catalogs.type` of the scheme kind), and its fields are named `scheme_code`, `scheme_name` and so on, so consumers see no change. + +| Column | Type | How it is computed | +|---|---|---| +| document_id | UUID | `documents.id`. | +| catalog_id | UUID | `documents.catalog_id`. | +| instance_id | UUID | `catalogs.instance_id`. | +| instance_name | TEXT | `instances.name`. | +| type | TEXT | `catalogs.type`. | +| code | TEXT | `documents.code`. | +| name | TEXT | `documents.name`. | +| aliases | TEXT[] | `documents.aliases`. | +| status | TEXT | `pending_reindex` if the document is live and `metadata.reindex_required` is true. Else `live` if live. Else `pending_prod` if the latest workflow's stage is `approval_for_prod` and the document is not disabled. Else `disabled`. | +| workflow_id | TEXT | `temporal_workflow_id` of the document's earliest `pipeline` workflow. | +| chunk_count | INTEGER | Count of the document's `chunks`. | +| network_visible | BOOLEAN | `metadata.network_visible`, true when absent. | +| promoted_at | TIMESTAMPTZ | `metadata.index_status.prod.last_indexed_at`, only for live documents. | + +"Live" means the latest workflow's `current_stage = 'completed'`, `is_disabled` is false and `metadata.catalog_visible` is true. There are no bootstrap rows: a scheme appears only once a document exists. + +--- + +## 12. `webhook_records` + +Records pushed to `POST /webhook` (`x-api-key` = `WEBHOOK_API_KEY`). Carried over from the previous store unchanged; not part of the instances and catalogs model. + +Unique: `(source, external_id)`; index on `(source, status)`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Record ID. | +| source | TEXT | NOT NULL DEFAULT 'webhook' | Sender. | +| external_id | TEXT | NOT NULL | `contentID`, else `id`, else `slug`, else a hash of the payload. | +| title | TEXT | | From the payload. | +| slug | TEXT | | From the payload. | +| status | TEXT | | From the payload. | +| is_published | BOOLEAN | | From `isPublished`. | +| category | TEXT | | From `domainName`. | +| payload | JSONB | NOT NULL, CHECK object | The full record. | +| payload_hash | TEXT | | SHA-256 of the canonical payload; an upsert with the same hash writes nothing. | +| indexed_hash | TEXT | | `payload_hash` last written to the DEV collection (migration 0002). | +| promoted_hash | TEXT | | `payload_hash` last written to the PROD collection (migration 0002). | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | The payload's `createdAt` when it has one. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | The payload's `updatedAt` when it has one. | + +The payload's own timestamps are stored as given, so these two columns are not maintained by the shared trigger. + +--- + +## `schema_migrations` + +Created by the runner, not by a migration file: `version` (the `NNNN` prefix, primary key), `name` (the file name) and `applied_at`. The runner takes a session advisory lock, so concurrent api and worker starts apply each file once, and runs each file in its own transaction. + +--- + +## Implementation notes + +- **Identifiers.** `db.py` functions take a `workflow_id` argument that may be the document UUID, a Temporal workflow id, the fingerprint, the filename, an index `doc_id` or the legacy `md5(fingerprint)` doc id (`find_document_by_doc_identifier`). Documents, jobs and artifacts come back as flat dicts; `id` is the UUID and `workflow_id` is the Temporal workflow id of the document's main run (the earliest `pipeline` or `ocr_only` workflow). The API rewrites `/documents/{id}/...` to that workflow id before routing, because handlers signal and query Temporal with it. +- **Jobs.** The API's "jobs" and "runs" are `workflows` rows; job ids and artifact ids are the UUIDs of `workflows` and `workflow_artifacts`. `latest_job_id`, `original_artifact_id` and `normalized_artifact_id` are derived by join, not stored. +- **Instance.** The API returns an `instance` string on documents, runs, queue rows and audit entries. It is the lowercase `instances.name` reached through `catalogs`; filters compare `lower(instances.name)`. +- **Catalog version.** `/catalog/v1/version` counts `audit_logs` rows with `entity_type = 'catalog'` and `action_type = 'catalog_version_bump'`, so it only grows. `updated_at` is the newest such row. +- **Document stage.** The stage a document shows is `document_current_workflow.current_stage`; `update_document_stage` writes to that workflow. +- **Page translations.** `save_pages` keeps stored translations unless the page carries `translations` (replaces them) or flat `translated_*` fields (merged into `translation_target_language`). A page whose OCR text changed loses its translations. +- **Users.** Rows are upserted from the auth dependency on write requests (at most once per five minutes per user) and from uploads, workflow starts and audit entries. `local-admin` is seeded by the migration. A user's `instance_id` is set from the tenant codes in the token. diff --git a/docs/SQLITE_SCHEMA.md b/docs/SQLITE_SCHEMA.md new file mode 100644 index 0000000..c3fa539 --- /dev/null +++ b/docs/SQLITE_SCHEMA.md @@ -0,0 +1,374 @@ +# SQLite Schema — knowledge-provider (legacy) + +> Legacy. The store is now Postgres, described in `docs/POSTGRES_SCHEMA.md`. This file documents the shape of the old SQLite file that `scripts/migrate_sqlite_to_postgres.py` reads. + +## Overview + +- One SQLite file is the canonical store for metadata, review state and editable content. MinIO holds blobs, Qdrant holds search projections, Temporal only orchestrates. +- 12 tables. `documents` is the root. `pages`, `chunks`, `document_jobs`, `document_artifacts`, `document_index_status` and `audit_logs` hang off it by `workflow_id`. +- Connection settings: WAL journal mode, `foreign_keys=ON`, `IMMEDIATE` transactions, 30s lock timeout. Writes are serialized by a process-local lock. +- Schema is created on startup. Changes are applied as idempotent `ALTER TABLE ... ADD COLUMN` calls, so the tables below show the final shape with migration-added columns merged in. +- Timestamps are ISO-8601 strings in TEXT. Booleans are INTEGER 0/1. Columns ending in `_json` hold JSON-encoded TEXT. +- `workflow_id` is the key used by the API, UI and Temporal. + +## Drawbacks + +### Integrity +- **No foreign keys.** Tables are linked by convention on `workflow_id`. Nothing stops orphan rows, and purging a document means deleting from each table by hand. +- **Pointer columns are unenforced.** `latest_job_id`, `original_artifact_id` and `normalized_artifact_id` on `documents` point into tables that point back, with no constraint. +- **No constraints on values.** Stages, statuses, job types, routing and flags are free TEXT or INTEGER. `created_at` and `updated_at` are nullable. Nothing checks that `valid_from <= valid_to` or that counts are non-negative. +- **Audit rows use sentinel strings** (`__system__`, `__settings__`, `__catalog__`) in key columns for non-document events, which a real foreign key could not hold. +- **Settings are untyped.** Every value is TEXT and cast on read. Nothing stops an out-of-range `alpha` or an invalid search method. Defaults disagree across three places (`alpha` is 0.7 in the model, 0.6 in the seed and reset). + +### Data modeling +- **Same fact stored in several places.** + - Uploader username, email and roles are copied onto each document and again into the audit log. + - Scheme name, aliases and routing are copied onto every document of a scheme. + - The scheme-to-document relationship is stored twice: a JSON list on the catalog entry and `scheme_code` on the document. + - The file's MD5 is stored as `document_id`, `canonical_document_id` and `source_file_fingerprint`; `filename` equals `source_filename`. +- **`documents` is too wide.** About 45 columns mix file facts, processing state, uploader identity, PROD request, validity and scheme data. +- **JSON in TEXT.** Lists and objects (`scheme_aliases_json`, `workflow_ids_json`, `source_page_numbers_json`, `config_json`, `details_json`) cannot be validated or queried. +- **Run metadata repeated on every chunk.** Chunking provider, model, config, run ID and version are identical for all chunks of a run. +- **Translation columns sit on `pages`.** Eight mostly-NULL columns for pages that need no translation, and only one target language per page. +- **Roles are comma-separated strings**, with no array semantics. +- **Dead or constant columns.** `marqo_doc_id`, `temporal_run_id`, `content_hash`, `collection_name` and `tool_routing` carry no used data. The manifest table has no writer in the code. +- **Hardcoded language.** The manifest table has a Gujarati title column and the search settings default to a Gujarati query profile. +- **`entity_id` in the audit log is not an ID.** It holds a page or chunk number. +- **Catalog table is a cache** that is fully recomputed from documents and can drift. + +## Tables + +| # | Table | Purpose | +|---|-------|---------| +| 1 | `documents` | One row per ingested document. Root table, holds stage and metadata. | +| 2 | `pages` | OCR and translated page content plus review state. | +| 3 | `chunks` | Chunked text plus review state. | +| 4 | `audit_logs` | Per-document change and action history. | +| 5 | `settings` | Key/value app config, including search defaults. | +| 6 | `document_jobs` | Temporal job runs per document. | +| 7 | `document_artifacts` | Pointers to blobs stored in MinIO. | +| 8 | `document_index_status` | Vector index state per document and index. | +| 9 | `document_manifest_entries` | Imported manifest (CSV) metadata. | +| 10 | `scheme_catalog_meta` | Single-row catalog version counter. | +| 11 | `scheme_catalog_entries` | Scheme master catalog (served by `/catalog/v1/*`). | +| 12 | `email_otps` | Email OTP login codes. | + +## 1. documents + +Primary key: `workflow_id` + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| workflow_id | TEXT | PRIMARY KEY | Temporal workflow ID. Unique key for the document and the join key used by every other table. | +| document_id | TEXT | NOT NULL | Document identifier carried into chunk payloads and index records. | +| canonical_document_id | TEXT | | Stable ID shared across re-uploads or re-ingestions of the same document. | +| filename | TEXT | NOT NULL | Name of the file as stored by the pipeline. | +| display_name | TEXT | | Human-friendly title shown in the UI. | +| source_filename | TEXT | | Original filename as uploaded by the user. | +| source_manifest_name | TEXT | | Filename of the matching entry in `document_manifest_entries`, used to link imported metadata. | +| source_file_fingerprint | TEXT | | Hash of the source file, used to detect duplicate uploads. | +| filepath | TEXT | NOT NULL | Local path of the uploaded file. | +| stage | TEXT | NOT NULL DEFAULT 'registered' | Current pipeline stage (registered, ocr_processing, ocr_review, translation_processing, translation_review, chunking, chunk_review, ready_for_ingestion, ingesting, approval_for_prod, ingesting_prod, completed, failed). | +| page_count | INTEGER | DEFAULT 0 | Number of pages extracted by OCR. | +| chunk_count | INTEGER | DEFAULT 0 | Number of chunks produced. | +| error_message | TEXT | | Failure reason when the stage is `failed`. | +| is_demo | INTEGER | DEFAULT 0 | Marks the document as demo content so it can be filtered out of summaries and index lists. | +| is_disabled | INTEGER | DEFAULT 0 | Soft-disable flag. A disabled document is hidden from lists and search. | +| created_at | TEXT | | When the row was registered. | +| updated_at | TEXT | | Last time the row changed. | +| ocr_completed_at | TEXT | | When OCR finished. | +| translation_completed_at | TEXT | | When translation finished. | +| chunks_completed_at | TEXT | | When chunking finished. | +| ingested_at | TEXT | | When the document was ingested into the vector index. | +| source_type | TEXT | | Input family derived from the file extension: `spreadsheet` (csv/xlsx), `document` (pdf, image, office) or `unknown`. | +| canonical_input_type | TEXT | | Form the input is normalized to: `spreadsheet` or `pdf`. Drives which pipeline path runs. | +| stop_after_ocr | INTEGER | DEFAULT 0 | When 1, the workflow halts after OCR (OCR-only batch runs). | +| reindex_required | INTEGER | DEFAULT 0 | Flags that approved content changed and the vector index is stale. | +| reindex_reason | TEXT | | Why a reindex is needed. | +| original_artifact_id | INTEGER | | Points to the `document_artifacts` row for the original upload. | +| normalized_artifact_id | INTEGER | | Points to the `document_artifacts` row for the normalized PDF or spreadsheet. | +| latest_job_id | INTEGER | | Points to the most recent `document_jobs` row. | +| instance | TEXT | | Tenant or state code (`bv` is platform-wide, others are state codes such as `mh`). Used for scoping and list filters. NULL or empty is backfilled with `DEFAULT_INSTANCE`. | +| uploaded_by_user_id | TEXT | | Keycloak user ID of the uploader. | +| uploaded_by_username | TEXT | | Username of the uploader. | +| uploaded_by_email | TEXT | | Email of the uploader. | +| uploaded_by_roles | TEXT | | Comma-separated roles of the uploader at upload time. | +| prod_ready_requested_at | TEXT | | When someone requested promotion to PROD. | +| prod_ready_requested_by_user_id | TEXT | | User ID of the PROD promotion requester. | +| prod_ready_requested_by_username | TEXT | | Username of the PROD promotion requester. | +| valid_from | TEXT | | Start of the period in which the document's chunks are searchable. Stamped with the upload day, editable by the approver. NULL means always current. | +| valid_to | TEXT | | End of the validity period. NULL means no end. | +| document_kind | TEXT | DEFAULT 'document' | Knowledge kind slug: `document`, `scheme`, `advisory`, `video` or a custom slug. Decides which Beckn catalog is announced. | +| scheme_code | TEXT | | Scheme identifier (for example `pmkisan`). Used when `document_kind` is `scheme`. Links to `scheme_catalog_entries`. | +| scheme_name | TEXT | | Human-readable scheme name. | +| scheme_aliases_json | TEXT | | JSON list of alternate names for the scheme, used for matching queries. | +| tool_routing | TEXT | | Which AI tool path serves the scheme: `qdrant`, `legacy` or `both`. | +| catalog_visible | INTEGER | DEFAULT 1 | Whether the document appears in the pull-based catalog API. | +| network_visible | INTEGER | DEFAULT 1 | Whether the document may be published to the Beckn network (Discovery Service). | + +Indexes: + +| Index | Columns | Purpose | +|-------|---------|---------| +| idx_documents_stage | stage | Filter and count documents by stage. | +| idx_documents_instance | instance | Tenant-scoped lists. | +| idx_documents_created | created_at DESC | Newest-first listing. | +| idx_documents_scheme_code | scheme_code | Look up documents by scheme. | + +## 2. pages + +Primary key: `id`. Unique: `(workflow_id, page_number)`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| id | INTEGER | PRIMARY KEY AUTOINCREMENT | Row ID. | +| workflow_id | TEXT | NOT NULL | Owning document (`documents.workflow_id`). | +| page_number | INTEGER | NOT NULL | 1-based page position in the document. | +| original_markdown | TEXT | | Raw OCR output for the page, kept unchanged. | +| edited_markdown | TEXT | | Reviewer-corrected OCR text. Takes precedence over the original when present. | +| is_reviewed | INTEGER | DEFAULT 0 | Whether the OCR text for this page has been reviewed. | +| reviewer_notes | TEXT | | Reviewer comments on the OCR text. | +| detected_language | TEXT | | Language detected for the page (via the `lang-detect` service). | +| translated_markdown | TEXT | | Machine translation of the page. | +| edited_translation | TEXT | | Reviewer-corrected translation. | +| translation_reviewed | INTEGER | DEFAULT 0 | Whether the translation has been reviewed. | +| translation_notes | TEXT | | Reviewer comments on the translation. | +| translation_provider | TEXT | | Provider that produced the translation. | +| translation_model | TEXT | | Model used for the translation. | +| translation_target_language | TEXT | | Language the page was translated into. | +| translated_at | TEXT | | When the translation was produced. | + +Indexes: `idx_pages_workflow (workflow_id)` — fetch all pages of a document. + +## 3. chunks + +Primary key: `id`. Unique: `(workflow_id, chunk_number)`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| id | INTEGER | PRIMARY KEY AUTOINCREMENT | Row ID. | +| workflow_id | TEXT | NOT NULL | Owning document (`documents.workflow_id`). | +| chunk_number | INTEGER | NOT NULL | Order of the chunk within the document. | +| original_text | TEXT | | Chunk text as produced by the chunker. | +| edited_text | TEXT | | Reviewer-edited chunk text. This is what gets indexed when present. | +| token_count | INTEGER | DEFAULT 0 | Token length of the chunk. | +| page_start | INTEGER | DEFAULT 1 | First page the chunk covers. | +| page_end | INTEGER | DEFAULT 1 | Last page the chunk covers. | +| source_page_numbers_json | TEXT | | JSON list of every page the chunk draws from. | +| source_spans_json | TEXT | | JSON list of text spans in the source pages, used to trace a chunk back to its provenance. | +| section_title | TEXT | | Heading of the section the chunk belongs to. | +| content_type | TEXT | | Kind of content in the chunk, for example `heading` or `body`. | +| is_reference | INTEGER | DEFAULT 0 | Marks reference material. Search can exclude these (`search_exclude_reference`). | +| chunking_provider | TEXT | | Chunking provider that produced the chunk. | +| chunking_model | TEXT | | Model used for chunking, if any. | +| chunking_config_json | TEXT | | JSON snapshot of the chunking settings used. | +| chunking_run_id | TEXT | | ID of the chunking run, to group chunks created together. | +| chunk_version | INTEGER | DEFAULT 1 | Version counter for the chunk. | +| is_reviewed | INTEGER | DEFAULT 0 | Whether the chunk has been reviewed. | +| is_excluded | INTEGER | DEFAULT 0 | Reviewer excluded the chunk. Excluded chunks are not indexed. | +| reviewer_notes | TEXT | | Reviewer comments on the chunk. | + +Indexes: `idx_chunks_workflow (workflow_id)` — fetch all chunks of a document. + +## 4. audit_logs + +Primary key: `id`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| id | INTEGER | PRIMARY KEY AUTOINCREMENT | Row ID. | +| workflow_id | TEXT | NOT NULL | Document the action applied to. | +| document_id | TEXT | NOT NULL | Document identifier at the time of the action. | +| action_type | TEXT | NOT NULL | What happened, for example `promote_to_prod`. | +| entity_type | TEXT | | Kind of record changed, for example `document`, page or chunk. | +| entity_id | INTEGER | | ID or number of the record changed. | +| field_name | TEXT | | Field that was changed. | +| old_value | TEXT | | Value before the change. | +| new_value | TEXT | | Value after the change. | +| metadata | TEXT | | Extra context, stored as JSON text. | +| timestamp | TEXT | NOT NULL | When the action happened. | +| actor_user_id | TEXT | | Keycloak user ID of whoever acted. | +| actor_username | TEXT | | Username of the actor. | +| actor_email | TEXT | | Email of the actor. | +| actor_roles | TEXT | | Roles of the actor at the time. | + +Indexes: + +| Index | Columns | Purpose | +|-------|---------|---------| +| idx_audit_workflow | workflow_id | Per-document audit trail. | +| idx_audit_timestamp | timestamp DESC | Newest-first global audit log. | + +## 5. settings + +Primary key: `key`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| key | TEXT | PRIMARY KEY | Setting name. | +| value | TEXT | NOT NULL | Setting value, always stored as text. | +| description | TEXT | | What the setting does. | +| updated_at | TEXT | NOT NULL | When the setting last changed. | + +Default rows (`INSERT OR IGNORE`): + +| key | default value | Purpose | +|-----|---------------|---------| +| search_method | HYBRID | Search mode: TENSOR, LEXICAL or HYBRID. | +| search_limit | 12 | Number of search results to return. | +| search_alpha | 0.6 | Hybrid weighting: 0 is lexical only, 1 is semantic only. | +| search_ranking_method | rrf | Hybrid ranking: `rrf` or `normalize_linear`. | +| search_show_highlights | true | Show highlighted matches in results. | +| search_ef_search | 256 | HNSW search accuracy parameter. | +| search_index_name | documents-index | Default vector DB collection name. | +| search_candidate_cap | 120 | Cap on the candidate retrieval pool. | +| search_candidate_multiplier | 10 | Candidate pool multiplier before the final cut. | +| search_max_chunks_per_doc | 2 | Result diversity cap per document. | +| search_use_e5_prefix | true | Prefix queries with `query:` for e5 embedding models. | +| search_exclude_reference | true | Exclude reference chunks when the index supports it. | +| search_query_expansion_profile | gu-v1 | Query expansion profile. | +| search_rerank_mode | none | Post-search reranking mode. | +| search_hybrid_rrfk | 60 | RRF tuning parameter for hybrid search. | + +## 6. document_jobs + +Primary key: `id`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| id | INTEGER | PRIMARY KEY AUTOINCREMENT | Job ID. Referenced by `documents.latest_job_id` and `document_artifacts.job_id`. | +| workflow_id | TEXT | NOT NULL | Document the job ran for. | +| job_type | TEXT | NOT NULL | Kind of run: `ocr_only`, `ocr_retry`, `translation_retry`, `chunking_retry`, `reingest`, `promote_prod`, plus the main pipeline run. | +| temporal_workflow_id | TEXT | | Temporal workflow ID, to find the run in the Temporal UI. | +| temporal_run_id | TEXT | | Temporal run ID. | +| status | TEXT | NOT NULL DEFAULT 'running' | Job state. | +| current_stage | TEXT | | Stage the job is in right now. | +| started_at | TEXT | NOT NULL | When the job started. | +| completed_at | TEXT | | When the job ended. | +| error_message | TEXT | | Failure reason. | +| config_json | TEXT | | JSON snapshot of the settings the job ran with. | + +Indexes: `idx_document_jobs_workflow (workflow_id, started_at DESC)` — latest jobs for a document. + +## 7. document_artifacts + +Primary key: `id`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| id | INTEGER | PRIMARY KEY AUTOINCREMENT | Artifact ID. Referenced by `documents.original_artifact_id` and `normalized_artifact_id`. | +| workflow_id | TEXT | NOT NULL | Owning document. | +| job_id | INTEGER | | Job that produced the artifact (`document_jobs.id`). | +| artifact_type | TEXT | NOT NULL | Kind of output, for example `original_upload`, `normalized_pdf`, `normalized_spreadsheet`, `ocr_pages_json`, `translation_pages_json`, `chunk_json_export`, `vector_payload_export`. | +| stage | TEXT | | Pipeline stage that created it. | +| storage_uri | TEXT | NOT NULL | MinIO location of the blob. | +| mime_type | TEXT | | MIME type of the blob. | +| filename | TEXT | | File name of the blob. | +| size_bytes | INTEGER | | Blob size in bytes. | +| metadata_json | TEXT | | Extra details as JSON, for example the source path or input type. | +| created_at | TEXT | NOT NULL | When the artifact was recorded. | + +Indexes: `idx_document_artifacts_workflow (workflow_id, created_at DESC)` — artifacts of a document, newest first. + +## 8. document_index_status + +Primary key: `(workflow_id, index_name)`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| workflow_id | TEXT | NOT NULL | Document that was indexed. | +| index_name | TEXT | NOT NULL | Vector index or collection (for example DEV or PROD). One row per document and index. | +| doc_id | TEXT | | Document ID as stored in the vector index. Replaces legacy `marqo_doc_id`. | +| chunk_count_indexed | INTEGER | DEFAULT 0 | Number of chunks currently in the index. | +| last_indexed_at | TEXT | | When the document was last written to the index. | +| last_verified_at | TEXT | | When the index state was last checked against the vector store. | +| schema_version | TEXT | | Index payload schema version, for spotting stale payloads. | +| status | TEXT | NOT NULL DEFAULT 'unknown' | Index state, for example `indexed` or `unknown`. | +| details_json | TEXT | | Extra details as JSON. | + +Older DBs may still carry `marqo_doc_id` until a cleanup script is run. Startup backfills `doc_id` from it when present. + +## 9. document_manifest_entries + +Primary key: `id`. Unique: `manifest_filename`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| id | INTEGER | PRIMARY KEY AUTOINCREMENT | Row ID. | +| manifest_filename | TEXT | NOT NULL UNIQUE | Filename the entry describes. Matches `documents.source_manifest_name`. | +| title_en | TEXT | | English title. | +| title_gu | TEXT | | Gujarati title. | +| doc_language | TEXT | | Language of the document. | +| category_tags | TEXT | | Category tags for the document. | +| description | TEXT | | Short description. | +| quality_score | TEXT | | Quality rating from the manifest. | +| priority_rank | TEXT | | Ingestion priority from the manifest. | +| ingestion_status | TEXT | | Ingestion status recorded in the manifest. | +| feedback | TEXT | | Reviewer feedback from the manifest. | +| source_csv_path | TEXT | | CSV file the row was imported from. | +| imported_at | TEXT | NOT NULL | When the row was imported. | + +Indexes: `idx_manifest_entries_filename (manifest_filename)` — look up by filename. + +## 10. scheme_catalog_meta + +Single-row table, enforced by `CHECK (id = 1)`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| id | INTEGER | PRIMARY KEY, CHECK (id = 1) | Fixed to 1 so only one row can exist. | +| version | INTEGER | NOT NULL DEFAULT 0 | Catalog version counter. Bumped on catalog changes so pull clients can detect updates. | +| updated_at | TEXT | | When the version last changed. | +| notes | TEXT | | Reason for the last bump. | + +## 11. scheme_catalog_entries + +Primary key: `scheme_code`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| scheme_code | TEXT | PRIMARY KEY | Scheme identifier (for example `pmkisan`). | +| scheme_name | TEXT | | Human-readable scheme name. | +| scheme_aliases_json | TEXT | | JSON list of alternate names for matching queries. | +| instances_json | TEXT | | JSON list of instances (tenants) that have this scheme. | +| workflow_ids_json | TEXT | | JSON list of `documents.workflow_id` values that back the scheme. | +| collection_name | TEXT | | Vector collection holding the scheme's chunks. | +| chunk_count | INTEGER | DEFAULT 0 | Number of chunks across the scheme's documents. | +| status | TEXT | NOT NULL DEFAULT 'disabled' | Catalog state: `live`, `pending_prod`, `pending_reindex` or `disabled`. Only live schemes are served as vector schemes. | +| network_visible | INTEGER | DEFAULT 0 | Whether the scheme is exposed on the network. | +| promoted_at | TEXT | | When the scheme was promoted to PROD. | +| content_hash | TEXT | | Hash of code, name, aliases, visibility and status, used to detect changes. | +| source | TEXT | | Where the entry came from: `pipeline` (built from scheme documents) or `import` (bootstrapped). | +| updated_at | TEXT | | When the entry last changed. | + +Indexes: `idx_scheme_catalog_status (status)` — filter entries by status. + +## 12. email_otps + +Primary key: `email`. One live code per email, overwritten on resend. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| email | TEXT | PRIMARY KEY | Email the code was sent to. | +| code_hash | TEXT | NOT NULL | Hash of the one-time code. The plain code is never stored. | +| expires_at | TEXT | NOT NULL | Expiry time of the code. | +| attempts | INTEGER | NOT NULL DEFAULT 0 | Failed verification attempts. Reset to 0 on resend. | +| created_at | TEXT | NOT NULL | When the code was first issued. | +| last_sent_at | TEXT | NOT NULL | When the code was last sent. Used to throttle resends. | + +## Relationships (by convention, not enforced) + +| From | Column | To | +|------|--------|----| +| pages | workflow_id | documents.workflow_id | +| chunks | workflow_id | documents.workflow_id | +| audit_logs | workflow_id | documents.workflow_id | +| document_jobs | workflow_id | documents.workflow_id | +| document_artifacts | workflow_id | documents.workflow_id | +| document_artifacts | job_id | document_jobs.id | +| document_index_status | workflow_id | documents.workflow_id | +| documents | original_artifact_id, normalized_artifact_id | document_artifacts.id | +| documents | latest_job_id | document_jobs.id | +| documents | source_manifest_name | document_manifest_entries.manifest_filename | +| documents | scheme_code | scheme_catalog_entries.scheme_code | +| scheme_catalog_entries | workflow_ids_json (JSON list) | documents.workflow_id | diff --git a/docs/SYSTEM_DESIGN.md b/docs/SYSTEM_DESIGN.md index d659603..7aa88e4 100644 --- a/docs/SYSTEM_DESIGN.md +++ b/docs/SYSTEM_DESIGN.md @@ -36,10 +36,10 @@ gate before **PROD**. |---|---| | Multilingual document → searchable chunks | OCR → translate → chunk → embed → index | | Human quality control | Temporal **signals** pause at OCR / translation / chunk / pre-ingest | -| Multi-state tenancy | `documents.instance` + JWT groups; lists/search scoped | +| Multi-state tenancy | Catalog-owned instance + JWT groups; document access scoped | | DEV before PROD | Stage 9 writes DEV index; stage 10 waits; Super Admin promotes to PROD | | Operability | Runs, queue, audit, reconcile, artifacts UI | -| Durable long work | Temporal retries + SQLite state mirror | +| Durable long work | Temporal retries + Postgres state mirror | ### Non-goals - Real-time chat / low-latency LLM product surface @@ -98,7 +98,7 @@ flowchart TB end subgraph Data - SQLITE[("SQLite WAL
documents.db")] + PGDB[("Postgres
documents, workflows, pages, chunks")] MINIO[("MinIO
objects")] QDEV[("Qdrant DEV index
vector + lexical")] QPROD[("Qdrant PROD
production vectors")] @@ -111,13 +111,13 @@ flowchart TB UI -->|"/api + Bearer JWT"| API UI -.->|OIDC PKCE| KC API -->|JWKS| KC - API --> SQLITE + API --> PGDB API --> MINIO API --> QDEV API -->|start / signal| TEMP TEMP --> TEMPDB TEMP --> WORKER - WORKER --> SQLITE + WORKER --> PGDB WORKER --> MINIO WORKER --> QDEV WORKER -->|promote_document_to_prod_qdrant| QPROD @@ -136,10 +136,10 @@ flowchart TB | **minio** | official | Source PDF + stage artifacts | 9000 | | **qdrant** | official image | DEV search index | 6333 | | **qdrant (prod)** | external / env | PROD vectors after Super Admin promote | `PROD_QDRANT_URL` | -| **sqlite** | volume file | Canonical document/page/chunk/job/audit state | path | +| **postgres** | official image | Canonical document/workflow/page/chunk/audit state | `DATABASE_URL` | | **keycloak** | official | Identity, groups `/states/{ST}/role`, `/global/super-admin` | 8082 | -API and worker share one **SQLite volume** and the same env for providers so +API and worker share one **Postgres database** and the same env for providers so read and write paths stay consistent. --- @@ -161,7 +161,7 @@ read and write paths stay consistent. │ │ ┌───────────────▼──────────────┐ ┌──────────▼────────────────────────────┐ │ ORCHESTRATION LAYER │ │ DOMAIN STATE LAYER │ -│ Temporal workflows │ │ SQLite (SoT for content) │ +│ Temporal workflows │ │ Postgres (SoT for content) │ │ + review signals │ │ MinIO (binaries) │ │ + activity retries │ │ Jobs / audit / index status │ └───────────────┬──────────────┘ └───────────────────────────────────────┘ @@ -245,38 +245,40 @@ Supporting workflows: `OcrOnly`, `TranslationOnly`, `ChunkingOnly`, | Store | System of record for | Not for | |---|---|---| -| **SQLite** | Document stage, pages, chunks, tags, jobs, audit, uploader, `instance` | Binary blobs | +| **Postgres** | Document stage, workflows, pages, chunks, artifacts, audit, uploader, `instance` | Binary blobs | | **MinIO** | PDF originals, exports, payloads | AuthZ decisions | | **Temporal DB** | Workflow history & timers | Business content | | **Qdrant DEV** | Searchable DEV vectors | Authoritative text | | **Qdrant PROD** | Production vectors | Draft/unapproved content | -### 7.2 Key entities (SQLite) +### 7.2 Key entities (Postgres) ```text -documents (workflow_id PK) - ├── instance # tenant / state code - ├── stage # DocumentStage enum - ├── uploaded_by_* # actor provenance - ├── page_count / chunk_count - ├── latest_job_id - ├── reindex_required - └── is_disabled / is_demo - -pages (workflow_id, page_number) - └── OCR + translation fields + review flags - -chunks (workflow_id, chunk_num) - └── text, lineage, exclusion, provenance - └── chunk_tags[] +documents (id UUID PK) + ├── catalog_id # → catalogs.instance_id → instances.name + ├── source (JSONB) # fingerprint, filename, storage_uri, page_count + ├── index_status (JSONB) # dev / prod vector index state + ├── metadata (JSONB) # chunk_count, reindex_required, is_demo, ... + ├── uploaded_by_user_id # → users + └── is_disabled + +workflows (one per processing run; the latest gives the document's stage) + └── workflow_type, status, current_stage, temporal_workflow_id → Runs UI + +workflow_artifacts → files each workflow produced (MinIO) -document_jobs - └── job_type, status, temporal ids → Runs UI +pages (document_id, page_number) + └── OCR text + translations (JSONB) + review flags + +chunks (document_id, chunk_number) + └── text, lineage, exclusion, provenance audit_logs - └── approvals, edits, promote_to_prod + └── approvals, edits, promote_to_prod (outlives the document) ``` +The [legacy SQLite schema](SQLITE_SCHEMA.md) describes the importer’s source database. + ### 7.3 Tenant rule - `instance` set **once** at create (not reassigned on upsert). - Restricted users: `list_*` / get / search filtered by allowed instances. @@ -337,7 +339,7 @@ UI always calls API via `/api` proxy — no direct Temporal/MinIO from browser. | Activity | Purpose | Failure handling | |---|---|---| -| `run_ocr_and_store` | Extract pages → SQLite + MinIO | OCR_RETRY | +| `run_ocr_and_store` | Extract pages → Postgres + MinIO | OCR_RETRY | | `detect_and_translate_pages_from_db` | Lang detect + translate | TRANSLATION_RETRY | | `create_chunks_from_db` | Chunk reviewed text | CHUNK_RETRY | | `auto_tag_chunks_from_db` | Domain tags | CHUNK_RETRY | @@ -353,12 +355,12 @@ Providers are **config-driven** (env URLs/models) so OCR/vLLM backends can chang ```mermaid flowchart LR - SQLITE[(SQLite chunks)] + PGDB[(Postgres chunks)] DEV[(DEV index
Qdrant)] PROD[(PROD Qdrant)] - SQLITE -->|approve_ingestion
ingest_document_from_db| DEV - SQLITE -->|approve_prod Super Admin
promote_document_to_prod_qdrant| PROD + PGDB -->|approve_ingestion
ingest_document_from_db| DEV + PGDB -->|approve_prod Super Admin
promote_document_to_prod_qdrant| PROD DEV -.->|not automatic| PROD ``` @@ -399,8 +401,8 @@ Routes: Dashboard, Documents, New Document, Document Ops, Runs, Search, Users (S ### Runtime properties - API + worker: **same image**, different entrypoint -- Shared SQLite volume (single-writer lock + WAL) -- Horizontal scale: Temporal workers scale out; SQLite remains single-primary (acceptable for ops volume; upgrade path = Postgres if needed) +- Shared Postgres database (pooled connections per process; schema applied from `migrations/` at startup) +- Horizontal scale: Temporal workers and API replicas scale out against the same database --- @@ -408,7 +410,7 @@ Routes: Dashboard, Documents, New Document, Document Ops, Runs, Search, Users (S | Concern | Design | |---|---| -| **Consistency** | SQLite SoT; Temporal for execution; search is eventual projection | +| **Consistency** | Postgres SoT; Temporal for execution; search is eventual projection | | **Idempotency** | Workflow ids; reingest overwrites by document id | | **Observability** | Runs UI, Temporal UI, audit_logs, stage mirror | | **Security** | JWT, least privilege, tenant 404, Super Admin prod gate | @@ -421,7 +423,7 @@ Routes: Dashboard, Documents, New Document, Document Ops, Runs, Search, Users (S ## 15. Threat & trust boundaries ```text -[ Browser ] --JWT--> [ API ] --internal net--> [ Worker / Temporal / MinIO / SQLite ] +[ Browser ] --JWT--> [ API ] --internal net--> [ Worker / Temporal / MinIO / Postgres ] | +--JWKS--> [ Keycloak ] +--HTTP--> [ Qdrant DEV ] @@ -439,7 +441,7 @@ Routes: Dashboard, Documents, New Document, Document Ops, Runs, Search, Users (S | Attribute | Target approach | |---|---| | Availability | Compose restart policies; Temporal retries | -| Durability | Workflow history + SQLite + MinIO objects | +| Durability | Workflow history + Postgres + MinIO objects | | Latency | Search interactive; OCR batch (minutes) | | Isolation | Per-state `instance` + Keycloak groups | | Auditability | append-only audit + Temporal history | @@ -450,7 +452,7 @@ Routes: Dashboard, Documents, New Document, Document Ops, Runs, Search, Users (S ## 17. Evolution / extension points 1. Explicit **“Request PROD approval”** action (today: auto-enter queue after DEV). -2. SQLite → Postgres if multi-writer scale is needed. +2. Moving `chunk_count` / `is_demo` out of `documents.metadata` into columns if list queries slow down. 3. Stricter per-state role on every mutation (not only global permission). 4. Automated quality checks before Super Admin sees queue. diff --git a/docs/architecture-pipeline-rbac.md b/docs/architecture-pipeline-rbac.md index 4495e9e..8721c8e 100644 --- a/docs/architecture-pipeline-rbac.md +++ b/docs/architecture-pipeline-rbac.md @@ -43,7 +43,7 @@ flowchart TB end subgraph DataPlane - SQLite["SQLite
documents, pages, chunks,
jobs, audit"] + Postgres["Postgres
documents, pages, chunks,
jobs, audit"] MinIO["MinIO / S3
source PDFs"] QdrantDev["Qdrant DEV
search index"] QdrantProd["Qdrant PROD
search index"] @@ -59,11 +59,11 @@ flowchart TB Browser -->|"OIDC PKCE login"| KC Browser -->|"Bearer JWT + /api/*"| API API -->|"validate JWT / JWKS"| KC - API --> SQLite + API --> Postgres API --> MinIO API -->|"start / signal workflows"| Temporal Temporal --> Worker - Worker --> SQLite + Worker --> Postgres Worker --> MinIO Worker --> OCR Worker --> TR @@ -81,7 +81,7 @@ flowchart TB | **Keycloak** | Source of truth for *who* is Super Admin vs which **state + role** a user has | | **JWT → API** | Map groups/roles → `permissions`, `instances`, `state_roles`; enforce tenancy + capability | | **Temporal workflow** | Durable pipeline with human gates (signals) | -| **SQLite** | System of record for document stage, pages, chunks, uploader, `instance` | +| **Postgres** | System of record for document stage, pages, chunks, uploader, `instance` | | **Qdrant DEV** | Searchable vectors after state-approved ingestion | | **Qdrant PROD** | Production vectors only after Super Admin `approve_prod` | @@ -213,8 +213,8 @@ stateDiagram-v2 | Stage | Who advances it | Effect | |---|---|---| -| Upload → Registered | Contributor / Super Admin (`upload`) | File in MinIO; SQLite row; `instance` set | -| OCR / Translation / Chunking | System (worker activities) | Pages/chunks in SQLite | +| Upload → Registered | Contributor / Super Admin (`upload`) | File in MinIO; Postgres row; `instance` set | +| OCR / Translation / Chunking | System (worker activities) | Pages/chunks in Postgres | | OCR / Translation / Chunk / **Pre-ingestion** approve | State **review** (+ pipeline as needed) or Super Admin | Temporal signal continues workflow | | **Ingesting in Dev** | System after `approve_ingestion` | `ingest_document_from_db` → **Qdrant DEV** | | **Approval for Prod** (queue) | Document waits | Still **DEV-only** in search until promote | @@ -355,14 +355,14 @@ flowchart LR ```mermaid flowchart TB PDF[Source PDF in MinIO] - SQLITE[(SQLite: pages + chunks
+ instance + uploader)] + PGDB[(Postgres: pages + chunks
+ instance + uploader)] DEV[(Qdrant DEV collection)] PROD[(Qdrant PROD collection)] - PDF --> SQLITE - SQLITE -->|"ingest_document_from_db
after approve_ingestion"| DEV + PDF --> PGDB + PGDB -->|"ingest_document_from_db
after approve_ingestion"| DEV DEV -->|"promote_document_to_prod_qdrant
after Super Admin approve_prod"| PROD - SQLITE -.->|"stage mirror"| SQLITE + PGDB -.->|"stage mirror"| PGDB ``` - **DEV ingest** activity: `ingest_document_from_db` (worker). diff --git a/docs/openapi-search.yaml b/docs/openapi-search.yaml index d9c578e..5115ef8 100644 --- a/docs/openapi-search.yaml +++ b/docs/openapi-search.yaml @@ -9,7 +9,7 @@ info: `pipeline/models.py`, and `pipeline/vector_store/qdrant_store.py`. Not the full API surface — only `/search` (vector/hybrid semantic search) - and `/chunks/search` (SQLite keyword search across chunk records). + and `/chunks/search` (keyword search across chunk records). version: "1.0.0" contact: name: knowledge-provider (docs-pipeline) @@ -83,9 +83,9 @@ paths: /chunks/search: get: operationId: searchChunksAcrossDocuments - summary: Keyword search across chunk records (SQLite), for KB maintainer workflows + summary: Keyword search across chunk records (Postgres), for KB maintainer workflows description: | - Distinct from `/search`: this queries chunk rows directly in SQLite by + Distinct from `/search`: this queries chunk rows directly in Postgres by substring/keyword match, not the vector index. Results are scoped to the caller's instance(s) unless the caller is an unrestricted admin. tags: [search] @@ -468,7 +468,7 @@ components: type: array items: type: object - description: Raw chunk record from SQLite (db.search_chunks); shape follows the chunks table, not formally typed here. + description: Raw chunk record from Postgres (db.search_chunks); shape follows the chunks table, not formally typed here. additionalProperties: true total: type: integer diff --git a/migrations/0001_initial_schema.sql b/migrations/0001_initial_schema.sql new file mode 100644 index 0000000..b20e900 --- /dev/null +++ b/migrations/0001_initial_schema.sql @@ -0,0 +1,262 @@ +CREATE EXTENSION IF NOT EXISTS pg_trgm; + +CREATE FUNCTION set_updated_at() RETURNS trigger AS $$ +BEGIN + NEW.updated_at := now(); + RETURN NEW; +END $$ LANGUAGE plpgsql; + +-- 1. instances: one row per tenant node (provider), mirrored from Keycloak +CREATE TABLE instances ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + keycloak_group_id TEXT UNIQUE, + name TEXT NOT NULL, + path TEXT NOT NULL UNIQUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE TRIGGER trg_instances_updated_at BEFORE UPDATE ON instances + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); + +-- 2. users +CREATE TABLE users ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + keycloak_user_id TEXT NOT NULL UNIQUE, + username TEXT, + email TEXT, + roles TEXT[] NOT NULL DEFAULT '{}', + instance_id UUID REFERENCES instances(id) ON DELETE SET NULL, + first_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_seen_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE UNIQUE INDEX users_email_lower_key ON users (lower(email)) WHERE email IS NOT NULL; + +INSERT INTO users (keycloak_user_id, username, roles) +VALUES ('local-admin', 'local-admin', '{superadmin}'); + +-- 3. catalogs: one row per provider instance and resource type +CREATE TABLE catalogs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + instance_id UUID NOT NULL REFERENCES instances(id) ON DELETE RESTRICT, + type TEXT NOT NULL CHECK (type ~ '^[a-z0-9]+:[A-Za-z0-9]+$'), + name TEXT NOT NULL, + metadata JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(metadata) = 'object'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (instance_id, type) +); +CREATE TRIGGER trg_catalogs_updated_at BEFORE UPDATE ON catalogs + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); + +-- 4. documents +CREATE TABLE documents ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + catalog_id UUID NOT NULL REFERENCES catalogs(id) ON DELETE RESTRICT, + code TEXT NOT NULL CHECK (code ~ '^[a-z0-9]+(-[a-z0-9]+)*$'), + name TEXT NOT NULL, + aliases TEXT[] NOT NULL DEFAULT '{}' CHECK (array_position(aliases, '') IS NULL), + source JSONB NOT NULL + CHECK (jsonb_typeof(source) = 'object') + CHECK (source->>'fingerprint' ~ '^[0-9a-f]{32}$'), + is_disabled BOOLEAN NOT NULL DEFAULT FALSE, + metadata JSONB NOT NULL + DEFAULT '{"index_status": {}, "catalog_visible": true, "network_visible": true, "reindex_required": false}' + CHECK (jsonb_typeof(metadata) = 'object'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (catalog_id, code) +); +CREATE INDEX documents_created_at_idx ON documents (created_at DESC); +CREATE INDEX documents_fingerprint_idx ON documents (catalog_id, (source->>'fingerprint')); +CREATE TRIGGER trg_documents_updated_at BEFORE UPDATE ON documents + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); + +-- 5. workflows +CREATE TABLE workflows ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + workflow_type TEXT NOT NULL, + temporal_workflow_id TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'running', + current_stage TEXT NOT NULL DEFAULT 'registered', + stage_timestamps JSONB NOT NULL DEFAULT '{}' + CHECK (jsonb_typeof(stage_timestamps) = 'object' + AND (stage_timestamps - ARRAY['ocr', 'translation', 'chunks']) = '{}'::jsonb), + stop_after_ocr BOOLEAN NOT NULL DEFAULT FALSE, + error_message TEXT, + config JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(config) = 'object'), + started_by_user_id UUID REFERENCES users(id) ON DELETE SET NULL, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + completed_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX workflows_document_started_idx ON workflows (document_id, started_at DESC); +CREATE INDEX workflows_status_started_idx ON workflows (status, started_at DESC); +CREATE INDEX workflows_temporal_id_idx ON workflows (temporal_workflow_id); +CREATE TRIGGER trg_workflows_updated_at BEFORE UPDATE ON workflows + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); + +CREATE VIEW document_current_workflow AS +SELECT DISTINCT ON (document_id) * +FROM workflows +ORDER BY document_id, started_at DESC, id DESC; + +-- 6. workflow_artifacts +CREATE TABLE workflow_artifacts ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + workflow_id UUID NOT NULL REFERENCES workflows(id) ON DELETE CASCADE, + artifact_type TEXT NOT NULL CHECK (btrim(artifact_type) <> ''), + stage TEXT, + storage_uri TEXT NOT NULL, + mime_type TEXT, + filename TEXT, + size_bytes BIGINT CHECK (size_bytes >= 0), + metadata JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(metadata) = 'object'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX workflow_artifacts_workflow_idx ON workflow_artifacts (workflow_id, created_at DESC); +CREATE INDEX workflow_artifacts_type_idx ON workflow_artifacts (workflow_id, artifact_type, created_at DESC); + +-- 7. pages +CREATE TABLE pages ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + page_number INTEGER NOT NULL CHECK (page_number >= 1), + original_markdown TEXT, + edited_markdown TEXT, + translations JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(translations) = 'object'), + metadata JSONB NOT NULL DEFAULT '{"is_reviewed": false}' CHECK (jsonb_typeof(metadata) = 'object'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (document_id, page_number) +); +CREATE TRIGGER trg_pages_updated_at BEFORE UPDATE ON pages + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); + +-- 8. chunks +CREATE TABLE chunks ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + chunk_number INTEGER NOT NULL CHECK (chunk_number >= 1), + original_text TEXT NOT NULL, + edited_text TEXT, + token_count INTEGER NOT NULL DEFAULT 0 CHECK (token_count >= 0), + page_start INTEGER NOT NULL DEFAULT 1 CHECK (page_start >= 1), + page_end INTEGER NOT NULL DEFAULT 1, + source_spans JSONB NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(source_spans) = 'array'), + section_title TEXT, + content_type TEXT, + is_reference BOOLEAN NOT NULL DEFAULT FALSE, + is_reviewed BOOLEAN NOT NULL DEFAULT FALSE, + is_excluded BOOLEAN NOT NULL DEFAULT FALSE, + reviewer_notes TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (document_id, chunk_number), + CHECK (page_end >= page_start) +); +CREATE INDEX chunks_text_trgm_idx ON chunks + USING gin (lower(coalesce(edited_text, original_text)) gin_trgm_ops); +CREATE TRIGGER trg_chunks_updated_at BEFORE UPDATE ON chunks + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); + +-- 9. audit_logs +CREATE TABLE audit_logs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + document_id UUID REFERENCES documents(id) ON DELETE SET NULL, + workflow_id TEXT, + instance_id UUID REFERENCES instances(id) ON DELETE SET NULL, + action_type TEXT NOT NULL CHECK (btrim(action_type) <> ''), + entity_type TEXT, + entity_number INTEGER, + field_name TEXT, + old_value TEXT, + new_value TEXT, + metadata JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(metadata) = 'object'), + actor_user_id UUID REFERENCES users(id) ON DELETE SET NULL, + actor JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(actor) = 'object'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK (entity_type IN ('setting', 'catalog') OR workflow_id IS NOT NULL) +); +CREATE INDEX audit_logs_document_idx ON audit_logs (document_id, created_at DESC); +CREATE INDEX audit_logs_instance_idx ON audit_logs (instance_id, created_at DESC); +CREATE INDEX audit_logs_created_idx ON audit_logs (created_at DESC); +CREATE INDEX audit_logs_action_idx ON audit_logs (action_type, created_at DESC); +CREATE INDEX audit_logs_settings_idx ON audit_logs (created_at DESC) WHERE entity_type = 'setting'; + +-- 10. search_settings +CREATE TABLE search_settings ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + singleton BOOLEAN NOT NULL DEFAULT TRUE UNIQUE CHECK (singleton), + retrieval JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(retrieval) = 'object'), + candidates JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(candidates) = 'object'), + results JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(results) = 'object'), + query JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(query) = 'object'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE TRIGGER trg_search_settings_updated_at BEFORE UPDATE ON search_settings + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); +INSERT INTO search_settings DEFAULT VALUES; + +-- 11. email_otps +CREATE TABLE email_otps ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + email TEXT NOT NULL UNIQUE CHECK (email = lower(email)), + code_hash TEXT NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0), + last_sent_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX email_otps_expires_idx ON email_otps (expires_at); + +-- webhook_records +CREATE TABLE webhook_records ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + source TEXT NOT NULL DEFAULT 'webhook', + external_id TEXT NOT NULL, + title TEXT, + slug TEXT, + status TEXT, + is_published BOOLEAN, + category TEXT, + payload JSONB NOT NULL CHECK (jsonb_typeof(payload) = 'object'), + payload_hash TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (source, external_id) +); +CREATE INDEX webhook_records_source_status_idx ON webhook_records (source, status); + +-- catalog_status view: one row per document, computed from catalogs, documents and workflows +CREATE VIEW catalog_status AS +SELECT + d.id AS document_id, + c.id AS catalog_id, + c.instance_id, + g.name AS instance_name, + c.type, + d.code, + d.name, + d.aliases, + CASE + WHEN x.is_live AND COALESCE((d.metadata->>'reindex_required')::boolean, FALSE) THEN 'pending_reindex' + WHEN x.is_live THEN 'live' + WHEN cw.current_stage = 'approval_for_prod' AND NOT d.is_disabled THEN 'pending_prod' + ELSE 'disabled' + END AS status, + (SELECT w.temporal_workflow_id FROM workflows w + WHERE w.document_id = d.id AND w.workflow_type = 'pipeline' + ORDER BY w.started_at ASC, w.id ASC LIMIT 1) AS workflow_id, + (SELECT count(*) FROM chunks ch WHERE ch.document_id = d.id)::INTEGER AS chunk_count, + COALESCE((d.metadata->>'network_visible')::boolean, TRUE) AS network_visible, + CASE WHEN x.is_live + THEN (d.metadata->'index_status'->'prod'->>'last_indexed_at')::timestamptz END AS promoted_at +FROM documents d +JOIN catalogs c ON c.id = d.catalog_id +JOIN instances g ON g.id = c.instance_id +LEFT JOIN document_current_workflow cw ON cw.document_id = d.id +CROSS JOIN LATERAL ( + SELECT (cw.current_stage = 'completed' AND NOT d.is_disabled + AND COALESCE((d.metadata->>'catalog_visible')::boolean, TRUE)) AS is_live +) x; diff --git a/migrations/0002_webhook_advisory_index_state.sql b/migrations/0002_webhook_advisory_index_state.sql new file mode 100644 index 0000000..328fc93 --- /dev/null +++ b/migrations/0002_webhook_advisory_index_state.sql @@ -0,0 +1,3 @@ +-- Advisory indexing state: the payload_hash last written to the dev / PROD Qdrant collection. +ALTER TABLE webhook_records ADD COLUMN indexed_hash TEXT; +ALTER TABLE webhook_records ADD COLUMN promoted_hash TEXT; diff --git a/pipeline/activities.py b/pipeline/activities.py index af119b2..cac2215 100644 --- a/pipeline/activities.py +++ b/pipeline/activities.py @@ -641,10 +641,10 @@ def _prepare_records( description: str | None = None, include_e5_prefix_field: bool = True, instance: str | None = None, - document_kind: str = "document", - scheme_code: str | None = None, - scheme_name: str | None = None, - scheme_aliases: list[str] | None = None, + kind: str = "document", + code: str | None = None, + name: str | None = None, + aliases: list[str] | None = None, valid_from: str | None = None, valid_to: str | None = None, provider_id: str | None = None, @@ -667,15 +667,13 @@ def _prepare_records( short_description = metadata.get("doc_short_description", "") effective_description = description or llm_doc_description or short_description - kind = (document_kind or "document").strip().lower() - is_scheme = kind == "scheme" and bool((scheme_code or "").strip()) - resolved_scheme_name = scheme_name or name_en or default_name + kind = (kind or "document").strip().lower() + is_scheme = kind == "scheme" and bool((code or "").strip()) + resolved_name = name or name_en or default_name # Merge curator-supplied, bootstrap and derived aliases so a scheme is never # indexed with an empty alias list (it would then only match its exact name). - aliases = ( - scheme_catalog.resolve_scheme_aliases( - scheme_code, resolved_scheme_name, scheme_aliases - ) + resolved_aliases = ( + scheme_catalog.resolve_scheme_aliases(code, resolved_name, aliases) if is_scheme else [] ) @@ -701,10 +699,10 @@ def _prepare_records( "_id": point_id, "doc_id": document_id, "workflow_id": workflow_id or "", - "instance": resolved_instance, + "instance": resolved_instance, # vector payload key, kept for consumers "instance_name": instance_name, "provider_id": provider_id, - "type": "scheme" if is_scheme else "document", + "type": kind, "source": "docs-pipeline", "filename": external_slug, "name_gu": name_gu, @@ -737,9 +735,9 @@ def _prepare_records( if validity.end_date is not None: record["end_date"] = validity.end_date if is_scheme: - record["scheme_code"] = (scheme_code or "").strip().lower() - record["scheme_name"] = resolved_scheme_name - record["scheme_aliases"] = aliases + record["scheme_code"] = (code or "").strip().lower() + record["scheme_name"] = resolved_name + record["scheme_aliases"] = resolved_aliases if include_e5_prefix_field: record["text_for_embedding"] = f"passage: {text}" if text else "passage:" records.append(record) @@ -748,26 +746,13 @@ def _prepare_records( def _scheme_fields_from_doc(doc: dict | None) -> dict: - """Extract scheme kwargs for _prepare_records from a documents row.""" - import json as _json - + """Extract kind/code/name/aliases kwargs for _prepare_records from a documents row.""" doc = doc or {} - aliases_raw = doc.get("scheme_aliases_json") - aliases: list[str] = [] - if isinstance(aliases_raw, list): - aliases = [str(a) for a in aliases_raw] - elif isinstance(aliases_raw, str) and aliases_raw.strip(): - try: - parsed = _json.loads(aliases_raw) - if isinstance(parsed, list): - aliases = [str(a) for a in parsed] - except Exception: - aliases = [] return { - "document_kind": (doc.get("document_kind") or "document"), - "scheme_code": doc.get("scheme_code"), - "scheme_name": doc.get("scheme_name"), - "scheme_aliases": aliases, + "kind": (doc.get("kind") or "document"), + "code": doc.get("code"), + "name": doc.get("name"), + "aliases": [str(a) for a in (doc.get("aliases") or [])], } @@ -775,7 +760,7 @@ def _provider_id_from_doc(doc: dict | None) -> str | None: doc = doc or {} group = named_group(group_for_instance(doc.get("instance"))) try: - provider = provider_for(load_provider_naming(), doc.get("document_kind"), group) + provider = provider_for(load_provider_naming(), doc.get("kind"), group) except InvalidProviderIdError as exc: raise ApplicationError(str(exc), non_retryable=True) from exc return provider["id"] if provider else None @@ -929,7 +914,7 @@ async def run_ocr(filepath: str) -> list[dict]: @activity.defn async def run_ocr_and_store(workflow_id: str, filepath: str) -> dict: - """Run OCR and persist pages to SQLite to avoid large Temporal payloads.""" + """Run OCR and persist pages to Postgres to avoid large Temporal payloads.""" from . import db local_path, cleanup_local = _resolve_local_path(filepath) @@ -1111,7 +1096,7 @@ async def create_chunks_from_db( chunk_overlap: int = 128, min_tokens: int = 100, ) -> dict: - """Create chunks from persisted pages and persist chunks in SQLite.""" + """Create chunks from persisted pages and persist chunks in Postgres.""" from . import db pages = db.get_pages(workflow_id) @@ -1209,7 +1194,7 @@ async def _persist_chunking_progress(event: dict) -> None: "stats": result.stats, }, ) - # Reconcile the document row immediately after chunk persistence so SQLite + # Reconcile the document row immediately after chunk persistence so Postgres # remains truthful even if the workflow fails before its final state update. db.reconcile_materialized_state(workflow_id) await _persist_chunking_progress( @@ -1250,7 +1235,7 @@ async def prepare_for_ingestion( instance=(doc or {}).get("instance"), **_validity_fields_from_doc(doc), provider_id=_provider_id_from_doc(doc), - display_name=(doc or {}).get("display_name"), + display_name=(doc or {}).get("name"), ) activity.logger.info(f"Prepared {len(records)} records") return records @@ -1290,7 +1275,7 @@ async def promote_document_to_prod_qdrant( filename: str, batch_size: int = 10, ) -> dict: - """Re-upsert document chunks into the PROD Qdrant collection (from SQLite).""" + """Re-upsert document chunks into the PROD Qdrant collection (from Postgres).""" from . import db from .vector_store.qdrant_store import QdrantVectorStore, get_qdrant_client @@ -1323,8 +1308,8 @@ async def promote_document_to_prod_qdrant( chunks = db.get_chunks(workflow_id, include_excluded=True) doc = db.get_document(workflow_id) or {} scheme_kwargs = _scheme_fields_from_doc(doc) - kind = (scheme_kwargs.get("document_kind") or "document").strip().lower() - is_scheme = kind == "scheme" and bool((scheme_kwargs.get("scheme_code") or "").strip()) + kind = (scheme_kwargs.get("kind") or "document").strip().lower() + is_scheme = kind == "scheme" and bool((scheme_kwargs.get("code") or "").strip()) if is_scheme: prod_url = ( @@ -1362,7 +1347,7 @@ async def promote_document_to_prod_qdrant( **scheme_kwargs, **_validity_fields_from_doc(doc), provider_id=_provider_id_from_doc(doc), - display_name=doc.get("display_name"), + display_name=doc.get("name"), ) activity.logger.info( "Promoting %s records to PROD Qdrant collection %s (kind=%s)", @@ -1375,6 +1360,7 @@ async def promote_document_to_prod_qdrant( store.upsert, prod_collection, records, batch_size=max(batch_size, 8) ) activity.logger.info("PROD promotion complete: %s", result.get("index_stats")) + await _drop_superseded_points("prod", workflow_id, doc, document_id, prod_collection, store=store) db.log_audit( workflow_id=workflow_id, document_id=document_id, @@ -1384,8 +1370,8 @@ async def promote_document_to_prod_qdrant( "collection": prod_collection, "records_ingested": result.get("records_ingested", len(records)), "prod_qdrant_url": prod_url, - "document_kind": kind, - "scheme_code": scheme_kwargs.get("scheme_code"), + "kind": kind, + "code": scheme_kwargs.get("code"), }, ) catalog_version = None @@ -1396,12 +1382,6 @@ async def promote_document_to_prod_qdrant( catalog_version = on_scheme_document_promoted(workflow_id) except Exception as exc: activity.logger.warning("Catalog update after promote failed: %s", exc) - try: - from .master_catalog_pg import sync_catalog_entry - - await asyncio.to_thread(sync_catalog_entry, workflow_id, "live") - except Exception as exc: - activity.logger.warning("Master catalog (Postgres/Redis) sync after promote failed: %s", exc) return { **result, "collection": prod_collection, @@ -1411,6 +1391,53 @@ async def promote_document_to_prod_qdrant( } +async def _drop_superseded_points( + env: str, + workflow_id: str, + doc: dict, + current_doc_id: str, + collection: str, + store=None, +) -> None: + """Delete points of earlier file versions (see ``metadata.superseded_doc_ids``). + + Runs after the new content is indexed, so search never has a gap. A failed + delete keeps the key for the next run and does not fail the ingest. + """ + from . import db + + stale = [ + doc_id + for doc_id in (doc.get("superseded_doc_ids") or {}).get(env, []) + if doc_id and doc_id != current_doc_id + ] + if not stale: + return + if store is None: + from .vector_store import get_vector_store + + store = get_vector_store() + deleted = [] + for doc_id in stale: + # Two documents holding the same file share one index key. While another + # live document is indexed under it the points stay, and the key stays on + # the list so a later ingest, disable or purge can still remove it. + if await asyncio.to_thread(db.doc_id_used_by_other_document, doc_id, doc.get("id"), env): + activity.logger.info( + "Keeping %s points for %s: another document is indexed under the same key", env, doc_id + ) + continue + result = await asyncio.to_thread(store.delete_by_doc_id, collection, doc_id) + if result.get("error"): + activity.logger.warning( + "Could not delete superseded %s points for %s: %s", env, doc_id, result["error"] + ) + else: + deleted.append(doc_id) + if deleted: + db.clear_superseded_doc_ids(workflow_id, env, deleted) + + @activity.defn async def ingest_document_from_db( workflow_id: str, @@ -1419,7 +1446,7 @@ async def ingest_document_from_db( index_name: str = "documents-index", batch_size: int = 10, ) -> dict: - """Prepare and ingest chunks directly from SQLite by workflow_id.""" + """Prepare and ingest chunks directly from Postgres by workflow_id.""" from . import db chunks = db.get_chunks(workflow_id, include_excluded=True) @@ -1433,7 +1460,7 @@ async def ingest_document_from_db( **_scheme_fields_from_doc(doc), **_validity_fields_from_doc(doc), provider_id=_provider_id_from_doc(doc), - display_name=doc.get("display_name"), + display_name=doc.get("name"), ) payload_path = _write_json_temp(records) try: @@ -1467,12 +1494,7 @@ async def ingest_document_from_db( status="indexed", details=result.get("index_stats"), ) - try: - from .master_catalog_pg import sync_catalog_entry - - await asyncio.to_thread(sync_catalog_entry, workflow_id, "dev") - except Exception as exc: - activity.logger.warning("Master catalog (Postgres/Redis) sync after DEV ingest failed: %s", exc) + await _drop_superseded_points("dev", workflow_id, doc, document_id, index_name) return result @@ -1578,7 +1600,7 @@ async def update_document_state( chunk_count: int = 0, error_message: str = None, ) -> dict: - """Update document state in SQLite.""" + """Update document state in Postgres.""" from . import db activity.logger.info(f"Updating state for {workflow_id}: stage={stage}") @@ -1615,7 +1637,7 @@ async def update_document_state( @activity.defn async def persist_document_content(workflow_id: str, pages: list[dict], chunks: list[dict]) -> dict: - """Persist pages and chunks to SQLite.""" + """Persist pages and chunks to Postgres.""" from . import db activity.logger.info(f"Persisting content for {workflow_id}: {len(pages)} pages, {len(chunks)} chunks") @@ -1648,7 +1670,7 @@ async def detect_and_translate_pages_from_db( target_language: str = "en", source_language: str = None, ) -> dict: - """Detect and translate pages loaded from SQLite; persist updated pages back to SQLite.""" + """Detect and translate pages loaded from Postgres; persist updated pages back to Postgres.""" from . import db pages = db.get_pages(workflow_id) @@ -1684,3 +1706,51 @@ async def detect_and_translate_pages_from_db( }, ) return {"page_count": len(translated), "translated_count": translated_count} + + +async def _run_advisory_activity(record_id: str, promote: bool) -> dict: + """Index (dev) or promote (PROD) one webhook advisory record; shared by both activities.""" + from . import advisory_index + from .vector_store import get_default_index_name, get_vector_store + from .vector_store.embeddings import count_tokens + + also_delete_from = None + if promote: + target = advisory_index.prod_target() + if target is None: + raise RuntimeError("PROD_VECTOR_DB_URL is required to promote advisory to prod") + store, collection = target + mark = "promoted" + else: + store, collection = get_vector_store(), get_default_index_name() + mark = "indexed" + try: + # An unpublished record must also disappear from PROD (deletion only). + also_delete_from = advisory_index.prod_target() + except Exception as exc: + activity.logger.warning("PROD target unavailable for advisory cleanup: %s", exc) + + # Embedding is CPU-bound and synchronous: keep it off the event loop. + result = await asyncio.to_thread( + advisory_index.index_advisory_record, + record_id, + store, + collection, + count_tokens, + mark, + also_delete_from, + ) + advisory_index.log_advisory_audit(record_id, mark, collection, result) + return result + + +@activity.defn +async def index_webhook_advisory(record_id: str) -> dict: + """Write one webhook advisory record's chunks to the dev collection (or delete if unpublished).""" + return await _run_advisory_activity(record_id, promote=False) + + +@activity.defn +async def promote_webhook_advisory(record_id: str) -> dict: + """Copy one webhook advisory record's chunks to the PROD collection (or delete if unpublished).""" + return await _run_advisory_activity(record_id, promote=True) diff --git a/pipeline/advisory_chunks.py b/pipeline/advisory_chunks.py new file mode 100644 index 0000000..cf496e2 --- /dev/null +++ b/pipeline/advisory_chunks.py @@ -0,0 +1,153 @@ +"""Turn a published advisory ``webhook_records`` row into Qdrant chunk records. + +One advisory entry (``popList`` item) becomes one chunk when its whole text fits the +embedding model's token limit, otherwise up to three chunks that each carry the advice +(overview + advice, symptoms + advice, conditions + advice). The stored ``text`` is +always the complete chunk text; the embedding model reads only its first tokens. +""" + +from __future__ import annotations + +import hashlib +from typing import Callable + +MAX_EMBEDDING_TOKENS = 512 + +# (section, label) for the question-side fields, in display order. +_FIELD_SECTIONS = ( + ("overview", "Overview"), + ("symptoms", "Symptoms"), + ("conditions", "Conditions"), +) + + +def clean_value(value: object) -> str: + """Return a trimmed string with surrounding quote characters removed ('' for None).""" + if value is None: + return "" + return str(value).strip().strip('"').strip() + + +def _clean_list(value: object) -> str: + """Join the non-empty cleaned items of a list (or a single value) with spaces.""" + if value is None: + return "" + items = value if isinstance(value, (list, tuple)) else [value] + return " ".join(part for part in (clean_value(item) for item in items) if part) + + +def _is_published(row: dict) -> bool: + return row.get("is_published") in (1, True) and clean_value(row.get("status")) == "Published" + + +def _header(title: str, crop: str, problem_type: str, season: str) -> list[str]: + lines = [title] if title else [] + parts = [] + if crop: + parts.append(f"Crop: {crop}") + if problem_type: + parts.append(f"Problem type: {problem_type}") + if season: + parts.append(f"Season: {season}") + if parts: + lines.append(" | ".join(parts)) + return lines + + +def _chunk_id(doc_id: str, pop_id: object, section: str) -> str: + return hashlib.md5(f"{doc_id}:{pop_id}:{section}".encode("utf-8")).hexdigest() + + +def build_chunks( + row: dict, + count_tokens: Callable[[str], int], + max_tokens: int = MAX_EMBEDDING_TOKENS, +) -> list[dict]: + """Build Qdrant records for one ``webhook_records`` row (empty list if unpublished).""" + if not _is_published(row): + return [] + payload = row.get("payload") or {} + pop_list = payload.get("popList") + if not isinstance(pop_list, list): + return [] + + doc_id = str(row["id"]) + title = clean_value(row.get("title")) or clean_value(payload.get("title")) + crop = clean_value(payload.get("commodityName")) + base = { + "doc_id": doc_id, + "workflow_id": "", + "type": "advisory", + "source": "webhook", + "title_en": title, + "slug": clean_value(row.get("slug")), + "external_id": clean_value(row.get("external_id")), + "crop": crop, + "domain": clean_value(payload.get("domainName")), + "subdomain": clean_value(payload.get("subdomainName")), + "institute": clean_value(payload.get("instituteName")), + "creator": clean_value(payload.get("creatorName")), + "updated_at": clean_value(row.get("updated_at")), + } + + records: list[dict] = [] + + def add(pop: dict, index: int, section: str, text: str, meta: dict) -> None: + pop_id = pop.get("popID") + key = pop_id if pop_id is not None else f"idx{index}" + chunk_id = _chunk_id(doc_id, key, section) + records.append( + { + **base, + **meta, + "_id": chunk_id, + "chunk_id": chunk_id, + "pop_id": pop_id, + "section": section, + "chunk_num": len(records), + "text": text, + "text_for_embedding": text, + } + ) + + for index, pop in enumerate(pop_list): + if not isinstance(pop, dict) or clean_value(pop.get("status")) != "Published": + continue + problem_type = clean_value(pop.get("problemType")) + season = clean_value(pop.get("season")) + meta = { + "problem_type": problem_type, + "problem": clean_value(pop.get("problemDetails")), + "season": season, + "start_period": clean_value(pop.get("startPeriod")), + "end_period": clean_value(pop.get("endPeriod")), + } + header = _header(title, crop, problem_type, season) + fields = { + "overview": clean_value(pop.get("description")), + "symptoms": _clean_list(pop.get("applicableSymptoms")), + "conditions": _clean_list(pop.get("applicableCondition")), + } + advisory = clean_value(pop.get("advisory")) + labelled = [(label, fields[section]) for section, label in _FIELD_SECTIONS if fields[section]] + if advisory: + labelled.append(("Advisory", advisory)) + if not labelled: + continue + + full_text = "\n".join(header + [f"{label}: {body}" for label, body in labelled]) + if count_tokens(full_text) <= max_tokens: + add(pop, index, "full", full_text, meta) + continue + + others = [(section, label) for section, label in _FIELD_SECTIONS if fields[section]] + if not others: + add(pop, index, "advisory", full_text, meta) + continue + for section, label in others: + lines = header + [f"{label}: {fields[section]}"] + if advisory: + lines.append(f"Advisory: {advisory}") + add(pop, index, section, "\n".join(lines), meta) + + return records diff --git a/pipeline/advisory_index.py b/pipeline/advisory_index.py new file mode 100644 index 0000000..c91f2f3 --- /dev/null +++ b/pipeline/advisory_index.py @@ -0,0 +1,84 @@ +"""Index (or remove) one advisory ``webhook_records`` row in a vector store collection.""" + +from __future__ import annotations + +import logging +import os +from typing import Callable, Literal, Optional + +from . import db +from .advisory_chunks import build_chunks + +logger = logging.getLogger(__name__) + + +def _delete_points(store, collection: str, doc_id: str) -> None: + """Delete a record's points; a missing collection is fine, any other failure is not.""" + result = store.delete_by_doc_id(collection, doc_id) or {} + error = str(result.get("error") or "") + if error and "doesn't exist" not in error and "not found" not in error.lower(): + raise RuntimeError(f"Could not delete points for {doc_id} from {collection}: {error}") + + +def index_advisory_record( + record_id: str, + store, + collection: str, + count_tokens: Callable[[str], int], + mark: Literal["indexed", "promoted"] = "indexed", + also_delete_from: Optional[tuple[object, str]] = None, +) -> dict: + """Replace a record's points in ``collection`` with its current chunks. + + Order: read row, delete the record's old points, write the new chunks, then store the + row's payload_hash (``indexed_hash`` or ``promoted_hash``). An unpublished record (no + chunks) only gets its points deleted; ``also_delete_from=(store, collection)`` removes + them from a second collection too (deletion only, never a content push). + """ + row = db.get_webhook_record(record_id) + if row is None: + return {"record_id": record_id, "status": "missing", "chunks": 0} + + doc_id = str(row["id"]) + chunks = build_chunks(row, count_tokens) + _delete_points(store, collection, doc_id) + if chunks: + store.upsert(collection, chunks, batch_size=8) + elif also_delete_from is not None: + other_store, other_collection = also_delete_from + _delete_points(other_store, other_collection, doc_id) + + marker = db.mark_webhook_record_promoted if mark == "promoted" else db.mark_webhook_record_indexed + marker(record_id, row["payload_hash"]) + logger.info("advisory %s record=%s collection=%s chunks=%s", mark, record_id, collection, len(chunks)) + return {"record_id": record_id, "status": "indexed" if chunks else "deleted", "chunks": len(chunks)} + + +def prod_target() -> Optional[tuple[object, str]]: + """The PROD (store, collection) pair from the PROD_VECTOR_DB_* env, or None when not configured.""" + url = (os.environ.get("PROD_VECTOR_DB_URL") or "").strip() + if not url: + return None + from .vector_store.qdrant_store import QdrantVectorStore, get_qdrant_client + + api_key = (os.environ.get("PROD_VECTOR_DB_API_KEY") or "").strip() or None + collection = ( + os.environ.get("PROD_VECTOR_DB_COLLECTION_NAME") + or os.environ.get("VECTOR_DB_COLLECTION_NAME") + or "documents-index" + ).strip() + return QdrantVectorStore(client=get_qdrant_client(url=url, api_key=api_key)), collection + + +def log_advisory_audit(record_id: str, mark: str, collection: str, result: dict) -> None: + """Write one audit_logs row for an indexed / deleted / promoted advisory record.""" + action = "advisory_promoted" if mark == "promoted" else ( + "advisory_deleted" if result.get("status") == "deleted" else "advisory_indexed" + ) + db.log_audit( + workflow_id=f"advisory-{record_id}", + document_id=record_id, + action_type=action, + entity_type="webhook_record", + metadata={"collection": collection, "chunks": result.get("chunks", 0), "status": result.get("status")}, + ) diff --git a/pipeline/advisory_start.py b/pipeline/advisory_start.py new file mode 100644 index 0000000..cc3ef07 --- /dev/null +++ b/pipeline/advisory_start.py @@ -0,0 +1,58 @@ +"""Start advisory index / promote workflows: one Temporal workflow per webhook record.""" + +from __future__ import annotations + +import logging +import os + +from temporalio.exceptions import WorkflowAlreadyStartedError + +from .workflows import AdvisoryIndexWorkflow, AdvisoryPromoteWorkflow + +logger = logging.getLogger(__name__) + +TASK_QUEUE = "ocr-pipeline" + + +async def _start_each(client, workflow_run, id_prefix: str, record_ids: list[str]) -> int: + started = 0 + for record_id in record_ids: + try: + await client.start_workflow( + workflow_run, + args=[record_id], + id=f"{id_prefix}{record_id}", + task_queue=TASK_QUEUE, + ) + started += 1 + except WorkflowAlreadyStartedError: + logger.info("advisory workflow %s%s already running; skipped", id_prefix, record_id) + except Exception: + logger.exception("could not start advisory workflow %s%s", id_prefix, record_id) + return started + + +async def start_advisory_index(client, record_ids: list[str]) -> int: + """Start ``AdvisoryIndexWorkflow`` (id ``advisory-``) for each id; returns how many started.""" + return await _start_each(client, AdvisoryIndexWorkflow.run, "advisory-", record_ids) + + +async def start_advisory_promote(client, record_ids: list[str]) -> int: + """Start ``AdvisoryPromoteWorkflow`` (id ``advisory-promote-``) for each id.""" + return await _start_each(client, AdvisoryPromoteWorkflow.run, "advisory-promote-", record_ids) + + +def _backfill_enabled() -> bool: + return os.environ.get("ADVISORY_BACKFILL_ON_START", "false").strip().lower() in {"1", "true", "yes", "on"} + + +async def backfill_on_start(client) -> int: + """Worker startup hook: when ADVISORY_BACKFILL_ON_START is on (default off), queue every unindexed row.""" + if not _backfill_enabled(): + return 0 + from . import db + + record_ids = db.get_unindexed_webhook_record_ids() + started = await start_advisory_index(client, record_ids) + logger.info("advisory startup backfill: %s unindexed rows, %s workflows started", len(record_ids), started) + return started diff --git a/pipeline/api/api.py b/pipeline/api/api.py index 6b5c5f0..4304a00 100644 --- a/pipeline/api/api.py +++ b/pipeline/api/api.py @@ -9,11 +9,12 @@ import json import logging import os +import re import time from contextlib import asynccontextmanager from datetime import datetime from pathlib import Path -from typing import Optional +from typing import Any, Dict, List, Optional # Load repo-root .env so KEYCLOAK_ADMIN_* etc. work without manual `source .env` try: @@ -28,7 +29,7 @@ from urllib.parse import quote -from fastapi import FastAPI, Header, HTTPException, Query, Request +from fastapi import Depends, FastAPI, Header, HTTPException, Query, Request from fastapi import Path as PathParam from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import StreamingResponse @@ -41,6 +42,7 @@ from .. import db, scheme_catalog from .. import display_name as display_name_rules +from ..advisory_start import start_advisory_index, start_advisory_promote from ..auth.config import load_auth_config, validate_auth_config from ..auth.deps import ( CurrentUser, @@ -99,7 +101,7 @@ SearchSettingsUpdate, SettingsAuditResponse, ) -from ..network_constants import RESULT_REJECTED +from ..network_constants import RESULT_REJECTED, catalog_type_for_kind from ..network_group import validate_network_provider_config from ..observability import configure_logs, reset_logs from ..workflows import ( @@ -123,6 +125,8 @@ datefmt='%H:%M:%S', ) +logger = logging.getLogger(__name__) + TASK_QUEUE = "ocr-pipeline" # Global clients @@ -168,8 +172,7 @@ async def _start_services(): auth_cfg.keycloak_jwks_url, ) - # Initialize SQLite database - logging.info("api startup: initializing SQLite database") + logging.info("api startup: initializing PostgreSQL database") db.init_db() # Temporal @@ -270,6 +273,51 @@ async def _start_services(): # own much tighter budget than the general default. RATE_LIMIT_OTP = os.environ.get("RATE_LIMIT_OTP", "10/minute") +_DOCUMENT_PATH_RE = re.compile(r"^/documents/([^/]+)(/.*)?$") +# Only identifier shapes with an indexed lookup are resolved before auth runs: +# the document UUID, the 32-hex fingerprint, or a ``doc-`` workflow id. +_RESOLVABLE_IDENTIFIER_RE = re.compile( + r"^(?:[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}|[0-9a-f]{32}|doc-[0-9a-f]{32}.*)$" +) +_static_document_segments: Optional[set[str]] = None + + +def _static_document_path_segments() -> set[str]: + """Literal second segments of ``/documents/...`` routes (``upload``, ``batch``...).""" + global _static_document_segments + if _static_document_segments is None: + segments = set() + for route in app.routes: + match = _DOCUMENT_PATH_RE.match(getattr(route, "path", "")) + if match and "{" not in match.group(1): + segments.add(match.group(1)) + _static_document_segments = segments + return _static_document_segments + + +@app.middleware("http") +async def canonicalize_document_identifier(request: Request, call_next): + """Let ``/documents/{id}/...`` take the document UUID or any legacy identifier. + + Handlers address a document by its Temporal workflow id (they signal and + query Temporal with it), so the path is rewritten to that id before routing. + An identifier of another shape, or one that matches no document, is left alone and + 404s in the handler; the lookup runs before auth, so it is limited to cheap shapes. + """ + match = _DOCUMENT_PATH_RE.match(request.scope["path"]) + if ( + match + and _RESOLVABLE_IDENTIFIER_RE.match(match.group(1)) + and match.group(1) not in _static_document_path_segments() + ): + document = await asyncio.to_thread(db.find_document_by_doc_identifier, match.group(1)) + if document and document["workflow_id"] and document["workflow_id"] != match.group(1): + new_path = f"/documents/{document['workflow_id']}{match.group(2) or ''}" + request.scope["path"] = new_path + request.scope["raw_path"] = quote(new_path, safe="/:").encode("ascii") + return await call_next(request) + + limiter = Limiter(key_func=get_remote_address) app.state.limiter = limiter app.add_exception_handler(RateLimitExceeded, request_logging.recording(_rate_limit_exceeded_handler)) @@ -360,7 +408,7 @@ def _compute_file_fingerprint(filepath: Path) -> str: def _instance_scope_for_user(user: AuthUser) -> Optional[list[str]]: - """None = unrestricted; otherwise only these instance ids.""" + """None = unrestricted; otherwise only these instance codes.""" allowed = allowed_instances(user) if allowed is None: return None @@ -368,7 +416,7 @@ def _instance_scope_for_user(user: AuthUser) -> Optional[list[str]]: def _document_cohorts_payload(summary: dict, by_instance: Optional[list[dict]] = None) -> dict: - """Shape SQLite aggregate counts for DocumentCohortsResponse (all ints).""" + """Shape Postgres aggregate counts for DocumentCohortsResponse (all ints).""" def _i(key: str, default: int = 0) -> int: val = summary.get(key, default) return int(val or 0) @@ -396,10 +444,78 @@ def _i(key: str, default: int = 0) -> int: def _resolve_create_instance(user: AuthUser, requested: Optional[str] = None) -> str: - """Plan 2: stamp document.instance from Keycloak-allowed states / portal.""" + """Resolve the upload owner using the existing Keycloak tenancy rules.""" return resolve_create_instance(user, requested) +def _document_title(filename: str) -> str: + """Human title for a new document: the file name without extension, separators as spaces.""" + stem = Path(filename or "").stem + return re.sub(r"[_\-\s]+", " ", stem).strip() or "Untitled document" + + +def _assert_can_replace(user: AuthUser, existing: dict) -> None: + """Replacing a document's file clears its pages and chunks, so guard it. + + Needs review rights in the document's instance; a disabled document or one with + a run in progress cannot be replaced; a document already promoted to PROD + can only be replaced by someone who may approve ingestion. + """ + doc_instance = existing["instance"] + if not user.has_permission_for_instance(Permission.REVIEW, doc_instance): + raise HTTPException(403, f"No permission to replace documents in instance: {doc_instance}") + if existing.get("is_disabled"): + raise HTTPException(409, "This document is disabled; restore it before replacing its file") + latest_job = db.get_latest_document_job(existing["workflow_id"]) + if latest_job and latest_job.get("status") == "running": + raise HTTPException(409, "A run is still in progress for this document") + if (existing.get("index_status") or {}).get("prod") and not user.has_permission_for_instance( + Permission.APPROVE_INGESTION, doc_instance + ): + raise HTTPException( + 403, "Replacing a document that is live in PROD needs approval rights in its instance" + ) + + +def _resolve_document_identity( + user: AuthUser, + instance: str, + kind: str, + code: str, + filename: str, +) -> tuple[str, str, Optional[dict]]: + """Kind, code and (when ``code`` already exists) the document an upload replaces. + + An explicit ``code`` that is already held in the tenant's catalog of that + kind means "update this document": the caller reuses it instead of inserting. + Without a code one is derived from the file name. + + Permissions are checked against the target instance, not just globally: a user + who may upload in one state must not be able to add to, or replace a + document of, a state where they only hold a view role. Replacing clears the + document's pages and chunks, so it also needs review rights in that instance. + """ + kind = (kind or "").strip().lower() + try: + catalog_type_for_kind(kind) + except ValueError as exc: + raise HTTPException(400, str(exc)) from exc + if not user.has_permission_for_instance(Permission.UPLOAD, instance): + raise HTTPException(403, f"No upload permission in instance: {instance}") + code = (code or "").strip() + if code: + try: + code = scheme_catalog.validate_scheme_code(code) + except ValueError as exc: + raise HTTPException(400, str(exc)) from exc + existing = db.find_document_by_code(instance, kind, code) + if existing: + existing = assert_document_instance_access(user, existing) + _assert_can_replace(user, existing) + return kind, code, existing + return kind, scheme_catalog.derive_scheme_code(_document_title(filename), kind=kind), None + + def _require_document_for_user(workflow_id: str, user: AuthUser) -> dict: """Load a document or 404 if missing / outside the caller's instance scope.""" return assert_document_instance_access(user, db.get_document(workflow_id)) @@ -453,11 +569,12 @@ def _actor_from_user(user) -> dict: def _document_summary_from_row(doc: dict, current_job: Optional[dict] = None) -> DocumentSummary: return DocumentSummary( + id=doc.get("id"), document_id=doc["document_id"], canonical_document_id=doc.get("canonical_document_id"), workflow_id=doc["workflow_id"], filename=doc["filename"], - display_name=doc.get("display_name"), + name=doc.get("name"), source_filename=doc.get("source_filename"), source_manifest_name=doc.get("source_manifest_name"), source_file_fingerprint=doc.get("source_file_fingerprint"), @@ -479,10 +596,9 @@ def _document_summary_from_row(doc: dict, current_job: Optional[dict] = None) -> uploaded_by_username=doc.get("uploaded_by_username"), uploaded_by_email=doc.get("uploaded_by_email"), uploaded_by_roles=_parse_roles_field(doc.get("uploaded_by_roles")), - document_kind=(doc.get("document_kind") or "document"), - scheme_code=doc.get("scheme_code"), - scheme_name=doc.get("scheme_name"), - scheme_aliases=_parse_scheme_aliases(doc), + kind=doc.get("kind"), + code=doc.get("code"), + aliases=list(doc.get("aliases") or []), tool_routing=doc.get("tool_routing"), catalog_visible=bool(int(doc["catalog_visible"])) if doc.get("catalog_visible") is not None else True, network_visible=bool(int(doc["network_visible"])) if doc.get("network_visible") is not None else True, @@ -710,7 +826,7 @@ async def auth_me(user: CurrentUser): "groups": list(user.groups or []), "state_roles": dict(user.state_roles or {}), "is_super_admin": bool(user.is_superadmin), - # Plan 2: portal instance code for BV / platform docs (not a state). + # Portal instance code for BV / platform docs (not a state). "portal_instance": PORTAL_INSTANCE, "auth_disabled": user.token_disabled_mode, } @@ -809,7 +925,6 @@ async def get_operations_queue( user: RequireSearch, limit: int = Query(100, le=500), offset: int = Query(0, ge=0), - x_include_demo: Optional[str] = Header(None, alias="X-Include-Demo"), x_include_disabled: Optional[str] = Header(None, alias="X-Include-Disabled"), ): """Return documents that currently need operator or agent action. @@ -817,12 +932,10 @@ async def get_operations_queue( Results are limited to instances the caller can access (state users never see portal ``bv`` or other states' documents). """ - include_demo = x_include_demo and x_include_demo.lower() == "true" include_disabled = x_include_disabled and x_include_disabled.lower() == "true" rows, total = db.list_operations_queue( limit=limit, offset=offset, - include_demo=include_demo, include_disabled=include_disabled, instances=_instance_scope_for_user(user), ) @@ -863,8 +976,9 @@ async def list_runs( ) + @app.get("/runs/{job_id}") -async def get_run(job_id: int, user: RequireSearch): +async def get_run(job_id: str, user: RequireSearch): """Get a specific document job/run (404 if missing or outside tenant scope).""" run = db.get_document_job(job_id) if not run: @@ -878,10 +992,6 @@ async def get_run(job_id: int, user: RequireSearch): return run -def _parse_scheme_aliases(doc: dict) -> list[str]: - return scheme_catalog.parse_aliases(doc.get("scheme_aliases_json")) - - NETWORK_PUBLISH_ARTIFACT_TYPES = {"network_publish_payload", "network_publish_skipped"} @@ -914,11 +1024,12 @@ def _build_document_detail(doc: dict) -> DocumentDetail: current_job = db.get_latest_document_job(workflow_id) artifacts = db.list_document_artifacts(workflow_id) return DocumentDetail( + id=doc.get("id"), document_id=doc["document_id"], canonical_document_id=doc.get("canonical_document_id"), workflow_id=workflow_id, filename=doc["filename"], - display_name=doc.get("display_name"), + name=doc.get("name"), source_filename=doc.get("source_filename"), source_manifest_name=doc.get("source_manifest_name"), source_file_fingerprint=doc.get("source_file_fingerprint"), @@ -953,10 +1064,9 @@ def _build_document_detail(doc: dict) -> DocumentDetail: uploaded_by_username=doc.get("uploaded_by_username"), uploaded_by_email=doc.get("uploaded_by_email"), uploaded_by_roles=_parse_roles_field(doc.get("uploaded_by_roles")), - document_kind=(doc.get("document_kind") or "document"), - scheme_code=doc.get("scheme_code"), - scheme_name=doc.get("scheme_name"), - scheme_aliases=_parse_scheme_aliases(doc), + kind=doc.get("kind"), + code=doc.get("code"), + aliases=list(doc.get("aliases") or []), tool_routing=doc.get("tool_routing"), catalog_visible=bool(int(doc["catalog_visible"])) if doc.get("catalog_visible") is not None else True, network_visible=bool(int(doc["network_visible"])) if doc.get("network_visible") is not None else True, @@ -1009,10 +1119,37 @@ def _require_catalog_read(request: Request, user: AuthUser) -> None: raise HTTPException(403, "Catalog access requires search or admin permission") +def _vector_doc_ids(doc: dict, env: str) -> list[str]: + """Index keys to delete for a document: its current key plus keys of earlier file versions in ``env``. + + A key another live document is indexed under in ``env`` (the same file + uploaded twice) is left out, since deleting it would remove that document's + vectors too. + """ + ids = [doc.get("document_id")] + ids.extend((doc.get("superseded_doc_ids") or {}).get(env, [])) + return [ + i + for i in dict.fromkeys(i for i in ids if i) + if not db.doc_id_used_by_other_document(i, doc.get("id"), env) + ] + + +def _delete_all_doc_ids(store, collection: str, doc_ids: list[str]) -> dict: + results = [store.delete_by_doc_id(collection, doc_id) for doc_id in doc_ids] + if len(results) == 1: + return results[0] + errors = [r["error"] for r in results if r.get("error")] + summary = {"deleted": 0 if errors else -1, "doc_ids": doc_ids, "results": results} + if errors: + summary["error"] = "; ".join(errors) + return summary + + def _delete_document_points_from_prod(doc: dict) -> dict: """Best-effort delete of a promoted (non-scheme) document's vectors from PROD documents-index.""" - doc_id = doc.get("document_id") - if not doc_id: + doc_ids = _vector_doc_ids(doc, "prod") + if not doc_ids: return {"deleted": False, "reason": "no_doc_id"} prod_url = (os.environ.get("PROD_VECTOR_DB_URL") or "").strip() if not prod_url: @@ -1028,16 +1165,16 @@ def _delete_document_points_from_prod(doc: dict) -> dict: client = get_qdrant_client(url=prod_url, api_key=prod_key) store = QdrantVectorStore(client=client) - return store.delete_by_doc_id(collection, doc_id) + return _delete_all_doc_ids(store, collection, doc_ids) except Exception as exc: - logging.exception("prod vector delete failed doc_id=%s collection=%s", doc_id, collection) + logging.exception("prod vector delete failed doc_ids=%s collection=%s", doc_ids, collection) return {"deleted": False, "error": str(exc), "collection": collection} def _delete_scheme_points_from_prod(doc: dict) -> dict: """Best-effort delete of scheme vectors from PROD schemes-index.""" - doc_id = doc.get("document_id") - if not doc_id: + doc_ids = _vector_doc_ids(doc, "prod") + if not doc_ids: return {"deleted": False, "reason": "no_doc_id"} prod_url = ( (os.environ.get("PROD_SCHEME_VECTOR_DB_URL") or "").strip() @@ -1058,9 +1195,9 @@ def _delete_scheme_points_from_prod(doc: dict) -> dict: client = get_qdrant_client(url=prod_url, api_key=prod_key) store = QdrantVectorStore(client=client) - return store.delete_by_doc_id(collection, doc_id) + return _delete_all_doc_ids(store, collection, doc_ids) except Exception as exc: - logging.exception("prod vector delete failed doc_id=%s collection=%s", doc_id, collection) + logging.exception("prod vector delete failed doc_ids=%s collection=%s", doc_ids, collection) return {"deleted": False, "error": str(exc), "collection": collection} @@ -1179,7 +1316,7 @@ async def get_workflow_error_details(workflow_id: str, user: RequireSearch): This endpoint queries Temporal directly to get the most detailed error information available, which may be more complete than what's - stored in SQLite. + stored in Postgres. """ import traceback @@ -1257,7 +1394,7 @@ async def get_workflow_error_details(workflow_id: str, user: RequireSearch): @app.get("/documents/{workflow_id}/runtime") async def get_document_runtime(workflow_id: str, user: RequireSearch): - """Return live runtime status by combining SQLite state and Temporal workflow state.""" + """Return live runtime status by combining Postgres state and Temporal workflow state.""" doc = _require_document_for_user(workflow_id, user) return await _get_runtime_payload(workflow_id, doc=doc) @@ -1269,7 +1406,7 @@ async def list_document_artifacts(workflow_id: str, user: RequireSearch): @app.get("/documents/{workflow_id}/artifacts/{artifact_id}") -async def get_document_artifact(workflow_id: str, user: RequireSearch, artifact_id: int): +async def get_document_artifact(workflow_id: str, user: RequireSearch, artifact_id: str): _require_document_for_user(workflow_id, user) artifact = db.get_document_artifact(workflow_id, artifact_id) if not artifact: @@ -1278,7 +1415,7 @@ async def get_document_artifact(workflow_id: str, user: RequireSearch, artifact_ @app.get("/documents/{workflow_id}/artifacts/{artifact_id}/content") -async def get_document_artifact_content(workflow_id: str, user: RequireSearch, artifact_id: int): +async def get_document_artifact_content(workflow_id: str, user: RequireSearch, artifact_id: str): _require_document_for_user(workflow_id, user) artifact = db.get_document_artifact(workflow_id, artifact_id) if not artifact: @@ -1348,7 +1485,7 @@ async def get_document_graph(workflow_id: str, user: RequireSearch): def _delete_artifacts_from_minio(workflow_id: str) -> dict: """Remove every MinIO object backing a document's artifacts. - Artifacts are the only part of a document that lives outside SQLite and the + Artifacts are the only part of a document that lives outside Postgres and the vector store — the original upload, the normalized PDF, and the OCR / translation / chunk JSON snapshots. Nothing else reads them once the document is gone, so a hard delete has to sweep them explicitly. @@ -1396,17 +1533,15 @@ async def disable_document( - Remove all chunks from the DEV vector index - Delete PROD Qdrant points too, if this document was ever promoted (schemes-index for scheme documents, documents-index otherwise) - - Remove the row from the Postgres master catalog and refresh the - Redis snapshots the AI layer reads, so a re-ingest starts clean - - For scheme documents, also rebuild the legacy SQLite scheme catalog + - For scheme documents, also rebuild the scheme catalog - Stop the workflow if it is still running - Soft delete (default) then marks the document disabled in SQLite and keeps + Soft delete (default) then marks the document disabled in Postgres and keeps every row and artifact, so ``POST /documents/{id}/restore`` can bring it back. Use X-Include-Disabled: true in list_documents to see it. ``purge=true`` additionally deletes the MinIO artifacts (original upload, - normalized PDF, OCR/translation/chunk JSON) and every SQLite row (pages, + normalized PDF, OCR/translation/chunk JSON) and every Postgres row (pages, chunks, artifacts, jobs, index status, the document itself). This is irreversible — restore has nothing left to restore. Audit rows are kept unless ``keep_audit=false``, since they record the deletion itself. @@ -1414,7 +1549,7 @@ async def disable_document( Args: workflow_id: The document workflow ID remove_from_search: If True (default), removes chunks from the index - purge: If True, hard delete — also drops MinIO artifacts and SQLite rows + purge: If True, hard delete — also drops MinIO artifacts and Postgres rows keep_audit: If False (and purge=True), also deletes the audit trail Requires ``admin`` (super admin — any document), or ``delete_own`` @@ -1449,7 +1584,6 @@ async def disable_document( "scheme_qdrant_deleted": None, "prod_qdrant_deleted": None, "catalog_version": None, - "master_catalog_version": None, "minio_artifacts_deleted": None, "sqlite_rows_deleted": None, } @@ -1468,44 +1602,43 @@ async def disable_document( except Exception: pass # Workflow already completed/cancelled - # Mark as disabled in SQLite (a purge deletes the row outright below, but + # Mark as disabled in Postgres (a purge deletes the row outright below, but # this keeps the document hidden if a later step fails partway through) db.set_document_disabled(workflow_id, True) # Remove from the DEV vector index if requested if remove_from_search: - doc_id = doc.get("document_id") - if doc_id: + # The current index key plus keys of earlier file versions still in DEV + # (see metadata.superseded_doc_ids). + for index, doc_id in enumerate(_vector_doc_ids(doc, "dev")): vector_result = delete_chunks_from_index(doc_id) - result["vector_deleted"] = vector_result.get("deleted", 0) + if index == 0: + result["vector_deleted"] = vector_result.get("deleted", 0) if "error" in vector_result: result["vector_error"] = vector_result["error"] - # PROD Qdrant cleanup, if this document was ever promoted - kind = (doc.get("document_kind") or "document").strip().lower() - if kind == "scheme" or doc.get("scheme_code"): + # PROD Qdrant cleanup, if this document was ever promoted. A document + # promoted as a scheme lives in the scheme index even if its kind was + # changed since, so the PROD index entry decides, not only the kind. + kind = (doc.get("kind") or "").strip().lower() + prod_entry = (doc.get("index_status") or {}).get("prod") or {} + prod_is_scheme = ( + kind == "scheme" + or prod_entry.get("index_name") == scheme_catalog.scheme_collection_name() + ) + if prod_is_scheme: q_result = _delete_scheme_points_from_prod(doc) result["scheme_qdrant_deleted"] = q_result try: result["catalog_version"] = scheme_catalog.on_scheme_document_disabled( - workflow_id, scheme_code=doc.get("scheme_code") + workflow_id, code=doc.get("code") ) except Exception as exc: result["catalog_error"] = str(exc) else: result["prod_qdrant_deleted"] = _delete_document_points_from_prod(doc) - # Master catalog (Postgres) + Redis snapshot cleanup — mirrors the sync - # done on ingest/promote so a deleted document stops appearing to the - # AI layer immediately, and a later re-ingest starts from a clean slate. - try: - from ..master_catalog_pg import remove_catalog_entry - - result["master_catalog_version"] = await asyncio.to_thread(remove_catalog_entry, workflow_id) - except Exception as exc: - result["master_catalog_error"] = str(exc) - - # Hard delete: drop the artifacts and the SQLite rows themselves. Ordered + # Hard delete: drop the artifacts and the Postgres rows themselves. Ordered # storage-then-database so a failure mid-way leaves rows pointing at # already-deleted objects (visible, fixable) rather than orphaned objects # no row references any more (invisible, unreclaimable). @@ -1520,7 +1653,7 @@ async def disable_document( entity_type="document", metadata={ "filename": doc.get("filename"), - "scheme_code": doc.get("scheme_code"), + "code": doc.get("code"), "stage": doc.get("stage"), "minio_artifacts_deleted": result["minio_artifacts_deleted"], "keep_audit": keep_audit, @@ -1541,9 +1674,8 @@ async def disable_document( "vector_deleted": result["vector_deleted"], "scheme_qdrant_deleted": result.get("scheme_qdrant_deleted"), "prod_qdrant_deleted": result.get("prod_qdrant_deleted"), - "scheme_code": doc.get("scheme_code"), + "code": doc.get("code"), "catalog_version": result.get("catalog_version"), - "master_catalog_version": result.get("master_catalog_version"), }, user=user, ) @@ -1558,19 +1690,18 @@ async def patch_scheme_metadata( user: RequireReview, ): """ - Set document_kind / scheme fields used by Master Catalog and Qdrant scheme promote. + Set code / name / aliases / validity used by Master Catalog and Qdrant scheme promote. - On completed docs, renaming scheme_code sets pending_reindex (excluded from live + On completed docs, renaming the code sets pending_reindex (excluded from live vector_schemes until re-promote). Other metadata edits bump catalog version only. """ _require_document_for_user(workflow_id, user) try: result = scheme_catalog.apply_scheme_metadata( workflow_id, - document_kind=body.document_kind, - scheme_code=body.scheme_code, - scheme_name=body.scheme_name, - scheme_aliases=body.scheme_aliases, + code=body.code, + name=body.name, + aliases=body.aliases, tool_routing=body.tool_routing, catalog_visible=body.catalog_visible, network_visible=body.network_visible, @@ -1599,16 +1730,15 @@ async def patch_scheme_metadata( doc = result["document"] return { "workflow_id": workflow_id, - "document_kind": doc.get("document_kind"), - "scheme_code": doc.get("scheme_code"), - "scheme_name": doc.get("scheme_name"), - "scheme_aliases": scheme_catalog.parse_aliases(doc.get("scheme_aliases_json")), + "kind": doc.get("kind"), + "code": doc.get("code"), + "name": doc.get("name"), + "aliases": list(doc.get("aliases") or []), "tool_routing": doc.get("tool_routing"), "catalog_visible": bool(int(doc["catalog_visible"])) if doc.get("catalog_visible") is not None else True, "network_visible": bool(int(doc["network_visible"])) if doc.get("network_visible") is not None else True, "valid_from": doc.get("valid_from"), "valid_to": doc.get("valid_to"), - "display_name": doc.get("display_name"), "catalog_version": result.get("catalog_version"), "requires_reindex": result.get("requires_reindex"), "pending_reindex": result.get("pending_reindex"), @@ -1625,8 +1755,6 @@ async def patch_scheme_metadata( async def catalog_version(request: Request, user: CurrentUser): """Cheap poll for cache warmers: { version, updated_at }.""" _require_catalog_read(request, user) - scheme_catalog.ensure_catalog_schema() - scheme_catalog.bootstrap_catalog_if_empty() meta = db.get_catalog_meta() return { "version": int(meta.get("version") or 0), @@ -1646,7 +1774,6 @@ async def catalog_schemes( ): """List schemes from the master catalog (vector-indexed + filters).""" _require_catalog_read(request, user) - scheme_catalog.bootstrap_catalog_if_empty() snap = scheme_catalog.build_snapshot( instance=instance, status=status, @@ -1678,7 +1805,6 @@ async def catalog_snapshot( collections metadata, and monotonic catalog_version. """ _require_catalog_read(request, user) - scheme_catalog.bootstrap_catalog_if_empty() return scheme_catalog.build_snapshot( instance=instance, status=status, @@ -1691,7 +1817,6 @@ async def catalog_snapshot( async def catalog_tool_prompt(request: Request, user: CurrentUser): """Prompt fragments only (for AI system prompt assembly).""" _require_catalog_read(request, user) - scheme_catalog.bootstrap_catalog_if_empty() snap = scheme_catalog.build_snapshot() return { "catalog_version": snap["catalog_version"], @@ -1705,15 +1830,12 @@ async def catalog_tool_prompt(request: Request, user: CurrentUser): @app.post("/catalog/v1/rebuild") async def catalog_rebuild(request: Request, user: RequireAdmin): - """Admin: recompute catalog entries from documents + preserve bootstrap codes.""" + """Admin: bump the catalog version (entries are computed from documents).""" from ..auth.permissions import Permission # Admin JWT always OK; service key also accepted for automation if not _service_key_ok(request) and not user.has_permission(Permission.ADMIN): raise HTTPException(403, "Rebuild requires admin") - scheme_catalog.ensure_catalog_schema() - # Ensure bootstrap seeds exist before rebuild merge - scheme_catalog.bootstrap_catalog_if_empty() version = scheme_catalog.rebuild_catalog_from_documents(reason="admin_rebuild") snap = scheme_catalog.build_snapshot() _log_audit( @@ -1731,19 +1853,12 @@ async def catalog_rebuild(request: Request, user: RequireAdmin): } -@app.post("/catalog/v1/bootstrap") -async def catalog_bootstrap(request: Request, user: RequireAdmin, force: bool = Query(False)): - """Admin: seed 13 vector schemes if empty (or force=true).""" - result = scheme_catalog.bootstrap_catalog_if_empty(force=force) - return result - - @app.post("/documents/{workflow_id}/restore") async def restore_document(workflow_id: str, user: RequireAdmin): """ Restore a soft-deleted (disabled) document. - Note: This only restores the document in SQLite. Chunks that were removed + Note: This only restores the document in Postgres. Chunks that were removed from the vector index will NOT be automatically re-indexed. To re-index, you would need to re-run the ingestion process. """ @@ -1773,33 +1888,15 @@ async def reingest_document( Use this to re-ingest documents that completed but weren't properly indexed (e.g., due to index schema changes). This starts a lightweight - workflow that uses chunks already stored in SQLite. + workflow that uses chunks already stored in Postgres. - The document must have chunks stored in SQLite (typically from a + The document must have chunks stored in Postgres (typically from a completed or previously ingested document). """ - # Get document from SQLite + # Get document from Postgres doc = _require_document_for_user(workflow_id, user) - # Reingest skips the entire review pipeline (including ready_for_ingestion, - # where document type / scheme name get set) and jumps straight to pushing - # chunks to Qdrant. That's fine for re-pushing already-classified content - # (e.g. after an embedding/index change), but a trap if it's used as a - # substitute for the real approval flow on a never-classified document — - # it would silently ingest with document_kind='document' and no scheme - # metadata, the exact raw-hash/filename-fallback master_catalog rows we've - # had to clean up by hand. 'document' is never offered as a choice in the - # classification panel, so seeing it here means classification was never done. - if (doc.get("document_kind") or "document") == "document" and not doc.get("scheme_code") and not doc.get("scheme_name"): - raise HTTPException( - 400, - "This document has never been classified (document type / scheme name not set). " - "Set it via PATCH /documents/{workflow_id}/scheme-metadata (or the classification " - "panel at the ready_for_ingestion stage) before reingesting — reingest skips that " - "review stage entirely and would otherwise ingest with no scheme metadata.", - ) - - # Get chunks from SQLite + # Get chunks from Postgres chunks = db.get_chunks(workflow_id, include_excluded=False) if not chunks: raise HTTPException(400, "No chunks found for document. The document may need to be reprocessed from scratch.") @@ -1818,7 +1915,7 @@ async def reingest_document( args=[ document_id, filename, - workflow_id, # original workflow_id for SQLite updates + workflow_id, # original workflow_id for Postgres updates page_count, len(chunks), index_name @@ -1833,6 +1930,7 @@ async def reingest_document( status="running", current_stage="ingesting", config={"index_name": index_name, "chunk_count": len(chunks)}, + actor=_actor_from_user(user), ) # Log audit @@ -1880,15 +1978,16 @@ async def retry_ocr(workflow_id: str, user: RequirePipeline): id=temporal_workflow_id, task_queue=TASK_QUEUE, ) - job_id = db.create_document_job( + db.create_document_job( workflow_id=workflow_id, job_type="ocr_retry", temporal_workflow_id=temporal_workflow_id, status="running", current_stage="ocr_processing", config={"source": "api_retry_ocr"}, + actor=_actor_from_user(user), ) - db.update_document_fields(workflow_id, latest_job_id=job_id, error_message=None) + db.update_document_fields(workflow_id, error_message=None) _log_audit( workflow_id=workflow_id, action_type="retry_ocr", @@ -1912,15 +2011,16 @@ async def retry_translation(workflow_id: str, user: RequirePipeline): id=temporal_workflow_id, task_queue=TASK_QUEUE, ) - job_id = db.create_document_job( + db.create_document_job( workflow_id=workflow_id, job_type="translation_retry", temporal_workflow_id=temporal_workflow_id, status="running", current_stage="translation_processing", config={"source": "api_retry_translation"}, + actor=_actor_from_user(user), ) - db.update_document_fields(workflow_id, latest_job_id=job_id, error_message=None) + db.update_document_fields(workflow_id, error_message=None) _log_audit( workflow_id=workflow_id, action_type="retry_translation", @@ -1958,7 +2058,7 @@ async def retry_chunking( id=temporal_workflow_id, task_queue=TASK_QUEUE, ) - job_id = db.create_document_job( + db.create_document_job( workflow_id=workflow_id, job_type="chunking_retry", temporal_workflow_id=temporal_workflow_id, @@ -1970,8 +2070,9 @@ async def retry_chunking( "chunk_overlap": chunk_overlap, "min_tokens": min_tokens, }, + actor=_actor_from_user(user), ) - db.update_document_fields(workflow_id, latest_job_id=job_id, error_message=None) + db.update_document_fields(workflow_id, error_message=None) _log_audit( workflow_id=workflow_id, action_type="retry_chunking", @@ -2021,19 +2122,6 @@ async def clear_reindex_required(workflow_id: str, user: RequirePipeline): } -@app.post("/documents/{workflow_id}/demo") -async def set_document_demo(workflow_id: str, user: RequireAdmin, is_demo: bool = Query(True)): - """ - Mark a document as demo. - - Demo documents are excluded from the UI by default but always available - for API testing via include_demo=true parameter. - """ - _require_document_for_user(workflow_id, user) - db.set_document_demo(workflow_id, is_demo) - return {"workflow_id": workflow_id, "is_demo": is_demo} - - async def _reconcile_single_document(doc: dict) -> dict: workflow_id = doc.get("workflow_id") current_stage = doc.get("stage") @@ -2114,7 +2202,7 @@ async def _reconcile_single_document(doc: dict) -> dict: @app.post("/documents/{workflow_id}/reconcile") async def reconcile_single_document(workflow_id: str, user: RequirePipeline): - """Reconcile SQLite stage with Temporal state for one document.""" + """Reconcile Postgres stage with Temporal state for one document.""" doc = _require_document_for_user(workflow_id, user) return await _reconcile_single_document(doc) @@ -2152,7 +2240,7 @@ async def _validate_approval_stage(workflow_id: str, expected_stage: str): except HTTPException: raise except Exception: - # Try SQLite fallback to check if workflow exists but is completed/failed + # Try Postgres fallback to check if workflow exists but is completed/failed doc = db.get_document(workflow_id) if doc: current = doc.get("stage") @@ -2492,6 +2580,7 @@ async def approve_prod( temporal_workflow_id=promote_workflow_id, status="running", current_stage="ingesting_prod", + actor=_actor_from_user(user), ) db.update_document_stage( workflow_id=workflow_id, @@ -2555,7 +2644,7 @@ def _log_audit( metadata: Additional context as dict user: Optional AuthUser whose email/roles are stored on the audit row """ - # Get document_id from SQLite + # Get document_id from Postgres doc = db.get_document(workflow_id) document_id = doc["document_id"] if doc else workflow_id @@ -2667,7 +2756,7 @@ async def get_document_audit_log( @app.get("/documents/{workflow_id}/pages") async def list_pages(workflow_id: str, user: RequireSearch): - """Get all pages for a document. SQLite-first for speed.""" + """Get all pages for a document. Postgres-first for speed.""" # Enforce tenant scope up front (404 hides other tenants' documents). _require_document_for_user(workflow_id, user) return db.get_pages(workflow_id) @@ -2675,7 +2764,7 @@ async def list_pages(workflow_id: str, user: RequireSearch): @app.get("/documents/{workflow_id}/pages/{page_num}") async def get_page(workflow_id: str, user: RequireSearch, page_num: int = PathParam(..., ge=1, le=10000, description="Page number (1-indexed)")): - """Get a specific page. SQLite-first for speed.""" + """Get a specific page. Postgres-first for speed.""" _require_document_for_user(workflow_id, user) page = db.get_page(workflow_id, page_num) if page: @@ -2872,14 +2961,14 @@ async def search_chunks_across_documents( @app.get("/documents/{workflow_id}/chunks") async def list_chunks(workflow_id: str, user: RequireSearch, include_excluded: bool = False): - """Get all chunks for a document. SQLite-first for speed.""" + """Get all chunks for a document. Postgres-first for speed.""" _require_document_for_user(workflow_id, user) return db.get_chunks(workflow_id, include_excluded=include_excluded) @app.get("/documents/{workflow_id}/chunks/{chunk_num}") async def get_chunk(workflow_id: str, user: RequireSearch, chunk_num: int = PathParam(..., ge=1, le=10000, description="Chunk number (1-indexed)")): - """Get a specific chunk. SQLite-first for speed.""" + """Get a specific chunk. Postgres-first for speed.""" _require_document_for_user(workflow_id, user) chunk = db.get_chunk(workflow_id, chunk_num) if chunk: @@ -3098,9 +3187,9 @@ def _inline_content_disposition(filename: str) -> str: async def get_document_pdf(workflow_id: str, user: RequireSearch): """ Get the original PDF file for a document. - Returns the PDF as a streaming response. SQLite-first for speed. + Returns the PDF as a streaming response. Postgres-first for speed. """ - # SQLite-first - instant lookup, tenant-scoped (404 hides other tenants). + # Postgres-first - instant lookup, tenant-scoped (404 hides other tenants). doc = _require_document_for_user(workflow_id, user) filepath = doc.get("filepath", "") @@ -3156,7 +3245,7 @@ def file_iterator(): async def resolve_provenance_chunk( request: Request, user: RequireSearch, - doc_id: Optional[str] = Query(None, description="workflow slug, SQLite document_id, or legacy vector-index doc_id"), + doc_id: Optional[str] = Query(None, description="workflow slug, Postgres document_id, or legacy vector-index doc_id"), chunk_num: Optional[int] = Query(None, alias="chunk_num"), ): """ @@ -3223,14 +3312,14 @@ async def get_document_qdrant_status( ) normalized_hit.setdefault("chunk_number", chunk_num) hits.append(normalized_hit) - sqlite_chunks = db.get_chunks(workflow_id, include_excluded=True) + db_chunks = db.get_chunks(workflow_id, include_excluded=True) status = { "workflow_id": workflow_id, "index_name": resolved_index, "backend": backend, "index_doc_id": index_doc_id, "doc_id": index_doc_id, - "sqlite_chunk_count": len([c for c in sqlite_chunks if not c.get("is_excluded")]), + "sqlite_chunk_count": len([c for c in db_chunks if not c.get("is_excluded")]), "indexed_chunk_count": len(hits), "status": "indexed" if hits else "missing", "hits": hits, @@ -3280,16 +3369,13 @@ async def get_vector_index_stats(index_name: str, user: RequireSearch): @app.get("/indexes/summary") async def get_vector_indexes_summary( user: RequireSearch, - x_include_demo: Optional[str] = Header(None, alias="X-Include-Demo"), x_include_disabled: Optional[str] = Header(None, alias="X-Include-Disabled"), ): - """Summarize index coverage from SQLite-backed index status plus live vector-store stats.""" + """Summarize index coverage from Postgres-backed index status plus live vector-store stats.""" from ..vector_store import get_vector_store - include_demo = x_include_demo and x_include_demo.lower() == "true" include_disabled = x_include_disabled and x_include_disabled.lower() == "true" summaries = db.list_index_summaries( - include_demo=include_demo, include_disabled=include_disabled, ) if not summaries: @@ -3326,10 +3412,118 @@ async def health(): """Health check.""" return { "status": "ok", - "temporal_connected": temporal_client is not None + "temporal_connected": temporal_client is not None, + "database": await asyncio.to_thread(db.check_health), } +# ============================================================================= +# Webhook (inbound pushes from external systems, x-api-key protected) +# ============================================================================= + +WEBHOOK_MAX_RECORDS = int(os.environ.get("WEBHOOK_MAX_RECORDS", "500")) + + +def require_webhook_key(request: Request) -> None: + """Dependency: x-api-key must equal WEBHOOK_API_KEY. Fails closed when unset.""" + import hmac + + expected = (os.environ.get("WEBHOOK_API_KEY") or "").strip() + if not expected: + raise HTTPException(503, "WEBHOOK_API_KEY is not configured") + provided = (request.headers.get("x-api-key") or "").strip() + if not provided or not hmac.compare_digest(provided, expected): + raise HTTPException(401, "Missing or invalid x-api-key") + + +@app.post("/webhook") +@limiter.limit(RATE_LIMIT_UPLOAD) +async def webhook( + request: Request, + records: List[Dict[str, Any]], + _: None = Depends(require_webhook_key), +): + """Accept a JSON array of records and upsert them into webhook_records.""" + if len(records) > WEBHOOK_MAX_RECORDS: + raise HTTPException(413, f"Too many records (max {WEBHOOK_MAX_RECORDS})") + try: + count = db.upsert_webhook_records("webhook", records) + except ValueError as e: + raise HTTPException(422, str(e)) + await _start_advisory_index_for(records) + return { + "success": True, + "message": "Webhook processed successfully.", + "recordCount": count, + } + + +async def _start_advisory_index_for(records: list) -> int: + """Start one advisory index workflow per record of this call that is not indexed yet. + + Best effort: the records are already saved, so a Temporal failure never fails the + webhook; the rows stay unindexed and the reindex endpoint (or worker startup backfill) + picks them up. + """ + if temporal_client is None: + logger.warning("Temporal client not ready; advisory indexing skipped for this webhook call") + return 0 + try: + external_ids = [db.webhook_external_id(r) for r in records if isinstance(r, dict)] + record_ids = db.get_unindexed_webhook_record_ids(external_ids=external_ids) + return await start_advisory_index(temporal_client, record_ids) + except Exception: + logger.exception("Could not start advisory indexing for this webhook call") + return 0 + + +class AdvisoryReindexBody(BaseModel): + record_ids: Optional[List[str]] = None + force: bool = False + + +class AdvisoryPromoteBody(BaseModel): + record_ids: Optional[List[str]] = None + + +def _require_temporal() -> Client: + if temporal_client is None: + raise HTTPException(503, "Temporal client is not ready") + return temporal_client + + +@app.post("/webhook/advisory/reindex") +async def advisory_reindex(body: AdvisoryReindexBody, user: RequireAdmin): + """Queue advisory indexing (dev collection): given record ids, else all unindexed (all with force).""" + client = _require_temporal() + if body.record_ids is not None: + record_ids = body.record_ids + elif body.force: + record_ids = db.get_webhook_record_ids() + else: + record_ids = db.get_unindexed_webhook_record_ids() + queued = await start_advisory_index(client, record_ids) + return {"success": True, "queued": queued} + + +@app.post("/webhook/advisory/promote") +async def advisory_promote(body: AdvisoryPromoteBody, user: RequireAdmin): + """Queue promotion to the PROD collection: given record ids, else all published records already indexed in dev.""" + client = _require_temporal() + if body.record_ids is not None: + record_ids = body.record_ids + else: + record_ids = db.get_webhook_record_ids(published_only=True, indexed_only=True) + queued = await start_advisory_promote(client, record_ids) + return {"success": True, "queued": queued} + + +@app.get("/webhook/advisory/status") +async def advisory_status(user: RequireAdmin): + """Counts of webhook advisory records: total, published, indexed, unindexed, promoted.""" + return db.advisory_status_counts() + + # ============================================================================= # Vector index (passage schema) # ============================================================================= @@ -3428,10 +3622,10 @@ async def get_ingest_info(user: RequireAdmin): @app.post("/documents/reconcile") async def reconcile_document_states(user: RequirePipeline): """ - Reconcile SQLite document states with Temporal workflow states. + Reconcile Postgres document states with Temporal workflow states. This endpoint checks all documents in processing/review stages and updates - SQLite if the Temporal workflow has terminated or failed. This fixes + Postgres if the Temporal workflow has terminated or failed. This fixes inconsistencies caused by external workflow termination or worker crashes. Returns a summary of documents checked and updated. @@ -3447,7 +3641,6 @@ async def reconcile_document_states(user: RequirePipeline): # Scope to caller's instances (None = unrestricted bypass / all tenants). docs = db.list_documents( limit=1000, - include_demo=True, include_disabled=True, instances=_instance_scope_for_user(user), ) @@ -3525,7 +3718,10 @@ async def update_search_settings_endpoint(settings: SearchSettingsUpdate, user: if not updates: raise HTTPException(400, "No settings provided to update") - return db.update_search_settings(updates) + try: + return db.update_search_settings(updates) + except ValueError as exc: + raise HTTPException(400, str(exc)) from exc @app.get("/settings/search/audit", response_model=SettingsAuditResponse) diff --git a/pipeline/api/documents.py b/pipeline/api/documents.py index 5eb43ef..b431755 100644 --- a/pipeline/api/documents.py +++ b/pipeline/api/documents.py @@ -37,6 +37,8 @@ async def start_document_workflow( index_name: str = "documents-index", stop_after_ocr: bool = False, instance: str = "", + kind: str = Query(..., description="scheme or advisory"), + code: str = "", ): """ Start a new document processing workflow. @@ -56,14 +58,17 @@ async def start_document_workflow( # Validate file path to prevent path traversal attacks filepath = _api.validate_file_path(data.filepath) source_filename = _api.get_filename_from_path(filepath) + doc_kind, doc_code, replace_doc = _api._resolve_document_identity( + user, create_instance, kind, code, source_filename + ) source_file_fingerprint = _api._compute_file_fingerprint(filepath) canonical_document_id = source_file_fingerprint workflow_id = _api.get_workflow_id(str(filepath)) document_id = canonical_document_id - # Reuse only when SQLite still tracks this workflow. - # If SQLite was purged, avoid returning stale Temporal state and create a fresh run ID. + # Reuse only when Postgres still tracks this workflow. + # If Postgres was purged, avoid returning stale Temporal state and create a fresh run ID. existing_doc = db.get_document(workflow_id) if existing_doc: # Same fingerprint/path must not leak or restart another tenant's doc. @@ -112,32 +117,48 @@ async def start_document_workflow( task_queue=_api.TASK_QUEUE, ) - # Save to SQLite for visibility during processing + # Save to Postgres for visibility during processing actor = _api._actor_from_user(user) # The period this document is searchable in starts here, at the one moment # that knows the upload day. A reviewer confirms or moves it later, on the # form that sets the document type. validity = document_validity.default_period() - db.upsert_document( - workflow_id=workflow_id, - document_id=document_id, - canonical_document_id=canonical_document_id, - filename=source_filename, - source_filename=source_filename, - source_file_fingerprint=source_file_fingerprint, - filepath=str(filepath), - stage="registered", - stop_after_ocr=stop_after_ocr, - instance=create_instance, - uploaded_by_user_id=actor.get("user_id") or None, - uploaded_by_username=actor.get("username") or None, - uploaded_by_email=actor.get("email") or None, - uploaded_by_roles=actor.get("roles_csv") or None, - valid_from=validity.start_date, - valid_to=validity.end_date, - ) - job_id = db.create_document_job( - workflow_id=workflow_id, + if replace_doc: + document_uuid = replace_doc["id"] + db.reset_document_for_reupload( + document_uuid, + filename=source_filename, + filepath=str(filepath), + fingerprint=source_file_fingerprint, + ) + else: + try: + document_uuid = db.upsert_document( + workflow_id=workflow_id, + document_id=document_id, + canonical_document_id=canonical_document_id, + filename=source_filename, + source_filename=source_filename, + source_file_fingerprint=source_file_fingerprint, + filepath=str(filepath), + stage="registered", + stop_after_ocr=stop_after_ocr, + instance=create_instance, + uploaded_by_user_id=actor.get("user_id") or None, + uploaded_by_username=actor.get("username") or None, + uploaded_by_email=actor.get("email") or None, + uploaded_by_roles=actor.get("roles_csv") or None, + valid_from=validity.start_date, + valid_to=validity.end_date, + with_workflow=False, + kind=doc_kind, + code=doc_code, + name=_api._document_title(source_filename), + ) + except db.DuplicateDocumentCode as exc: + raise HTTPException(409, str(exc)) from exc + db.create_document_job( + workflow_id=document_uuid, job_type="ocr_only" if stop_after_ocr else "pipeline", temporal_workflow_id=workflow_id, status="running", @@ -151,15 +172,21 @@ async def start_document_workflow( "stop_after_ocr": stop_after_ocr, "uploaded_by": actor, }, + actor=actor, ) - db.update_document_fields(workflow_id, latest_job_id=job_id) _api._log_audit( workflow_id=workflow_id, action_type="document_upload", entity_type="document", field_name="registered", new_value=source_filename, - metadata={"source": "register_endpoint", "instance": create_instance}, + metadata={ + "source": "register_endpoint", + "instance": create_instance, + "kind": doc_kind, + "code": doc_code, + "replaced_existing": bool(replace_doc), + }, user=user, ) @@ -194,6 +221,7 @@ async def start_batch_workflows( min_tokens: int = 100, stop_after_ocr: bool = False, instance: str = "", + kind: str = Query(..., description="scheme or advisory"), ): """Start workflows for all supported documents in a directory.""" create_instance = _api._resolve_create_instance(user, instance) @@ -218,6 +246,8 @@ async def start_batch_workflows( min_tokens=min_tokens, stop_after_ocr=stop_after_ocr, instance=create_instance, + kind=kind, + code="", ) results.append(result) except Exception as e: @@ -243,14 +273,12 @@ async def list_documents( stage: Optional[DocumentStage] = None, limit: int = Query(100, le=500), offset: int = Query(0, ge=0), - x_include_demo: Optional[str] = Header(None, alias="X-Include-Demo"), x_include_disabled: Optional[str] = Header(None, alias="X-Include-Disabled") ): """ List all document workflows. - Uses SQLite for fast listing (no Temporal queries for performance). - Demo documents are excluded by default - use X-Include-Demo: true header to show them. + Uses Postgres for fast listing (no Temporal queries for performance). Disabled (soft-deleted) documents are excluded by default - use X-Include-Disabled: true to show them. Results are limited to instances the caller can access. @@ -259,15 +287,13 @@ async def list_documents( - offset: Skip first N documents (default 0) """ stage_filter = stage.value if stage else None - include_demo = x_include_demo and x_include_demo.lower() == "true" - include_disabled = x_include_disabled and x_include_disabled.lower() == "true" + include_disabled = bool(x_include_disabled and x_include_disabled.lower() == "true") - # Use SQLite only for fast listing - no Temporal queries + # Use Postgres only for fast listing - no Temporal queries docs = db.list_documents( stage=stage_filter, limit=limit, offset=offset, - include_demo=include_demo, include_disabled=include_disabled, instances=_api._instance_scope_for_user(user), ) @@ -278,20 +304,16 @@ async def list_documents( @router.get("/summary", response_model=DocumentCohortsResponse) async def get_documents_summary( user: CurrentUser, - x_include_demo: Optional[str] = Header(None, alias="X-Include-Demo"), x_include_disabled: Optional[str] = Header(None, alias="X-Include-Disabled") ): - """Return aggregate SQLite counts for dashboard totals and migration planning.""" - include_demo = x_include_demo and x_include_demo.lower() == "true" - include_disabled = x_include_disabled and x_include_disabled.lower() == "true" + """Return aggregate Postgres counts for dashboard totals and migration planning.""" + include_disabled = bool(x_include_disabled and x_include_disabled.lower() == "true") instance_scope = _api._instance_scope_for_user(user) summary = db.get_document_summary_counts( - include_demo=include_demo, include_disabled=include_disabled, instances=instance_scope, ) by_instance = db.get_document_counts_by_instance( - include_demo=include_demo, include_disabled=include_disabled, instances=instance_scope, ) @@ -301,20 +323,16 @@ async def get_documents_summary( @router.get("/cohorts", response_model=DocumentCohortsResponse) async def get_document_cohorts( user: CurrentUser, - x_include_demo: Optional[str] = Header(None, alias="X-Include-Demo"), x_include_disabled: Optional[str] = Header(None, alias="X-Include-Disabled") ): """Return machine-friendly cohort counts for queueing and orchestration.""" - include_demo = x_include_demo and x_include_demo.lower() == "true" - include_disabled = x_include_disabled and x_include_disabled.lower() == "true" + include_disabled = bool(x_include_disabled and x_include_disabled.lower() == "true") instance_scope = _api._instance_scope_for_user(user) summary = db.get_document_summary_counts( - include_demo=include_demo, include_disabled=include_disabled, instances=instance_scope, ) by_instance = db.get_document_counts_by_instance( - include_demo=include_demo, include_disabled=include_disabled, instances=instance_scope, ) diff --git a/pipeline/api/upload.py b/pipeline/api/upload.py index 543d4a7..b7d4be1 100644 --- a/pipeline/api/upload.py +++ b/pipeline/api/upload.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Optional -from fastapi import APIRouter, File, Form, HTTPException, Request, UploadFile +from fastapi import APIRouter, File, Form, HTTPException, Query, Request, UploadFile from .. import db, document_validity from .. import display_name as display_name_rules @@ -32,6 +32,8 @@ async def upload_and_process( index_name: str = "documents-index", stop_after_ocr: bool = False, instance: str = "", + kind: str = Query(..., description="scheme or advisory"), + code: str = "", display_name: Optional[str] = Form(None), ): """ @@ -43,6 +45,9 @@ async def upload_and_process( Requires permission: upload (no-op while AUTH_DISABLED=true). """ create_instance = _api._resolve_create_instance(user, instance) + doc_kind, doc_code, replace_doc = _api._resolve_document_identity( + user, create_instance, kind, code, file.filename or "" + ) suffix = Path(file.filename or "").suffix.lower() if suffix not in _api.ALLOWED_EXTENSIONS: raise HTTPException(400, f"Unsupported file type: {suffix}") @@ -82,8 +87,8 @@ async def upload_and_process( document_id = file_hash canonical_document_id = file_hash - # Reuse only when SQLite still tracks this workflow. - # If SQLite was purged, avoid returning stale Temporal state and create a fresh run ID. + # Reuse only when Postgres still tracks this workflow. + # If Postgres was purged, avoid returning stale Temporal state and create a fresh run ID. existing_doc = db.get_document(workflow_id) if existing_doc: existing_doc = assert_document_instance_access(user, existing_doc) @@ -131,31 +136,47 @@ async def upload_and_process( task_queue=_api.TASK_QUEUE, ) - # Save to SQLite for visibility during processing + # Save to Postgres for visibility during processing actor = _api._actor_from_user(user) # See the manifest upload path above: the period starts at the upload. validity = document_validity.default_period() - db.upsert_document( - workflow_id=workflow_id, - document_id=document_id, - canonical_document_id=canonical_document_id, - filename=file.filename, - source_filename=file.filename, - source_file_fingerprint=file_hash, - filepath=minio_path, - display_name=display_name, - stage="registered", - stop_after_ocr=stop_after_ocr, - instance=create_instance, - uploaded_by_user_id=actor.get("user_id") or None, - uploaded_by_username=actor.get("username") or None, - uploaded_by_email=actor.get("email") or None, - uploaded_by_roles=actor.get("roles_csv") or None, - valid_from=validity.start_date, - valid_to=validity.end_date, - ) + if replace_doc: + document_uuid = replace_doc["id"] + db.reset_document_for_reupload( + document_uuid, + filename=file.filename, + filepath=minio_path, + fingerprint=file_hash, + ) + else: + try: + document_uuid = db.upsert_document( + workflow_id=workflow_id, + document_id=document_id, + canonical_document_id=canonical_document_id, + filename=file.filename, + source_filename=file.filename, + source_file_fingerprint=file_hash, + filepath=minio_path, + stage="registered", + stop_after_ocr=stop_after_ocr, + instance=create_instance, + uploaded_by_user_id=actor.get("user_id") or None, + uploaded_by_username=actor.get("username") or None, + uploaded_by_email=actor.get("email") or None, + uploaded_by_roles=actor.get("roles_csv") or None, + valid_from=validity.start_date, + valid_to=validity.end_date, + with_workflow=False, + kind=doc_kind, + code=doc_code, + # v2: the contributor's Display Name is documents.name. + name=display_name or _api._document_title(file.filename or ""), + ) + except db.DuplicateDocumentCode as exc: + raise HTTPException(409, str(exc)) from exc job_id = db.create_document_job( - workflow_id=workflow_id, + workflow_id=document_uuid, job_type="ocr_only" if stop_after_ocr else "pipeline", temporal_workflow_id=workflow_id, status="running", @@ -169,8 +190,9 @@ async def upload_and_process( "stop_after_ocr": stop_after_ocr, "uploaded_by": actor, }, + actor=actor, ) - original_artifact_id = db.add_document_artifact( + db.add_document_artifact( workflow_id=workflow_id, job_id=job_id, artifact_type="original_upload", @@ -190,6 +212,9 @@ async def upload_and_process( metadata={ "source": "upload_endpoint", "instance": create_instance, + "kind": doc_kind, + "code": doc_code, + "replaced_existing": bool(replace_doc), "size_bytes": file_size, "auto_approve": auto_approve, }, @@ -199,8 +224,6 @@ async def upload_and_process( canonical_input_type = "spreadsheet" if suffix in {".csv", ".xlsx"} else "pdf" db.update_document_fields( workflow_id, - latest_job_id=job_id, - original_artifact_id=original_artifact_id, source_type=source_type, canonical_input_type=canonical_input_type, stop_after_ocr=1 if stop_after_ocr else 0, diff --git a/pipeline/auth/deps.py b/pipeline/auth/deps.py index c09dae9..5a3af81 100644 --- a/pipeline/auth/deps.py +++ b/pipeline/auth/deps.py @@ -5,10 +5,13 @@ import asyncio import base64 import json +import logging +import time from typing import Annotated, Any, Callable from fastapi import Depends, HTTPException, Request +from .. import db from .config import load_auth_config from .jwt import claims_to_user, decode_and_validate_token from .models import AuthUser, local_bypass_user @@ -72,6 +75,28 @@ def _bypass_user_with_optional_jwt(token: str | None) -> AuthUser: ) +_USER_SYNC_INTERVAL_SECONDS = 300 +_user_synced_at: dict[str, float] = {} +_WRITE_METHODS = {"POST", "PUT", "PATCH", "DELETE"} + + +def _sync_user_row(user: AuthUser) -> None: + """Mirror the caller into ``users`` (at most once per interval per user).""" + now = time.monotonic() + last = _user_synced_at.get(user.user_id) + if last is not None and now - last < _USER_SYNC_INTERVAL_SECONDS: + return + try: + db.upsert_user( + user.user_id, user.username, user.email, user.roles, + tenants=list(user.instances or []), + ) + except Exception: + logging.warning("users upsert failed for %s", user.user_id, exc_info=True) + return + _user_synced_at[user.user_id] = now + + async def get_current_user(request: Request) -> AuthUser: """Resolve the caller. When AUTH_DISABLED=true, returns a local bypass user. @@ -83,12 +108,16 @@ async def get_current_user(request: Request) -> AuthUser: token = _bearer_token(request, required=False) if config.disabled: - return _bypass_user_with_optional_jwt(token) + user = _bypass_user_with_optional_jwt(token) + else: + if not token: + raise HTTPException(401, "Missing Bearer token") + # JWKS fetch/refresh is sync and can block; keep the event loop free. + user = await asyncio.to_thread(decode_and_validate_token, token, config) - if not token: - raise HTTPException(401, "Missing Bearer token") - # JWKS fetch/refresh is sync and can block; keep the event loop free. - return await asyncio.to_thread(decode_and_validate_token, token, config) + if request.method in _WRITE_METHODS and user.user_id: + await asyncio.to_thread(_sync_user_row, user) + return user def require_permission(permission: Permission | str) -> Callable[..., AuthUser]: diff --git a/pipeline/auth/models.py b/pipeline/auth/models.py index dd7fcdd..947d21c 100644 --- a/pipeline/auth/models.py +++ b/pipeline/auth/models.py @@ -85,7 +85,7 @@ def is_instance_unrestricted(self) -> bool: def has_instance(self, instance: str) -> bool: if not self.instances: - # Empty instance list means "no tenant restriction yet" only in disabled mode. + # Empty list means "no tenant restriction yet" only in disabled mode. return self.token_disabled_mode return instance.strip().lower() in {i.lower() for i in self.instances} diff --git a/pipeline/auth/tenancy.py b/pipeline/auth/tenancy.py index e7813f6..e507bac 100644 --- a/pipeline/auth/tenancy.py +++ b/pipeline/auth/tenancy.py @@ -1,9 +1,11 @@ -"""Instance (tenant / state) access helpers — Plan 2 (Keycloak + document.instance). +"""Instance (tenant / state) access helpers — Keycloak groups + document instances. -Access who-sees-what-state comes from the JWT (Keycloak groups). The database -only stores which state a *document* belongs to (``documents.instance``). +Who-sees-what-state comes from the JWT (Keycloak groups). The database only +stores which instance a *document* belongs to (``instances`` via +``documents.catalog_id -> catalogs.instance_id``); the document dict carries its +tenant code as ``instance``. -Plan 2 conventions: +Conventions: - State codes: lowercase (``mh``, ``up``, …) matching Keycloak ``/states/MH/...`` - Portal / BV platform docs: ``bv`` (Bharat Vistaar) - Super admin (``/global/super-admin``): unrestricted instances @@ -22,7 +24,7 @@ def default_instance() -> str: - """Fallback instance id for new docs and legacy rows without a value.""" + """Fallback instance code for new docs and legacy rows without a value.""" return (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" @@ -31,6 +33,7 @@ def portal_instance() -> str: def group_for_instance(instance: str | None) -> str | None: + """The named network instance of a tenant code, or None for the portal / default tenant.""" code = (instance or "").strip().lower() if not code or code in {portal_instance(), default_instance()}: return None @@ -49,7 +52,7 @@ def unrestricted(user: AuthUser) -> bool: def allowed_instances(user: AuthUser) -> set[str] | None: """ - Return the set of instance ids the user may access, or None if unrestricted. + Return the set of instance codes the user may access, or None if unrestricted. """ if unrestricted(user): return None @@ -66,7 +69,7 @@ def user_can_access_instance(user: AuthUser, instance: str | None) -> bool: def assert_instance_access(user: AuthUser, instance: str | None) -> str: - """Raise 403 if user cannot access instance; return normalized instance id.""" + """Raise 403 if user cannot access instance; return normalized instance code.""" normalized = normalize_instance(instance) if not user_can_access_instance(user, normalized): raise HTTPException(403, f"No access to instance: {normalized}") @@ -74,7 +77,7 @@ def assert_instance_access(user: AuthUser, instance: str | None) -> str: def resolve_create_instance(user: AuthUser, requested: str | None = None) -> str: - """Pick document ``instance`` at create/upload time (Plan 2). + """Pick the document's ``instance`` at create/upload time. - Super admin / bypass: use requested, else portal ``bv``, else DEFAULT_INSTANCE. - Single-state user: that state (requested must match if provided). diff --git a/pipeline/config.py b/pipeline/config.py index 7754851..426e41c 100644 --- a/pipeline/config.py +++ b/pipeline/config.py @@ -20,7 +20,10 @@ class Config: temporal_host: str = "localhost:7233" minio_endpoint: str = "localhost:9000" minio_bucket: str = "documents" - document_db_path: str = "/data/documents.db" + database_url: str = "postgresql://postgres:postgres@localhost:5432/knowledge_provider" + db_pool_min_size: int = 1 + db_pool_max_size: int = 10 + db_pool_timeout_seconds: float = 30.0 translation_provider: str = "gemma_vllm" translation_model: str = "google/gemma-4-31b-it" translation_vllm_base_url: str = "http://localhost:8020/v1" @@ -52,22 +55,6 @@ class Config: # File access allowed_file_paths: list[str] = None - # Master Catalog (Postgres) — code/name/tool_name/prompt_snippet, generic - # across content types (scheme, advisory, ...), not scheme-specific - master_catalog_pg_host: str = "localhost" - master_catalog_pg_port: int = 5432 - master_catalog_pg_db: str = "master_catalog" - master_catalog_pg_user: str = "master_catalog" - master_catalog_pg_password: str = "" - master_catalog_pg_sslmode: str = "disable" - - # AI layer Redis (bharat-oan-api) — direct write of the catalog snapshot - ai_layer_redis_host: str = "" - ai_layer_redis_port: int = 6379 - ai_layer_redis_db: int = 0 - ai_layer_redis_password: str = "" - master_catalog_redis_ttl_seconds: int = 172800 - def __post_init__(self): if self.cors_origins is None: self.cors_origins = ["http://localhost:3000"] @@ -121,7 +108,10 @@ def load_config() -> Config: temporal_host=os.environ.get("TEMPORAL_HOST", "localhost:7233"), minio_endpoint=os.environ.get("MINIO_ENDPOINT", "localhost:9000"), minio_bucket=os.environ.get("MINIO_BUCKET", "documents"), - document_db_path=os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db"), + database_url=os.environ.get("DATABASE_URL", Config.database_url), + db_pool_min_size=int(os.environ.get("DB_POOL_MIN_SIZE", "1")), + db_pool_max_size=int(os.environ.get("DB_POOL_MAX_SIZE", "10")), + db_pool_timeout_seconds=float(os.environ.get("DB_POOL_TIMEOUT_SECONDS", "30")), translation_provider=os.environ.get("TRANSLATION_PROVIDER", "gemma_vllm"), translation_model=os.environ.get("TRANSLATION_MODEL", "google/gemma-4-31b-it"), translation_vllm_base_url=os.environ.get("TRANSLATION_VLLM_BASE_URL", "http://localhost:8020/v1"), @@ -152,21 +142,6 @@ def load_config() -> Config: # File access allowed_file_paths=os.environ.get("ALLOWED_FILE_PATHS", "").split(",") if os.environ.get("ALLOWED_FILE_PATHS") else None, - - # Master Catalog (Postgres) - master_catalog_pg_host=os.environ.get("MASTER_CATALOG_PG_HOST", "localhost"), - master_catalog_pg_port=int(os.environ.get("MASTER_CATALOG_PG_PORT", "5432")), - master_catalog_pg_db=os.environ.get("MASTER_CATALOG_PG_DB", "master_catalog"), - master_catalog_pg_user=os.environ.get("MASTER_CATALOG_PG_USER", "master_catalog"), - master_catalog_pg_password=os.environ.get("MASTER_CATALOG_PG_PASSWORD", ""), - master_catalog_pg_sslmode=os.environ.get("MASTER_CATALOG_PG_SSLMODE", "disable"), - - # AI layer Redis - ai_layer_redis_host=os.environ.get("AI_LAYER_REDIS_HOST", ""), - ai_layer_redis_port=int(os.environ.get("AI_LAYER_REDIS_PORT", "6379")), - ai_layer_redis_db=int(os.environ.get("AI_LAYER_REDIS_DB", "0")), - ai_layer_redis_password=os.environ.get("AI_LAYER_REDIS_PASSWORD", ""), - master_catalog_redis_ttl_seconds=int(os.environ.get("MASTER_CATALOG_REDIS_TTL_SECONDS", "172800")), ) @@ -189,7 +164,10 @@ def print_config_status(): ("TEMPORAL_HOST", "localhost:7233"), ("MINIO_ENDPOINT", "localhost:9000"), ("MINIO_BUCKET", "documents"), - ("DOCUMENT_DB_PATH", "/data/documents.db"), + ("DATABASE_URL", "postgresql://postgres:postgres@localhost:5432/knowledge_provider"), + ("DB_POOL_MIN_SIZE", "1"), + ("DB_POOL_MAX_SIZE", "10"), + ("DB_POOL_TIMEOUT_SECONDS", "30"), ("TRANSLATION_PROVIDER", "gemma_vllm"), ("TRANSLATION_MODEL", "gemma-4"), ("TRANSLATION_VLLM_BASE_URL", "http://localhost:8020/v1"), @@ -213,9 +191,6 @@ def print_config_status(): ("CORS_ORIGINS", "(default)"), ("RATE_LIMIT_DEFAULT", "100/minute"), ("RATE_LIMIT_UPLOAD", "10/minute"), - ("MASTER_CATALOG_PG_HOST", "localhost"), - ("MASTER_CATALOG_PG_DB", "master_catalog"), - ("AI_LAYER_REDIS_HOST", "(unset — Redis push disabled)"), ] for var_name, default in optional: diff --git a/pipeline/db.py b/pipeline/db.py index 30b3ff0..b8a70e8 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -1,658 +1,987 @@ """ -SQLite state persistence for document visibility. +Postgres persistence for documents, workflows, pages, chunks, audit and settings. -This provides a fallback when Temporal workflow queries fail during -long-running activities, ensuring the dashboard always shows document status. +Schema: ``migrations/`` (see docs/POSTGRES_SCHEMA.md). One function per query. + +A document is content and metadata; a workflow is one processing run of a +document. Callers still address a document by ``workflow_id`` - the Temporal +workflow id of its main run (``doc-``) - and every function here also +accepts the document UUID, the fingerprint or the filename (see +``find_document_by_doc_identifier``). Documents, jobs and artifacts come back +as flat dicts so call sites need not know how the tables are split; ``id`` +carries the UUID. """ import hashlib import json +import logging import os -import sqlite3 +import re +import threading +import uuid from contextlib import contextmanager -from datetime import datetime -from pathlib import Path -from threading import Lock -from typing import Optional +from datetime import date, datetime, timezone +from typing import Any, Optional + +import psycopg +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb +from psycopg_pool import ConnectionPool + +from . import migrate +from .models import ( + CandidateSettings, + ChunkingRun, + DocumentMetadata, + DocumentSource, + IndexState, + PageMetadata, + PageTranslation, + QuerySettings, + ResultSettings, + RetrievalSettings, +) +from .network_constants import KIND_TO_CATALOG_TYPE, catalog_type_for_kind, kind_for_catalog_type + +logger = logging.getLogger(__name__) + +DEFAULT_DATABASE_URL = "postgresql://postgres:postgres@localhost:5432/knowledge_provider" + +_pool: Optional[ConnectionPool] = None +_pool_url: Optional[str] = None +_pool_lock = threading.Lock() + + +def database_url() -> str: + return os.environ.get("DATABASE_URL") or DEFAULT_DATABASE_URL + + +def get_pool() -> ConnectionPool: + global _pool, _pool_url + url = database_url() + if _pool is not None and _pool_url == url: + return _pool + with _pool_lock: + if _pool is not None and _pool_url != url: + _pool.close() + _pool = None + if _pool is None: + pool = ConnectionPool( + conninfo=url, + min_size=int(os.environ.get("DB_POOL_MIN_SIZE", "1")), + max_size=int(os.environ.get("DB_POOL_MAX_SIZE", "10")), + timeout=float(os.environ.get("DB_POOL_TIMEOUT_SECONDS", "30")), + kwargs={"row_factory": dict_row, "options": "-c timezone=UTC"}, + open=False, + ) + pool.open(wait=True) + _pool = pool + _pool_url = url + logger.info("db pool opened: %s", pool.get_stats()) + return _pool + -# Database path - can be configured via environment -DB_PATH = os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db") +def close_pool() -> None: + global _pool, _pool_url + with _pool_lock: + if _pool is not None: + _pool.close() + _pool = None + _pool_url = None -# Lock for thread-safe operations -_db_lock = Lock() +@contextmanager +def get_connection(): + """A pooled connection; the transaction commits on exit and rolls back on error.""" + with get_pool().connection() as conn: + yield conn -def _add_column_if_missing(conn: sqlite3.Connection, table: str, column: str, definition: str) -> None: - """Best-effort SQLite migration helper.""" + +def init_db() -> None: + """Apply pending migrations and open the connection pool.""" + applied = migrate.apply_migrations(database_url()) + current = migrate.current_version(database_url()) + logger.info("db migrations current=%s applied_now=%s", current, applied) + get_pool() + created = sync_catalogs_for_all_instances() + if created: + logger.info("catalog sync created %d missing catalogs", created) + + +def check_health() -> dict: + """Reachability and migration state, for the /health endpoint.""" try: - conn.execute(f"ALTER TABLE {table} ADD COLUMN {column} {definition}") - except sqlite3.OperationalError: - pass + with get_connection() as conn: + conn.execute("SELECT 1") + return {"reachable": True, "migrations_current": migrate.is_current(database_url())} + except Exception: + logger.exception("database health check failed") + return {"reachable": False, "migrations_current": False} + + +# ============================================================================= +# Helpers +# ============================================================================= +_UUID_RE = re.compile( + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" +) +_FINGERPRINT_RE = re.compile(r"^[0-9a-f]{32}$") + +_REVIEW_STAGES = { + "ocr_review", + "translation_review", + "chunk_review", + "ready_for_ingestion", + "approval_for_prod", +} -def _ensure_db_dir(): - """Ensure the database directory exists.""" - db_dir = Path(DB_PATH).parent - if not db_dir.exists(): - db_dir.mkdir(parents=True, exist_ok=True) +PROD_INDEX_KEY = "prod" +DEV_INDEX_KEY = "dev" -@contextmanager -def get_connection(): - """Get a database connection with proper cleanup and isolation.""" - _ensure_db_dir() - conn = sqlite3.connect( - DB_PATH, - check_same_thread=False, - isolation_level="IMMEDIATE", # Acquire lock immediately on write - timeout=30.0 # Wait up to 30s for locks +def _default_instance() -> str: + return (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + + +def _normalize_instances(instances: list[str]) -> list[str]: + default = _default_instance() + return sorted({(i or "").strip().lower() or default for i in instances}) + + +def _iso_now() -> str: + return datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + + +def _is_uuid(value: Any) -> bool: + return isinstance(value, str) and bool(_UUID_RE.match(value)) + + +def _clean(value: Any) -> Any: + if isinstance(value, uuid.UUID): + return str(value) + if isinstance(value, datetime): + if value.tzinfo is not None: + value = value.astimezone(timezone.utc).replace(tzinfo=None) + return value.isoformat() + if isinstance(value, date): + return value.isoformat() + return value + + +def _clean_row(row: Optional[dict]) -> Optional[dict]: + if row is None: + return None + return {k: _clean(v) for k, v in row.items()} + + +def _csv_roles(roles: Any) -> list[str]: + if roles is None: + return [] + if isinstance(roles, (list, tuple, set)): + return [str(r).strip() for r in roles if str(r).strip()] + return [part.strip() for part in str(roles).split(",") if part.strip()] + + +def _fingerprint_for( + source_file_fingerprint: Optional[str], + document_id: Optional[str], + workflow_id: Optional[str], +) -> str: + for candidate in (source_file_fingerprint, document_id): + if candidate and _FINGERPRINT_RE.match(candidate): + return candidate + return hashlib.md5((document_id or workflow_id or "").encode()).hexdigest() + + +def _prod_collection_names() -> set[str]: + names = { + os.environ.get("PROD_VECTOR_DB_COLLECTION_NAME"), + os.environ.get("PROD_SCHEME_VECTOR_DB_COLLECTION_NAME"), + } + return {n.strip() for n in names if n and n.strip()} + + +def _index_key_for(index_name: str) -> str: + return PROD_INDEX_KEY if index_name in _prod_collection_names() else DEV_INDEX_KEY + + +# ============================================================================= +# Identifier resolution +# ============================================================================= + +def _resolve_document_uuid(conn, identifier: Optional[str]) -> Optional[str]: + """Document UUID for a UUID, Temporal workflow id, fingerprint or filename.""" + if not identifier: + return None + identifier = str(identifier) + + if _is_uuid(identifier): + row = conn.execute( + "SELECT id FROM documents WHERE id = %s::uuid", (identifier,) + ).fetchone() + if row: + return str(row["id"]) + + row = conn.execute( + """ + SELECT document_id FROM workflows + WHERE temporal_workflow_id = %s + ORDER BY started_at DESC LIMIT 1 + """, + (identifier,), + ).fetchone() + if row: + return str(row["document_id"]) + + row = conn.execute( + """ + SELECT id FROM documents + WHERE source->>'fingerprint' = %s OR source->>'original_filename' = %s + ORDER BY created_at DESC LIMIT 1 + """, + (identifier, identifier), + ).fetchone() + if row: + return str(row["id"]) + + row = conn.execute( + """ + SELECT id FROM documents d + WHERE EXISTS ( + SELECT 1 FROM jsonb_each(COALESCE(d.metadata->'index_status', '{}'::jsonb)) e + WHERE e.value->>'doc_id' = %s + ) OR md5(d.source->>'fingerprint') = %s + ORDER BY created_at DESC LIMIT 1 + """, + (identifier, identifier), + ).fetchone() + return str(row["id"]) if row else None + + +def _require_document_uuid(conn, identifier: str) -> str: + doc_uuid = _resolve_document_uuid(conn, identifier) + if doc_uuid is None: + raise LookupError(f"Document not found: {identifier}") + return doc_uuid + + +# ============================================================================= +# Document row <-> flat dict +# ============================================================================= + +_DOC_SELECT = """ + SELECT + d.id, d.catalog_id, d.code, d.name, d.aliases, d.source, d.is_disabled, d.metadata, + d.created_at, d.updated_at, + c.type AS catalog_type, g.name AS instance_name, + (SELECT count(*) FROM pages p WHERE p.document_id = d.id) AS page_total, + (SELECT count(*) FROM chunks ch WHERE ch.document_id = d.id) AS chunk_total, + root.temporal_workflow_id AS root_workflow_id, + root.stop_after_ocr AS root_stop_after_ocr, + cw.id AS current_workflow_pk, + cw.current_stage AS current_stage, + cw.error_message AS current_error_message, + u.keycloak_user_id AS up_user_id, u.username AS up_username, + u.email AS up_email, u.roles AS up_roles, + ts.ocr_completed_at, ts.translation_completed_at, ts.chunks_completed_at, + ts.ingested_at, + req.started_at AS prod_ready_requested_at, + ru.keycloak_user_id AS prod_ready_requested_by_user_id, + ru.username AS prod_ready_requested_by_username, + oa.id AS original_artifact_id, + na.id AS normalized_artifact_id + FROM documents d + JOIN catalogs c ON c.id = d.catalog_id + JOIN instances g ON g.id = c.instance_id + LEFT JOIN document_current_workflow cw ON cw.document_id = d.id + LEFT JOIN LATERAL ( + SELECT w.temporal_workflow_id, w.stop_after_ocr FROM workflows w + WHERE w.document_id = d.id + ORDER BY (w.workflow_type IN ('pipeline', 'ocr_only')) DESC, w.started_at ASC, w.id ASC + LIMIT 1 + ) root ON TRUE + LEFT JOIN users u ON u.id = NULLIF(d.metadata->>'uploaded_by_user_id', '')::uuid + LEFT JOIN LATERAL ( + SELECT min(w.stage_timestamps->>'ocr') AS ocr_completed_at, + min(w.stage_timestamps->>'translation') AS translation_completed_at, + min(w.stage_timestamps->>'chunks') AS chunks_completed_at, + max(COALESCE(w.completed_at, w.updated_at)) + FILTER (WHERE w.current_stage = 'completed') AS ingested_at + FROM workflows w WHERE w.document_id = d.id + ) ts ON TRUE + LEFT JOIN LATERAL ( + SELECT w.started_at, w.started_by_user_id FROM workflows w + WHERE w.document_id = d.id AND w.workflow_type = 'promote_prod' + AND w.status = 'waiting_review' + ORDER BY w.started_at DESC LIMIT 1 + ) req ON TRUE + LEFT JOIN users ru ON ru.id = req.started_by_user_id + LEFT JOIN LATERAL ( + SELECT a.id FROM workflow_artifacts a JOIN workflows w ON w.id = a.workflow_id + WHERE w.document_id = d.id AND a.artifact_type = 'original_upload' + ORDER BY a.created_at DESC, a.id DESC LIMIT 1 + ) oa ON TRUE + LEFT JOIN LATERAL ( + SELECT a.id FROM workflow_artifacts a JOIN workflows w ON w.id = a.workflow_id + WHERE w.document_id = d.id + AND a.artifact_type IN ('normalized_pdf', 'normalized_spreadsheet') + ORDER BY a.created_at DESC, a.id DESC LIMIT 1 + ) na ON TRUE +""" + + +def _kind_of(catalog_type: Optional[str]) -> Optional[str]: + """Knowledge kind a catalog type stands for; an unmapped type is returned as is.""" + if not catalog_type: + return None + try: + return kind_for_catalog_type(catalog_type) + except ValueError: + return catalog_type + + +def _document_from_row(row: Optional[dict]) -> Optional[dict]: + if row is None: + return None + source = row["source"] or {} + meta = row["metadata"] or {} + fingerprint = source.get("fingerprint") + roles = row["up_roles"] or [] + entry_aliases = row["aliases"] + doc = { + "id": row["id"], + "workflow_id": row["root_workflow_id"], + "document_id": fingerprint, + "canonical_document_id": fingerprint, + "filename": source.get("original_filename") or "", + "name": row["name"], + "source_filename": source.get("original_filename"), + "source_manifest_name": None, + "source_file_fingerprint": fingerprint, + "filepath": source.get("storage_uri") or "", + "instance": (row["instance_name"] or "").lower(), + "stage": row["current_stage"] or "registered", + "page_count": int(row["page_total"] or 0), + "chunk_count": int(row["chunk_total"] or 0), + "error_message": row["current_error_message"], + "is_disabled": bool(row["is_disabled"]), + "source_type": source.get("source_type"), + "canonical_input_type": source.get("canonical_input_type"), + "stop_after_ocr": bool(row["root_stop_after_ocr"]), + "reindex_required": bool(meta.get("reindex_required")), + "reindex_reason": meta.get("reindex_reason"), + "kind": _kind_of(row["catalog_type"]), + "code": row["code"], + "aliases": list(entry_aliases or []), + "tool_routing": "qdrant", + "catalog_visible": bool(meta.get("catalog_visible", True)), + "network_visible": bool(meta.get("network_visible", True)), + "valid_from": meta.get("valid_from"), + "valid_to": meta.get("valid_to"), + "uploaded_by_user_id": row["up_user_id"], + "uploaded_by_username": row["up_username"], + "uploaded_by_email": row["up_email"], + "uploaded_by_roles": ",".join(roles) if roles else None, + "prod_ready_requested_at": row["prod_ready_requested_at"], + "prod_ready_requested_by_user_id": row["prod_ready_requested_by_user_id"], + "prod_ready_requested_by_username": row["prod_ready_requested_by_username"], + "ocr_completed_at": row["ocr_completed_at"], + "translation_completed_at": row["translation_completed_at"], + "chunks_completed_at": row["chunks_completed_at"], + "ingested_at": row["ingested_at"], + "original_artifact_id": row["original_artifact_id"], + "normalized_artifact_id": row["normalized_artifact_id"], + "latest_job_id": row["current_workflow_pk"], + "catalog_id": row["catalog_id"], + "index_status": meta.get("index_status") or {}, + "superseded_doc_ids": meta.get("superseded_doc_ids") or {}, + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } + return {k: _clean(v) for k, v in doc.items()} + + +def _fetch_document(conn, doc_uuid: Optional[str]) -> Optional[dict]: + if doc_uuid is None: + return None + row = conn.execute(_DOC_SELECT + " WHERE d.id = %s::uuid", (doc_uuid,)).fetchone() + return _document_from_row(row) + + +def _document_filters( + include_disabled: bool, + instances: Optional[list[str]], + alias: str = "d", + instance_alias: str = "g", +) -> tuple[list[str], list]: + """WHERE clauses for document lists. The query must join ``instances`` as ``instance_alias``.""" + clauses: list[str] = [] + params: list = [] + if not include_disabled: + clauses.append(f"NOT {alias}.is_disabled") + if instances is not None: + clauses.append(f"lower({instance_alias}.name) = ANY(%s)") + params.append(_normalize_instances(instances)) + return clauses, params + + +# ============================================================================= +# Instances +# ============================================================================= + +def _tenant_path(instance: str) -> str: + """Keycloak group path of a tenant code: ``bv`` is ``/global``, others ``/states/``.""" + return "/global" if instance == "bv" else f"/states/{instance.upper()}" + + +def _tenant_code_from_path(path: str) -> str: + return "bv" if path == "/global" else path.rstrip("/").rsplit("/", 1)[-1].lower() + + +def upsert_instance(path: str, keycloak_group_id: Optional[str] = None, conn=None) -> str: + """Create or refresh the ``instances`` row for a Keycloak group path; returns its UUID. + + Rows made by the SQLite move have no Keycloak id yet; the first sync that + sees the path fills it in. The instance's catalogs (one per supported type) are + ensured in the same transaction, so an instance never exists without them. + """ + if conn is None: + with get_connection() as own: + return upsert_instance(path, keycloak_group_id, conn=own) + row = conn.execute( + """ + INSERT INTO instances (keycloak_group_id, name, path) VALUES (%s, %s, %s) + ON CONFLICT (path) DO UPDATE SET + keycloak_group_id = COALESCE(EXCLUDED.keycloak_group_id, instances.keycloak_group_id) + RETURNING id + """, + (keycloak_group_id, _tenant_code_from_path(path), path), + ).fetchone() + instance_id = str(row["id"]) + ensure_instance_catalogs(instance_id, conn=conn) + return instance_id + + +def get_or_create_instance(instance: Optional[str], conn=None) -> str: + """UUID of the instance for a tenant code (``bv``, ``mh``, ...), created when missing.""" + code = (instance or _default_instance()).strip().lower() or _default_instance() + return upsert_instance(_tenant_path(code), conn=conn) + + +def _catalog_seed_rows() -> tuple[list[str], list[str]]: + """Parallel ``(types, kinds)`` lists for every supported catalog type.""" + kinds = list(KIND_TO_CATALOG_TYPE) + return [KIND_TO_CATALOG_TYPE[k] for k in kinds], kinds + + +def ensure_instance_catalogs(instance_id: str, conn=None) -> None: + """Create the instance's missing catalogs, one per supported type; existing rows are untouched.""" + if conn is None: + with get_connection() as own: + return ensure_instance_catalogs(instance_id, conn=own) + types, kinds = _catalog_seed_rows() + conn.execute( + """ + INSERT INTO catalogs (instance_id, type, name) + SELECT g.id, t.type, upper(g.name) || ' ' || t.kind || ' catalog' + FROM instances g, unnest(%s::text[], %s::text[]) AS t(type, kind) + WHERE g.id = %s::uuid + ON CONFLICT (instance_id, type) DO NOTHING + """, + (types, kinds, instance_id), ) - conn.row_factory = sqlite3.Row - # Enable WAL mode for better concurrent access - conn.execute("PRAGMA journal_mode=WAL") - # Ensure foreign keys are enforced - conn.execute("PRAGMA foreign_keys=ON") + + +def sync_catalogs_for_all_instances(conn=None) -> int: + """Add every missing ``(instance, type)`` catalog; returns how many rows were created. + + Idempotent. Run at startup so a type added to the kind map reaches instances + that already exist. + """ + if conn is None: + with get_connection() as own: + return sync_catalogs_for_all_instances(conn=own) + types, kinds = _catalog_seed_rows() + cur = conn.execute( + """ + INSERT INTO catalogs (instance_id, type, name) + SELECT g.id, t.type, upper(g.name) || ' ' || t.kind || ' catalog' + FROM instances g, unnest(%s::text[], %s::text[]) AS t(type, kind) + ON CONFLICT (instance_id, type) DO NOTHING + """, + (types, kinds), + ) + return cur.rowcount + + +def get_or_create_catalog(instance_id: str, kind: str, conn=None) -> dict: + """The ``(instance, kind)`` catalog row; created when missing (normally seeded with the instance).""" + if conn is None: + with get_connection() as own: + return get_or_create_catalog(instance_id, kind, conn=own) + catalog_type = catalog_type_for_kind(kind) + row = conn.execute( + """ + INSERT INTO catalogs (instance_id, type, name) + SELECT g.id, %s, upper(g.name) || ' ' || %s || ' catalog' FROM instances g WHERE g.id = %s::uuid + ON CONFLICT (instance_id, type) DO UPDATE SET type = EXCLUDED.type + RETURNING id, instance_id, type, name, metadata + """, + (catalog_type, kind, instance_id), + ).fetchone() + return _clean_row(row) + + +# ============================================================================= +# Users +# ============================================================================= + +def upsert_user( + keycloak_user_id: str, + username: Optional[str] = None, + email: Optional[str] = None, + roles: Any = None, + conn=None, + tenants: Optional[list[str]] = None, +) -> Optional[str]: + """Create or refresh the ``users`` row for a Keycloak subject; returns its UUID. + + ``tenants`` are the tenant codes in the user's token. A user has one + ``instance_id``, so the first code (sorted) wins and a warning is logged when + there are several. Authorization still reads the token, not this column. + """ + keycloak_user_id = (keycloak_user_id or "").strip() + if not keycloak_user_id: + return None + if conn is None: + with get_connection() as own: + return upsert_user(keycloak_user_id, username, email, roles, conn=own, tenants=tenants) + + role_list = _csv_roles(roles) + tenant_codes = sorted({(t or "").strip().lower() for t in (tenants or []) if (t or "").strip()}) + if len(tenant_codes) > 1: + logger.warning( + "user %s has several tenants %s; users.instance_id uses %s", + keycloak_user_id, tenant_codes, tenant_codes[0], + ) + instance_uuid = get_or_create_instance(tenant_codes[0], conn=conn) if tenant_codes else None + email = (email or "").strip() or None + username = (username or "").strip() or None + for candidate_email in ([email, None] if email else [None]): + try: + with conn.transaction(): + row = conn.execute( + """ + INSERT INTO users (keycloak_user_id, username, email, roles, instance_id) + VALUES (%s, %s, %s, %s, %s::uuid) + ON CONFLICT (keycloak_user_id) DO UPDATE SET + username = COALESCE(EXCLUDED.username, users.username), + email = COALESCE(EXCLUDED.email, users.email), + roles = CASE WHEN cardinality(EXCLUDED.roles) > 0 + THEN EXCLUDED.roles ELSE users.roles END, + instance_id = COALESCE(EXCLUDED.instance_id, users.instance_id), + last_seen_at = now() + RETURNING id + """, + (keycloak_user_id, username, candidate_email, role_list, instance_uuid), + ).fetchone() + return str(row["id"]) + except psycopg.errors.UniqueViolation: + continue + return None + + +def get_user_by_keycloak_id(keycloak_user_id: str) -> Optional[dict]: + with get_connection() as conn: + return _clean_row( + conn.execute( + "SELECT * FROM users WHERE keycloak_user_id = %s", (keycloak_user_id,) + ).fetchone() + ) + + +# ============================================================================= +# Webhook records +# ============================================================================= + +def _parse_timestamp(value: Any) -> Optional[datetime]: + if not value: + return None try: - yield conn - finally: - conn.close() + parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed -def init_db(): - """Initialize the database schema.""" - with _db_lock: - with get_connection() as conn: - conn.execute(""" - CREATE TABLE IF NOT EXISTS documents ( - workflow_id TEXT PRIMARY KEY, - document_id TEXT NOT NULL, - canonical_document_id TEXT, - filename TEXT NOT NULL, - display_name TEXT, - source_filename TEXT, - source_manifest_name TEXT, - source_file_fingerprint TEXT, - filepath TEXT NOT NULL, - stage TEXT NOT NULL DEFAULT 'registered', - page_count INTEGER DEFAULT 0, - chunk_count INTEGER DEFAULT 0, - error_message TEXT, - is_demo INTEGER DEFAULT 0, - created_at TEXT, - updated_at TEXT, - ocr_completed_at TEXT, - translation_completed_at TEXT, - chunks_completed_at TEXT, - ingested_at TEXT, - source_type TEXT, - canonical_input_type TEXT, - stop_after_ocr INTEGER DEFAULT 0, - reindex_required INTEGER DEFAULT 0, - reindex_reason TEXT, - original_artifact_id INTEGER, - normalized_artifact_id INTEGER, - latest_job_id INTEGER - ) - """) - _add_column_if_missing(conn, "documents", "is_demo", "INTEGER DEFAULT 0") - _add_column_if_missing(conn, "documents", "is_disabled", "INTEGER DEFAULT 0") - _add_column_if_missing(conn, "documents", "display_name", "TEXT") - _add_column_if_missing(conn, "documents", "canonical_document_id", "TEXT") - _add_column_if_missing(conn, "documents", "source_filename", "TEXT") - _add_column_if_missing(conn, "documents", "source_manifest_name", "TEXT") - _add_column_if_missing(conn, "documents", "source_file_fingerprint", "TEXT") - _add_column_if_missing(conn, "documents", "translation_completed_at", "TEXT") - _add_column_if_missing(conn, "documents", "source_type", "TEXT") - _add_column_if_missing(conn, "documents", "canonical_input_type", "TEXT") - _add_column_if_missing(conn, "documents", "stop_after_ocr", "INTEGER DEFAULT 0") - _add_column_if_missing(conn, "documents", "reindex_required", "INTEGER DEFAULT 0") - _add_column_if_missing(conn, "documents", "reindex_reason", "TEXT") - _add_column_if_missing(conn, "documents", "original_artifact_id", "INTEGER") - _add_column_if_missing(conn, "documents", "normalized_artifact_id", "INTEGER") - _add_column_if_missing(conn, "documents", "latest_job_id", "INTEGER") - _add_column_if_missing(conn, "documents", "instance", "TEXT") - _add_column_if_missing(conn, "documents", "uploaded_by_user_id", "TEXT") - _add_column_if_missing(conn, "documents", "uploaded_by_username", "TEXT") - _add_column_if_missing(conn, "documents", "uploaded_by_email", "TEXT") - _add_column_if_missing(conn, "documents", "uploaded_by_roles", "TEXT") - _add_column_if_missing(conn, "documents", "prod_ready_requested_at", "TEXT") - _add_column_if_missing(conn, "documents", "prod_ready_requested_by_user_id", "TEXT") - _add_column_if_missing(conn, "documents", "prod_ready_requested_by_username", "TEXT") - # Period this document's chunks are searchable in, stamped on - # upload and editable by the approver. NULL on documents uploaded - # before validity existed - search reads that as always current - # (see pipeline/document_validity.period_from_row). - _add_column_if_missing(conn, "documents", "valid_from", "TEXT") - _add_column_if_missing(conn, "documents", "valid_to", "TEXT") - # Stamp NULL/empty rows with the configured default so list filters - # (which coalesce to DEFAULT_INSTANCE) match migrated data. - default_instance = ( - (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" - ) +def _webhook_payload_and_hash(rec: dict) -> tuple[str, str]: + payload = json.dumps(rec, sort_keys=True, ensure_ascii=False, default=str) + return payload, hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def webhook_external_id(rec: dict) -> str: + """The (source, external_id) key of a webhook record: contentID, else id, else slug, else payload hash.""" + _, payload_hash = _webhook_payload_and_hash(rec) + for key in ("contentID", "id", "slug"): + value = rec.get(key) + if value not in (None, ""): + return str(value) + return payload_hash + + +def upsert_webhook_records(source: str, records: list) -> int: + """Upsert records by (source, external_id) in one transaction; returns len(records). + + external_id comes from contentID, else id, else slug, else a hash of the payload. + Unchanged payloads (same payload_hash) are not rewritten but still counted. + """ + now = datetime.now(timezone.utc) + rows = [] + for rec in records: + if not isinstance(rec, dict): + raise ValueError("Each record must be a JSON object") + payload_text, payload_hash = _webhook_payload_and_hash(rec) + ext = webhook_external_id(rec) + pub = rec.get("isPublished") + rows.append(( + source, str(ext), rec.get("title"), rec.get("slug"), rec.get("status"), + bool(pub) if isinstance(pub, (bool, int)) else None, + rec.get("domainName"), Jsonb(json.loads(payload_text)), payload_hash, + _parse_timestamp(rec.get("createdAt")) or now, + _parse_timestamp(rec.get("updatedAt")) or now, + )) + with get_connection() as conn: + for r in rows: conn.execute( """ - UPDATE documents - SET instance = ? - WHERE instance IS NULL OR trim(instance) = '' + INSERT INTO webhook_records + (source, external_id, title, slug, status, is_published, category, + payload, payload_hash, created_at, updated_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (source, external_id) DO UPDATE SET + title = EXCLUDED.title, + slug = EXCLUDED.slug, + status = EXCLUDED.status, + is_published = EXCLUDED.is_published, + category = EXCLUDED.category, + payload = EXCLUDED.payload, + payload_hash = EXCLUDED.payload_hash, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at + WHERE webhook_records.payload_hash IS DISTINCT FROM EXCLUDED.payload_hash """, - (default_instance,), + r, ) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_documents_stage - ON documents(stage) - """) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_documents_instance - ON documents(instance) - """) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_documents_created - ON documents(created_at DESC) - """) - # Audit logs table - conn.execute(""" - CREATE TABLE IF NOT EXISTS audit_logs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - workflow_id TEXT NOT NULL, - document_id TEXT NOT NULL, - action_type TEXT NOT NULL, - entity_type TEXT, - entity_id INTEGER, - field_name TEXT, - old_value TEXT, - new_value TEXT, - metadata TEXT, - timestamp TEXT NOT NULL, - actor_user_id TEXT, - actor_username TEXT, - actor_email TEXT, - actor_roles TEXT - ) - """) - _add_column_if_missing(conn, "audit_logs", "actor_user_id", "TEXT") - _add_column_if_missing(conn, "audit_logs", "actor_username", "TEXT") - _add_column_if_missing(conn, "audit_logs", "actor_email", "TEXT") - _add_column_if_missing(conn, "audit_logs", "actor_roles", "TEXT") - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_audit_workflow - ON audit_logs(workflow_id) - """) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_audit_timestamp - ON audit_logs(timestamp DESC) - """) - # Pages table for persistence after workflow completion - conn.execute(""" - CREATE TABLE IF NOT EXISTS pages ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - workflow_id TEXT NOT NULL, - page_number INTEGER NOT NULL, - original_markdown TEXT, - edited_markdown TEXT, - is_reviewed INTEGER DEFAULT 0, - reviewer_notes TEXT, - detected_language TEXT, - translated_markdown TEXT, - edited_translation TEXT, - translation_reviewed INTEGER DEFAULT 0, - translation_notes TEXT, - translation_provider TEXT, - translation_model TEXT, - translation_target_language TEXT, - translated_at TEXT, - UNIQUE(workflow_id, page_number) - ) - """) - _add_column_if_missing(conn, "pages", "translation_provider", "TEXT") - _add_column_if_missing(conn, "pages", "translation_model", "TEXT") - _add_column_if_missing(conn, "pages", "translation_target_language", "TEXT") - _add_column_if_missing(conn, "pages", "translated_at", "TEXT") - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_pages_workflow - ON pages(workflow_id) - """) - # Chunks table for persistence after workflow completion - conn.execute(""" - CREATE TABLE IF NOT EXISTS chunks ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - workflow_id TEXT NOT NULL, - chunk_number INTEGER NOT NULL, - original_text TEXT, - edited_text TEXT, - token_count INTEGER DEFAULT 0, - page_start INTEGER DEFAULT 1, - page_end INTEGER DEFAULT 1, - source_page_numbers_json TEXT, - source_spans_json TEXT, - section_title TEXT, - content_type TEXT, - is_reference INTEGER DEFAULT 0, - chunking_provider TEXT, - chunking_model TEXT, - chunking_config_json TEXT, - chunking_run_id TEXT, - chunk_version INTEGER DEFAULT 1, - is_reviewed INTEGER DEFAULT 0, - is_excluded INTEGER DEFAULT 0, - reviewer_notes TEXT, - UNIQUE(workflow_id, chunk_number) - ) - """) - _add_column_if_missing(conn, "chunks", "source_page_numbers_json", "TEXT") - _add_column_if_missing(conn, "chunks", "source_spans_json", "TEXT") - _add_column_if_missing(conn, "chunks", "section_title", "TEXT") - _add_column_if_missing(conn, "chunks", "content_type", "TEXT") - _add_column_if_missing(conn, "chunks", "is_reference", "INTEGER DEFAULT 0") - _add_column_if_missing(conn, "chunks", "chunking_provider", "TEXT") - _add_column_if_missing(conn, "chunks", "chunking_model", "TEXT") - _add_column_if_missing(conn, "chunks", "chunking_config_json", "TEXT") - _add_column_if_missing(conn, "chunks", "chunking_run_id", "TEXT") - _add_column_if_missing(conn, "chunks", "chunk_version", "INTEGER DEFAULT 1") - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_chunks_workflow - ON chunks(workflow_id) - """) - # Settings table for application configuration - conn.execute(""" - CREATE TABLE IF NOT EXISTS settings ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL, - description TEXT, - updated_at TEXT NOT NULL - ) - """) - conn.execute(""" - CREATE TABLE IF NOT EXISTS document_jobs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - workflow_id TEXT NOT NULL, - job_type TEXT NOT NULL, - temporal_workflow_id TEXT, - temporal_run_id TEXT, - status TEXT NOT NULL DEFAULT 'running', - current_stage TEXT, - started_at TEXT NOT NULL, - completed_at TEXT, - error_message TEXT, - config_json TEXT - ) - """) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_document_jobs_workflow - ON document_jobs(workflow_id, started_at DESC) - """) - conn.execute(""" - CREATE TABLE IF NOT EXISTS document_artifacts ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - workflow_id TEXT NOT NULL, - job_id INTEGER, - artifact_type TEXT NOT NULL, - stage TEXT, - storage_uri TEXT NOT NULL, - mime_type TEXT, - filename TEXT, - size_bytes INTEGER, - metadata_json TEXT, - created_at TEXT NOT NULL - ) - """) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_document_artifacts_workflow - ON document_artifacts(workflow_id, created_at DESC) - """) - conn.execute(""" - CREATE TABLE IF NOT EXISTS document_index_status ( - workflow_id TEXT NOT NULL, - index_name TEXT NOT NULL, - doc_id TEXT, - chunk_count_indexed INTEGER DEFAULT 0, - last_indexed_at TEXT, - last_verified_at TEXT, - schema_version TEXT, - status TEXT NOT NULL DEFAULT 'unknown', - details_json TEXT, - PRIMARY KEY (workflow_id, index_name) - ) - """) - # doc_id supersedes the legacy marqo_doc_id column (generic - # naming, no DB name baked into the schema) on databases that - # predate this change. marqo_doc_id is left in place, unwritten, - # until scripts/drop_legacy_marqo_columns.py is run — so this - # backfill is best-effort: a fresh database, or one that's already - # had the column dropped, simply has nothing to migrate. - _add_column_if_missing(conn, "document_index_status", "doc_id", "TEXT") - try: - conn.execute(""" - UPDATE document_index_status - SET doc_id = marqo_doc_id - WHERE doc_id IS NULL AND marqo_doc_id IS NOT NULL - """) - except sqlite3.OperationalError: - pass - conn.execute(""" - CREATE TABLE IF NOT EXISTS document_manifest_entries ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - manifest_filename TEXT NOT NULL UNIQUE, - title_en TEXT, - title_gu TEXT, - doc_language TEXT, - category_tags TEXT, - description TEXT, - quality_score TEXT, - priority_rank TEXT, - ingestion_status TEXT, - feedback TEXT, - source_csv_path TEXT, - imported_at TEXT NOT NULL - ) - """) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_manifest_entries_filename - ON document_manifest_entries(manifest_filename) - """) - # Insert default search settings if not exists - default_settings = [ - ("search_method", "HYBRID", "Search method: TENSOR, LEXICAL, or HYBRID"), - ("search_limit", "12", "Number of search results to return"), - ("search_alpha", "0.6", "Hybrid search alpha: 0=lexical, 1=semantic"), - ("search_ranking_method", "rrf", "Hybrid ranking: rrf or normalize_linear"), - ("search_show_highlights", "true", "Show highlighted matches in results"), - ("search_ef_search", "256", "HNSW search accuracy parameter"), - ("search_index_name", "documents-index", "Default vector DB collection name"), - ("search_candidate_cap", "120", "Candidate retrieval pool cap"), - ("search_candidate_multiplier", "10", "Candidate pool multiplier before final cut"), - ("search_max_chunks_per_doc", "2", "Final result diversity cap per document"), - ("search_use_e5_prefix", "true", "Prefix search queries with e5 query:"), - ("search_exclude_reference", "true", "Exclude reference chunks when index supports it"), - ("search_query_expansion_profile", "gu-v1", "Query expansion profile"), - ("search_rerank_mode", "none", "Post-search reranking mode"), - ("search_hybrid_rrfk", "60", "RRF tuning parameter for hybrid search"), - ] - for key, value, description in default_settings: - conn.execute(""" - INSERT OR IGNORE INTO settings (key, value, description, updated_at) - VALUES (?, ?, ?, ?) - """, (key, value, description, datetime.utcnow().isoformat())) - # Scheme metadata + master catalog tables (AI tool sync) - init_scheme_catalog_schema(conn=conn) - # Email-OTP login codes (one live code per email; overwritten on resend). - conn.execute(""" - CREATE TABLE IF NOT EXISTS email_otps ( - email TEXT PRIMARY KEY, - code_hash TEXT NOT NULL, - expires_at TEXT NOT NULL, - attempts INTEGER NOT NULL DEFAULT 0, - created_at TEXT NOT NULL, - last_sent_at TEXT NOT NULL - ) - """) - conn.commit() + return len(rows) + + +_WEBHOOK_RECORD_COLUMNS = ( + "id, source, external_id, title, slug, status, is_published, category, payload, " + "payload_hash, created_at, updated_at, indexed_hash, promoted_hash" +) +def get_webhook_record(record_id: str) -> Optional[dict]: + """One webhook_records row as a dict (``payload`` already parsed from JSONB); None if missing.""" + if not _is_uuid(record_id): + return None + with get_connection() as conn: + row = conn.execute( + f"SELECT {_WEBHOOK_RECORD_COLUMNS} FROM webhook_records WHERE id = %s", (record_id,) + ).fetchone() + if row is None: + return None + record = dict(row) + record["id"] = str(record["id"]) + if not isinstance(record["payload"], dict): + record["payload"] = {} + return record + + +def get_unindexed_webhook_record_ids( + source: str = "webhook", external_ids: Optional[list] = None +) -> list[str]: + """Ids of rows whose payload changed since they were last indexed (or never indexed). + + ``external_ids`` limits the check to the records of one webhook call. + """ + sql = ( + "SELECT id FROM webhook_records WHERE source = %s " + "AND indexed_hash IS DISTINCT FROM payload_hash" + ) + params: list = [source] + if external_ids is not None: + if not external_ids: + return [] + sql += " AND external_id = ANY(%s)" + params.append([str(e) for e in external_ids]) + with get_connection() as conn: + return [str(r["id"]) for r in conn.execute(sql + " ORDER BY created_at, id", params)] + + +def get_webhook_record_ids( + source: str = "webhook", published_only: bool = False, indexed_only: bool = False +) -> list[str]: + """All record ids for a source, optionally only published and/or dev-indexed ones. + + ``indexed_only`` keeps rows whose current payload is what was last written to the dev + collection (``indexed_hash = payload_hash``), so a bulk promote never pushes a payload + that has not been indexed in dev. + """ + sql = "SELECT id FROM webhook_records WHERE source = %s" + if published_only: + sql += " AND is_published AND status = 'Published'" + if indexed_only: + sql += " AND indexed_hash = payload_hash" + with get_connection() as conn: + return [str(r["id"]) for r in conn.execute(sql + " ORDER BY created_at, id", (source,))] + + +def _mark_webhook_record(column: str, record_id: str, payload_hash: str) -> None: + with get_connection() as conn: + conn.execute(f"UPDATE webhook_records SET {column} = %s WHERE id = %s", (payload_hash, record_id)) + + +def mark_webhook_record_indexed(record_id: str, payload_hash: str) -> None: + """Record the payload_hash that was written to the dev collection.""" + _mark_webhook_record("indexed_hash", record_id, payload_hash) + + +def mark_webhook_record_promoted(record_id: str, payload_hash: str) -> None: + """Record the payload_hash that was written to the PROD collection.""" + _mark_webhook_record("promoted_hash", record_id, payload_hash) + + +def advisory_status_counts(source: str = "webhook") -> dict: + """Counts for the advisory status endpoint.""" + with get_connection() as conn: + row = conn.execute( + """ + SELECT + COUNT(*) AS total, + COUNT(*) FILTER (WHERE is_published AND status = 'Published') AS published, + COUNT(*) FILTER (WHERE indexed_hash = payload_hash) AS indexed, + COUNT(*) FILTER (WHERE promoted_hash = payload_hash) AS promoted + FROM webhook_records WHERE source = %s + """, + (source,), + ).fetchone() + total = int(row["total"]) + indexed = int(row["indexed"]) + return { + "total": total, + "published": int(row["published"]), + "indexed": indexed, + "unindexed": total - indexed, + "promoted": int(row["promoted"]), + } + + +# ============================================================================= +# Email OTPs +# ============================================================================= + def store_email_otp( *, email: str, code_hash: str, expires_at: str, created_at: str, last_sent_at: str ) -> None: """Upsert the one live OTP for an email, resetting attempts on resend.""" - with _db_lock: - with get_connection() as conn: - conn.execute( - """ - INSERT INTO email_otps (email, code_hash, expires_at, attempts, created_at, last_sent_at) - VALUES (?, ?, ?, 0, ?, ?) - ON CONFLICT(email) DO UPDATE SET - code_hash = excluded.code_hash, - expires_at = excluded.expires_at, - attempts = 0, - created_at = excluded.created_at, - last_sent_at = excluded.last_sent_at - """, - (email, code_hash, expires_at, created_at, last_sent_at), - ) - conn.commit() + with get_connection() as conn: + conn.execute( + """ + INSERT INTO email_otps (email, code_hash, expires_at, attempts, last_sent_at) + VALUES (%s, %s, %s, 0, %s) + ON CONFLICT (email) DO UPDATE SET + code_hash = EXCLUDED.code_hash, + expires_at = EXCLUDED.expires_at, + attempts = 0, + last_sent_at = EXCLUDED.last_sent_at + """, + (email.strip().lower(), code_hash, expires_at, last_sent_at), + ) -def get_email_otp(email: str) -> Optional[sqlite3.Row]: +def get_email_otp(email: str) -> Optional[dict]: + """The live OTP row; timestamps are timezone-aware ISO strings (UTC).""" with get_connection() as conn: - return conn.execute( - "SELECT * FROM email_otps WHERE email = ?", (email,) + row = conn.execute( + "SELECT * FROM email_otps WHERE email = %s", (email.strip().lower(),) ).fetchone() + if row is None: + return None + row = _clean_row(row) | { + "expires_at": row["expires_at"].astimezone(timezone.utc).isoformat(), + "last_sent_at": row["last_sent_at"].astimezone(timezone.utc).isoformat(), + } + return row -def increment_email_otp_attempts(email: str) -> None: - with _db_lock: - with get_connection() as conn: - conn.execute( - "UPDATE email_otps SET attempts = attempts + 1 WHERE email = ?", (email,) - ) - conn.commit() +def increment_email_otp_attempts(email: str) -> Optional[int]: + """Atomically count one failed attempt; returns the new total.""" + with get_connection() as conn: + row = conn.execute( + "UPDATE email_otps SET attempts = attempts + 1 WHERE email = %s RETURNING attempts", + (email.strip().lower(),), + ).fetchone() + return int(row["attempts"]) if row else None def delete_email_otp(email: str) -> None: - with _db_lock: - with get_connection() as conn: - conn.execute("DELETE FROM email_otps WHERE email = ?", (email,)) - conn.commit() - - -def init_scheme_catalog_schema(conn: Optional[sqlite3.Connection] = None) -> None: - """Idempotent: scheme document columns + scheme_catalog_* tables.""" - - def _apply(c: sqlite3.Connection) -> None: - for column, definition in ( - ("document_kind", "TEXT DEFAULT 'document'"), - ("scheme_code", "TEXT"), - ("scheme_name", "TEXT"), - ("scheme_aliases_json", "TEXT"), - ("tool_routing", "TEXT"), - ("catalog_visible", "INTEGER DEFAULT 1"), - ("network_visible", "INTEGER DEFAULT 1"), - ): - _add_column_if_missing(c, "documents", column, definition) - - c.execute(""" - CREATE TABLE IF NOT EXISTS scheme_catalog_meta ( - id INTEGER PRIMARY KEY CHECK (id = 1), - version INTEGER NOT NULL DEFAULT 0, - updated_at TEXT, - notes TEXT - ) - """) - c.execute(""" - INSERT OR IGNORE INTO scheme_catalog_meta (id, version, updated_at, notes) - VALUES (1, 0, ?, ?) - """, (datetime.utcnow().isoformat(), "init")) - - c.execute(""" - CREATE TABLE IF NOT EXISTS scheme_catalog_entries ( - scheme_code TEXT PRIMARY KEY, - scheme_name TEXT, - scheme_aliases_json TEXT, - instances_json TEXT, - workflow_ids_json TEXT, - collection_name TEXT, - chunk_count INTEGER DEFAULT 0, - status TEXT NOT NULL DEFAULT 'disabled', - network_visible INTEGER DEFAULT 0, - promoted_at TEXT, - content_hash TEXT, - source TEXT, - updated_at TEXT - ) - """) - c.execute(""" - CREATE INDEX IF NOT EXISTS idx_scheme_catalog_status - ON scheme_catalog_entries(status) - """) - c.execute(""" - CREATE INDEX IF NOT EXISTS idx_documents_scheme_code - ON documents(scheme_code) - """) - - if conn is not None: - _apply(conn) - return - - with _db_lock: - with get_connection() as c: - _apply(c) - c.commit() + with get_connection() as conn: + conn.execute("DELETE FROM email_otps WHERE email = %s", (email.strip().lower(),)) + + +# ============================================================================= +# Catalog +# ============================================================================= + +CATALOG_VERSION_ACTION = "catalog_version_bump" def get_catalog_meta() -> dict: - init_scheme_catalog_schema() + """Catalog version: the number of version bumps recorded in ``audit_logs``.""" with get_connection() as conn: - row = conn.execute("SELECT * FROM scheme_catalog_meta WHERE id = 1").fetchone() - if not row: - return {"id": 1, "version": 0, "updated_at": None, "notes": None} - return dict(row) + row = conn.execute( + """ + SELECT count(*) AS version, max(created_at) AS updated_at, + (array_agg(metadata->>'reason' ORDER BY created_at DESC))[1] AS notes + FROM audit_logs + WHERE entity_type = 'catalog' AND action_type = %s + """, + (CATALOG_VERSION_ACTION,), + ).fetchone() + return { + "id": 1, + "version": int(row["version"] or 0), + "updated_at": _clean(row["updated_at"]), + "notes": row["notes"], + } def bump_catalog_version(reason: str = "") -> int: """Monotonic catalog version bump. Returns new version.""" - now = datetime.utcnow().isoformat() - init_scheme_catalog_schema() - with _db_lock: - with get_connection() as conn: - conn.execute( - """ - UPDATE scheme_catalog_meta - SET version = version + 1, updated_at = ?, notes = ? - WHERE id = 1 - """, - (now, (reason or "")[:500]), - ) - conn.commit() - row = conn.execute( - "SELECT version FROM scheme_catalog_meta WHERE id = 1" - ).fetchone() - return int(row["version"]) if row else 0 + with get_connection() as conn: + conn.execute( + """ + INSERT INTO audit_logs (action_type, entity_type, metadata) + VALUES (%s, 'catalog', %s) + """, + (CATALOG_VERSION_ACTION, Jsonb({"reason": (reason or "")[:500]})), + ) + row = conn.execute( + "SELECT count(*) AS version FROM audit_logs " + "WHERE entity_type = 'catalog' AND action_type = %s", + (CATALOG_VERSION_ACTION,), + ).fetchone() + return int(row["version"]) -def upsert_catalog_entry( - *, - scheme_code: str, - scheme_name: str, - scheme_aliases: list, - instances: list, - workflow_ids: list, - collection_name: str, - chunk_count: int, - status: str, - network_visible: int, - promoted_at: Optional[str], - content_hash: str, - source: str, - updated_at: str, -) -> None: - init_scheme_catalog_schema() - with _db_lock: - with get_connection() as conn: - conn.execute( - """ - INSERT INTO scheme_catalog_entries ( - scheme_code, scheme_name, scheme_aliases_json, instances_json, - workflow_ids_json, collection_name, chunk_count, status, - network_visible, promoted_at, content_hash, source, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(scheme_code) DO UPDATE SET - scheme_name = excluded.scheme_name, - scheme_aliases_json = excluded.scheme_aliases_json, - instances_json = excluded.instances_json, - workflow_ids_json = excluded.workflow_ids_json, - collection_name = excluded.collection_name, - chunk_count = excluded.chunk_count, - status = excluded.status, - network_visible = excluded.network_visible, - promoted_at = excluded.promoted_at, - content_hash = excluded.content_hash, - source = excluded.source, - updated_at = excluded.updated_at - """, - ( - scheme_code, - scheme_name, - json.dumps(scheme_aliases or []), - json.dumps(instances or []), - json.dumps(workflow_ids or []), - collection_name, - int(chunk_count or 0), - status, - int(network_visible or 0), - promoted_at, - content_hash, - source, - updated_at, - ), - ) - conn.commit() +def list_document_codes(kind: str) -> set[str]: + """Every document code in catalogs of one knowledge kind.""" + with get_connection() as conn: + rows = conn.execute( + "SELECT d.code FROM documents d JOIN catalogs c ON c.id = d.catalog_id WHERE c.type = %s", + (catalog_type_for_kind(kind),), + ).fetchall() + return {r["code"] for r in rows} + + +def _catalog_row(row: dict) -> dict: + cleaned = _clean_row(row) + cleaned.pop("instance_name", None) + cleaned["aliases"] = list(row["aliases"] or []) + cleaned["instances"] = [(row["instance_name"] or "").lower()] + cleaned["workflow_ids"] = [row["workflow_id"]] if row["workflow_id"] else [] + cleaned["source"] = "pipeline" + return cleaned -def list_catalog_entries(status: Optional[str] = None) -> list[dict]: - init_scheme_catalog_schema() +def list_catalog_entries(status: Optional[str] = None, kind: str = "scheme") -> list[dict]: + """Rows of the ``catalog_status`` view (one per document) for one knowledge kind.""" + catalog_type = catalog_type_for_kind(kind) with get_connection() as conn: if status: rows = conn.execute( - "SELECT * FROM scheme_catalog_entries WHERE status = ? ORDER BY scheme_code", - (status,), + "SELECT * FROM catalog_status WHERE type = %s AND status = %s ORDER BY code", + (catalog_type, status), ).fetchall() else: rows = conn.execute( - "SELECT * FROM scheme_catalog_entries ORDER BY scheme_code" + "SELECT * FROM catalog_status WHERE type = %s ORDER BY code", (catalog_type,) ).fetchall() - return [dict(r) for r in rows] + return [_catalog_row(r) for r in rows] -def get_catalog_entry(scheme_code: str) -> Optional[dict]: - init_scheme_catalog_schema() +def get_catalog_entry(code: str, kind: str = "scheme") -> Optional[dict]: with get_connection() as conn: row = conn.execute( - "SELECT * FROM scheme_catalog_entries WHERE scheme_code = ?", - (scheme_code,), + "SELECT * FROM catalog_status WHERE type = %s AND code = %s LIMIT 1", + (catalog_type_for_kind(kind), code), ).fetchone() - return dict(row) if row else None + return _catalog_row(row) if row else None -def list_scheme_documents_for_catalog() -> list[dict]: - """All documents that look scheme-related (for rebuild).""" - init_scheme_catalog_schema() +def list_used_scheme_codes(exclude_workflow_id: Optional[str] = None, kind: str = "scheme") -> set[str]: + """Every document code already taken in a kind - the collision domain for auto-generated codes. + + The excluded document's own code is not counted, so re-deriving a + document's code does not collide with itself. + """ with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, exclude_workflow_id) if exclude_workflow_id else None rows = conn.execute( """ - SELECT * FROM documents - WHERE document_kind = 'scheme' - OR (scheme_code IS NOT NULL AND trim(scheme_code) != '') - """ + SELECT d.code FROM documents d JOIN catalogs c ON c.id = d.catalog_id + WHERE c.type = %s AND (%s::uuid IS NULL OR d.id <> %s::uuid) + """, + (catalog_type_for_kind(kind), doc_uuid, doc_uuid), ).fetchall() - return [dict(r) for r in rows] + return {r["code"] for r in rows} -def list_used_scheme_codes(exclude_workflow_id: Optional[str] = None) -> set[str]: - """Every scheme_code currently in use, across both the live documents table - and the scheme_catalog_entries registry (PROD-published codes may outlive - the document row that first defined them) — the full collision domain for - auto-generating a new code from a title.""" - init_scheme_catalog_schema() +def list_scheme_catalog_entries_for_workflow(workflow_id: str) -> list[dict]: + """Catalog rows whose documents still include this workflow.""" with get_connection() as conn: - doc_rows = conn.execute( - "SELECT scheme_code, workflow_id FROM documents " - "WHERE scheme_code IS NOT NULL AND trim(scheme_code) != ''" + rows = conn.execute( + "SELECT code, workflow_id FROM catalog_status WHERE type = %s AND workflow_id = %s", + (catalog_type_for_kind("scheme"), workflow_id), ).fetchall() - entry_rows = conn.execute("SELECT scheme_code FROM scheme_catalog_entries").fetchall() - codes = { - row["scheme_code"].strip().lower() - for row in doc_rows - if row["workflow_id"] != exclude_workflow_id - } - codes.update(row["scheme_code"].strip().lower() for row in entry_rows) - return codes + return [{"code": r["code"], "workflow_ids": [r["workflow_id"]]} for r in rows] +def find_document_by_code(instance: str, kind: str, code: str) -> Optional[dict]: + """The document holding ``code`` in the instance's ``kind`` catalog, if any.""" + tenant = (instance or _default_instance()).strip().lower() or _default_instance() + with get_connection() as conn: + row = conn.execute( + """ + SELECT d.id FROM documents d + JOIN catalogs c ON c.id = d.catalog_id + JOIN instances g ON g.id = c.instance_id + WHERE lower(g.name) = %s AND c.type = %s AND d.code = %s + """, + (tenant, catalog_type_for_kind(kind), code), + ).fetchone() + return _fetch_document(conn, str(row["id"])) if row else None + + +class DuplicateDocumentCode(ValueError): + """The ``(catalog, code)`` pair is already held by another document.""" + + +# ============================================================================= +# Documents +# ============================================================================= + def upsert_document( workflow_id: str, document_id: str, filename: str, filepath: str, canonical_document_id: Optional[str] = None, - display_name: Optional[str] = None, + name: Optional[str] = None, source_filename: Optional[str] = None, source_manifest_name: Optional[str] = None, source_file_fingerprint: Optional[str] = None, @@ -667,9 +996,9 @@ def upsert_document( source_type: Optional[str] = None, canonical_input_type: Optional[str] = None, stop_after_ocr: bool = False, - original_artifact_id: Optional[int] = None, - normalized_artifact_id: Optional[int] = None, - latest_job_id: Optional[int] = None, + original_artifact_id: Optional[str] = None, + normalized_artifact_id: Optional[str] = None, + latest_job_id: Optional[str] = None, instance: Optional[str] = None, uploaded_by_user_id: Optional[str] = None, uploaded_by_username: Optional[str] = None, @@ -677,11 +1006,21 @@ def upsert_document( uploaded_by_roles: Optional[str] = None, valid_from: Optional[str] = None, valid_to: Optional[str] = None, -): - """Insert or update a document record. - - ``instance`` is applied on INSERT only. Updates leave the existing tenant - stamp alone so a re-upload / restart cannot silently reassign tenants. + with_workflow: bool = True, + kind: Optional[str] = None, + code: Optional[str] = None, + aliases: Optional[list[str]] = None, +) -> str: + """Insert or update a document record; returns the document UUID. + + A new document needs ``kind`` (``scheme`` / ``advisory``) and ``code``: + ``instance`` (a tenant code) and the kind pick its ``(instance, kind)`` catalog, + which is created when missing. ``name`` becomes ``documents.name`` (the + code when absent). ``DuplicateDocumentCode`` is raised when another + document already holds ``code`` in that catalog. + + ``instance`` and the catalog are applied on INSERT only. Updates leave them + alone so a re-upload / restart cannot silently reassign tenants. Uploader identity is set on INSERT, and filled on UPDATE only when currently null. ``valid_from``/``valid_to`` are the document's validity period, stored as @@ -689,94 +1028,218 @@ def upsert_document( restart that re-registers the same workflow. What the period should default to is the caller's call (see ``pipeline/document_validity.py``); omitting both stores NULL, which search reads as "no stated period". - """ - now = datetime.utcnow().isoformat() - instance_value = (instance or os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" - with _db_lock: - with get_connection() as conn: - # Check if exists - row = conn.execute( - "SELECT created_at FROM documents WHERE workflow_id = ?", - (workflow_id,) + ``page_count`` and ``chunk_count`` are accepted for call-site compatibility + and ignored: both are counted from ``pages`` and ``chunks``. + + A new document also gets its first workflow row (``temporal_workflow_id`` = + ``workflow_id``) unless ``with_workflow`` is False, for callers that create + it themselves with ``create_document_job`` (addressing the document by the + returned UUID, since no workflow id resolves to it yet). + """ + instance_value = (instance or _default_instance()).strip().lower() or _default_instance() + fingerprint = _fingerprint_for( + source_file_fingerprint or canonical_document_id, document_id, workflow_id + ) + source_patch = { + "fingerprint": fingerprint, + "original_filename": filename, + "storage_uri": filepath, + "source_type": source_type, + "canonical_input_type": canonical_input_type, + } + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) if workflow_id else None + if doc_uuid: + patch = {k: v for k, v in source_patch.items() if v is not None} + DocumentSource.model_validate({**_load(conn, doc_uuid, "source"), **patch}) + uploader_uuid = ( + upsert_user(uploaded_by_user_id, uploaded_by_username, uploaded_by_email, + uploaded_by_roles, conn=conn, tenants=[instance_value]) + if uploaded_by_user_id else None + ) + conn.execute( + """ + UPDATE documents SET + source = source || %s::jsonb, + name = COALESCE(%s, name), + metadata = CASE + WHEN %s::text IS NULL OR metadata ? 'uploaded_by_user_id' THEN metadata + ELSE metadata || jsonb_build_object('uploaded_by_user_id', %s::text) + END + WHERE id = %s::uuid + """, + (Jsonb(patch), name or None, uploader_uuid, uploader_uuid, doc_uuid), + ) + cw = conn.execute( + "SELECT id FROM document_current_workflow WHERE document_id = %s::uuid", (doc_uuid,) ).fetchone() + if cw: + conn.execute( + """ + UPDATE workflows SET current_stage = %s, error_message = %s, + stop_after_ocr = %s + WHERE id = %s::uuid + """, + (stage, error_message, bool(stop_after_ocr), str(cw["id"])), + ) + return doc_uuid + + if not kind or not code: + raise ValueError("A new document needs kind and code") + source = DocumentSource.model_validate( + {k: v for k, v in source_patch.items() if v is not None} + ).model_dump(exclude_none=True) + user_uuid = ( + upsert_user(uploaded_by_user_id, uploaded_by_username, uploaded_by_email, + uploaded_by_roles, conn=conn, tenants=[instance_value]) + if uploaded_by_user_id else None + ) + metadata = DocumentMetadata( + valid_from=valid_from, valid_to=valid_to, uploaded_by_user_id=user_uuid + ).model_dump(exclude_none=True) + catalog = get_or_create_catalog(get_or_create_instance(instance_value, conn=conn), kind, conn=conn) + try: + with conn.transaction(): + row = conn.execute( + """ + INSERT INTO documents (catalog_id, code, name, aliases, source, metadata) + VALUES (%s::uuid, %s, %s, %s, %s, %s) + RETURNING id + """, + ( + catalog["id"], code, name or code, + [a for a in (aliases or []) if a], Jsonb(source), Jsonb(metadata), + ), + ).fetchone() + except psycopg.errors.UniqueViolation as exc: + if exc.diag.constraint_name == "documents_catalog_id_code_key": + raise DuplicateDocumentCode( + f"Code '{code}' is already used in the {kind} catalog of {instance_value}" + ) from exc + raise + if with_workflow: + conn.execute( + """ + INSERT INTO workflows (document_id, workflow_type, temporal_workflow_id, + current_stage, error_message, stop_after_ocr, + started_by_user_id) + VALUES (%s::uuid, %s, %s, %s, %s, %s, %s::uuid) + """, + ( + str(row["id"]), + "ocr_only" if stop_after_ocr else "pipeline", + workflow_id, stage, error_message, bool(stop_after_ocr), user_uuid, + ), + ) + return str(row["id"]) - if row: - # Update existing — do not overwrite instance (tenant ownership). - conn.execute(""" - UPDATE documents SET - document_id = ?, - canonical_document_id = COALESCE(?, canonical_document_id), - filename = ?, - display_name = COALESCE(?, display_name), - source_filename = COALESCE(?, source_filename), - source_manifest_name = COALESCE(?, source_manifest_name), - source_file_fingerprint = COALESCE(?, source_file_fingerprint), - filepath = ?, - stage = ?, - page_count = ?, - chunk_count = ?, - error_message = ?, - updated_at = ?, - ocr_completed_at = COALESCE(?, ocr_completed_at), - translation_completed_at = COALESCE(?, translation_completed_at), - chunks_completed_at = COALESCE(?, chunks_completed_at), - ingested_at = COALESCE(?, ingested_at), - source_type = COALESCE(?, source_type), - canonical_input_type = COALESCE(?, canonical_input_type), - stop_after_ocr = COALESCE(?, stop_after_ocr), - original_artifact_id = COALESCE(?, original_artifact_id), - normalized_artifact_id = COALESCE(?, normalized_artifact_id), - latest_job_id = COALESCE(?, latest_job_id), - uploaded_by_user_id = COALESCE(uploaded_by_user_id, ?), - uploaded_by_username = COALESCE(uploaded_by_username, ?), - uploaded_by_email = COALESCE(uploaded_by_email, ?), - uploaded_by_roles = COALESCE(uploaded_by_roles, ?) - WHERE workflow_id = ? - """, ( - document_id, canonical_document_id, filename, display_name, - source_filename, source_manifest_name, source_file_fingerprint, - filepath, stage, - page_count, chunk_count, error_message, now, - ocr_completed_at, translation_completed_at, chunks_completed_at, ingested_at, - source_type, canonical_input_type, 1 if stop_after_ocr else 0, - original_artifact_id, normalized_artifact_id, latest_job_id, - uploaded_by_user_id, uploaded_by_username, uploaded_by_email, uploaded_by_roles, - workflow_id - )) - else: - # Insert new - conn.execute(""" - INSERT INTO documents ( - workflow_id, document_id, filename, filepath, - canonical_document_id, display_name, source_filename, source_manifest_name, - source_file_fingerprint, - stage, page_count, chunk_count, error_message, - created_at, updated_at, - ocr_completed_at, translation_completed_at, chunks_completed_at, ingested_at, - source_type, canonical_input_type, stop_after_ocr, - reindex_required, reindex_reason, - original_artifact_id, normalized_artifact_id, latest_job_id, - instance, - uploaded_by_user_id, uploaded_by_username, uploaded_by_email, uploaded_by_roles, - valid_from, valid_to - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, ( - workflow_id, document_id, filename, filepath, - canonical_document_id, display_name, source_filename, source_manifest_name, - source_file_fingerprint, - stage, page_count, chunk_count, error_message, - now, now, - ocr_completed_at, translation_completed_at, chunks_completed_at, ingested_at, - source_type, canonical_input_type, 1 if stop_after_ocr else 0, - 0, None, - original_artifact_id, normalized_artifact_id, latest_job_id, - instance_value, - uploaded_by_user_id, uploaded_by_username, uploaded_by_email, uploaded_by_roles, - valid_from, valid_to, - )) - - conn.commit() + +def reset_document_for_reupload( + doc_uuid: str, + *, + filename: str, + filepath: str, + fingerprint: str, + source_type: Optional[str] = None, + canonical_input_type: Optional[str] = None, +) -> None: + """Point an existing document at a re-uploaded file. + + ``source`` is replaced, pages and chunks are cleared for the new run, and + the document is flagged ``reindex_required`` because its vector index is now + stale. Earlier workflows and artifacts stay as history. + + The fingerprint is the document's key in the vector index, so the old key + is remembered in ``metadata.superseded_doc_ids`` for every index (dev/prod) + that holds it. The next ingest/promote and disable/purge delete those points; + without this they would stay searchable with nothing pointing at them. + """ + source = DocumentSource.model_validate({ + "fingerprint": fingerprint, + "original_filename": filename, + "storage_uri": filepath, + "source_type": source_type, + "canonical_input_type": canonical_input_type, + }).model_dump(exclude_none=True) + with get_connection() as conn: + current = conn.execute( + "SELECT source->>'fingerprint' AS fingerprint, metadata FROM documents " + "WHERE id = %s::uuid FOR UPDATE", + (doc_uuid,), + ).fetchone() + if current is None: + raise LookupError(f"Document not found: {doc_uuid}") + meta = current["metadata"] or {} + superseded = {k: list(v) for k, v in (meta.get("superseded_doc_ids") or {}).items()} + for env, entry in (meta.get("index_status") or {}).items(): + old_key = (entry or {}).get("doc_id") or current["fingerprint"] + if old_key and old_key != fingerprint and old_key not in superseded.get(env, []): + superseded.setdefault(env, []).append(old_key) + conn.execute("DELETE FROM pages WHERE document_id = %s::uuid", (doc_uuid,)) + conn.execute("DELETE FROM chunks WHERE document_id = %s::uuid", (doc_uuid,)) + conn.execute( + """ + UPDATE documents SET + source = %s::jsonb, + metadata = (metadata - 'chunking') + || jsonb_build_object('reindex_required', TRUE, 'reindex_reason', 'source_replaced', + 'superseded_doc_ids', %s::jsonb) + WHERE id = %s::uuid + """, + (Jsonb(source), Jsonb(superseded), doc_uuid), + ) + + +def doc_id_used_by_other_document(doc_id: str, exclude_doc_uuid: Optional[str], env: str) -> bool: + """True when another *live* document is indexed in ``env`` (dev/prod) under ``doc_id``. + + The index key is the file fingerprint, so two documents holding the same file + share points in the vector index. Deleting a key another document still needs + would remove that document's vectors too. Only a document that is not + disabled and whose own ``index_status[env]`` holds this key counts: one that + was merely uploaded, or is disabled, has nothing in the index to protect. + """ + with get_connection() as conn: + row = conn.execute( + """ + SELECT 1 FROM documents d + WHERE (%s::uuid IS NULL OR d.id <> %s::uuid) + AND NOT d.is_disabled + AND d.metadata->'index_status'->%s::text->>'doc_id' = %s + LIMIT 1 + """, + (exclude_doc_uuid, exclude_doc_uuid, env, doc_id), + ).fetchone() + return row is not None + + +def clear_superseded_doc_ids(workflow_id: str, env: str, doc_ids: list[str]) -> None: + """Forget superseded index keys once their points are deleted from ``env`` (dev/prod).""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return + row = conn.execute( + "SELECT metadata->'superseded_doc_ids' AS sup FROM documents WHERE id = %s::uuid FOR UPDATE", + (doc_uuid,), + ).fetchone() + superseded = {k: list(v) for k, v in (row["sup"] or {}).items()} + remaining = [i for i in superseded.get(env, []) if i not in set(doc_ids)] + if remaining: + superseded[env] = remaining + else: + superseded.pop(env, None) + conn.execute( + "UPDATE documents SET metadata = metadata || jsonb_build_object('superseded_doc_ids', %s::jsonb) " + "WHERE id = %s::uuid", + (Jsonb(superseded), doc_uuid), + ) + + +def _load(conn, doc_uuid: str, column: str) -> dict: + row = conn.execute(f"SELECT {column} FROM documents WHERE id = %s::uuid", (doc_uuid,)).fetchone() + return dict(row[column]) if row else {} def update_document_stage( @@ -786,64 +1249,54 @@ def update_document_stage( chunk_count: Optional[int] = None, error_message: Optional[str] = None ): - """Update just the stage and counts for a document.""" - now = datetime.utcnow().isoformat() + """Update the stage (of the document's current workflow). - with _db_lock: - with get_connection() as conn: - # Get previous state for audit logging - row = conn.execute( - "SELECT stage, document_id FROM documents WHERE workflow_id = ?", - (workflow_id,) - ).fetchone() - old_stage = row["stage"] if row else None - document_id = row["document_id"] if row else workflow_id - - # Build dynamic update - updates = ["stage = ?", "updated_at = ?"] - values = [stage, now] - - if page_count is not None: - updates.append("page_count = ?") - values.append(page_count) - - if chunk_count is not None: - updates.append("chunk_count = ?") - values.append(chunk_count) - - if error_message is not None: - updates.append("error_message = ?") - values.append(error_message) - - # Set timestamp based on stage - if stage == "ocr_review": - updates.append("ocr_completed_at = ?") - values.append(now) - elif stage == "translation_review": - updates.append("translation_completed_at = ?") - values.append(now) - elif stage == "chunk_review": - updates.append("chunks_completed_at = ?") - values.append(now) - elif stage == "completed": - updates.append("ingested_at = ?") - values.append(now) - - values.append(workflow_id) - - # Note: updates list contains only hardcoded field names, not user input - # Values are properly parameterized via ? - conn.execute( - f"UPDATE documents SET {', '.join(updates)} WHERE workflow_id = ?", - values + ``page_count`` and ``chunk_count`` are accepted for call-site compatibility; + both are counted from ``pages`` and ``chunks``, so only the audit entry keeps them. + """ + old_stage = None + audit_workflow_id = workflow_id + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return + cw = conn.execute( + "SELECT id, current_stage FROM document_current_workflow WHERE document_id = %s::uuid", + (doc_uuid,), + ).fetchone() + if cw is None: + return + old_stage = cw["current_stage"] + + assignments = ["current_stage = %s"] + params: list = [stage] + if error_message is not None: + assignments.append("error_message = %s") + params.append(error_message) + stamp_key = { + "ocr_review": "ocr", + "translation_review": "translation", + "chunk_review": "chunks", + }.get(stage) + if stamp_key: + assignments.append( + "stage_timestamps = jsonb_build_object(%s::text, %s::text) || stage_timestamps" ) - conn.commit() + params.extend([stamp_key, _iso_now()]) + if stage == "completed": + assignments.append("completed_at = now()") + params.append(str(cw["id"])) + conn.execute( + f"UPDATE workflows SET {', '.join(assignments)} WHERE id = %s::uuid", params + ) + + conn.execute("UPDATE documents SET updated_at = now() WHERE id = %s::uuid", (doc_uuid,)) + audit_workflow_id = _root_workflow_id(conn, doc_uuid) or workflow_id - # Log stage transition if stage changed (outside lock to avoid deadlock) if old_stage and old_stage != stage: log_audit( - workflow_id=workflow_id, - document_id=document_id, + workflow_id=audit_workflow_id, + document_id=audit_workflow_id, action_type="stage_change", field_name="stage", old_value=old_stage, @@ -852,6 +1305,18 @@ def update_document_stage( ) +def _root_workflow_id(conn, doc_uuid: str) -> Optional[str]: + row = conn.execute( + """ + SELECT temporal_workflow_id FROM workflows WHERE document_id = %s::uuid + ORDER BY (workflow_type IN ('pipeline', 'ocr_only')) DESC, started_at ASC, id ASC + LIMIT 1 + """, + (doc_uuid,), + ).fetchone() + return row["temporal_workflow_id"] if row else None + + def reconcile_materialized_state(workflow_id: str) -> Optional[dict]: """ Reconcile a document row against already-persisted page/chunk content. @@ -862,23 +1327,19 @@ def reconcile_materialized_state(workflow_id: str) -> Optional[dict]: doc = get_document(workflow_id) if not doc: return None + workflow_id = doc["workflow_id"] with get_connection() as conn: page_stats = conn.execute( """ SELECT COUNT(*) AS page_rows, COALESCE(MAX(page_number), 0) AS max_page_number - FROM pages - WHERE workflow_id = ? + FROM pages WHERE document_id = %s::uuid """, - (workflow_id,), + (doc["id"],), ).fetchone() chunk_stats = conn.execute( - """ - SELECT COUNT(*) AS chunk_rows - FROM chunks - WHERE workflow_id = ? - """, - (workflow_id,), + "SELECT COUNT(*) AS chunk_rows FROM chunks WHERE document_id = %s::uuid", + (doc["id"],), ).fetchone() page_count = int((page_stats["page_rows"] if page_stats else 0) or 0) @@ -889,7 +1350,7 @@ def reconcile_materialized_state(workflow_id: str) -> Optional[dict]: current_stage = doc.get("stage") latest_job = get_latest_document_job(workflow_id) updated = False - now = datetime.utcnow().isoformat() + now = _iso_now() # Highest-confidence reconciliation first: if chunks exist, the document has # already reached chunk review materiality even if the workflow stalled. @@ -972,21 +1433,15 @@ def reconcile_materialized_state(workflow_id: str) -> Optional[dict]: def get_document(workflow_id: str) -> Optional[dict]: - """Get a single document by workflow ID.""" + """Get a single document by workflow id, UUID or any other identifier.""" with get_connection() as conn: - row = conn.execute( - "SELECT * FROM documents WHERE workflow_id = ?", - (workflow_id,) - ).fetchone() - - return dict(row) if row else None + return _fetch_document(conn, _resolve_document_uuid(conn, workflow_id)) def list_documents( stage: Optional[str] = None, limit: int = 100, offset: int = 0, - include_demo: bool = False, include_disabled: bool = False, instances: Optional[list[str]] = None, ) -> list[dict]: @@ -996,197 +1451,123 @@ def list_documents( stage: Filter by document stage limit: Max documents to return offset: Pagination offset - include_demo: If False (default), excludes demo documents from results include_disabled: If False (default), excludes soft-deleted documents from results instances: If provided, only return docs whose instance is in this list - (NULL/empty instance treated as DEFAULT_INSTANCE / 'default') """ - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + if instances is not None and not _normalize_instances(instances): + return [] + clauses, params = _document_filters(include_disabled, instances) + if stage: + clauses.append("cw.current_stage = %s") + params.append(stage) + where = f"WHERE {' AND '.join(clauses)}" if clauses else "" with get_connection() as conn: - # Note: filters are hardcoded SQL fragments based on boolean flags, not user input - demo_filter = "" if include_demo else "AND (is_demo = 0 OR is_demo IS NULL)" - disabled_filter = "" if include_disabled else "AND (is_disabled = 0 OR is_disabled IS NULL)" - instance_filter = "" - params: list = [] - - if instances is not None: - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) - if not normalized: - return [] - placeholders = ",".join("?" for _ in normalized) - instance_filter = ( - f"AND lower(COALESCE(NULLIF(trim(instance), ''), ?)) IN ({placeholders})" - ) - params.append(default_instance) - params.extend(normalized) - - if stage: - rows = conn.execute(f""" - SELECT * FROM documents - WHERE stage = ? {demo_filter} {disabled_filter} {instance_filter} - ORDER BY created_at DESC - LIMIT ? OFFSET ? - """, (stage, *params, limit, offset)).fetchall() - else: - rows = conn.execute(f""" - SELECT * FROM documents - WHERE 1=1 {demo_filter} {disabled_filter} {instance_filter} - ORDER BY created_at DESC - LIMIT ? OFFSET ? - """, (*params, limit, offset)).fetchall() - - return [dict(row) for row in rows] + rows = conn.execute( + f"{_DOC_SELECT} {where} ORDER BY d.created_at DESC, d.id LIMIT %s OFFSET %s", + (*params, limit, offset), + ).fetchall() + return [_document_from_row(r) for r in rows] + + +_REVIEW_QUEUE_STAGES = ("ocr_review", "translation_review", "chunk_review", "approval_for_prod") + +_SUMMARY_KEYS = ( + "total_documents", + "authoritative_documents", + "legacy_documents", + "completed_documents", + "review_queue", + "ocr_review_documents", + "translation_review_documents", + "chunk_review_documents", + "translation_processing_documents", + "chunking_documents", + "ready_for_ingestion_documents", + "failed_documents", + "needs_reindex", + "running_jobs", +) def get_document_summary_counts( - include_demo: bool = False, include_disabled: bool = False, instances: Optional[list[str]] = None, ) -> dict: """Return aggregate document counts for dashboard and migration planning.""" - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" - with get_connection() as conn: - demo_filter = "" if include_demo else "AND (is_demo = 0 OR is_demo IS NULL)" - disabled_filter = "" if include_disabled else "AND (is_disabled = 0 OR is_disabled IS NULL)" - job_demo_filter = "" if include_demo else "AND (d.is_demo = 0 OR d.is_demo IS NULL)" - job_disabled_filter = "" if include_disabled else "AND (d.is_disabled = 0 OR d.is_disabled IS NULL)" - instance_filter = "" - job_instance_filter = "" - params: list = [] - job_params: list = [] - - if instances is not None: - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) - if not normalized: - return { - "total_documents": 0, - "authoritative_documents": 0, - "legacy_documents": 0, - "completed_documents": 0, - "review_queue": 0, - "ocr_review_documents": 0, - "translation_review_documents": 0, - "chunk_review_documents": 0, - "translation_processing_documents": 0, - "chunking_documents": 0, - "ready_for_ingestion_documents": 0, - "failed_documents": 0, - "needs_reindex": 0, - "running_jobs": 0, - } - placeholders = ",".join("?" for _ in normalized) - instance_filter = ( - f"AND lower(COALESCE(NULLIF(trim(instance), ''), ?)) IN ({placeholders})" - ) - job_instance_filter = ( - f"AND lower(COALESCE(NULLIF(trim(d.instance), ''), ?)) IN ({placeholders})" - ) - params = [default_instance, *normalized] - job_params = [default_instance, *normalized] - - row = conn.execute(f""" + if instances is not None and not _normalize_instances(instances): + return {key: 0 for key in _SUMMARY_KEYS} + clauses, params = _document_filters(include_disabled, instances) + where = f"WHERE {' AND '.join(clauses)}" if clauses else "" + with get_connection() as conn: + row = conn.execute( + f""" SELECT COUNT(*) AS total_documents, - -- SQLite SUM() returns NULL when no rows match; COALESCE keeps response ints valid. - COALESCE(SUM(CASE WHEN source_manifest_name IS NOT NULL THEN 1 ELSE 0 END), 0) AS authoritative_documents, - COALESCE(SUM(CASE WHEN source_manifest_name IS NULL THEN 1 ELSE 0 END), 0) AS legacy_documents, - COALESCE(SUM(CASE WHEN stage = 'completed' THEN 1 ELSE 0 END), 0) AS completed_documents, - COALESCE(SUM(CASE WHEN stage IN ('ocr_review', 'translation_review', 'chunk_review', 'approval_for_prod') THEN 1 ELSE 0 END), 0) AS review_queue, - COALESCE(SUM(CASE WHEN stage = 'ocr_review' THEN 1 ELSE 0 END), 0) AS ocr_review_documents, - COALESCE(SUM(CASE WHEN stage = 'translation_review' THEN 1 ELSE 0 END), 0) AS translation_review_documents, - COALESCE(SUM(CASE WHEN stage = 'chunk_review' THEN 1 ELSE 0 END), 0) AS chunk_review_documents, - COALESCE(SUM(CASE WHEN stage = 'translation_processing' THEN 1 ELSE 0 END), 0) AS translation_processing_documents, - COALESCE(SUM(CASE WHEN stage = 'chunking' THEN 1 ELSE 0 END), 0) AS chunking_documents, - COALESCE(SUM(CASE WHEN stage = 'ready_for_ingestion' THEN 1 ELSE 0 END), 0) AS ready_for_ingestion_documents, - COALESCE(SUM(CASE WHEN stage = 'failed' THEN 1 ELSE 0 END), 0) AS failed_documents, - COALESCE(SUM(CASE WHEN reindex_required = 1 THEN 1 ELSE 0 END), 0) AS needs_reindex, - ( - SELECT COUNT(*) - FROM document_jobs j - JOIN documents d ON d.workflow_id = j.workflow_id - WHERE j.status = 'running' {job_demo_filter} {job_disabled_filter} {job_instance_filter} - ) AS running_jobs - FROM documents - WHERE 1=1 {demo_filter} {disabled_filter} {instance_filter} - """, (*job_params, *params)).fetchone() - result = dict(row) if row else {} - # Defensive: every count field must be an int for DocumentCohortsResponse. - int_keys = ( - "total_documents", - "authoritative_documents", - "legacy_documents", - "completed_documents", - "review_queue", - "ocr_review_documents", - "translation_review_documents", - "chunk_review_documents", - "translation_processing_documents", - "chunking_documents", - "ready_for_ingestion_documents", - "failed_documents", - "needs_reindex", - "running_jobs", - ) - return {key: int(result.get(key) or 0) for key in int_keys} + 0 AS authoritative_documents, + COUNT(*) AS legacy_documents, + COUNT(*) FILTER (WHERE cw.current_stage = 'completed') AS completed_documents, + COUNT(*) FILTER (WHERE cw.current_stage IN + ('ocr_review', 'translation_review', 'chunk_review', 'approval_for_prod')) AS review_queue, + COUNT(*) FILTER (WHERE cw.current_stage = 'ocr_review') AS ocr_review_documents, + COUNT(*) FILTER (WHERE cw.current_stage = 'translation_review') AS translation_review_documents, + COUNT(*) FILTER (WHERE cw.current_stage = 'chunk_review') AS chunk_review_documents, + COUNT(*) FILTER (WHERE cw.current_stage = 'translation_processing') AS translation_processing_documents, + COUNT(*) FILTER (WHERE cw.current_stage = 'chunking') AS chunking_documents, + COUNT(*) FILTER (WHERE cw.current_stage = 'ready_for_ingestion') AS ready_for_ingestion_documents, + COUNT(*) FILTER (WHERE cw.current_stage = 'failed') AS failed_documents, + COUNT(*) FILTER (WHERE COALESCE((d.metadata->>'reindex_required')::boolean, FALSE)) AS needs_reindex, + COALESCE(SUM((SELECT COUNT(*) FROM workflows j + WHERE j.document_id = d.id AND j.status = 'running')), 0) AS running_jobs + FROM documents d + JOIN catalogs c ON c.id = d.catalog_id + JOIN instances g ON g.id = c.instance_id + LEFT JOIN document_current_workflow cw ON cw.document_id = d.id + {where} + """, + params, + ).fetchone() + return {key: int(row[key] or 0) for key in _SUMMARY_KEYS} def get_document_counts_by_instance( - include_demo: bool = False, include_disabled: bool = False, instances: Optional[list[str]] = None, ) -> list[dict]: """Return document counts grouped by instance (state), for dashboard breakdowns.""" - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" - with get_connection() as conn: - demo_filter = "" if include_demo else "AND (is_demo = 0 OR is_demo IS NULL)" - disabled_filter = "" if include_disabled else "AND (is_disabled = 0 OR is_disabled IS NULL)" - instance_filter = "" - params: list = [default_instance] - - if instances is not None: - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) - if not normalized: - return [] - placeholders = ",".join("?" for _ in normalized) - instance_filter = ( - f"AND lower(COALESCE(NULLIF(trim(instance), ''), ?)) IN ({placeholders})" - ) - params.append(default_instance) - params.extend(normalized) - - rows = conn.execute(f""" + if instances is not None and not _normalize_instances(instances): + return [] + clauses, params = _document_filters(include_disabled, instances) + where = f"WHERE {' AND '.join(clauses)}" if clauses else "" + with get_connection() as conn: + rows = conn.execute( + f""" SELECT - lower(COALESCE(NULLIF(trim(instance), ''), ?)) AS instance, + lower(g.name) AS "instance", COUNT(*) AS count, - COALESCE(SUM(CASE WHEN stage = 'completed' THEN 1 ELSE 0 END), 0) AS success, - COALESCE(SUM(CASE WHEN stage = 'failed' THEN 1 ELSE 0 END), 0) AS failed, - COALESCE(SUM(CASE WHEN stage IN ('ocr_review', 'translation_review', 'chunk_review', 'approval_for_prod') THEN 1 ELSE 0 END), 0) AS dev_approval - FROM documents - WHERE 1=1 {demo_filter} {disabled_filter} {instance_filter} - GROUP BY instance - ORDER BY count DESC, instance ASC - """, params).fetchall() - return [ - { - "instance": row["instance"], - "count": int(row["count"] or 0), - "success": int(row["success"] or 0), - "failed": int(row["failed"] or 0), - "dev_approval": int(row["dev_approval"] or 0), - } - for row in rows - ] - - -def set_document_demo(workflow_id: str, is_demo: bool = True): - """Mark a document as demo (filtered from UI by default).""" - with _db_lock: - with get_connection() as conn: - conn.execute( - "UPDATE documents SET is_demo = ? WHERE workflow_id = ?", - (1 if is_demo else 0, workflow_id) - ) - conn.commit() + COUNT(*) FILTER (WHERE cw.current_stage = 'completed') AS success, + COUNT(*) FILTER (WHERE cw.current_stage = 'failed') AS failed, + COUNT(*) FILTER (WHERE cw.current_stage IN + ('ocr_review', 'translation_review', 'chunk_review', 'approval_for_prod')) AS dev_approval + FROM documents d + JOIN catalogs c ON c.id = d.catalog_id + JOIN instances g ON g.id = c.instance_id + LEFT JOIN document_current_workflow cw ON cw.document_id = d.id + {where} + GROUP BY lower(g.name) + ORDER BY count DESC, "instance" ASC + """, + params, + ).fetchall() + return [ + { + "instance": row["instance"], + "count": int(row["count"] or 0), + "success": int(row["success"] or 0), + "failed": int(row["failed"] or 0), + "dev_approval": int(row["dev_approval"] or 0), + } + for row in rows + ] def set_document_disabled(workflow_id: str, is_disabled: bool = True): @@ -1195,98 +1576,67 @@ def set_document_disabled(workflow_id: str, is_disabled: bool = True): Unlike hard delete, this preserves the document record for audit purposes. Use X-Include-Disabled: true header in API to see disabled documents. """ - with _db_lock: - with get_connection() as conn: + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid: conn.execute( - "UPDATE documents SET is_disabled = ?, updated_at = ? WHERE workflow_id = ?", - (1 if is_disabled else 0, datetime.utcnow().isoformat(), workflow_id) + "UPDATE documents SET is_disabled = %s WHERE id = %s::uuid", + (bool(is_disabled), doc_uuid), ) - conn.commit() def delete_document(workflow_id: str): - """Delete a document record.""" - with _db_lock: - with get_connection() as conn: - conn.execute( - "DELETE FROM documents WHERE workflow_id = ?", - (workflow_id,) - ) - conn.commit() - - -# Every table holding per-document rows, child-first so a partial failure never -# strands parent rows. `chunk_tags` is legacy — created by an older tagging -# build and absent from fresh databases — so it is probed rather than assumed. -_PURGE_TABLES = ( - "chunk_tags", - "chunks", - "pages", - "document_artifacts", - "document_index_status", - "document_jobs", - "documents", -) + """Delete a document record and everything that cascades from it.""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid: + conn.execute("DELETE FROM documents WHERE id = %s::uuid", (doc_uuid,)) def purge_document(workflow_id: str, keep_audit: bool = True) -> dict: - """Hard delete every SQLite row belonging to a document. - - Unlike :func:`set_document_disabled` this is irreversible — there is - nothing left for ``POST /documents/{id}/restore`` to bring back. + """Hard delete every row belonging to a document. - Audit rows are kept by default: they are the record that the deletion - happened, not part of the document itself. - - Returns a per-table count of deleted rows. - """ - deleted: dict[str, int] = {} - with _db_lock: - with get_connection() as conn: - existing = { - row[0] - for row in conn.execute( - "SELECT name FROM sqlite_master WHERE type = 'table'" - ) - } - for table in _PURGE_TABLES: - if table not in existing: - continue - columns = {r[1] for r in conn.execute(f"PRAGMA table_info([{table}])")} - if "workflow_id" not in columns: - continue - cursor = conn.execute( - f"DELETE FROM [{table}] WHERE workflow_id = ?", (workflow_id,) - ) - if cursor.rowcount > 0: - deleted[table] = cursor.rowcount - - if not keep_audit and "audit_logs" in existing: - cursor = conn.execute( - "DELETE FROM audit_logs WHERE workflow_id = ?", (workflow_id,) - ) - if cursor.rowcount > 0: - deleted["audit_logs"] = cursor.rowcount - - conn.commit() - return deleted + Unlike :func:`set_document_disabled` this is irreversible - there is + nothing left for ``POST /documents/{id}/restore`` to bring back. + Audit rows are kept by default: they are the record that the deletion + happened, not part of the document itself. Their ``document_id`` becomes + NULL; the ``workflow_id`` and ``instance`` snapshots stay. -def list_scheme_catalog_entries_for_workflow(workflow_id: str) -> list[dict]: - """Scheme catalog rows that still reference this workflow.""" + Returns a per-table count of deleted rows. + """ + deleted: dict[str, int] = {} with get_connection() as conn: - rows = conn.execute( - "SELECT scheme_code, workflow_ids_json FROM scheme_catalog_entries" - ).fetchall() - matches = [] - for row in rows: - try: - ids = json.loads(row["workflow_ids_json"] or "[]") - except (ValueError, TypeError): - ids = [] - if workflow_id in ids: - matches.append({"scheme_code": row["scheme_code"], "workflow_ids": ids}) - return matches + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return deleted + root = _root_workflow_id(conn, doc_uuid) or workflow_id + + def _count(sql: str) -> int: + return int(conn.execute(sql, (doc_uuid,)).fetchone()["n"]) + + counts = { + "chunks": _count("SELECT count(*) AS n FROM chunks WHERE document_id = %s::uuid"), + "pages": _count("SELECT count(*) AS n FROM pages WHERE document_id = %s::uuid"), + "document_artifacts": _count( + "SELECT count(*) AS n FROM workflow_artifacts a JOIN workflows w ON w.id = a.workflow_id " + "WHERE w.document_id = %s::uuid" + ), + "document_jobs": _count("SELECT count(*) AS n FROM workflows WHERE document_id = %s::uuid"), + } + if not keep_audit: + row = conn.execute( + "DELETE FROM audit_logs WHERE document_id = %s::uuid OR workflow_id = %s", + (doc_uuid, root), + ) + if row.rowcount > 0: + deleted["audit_logs"] = row.rowcount + conn.execute("DELETE FROM documents WHERE id = %s::uuid", (doc_uuid,)) + for table, count in counts.items(): + if count > 0: + deleted[table] = count + deleted["documents"] = 1 + return deleted def get_document_count(stage: Optional[str] = None) -> int: @@ -1294,121 +1644,249 @@ def get_document_count(stage: Optional[str] = None) -> int: with get_connection() as conn: if stage: row = conn.execute( - "SELECT COUNT(*) as cnt FROM documents WHERE stage = ?", - (stage,) + """ + SELECT COUNT(*) AS cnt FROM documents d + JOIN document_current_workflow cw ON cw.document_id = d.id + WHERE cw.current_stage = %s + """, + (stage,), ).fetchone() else: - row = conn.execute("SELECT COUNT(*) as cnt FROM documents").fetchone() - - return row["cnt"] if row else 0 - - -def upsert_manifest_entry( - manifest_filename: str, - title_en: Optional[str] = None, - title_gu: Optional[str] = None, - doc_language: Optional[str] = None, - category_tags: Optional[str] = None, - description: Optional[str] = None, - quality_score: Optional[str] = None, - priority_rank: Optional[str] = None, - ingestion_status: Optional[str] = None, - feedback: Optional[str] = None, - source_csv_path: Optional[str] = None, -) -> dict: - imported_at = datetime.utcnow().isoformat() - with _db_lock: - with get_connection() as conn: - conn.execute(""" - INSERT INTO document_manifest_entries ( - manifest_filename, title_en, title_gu, doc_language, - category_tags, description, quality_score, priority_rank, - ingestion_status, feedback, source_csv_path, imported_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(manifest_filename) DO UPDATE SET - title_en = COALESCE(excluded.title_en, document_manifest_entries.title_en), - title_gu = COALESCE(excluded.title_gu, document_manifest_entries.title_gu), - doc_language = COALESCE(excluded.doc_language, document_manifest_entries.doc_language), - category_tags = COALESCE(excluded.category_tags, document_manifest_entries.category_tags), - description = COALESCE(excluded.description, document_manifest_entries.description), - quality_score = COALESCE(excluded.quality_score, document_manifest_entries.quality_score), - priority_rank = COALESCE(excluded.priority_rank, document_manifest_entries.priority_rank), - ingestion_status = COALESCE(excluded.ingestion_status, document_manifest_entries.ingestion_status), - feedback = COALESCE(excluded.feedback, document_manifest_entries.feedback), - source_csv_path = COALESCE(excluded.source_csv_path, document_manifest_entries.source_csv_path), - imported_at = excluded.imported_at - """, ( - manifest_filename, title_en, title_gu, doc_language, - category_tags, description, quality_score, priority_rank, - ingestion_status, feedback, source_csv_path, imported_at - )) - conn.commit() - return get_manifest_entry(manifest_filename) or {} - - -def get_manifest_entry(manifest_filename: str) -> Optional[dict]: + row = conn.execute("SELECT COUNT(*) AS cnt FROM documents").fetchone() + return int(row["cnt"]) if row else 0 + + +def update_document_fields(workflow_id: str, **updates: object) -> Optional[dict]: + """Update selected document fields and return the document. + + Flat names are mapped onto the tables: ``stage``/``error_message`` act on the + current workflow, ``code``/``name``/``aliases`` on the document columns, + a document's kind is fixed by its catalog at upload, and flags, + period and uploader go into ``metadata``. Names derived from other rows (counts, artifact and job + pointers, completion timestamps) are accepted and ignored. + """ with get_connection() as conn: - row = conn.execute( - "SELECT * FROM document_manifest_entries WHERE manifest_filename = ?", - (manifest_filename,), - ).fetchone() - return dict(row) if row else None + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + if not updates: + return _fetch_document(conn, doc_uuid) + + doc_sets: list[str] = [] + doc_params: list = [] + source_patch: dict[str, Any] = {} + meta_patch: dict[str, Any] = {} + meta_remove: list[str] = [] + wf_sets: list[str] = [] + wf_params: list = [] + + for key, value in updates.items(): + if key == "stage": + wf_sets.append("current_stage = %s") + wf_params.append(value) + elif key == "error_message": + wf_sets.append("error_message = %s") + wf_params.append(value) + elif key == "source_type": + source_patch["source_type"] = value + elif key == "canonical_input_type": + source_patch["canonical_input_type"] = value + elif key == "filepath": + source_patch["storage_uri"] = value + elif key in ("filename", "source_filename"): + if key == "filename" or "filename" not in updates: + source_patch["original_filename"] = value + elif key in ("document_id", "canonical_document_id", "source_file_fingerprint"): + if isinstance(value, str) and _FINGERPRINT_RE.match(value): + source_patch["fingerprint"] = value + elif key == "name": + if value: + doc_sets.append("name = %s") + doc_params.append(value) + elif key == "code": + if value: + doc_sets.append("code = %s") + doc_params.append(value) + elif key == "aliases": + doc_sets.append("aliases = %s") + doc_params.append([a for a in (value or []) if a]) + elif key in ("reindex_required", "catalog_visible", "network_visible"): + meta_patch[key] = bool(value) + elif key == "reindex_reason": + if value: + meta_patch["reindex_reason"] = value + else: + meta_remove.append("reindex_reason") + elif key == "stop_after_ocr": + conn.execute( + "UPDATE workflows SET stop_after_ocr = %s WHERE id = (" + "SELECT id FROM workflows WHERE document_id = %s::uuid " + "ORDER BY (workflow_type IN ('pipeline', 'ocr_only')) DESC, started_at ASC, id ASC LIMIT 1)", + (bool(value), doc_uuid), + ) + elif key in ("valid_from", "valid_to"): + meta_patch[key] = value.isoformat() if isinstance(value, date) else value + elif key == "uploaded_by_user_id" and value: + has_uploader = conn.execute( + "SELECT metadata ? 'uploaded_by_user_id' AS has FROM documents WHERE id = %s::uuid", + (doc_uuid,), + ).fetchone()["has"] + if not has_uploader: + meta_patch["uploaded_by_user_id"] = upsert_user( + str(value), + updates.get("uploaded_by_username"), + updates.get("uploaded_by_email"), + updates.get("uploaded_by_roles"), + conn=conn, + ) + + if source_patch: + doc_sets.append("source = source || %s::jsonb") + doc_params.append(Jsonb({k: v for k, v in source_patch.items() if v is not None})) + if meta_patch or meta_remove: + expr = "metadata" + for removed in meta_remove: + expr = f"({expr} - '{removed}')" + if meta_patch: + expr = f"{expr} || %s::jsonb" + doc_params.append(Jsonb(meta_patch)) + doc_sets.append(f"metadata = {expr}") + + if wf_sets: + conn.execute( + f"UPDATE workflows SET {', '.join(wf_sets)} WHERE id = (" + "SELECT id FROM document_current_workflow WHERE document_id = %s::uuid)", + (*wf_params, doc_uuid), + ) + if doc_sets: + try: + with conn.transaction(): + conn.execute( + f"UPDATE documents SET {', '.join(doc_sets)} WHERE id = %s::uuid", + (*doc_params, doc_uuid), + ) + except psycopg.errors.UniqueViolation as exc: + if exc.diag.constraint_name == "documents_catalog_id_code_key": + raise DuplicateDocumentCode( + "Another document in the target catalog already uses this code" + ) from exc + raise + elif wf_sets: + conn.execute("UPDATE documents SET updated_at = now() WHERE id = %s::uuid", (doc_uuid,)) + return _fetch_document(conn, doc_uuid) + + +def mark_document_reindex_required( + workflow_id: str, + required: bool = True, + reason: Optional[str] = None, +) -> Optional[dict]: + return update_document_fields( + workflow_id, + reindex_required=bool(required), + reindex_reason=reason if required else None, + ) + + +def set_document_validity( + workflow_id: str, + valid_from: str, + valid_to: str, +) -> Optional[dict]: + """Record the period this document's chunks are searchable in. Both ends + are `YYYY-MM-DD`; validation belongs to the caller + (`pipeline/document_validity.parse_period`), not to this SQL layer.""" + return update_document_fields(workflow_id, valid_from=valid_from, valid_to=valid_to) -def list_manifest_entries(limit: int = 1000, offset: int = 0) -> list[dict]: +def mark_prod_ready_requested( + workflow_id: str, + user_id: Optional[str] = None, + username: Optional[str] = None, +) -> Optional[dict]: + """Flag that a state_admin has asked a super_admin to review this document + for prod promotion. Purely informational - does not fire the approve_prod + signal itself; the super_admin still makes that call separately. + + Recorded as a ``promote_prod`` workflow in ``waiting_review``.""" with get_connection() as conn: - rows = conn.execute(""" - SELECT * FROM document_manifest_entries - ORDER BY manifest_filename - LIMIT ? OFFSET ? - """, (limit, offset)).fetchall() - return [dict(row) for row in rows] + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + root = _root_workflow_id(conn, doc_uuid) or workflow_id + user_uuid = upsert_user(user_id, username, None, None, conn=conn) if user_id else None + conn.execute( + """ + UPDATE workflows SET status = 'running' + WHERE document_id = %s::uuid AND workflow_type = 'promote_prod' AND status = 'waiting_review' + """, + (doc_uuid,), + ) + conn.execute( + """ + INSERT INTO workflows (document_id, workflow_type, temporal_workflow_id, status, + current_stage, started_by_user_id) + VALUES (%s::uuid, 'promote_prod', %s, 'waiting_review', 'approval_for_prod', %s::uuid) + """, + (doc_uuid, f"{root}-prod-request", user_uuid), + ) + return _fetch_document(conn, doc_uuid) -def update_document_fields(workflow_id: str, **updates: object) -> Optional[dict]: - """Update selected document fields and return the document.""" - if not updates: - return get_document(workflow_id) +def clear_prod_ready_requested(workflow_id: str) -> Optional[dict]: + """Reset the request flag once prod approval actually happens (or the + document leaves the approval_for_prod gate for any other reason).""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + conn.execute( + """ + UPDATE workflows SET status = 'running' + WHERE document_id = %s::uuid AND workflow_type = 'promote_prod' AND status = 'waiting_review' + """, + (doc_uuid,), + ) + return _fetch_document(conn, doc_uuid) - allowed_fields = { - "stage", "page_count", "chunk_count", "error_message", "updated_at", - "ocr_completed_at", "translation_completed_at", "chunks_completed_at", "ingested_at", - "source_type", "canonical_input_type", "original_artifact_id", "normalized_artifact_id", - "latest_job_id", "filepath", "filename", "display_name", "document_id", - "canonical_document_id", "source_filename", "source_manifest_name", - "source_file_fingerprint", "stop_after_ocr", "reindex_required", "reindex_reason", - "uploaded_by_user_id", "uploaded_by_username", "uploaded_by_email", "uploaded_by_roles", - # Scheme / master catalog fields - "document_kind", "scheme_code", "scheme_name", "scheme_aliases_json", - "tool_routing", "catalog_visible", "network_visible", - "prod_ready_requested_at", "prod_ready_requested_by_user_id", "prod_ready_requested_by_username", - # Document validity period (searchable-from / searchable-to) - "valid_from", "valid_to", - } - set_clauses = [] - values: list[object] = [] - for key, value in updates.items(): - if key not in allowed_fields: - continue - set_clauses.append(f"{key} = ?") - values.append(value) - if not set_clauses: - return get_document(workflow_id) +# ============================================================================= +# Workflows (exposed as "jobs") +# ============================================================================= - if "updated_at" not in updates: - set_clauses.append("updated_at = ?") - values.append(datetime.utcnow().isoformat()) +_JOB_SELECT = """ + SELECT w.id, w.document_id, w.workflow_type, w.temporal_workflow_id, w.status, + w.current_stage, w.config, w.error_message, w.started_at, w.completed_at, + root.temporal_workflow_id AS doc_workflow_id + FROM workflows w + LEFT JOIN LATERAL ( + SELECT r.temporal_workflow_id FROM workflows r + WHERE r.document_id = w.document_id + ORDER BY (r.workflow_type IN ('pipeline', 'ocr_only')) DESC, r.started_at ASC, r.id ASC + LIMIT 1 + ) root ON TRUE +""" - values.append(workflow_id) - with _db_lock: - with get_connection() as conn: - conn.execute( - f"UPDATE documents SET {', '.join(set_clauses)} WHERE workflow_id = ?", - values, - ) - conn.commit() - return get_document(workflow_id) + +def _job_from_row(row: Optional[dict]) -> Optional[dict]: + if row is None: + return None + config = row["config"] or {} + job = { + "id": row["id"], + "workflow_id": row["doc_workflow_id"], + "job_type": row["workflow_type"], + "temporal_workflow_id": row["temporal_workflow_id"], + "temporal_run_id": config.get("temporal_run_id"), + "status": row["status"], + "current_stage": row["current_stage"], + "started_at": row["started_at"], + "completed_at": row["completed_at"], + "error_message": row["error_message"], + "config": config, + "config_json": json.dumps(config) if config else None, + } + return {k: _clean(v) for k, v in job.items()} def create_document_job( @@ -1419,32 +1897,65 @@ def create_document_job( status: str = "running", current_stage: Optional[str] = None, config: Optional[dict] = None, -) -> int: - now = datetime.utcnow().isoformat() - config_json = json.dumps(config) if config else None - with _db_lock: - with get_connection() as conn: - cursor = conn.execute(""" - INSERT INTO document_jobs ( - workflow_id, job_type, temporal_workflow_id, temporal_run_id, - status, current_stage, started_at, config_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) - """, ( - workflow_id, job_type, temporal_workflow_id, temporal_run_id, - status, current_stage, now, config_json - )) - job_id = cursor.lastrowid - conn.execute( - "UPDATE documents SET latest_job_id = ?, updated_at = ? WHERE workflow_id = ?", - (job_id, now, workflow_id), + actor: Optional[dict] = None, +) -> str: + """Record a workflow run for a document; returns the workflow UUID. + + A ``promote_prod`` run adopts the document's pending request row (see + ``mark_prod_ready_requested``) instead of adding a second one. + """ + cfg = dict(config or {}) + if temporal_run_id: + cfg["temporal_run_id"] = temporal_run_id + with get_connection() as conn: + doc_uuid = _require_document_uuid(conn, workflow_id) + root = _root_workflow_id(conn, doc_uuid) or workflow_id + user_uuid = None + if actor and actor.get("user_id"): + user_uuid = upsert_user( + actor["user_id"], actor.get("username"), actor.get("email"), actor.get("roles"), + conn=conn, ) - conn.commit() - return job_id + if job_type == "promote_prod": + adopted = conn.execute( + """ + UPDATE workflows SET temporal_workflow_id = %s, status = %s, + current_stage = COALESCE(%s, current_stage), + config = config || %s::jsonb + WHERE id = ( + SELECT id FROM workflows + WHERE document_id = %s::uuid AND workflow_type = 'promote_prod' + AND status = 'waiting_review' + ORDER BY started_at DESC LIMIT 1) + RETURNING id + """, + (temporal_workflow_id or root, status, current_stage, Jsonb(cfg), doc_uuid), + ).fetchone() + if adopted: + conn.execute("UPDATE documents SET updated_at = now() WHERE id = %s::uuid", (doc_uuid,)) + return str(adopted["id"]) + row = conn.execute( + """ + INSERT INTO workflows (document_id, workflow_type, temporal_workflow_id, status, + current_stage, stop_after_ocr, config, started_by_user_id) + VALUES (%s::uuid, %s, %s, %s, + COALESCE(%s, 'registered'), %s, %s, %s::uuid) + RETURNING id + """, + ( + doc_uuid, job_type, temporal_workflow_id or root, status, current_stage, + bool(cfg.get("stop_after_ocr")), Jsonb(cfg), user_uuid, + ), + ).fetchone() + conn.execute("UPDATE documents SET updated_at = now() WHERE id = %s::uuid", (doc_uuid,)) + return str(row["id"]) -def update_document_job(job_id: int, **updates: object) -> Optional[dict]: +def update_document_job(job_id: str, **updates: object) -> Optional[dict]: if not updates: return get_document_job(job_id) + if not _is_uuid(str(job_id)): + return None allowed_fields = { "status", "current_stage", "completed_at", "error_message", "temporal_run_id", "config_json" } @@ -1453,122 +1964,120 @@ def update_document_job(job_id: int, **updates: object) -> Optional[dict]: for key, value in updates.items(): if key not in allowed_fields: continue - if key == "config_json" and isinstance(value, dict): - value = json.dumps(value) - set_clauses.append(f"{key} = ?") + if key == "status": + set_clauses.append("status = %s") + elif key == "current_stage": + set_clauses.append("current_stage = %s") + elif key == "completed_at": + set_clauses.append("completed_at = %s::timestamptz") + elif key == "temporal_run_id": + set_clauses.append("config = config || jsonb_build_object('temporal_run_id', %s::text)") + elif key == "config_json": + if isinstance(value, str): + value = json.loads(value) + set_clauses.append("config = config || %s::jsonb") + value = Jsonb(value or {}) + else: + set_clauses.append(f"{key} = %s") values.append(value) if not set_clauses: return get_document_job(job_id) - values.append(job_id) - with _db_lock: - with get_connection() as conn: - conn.execute( - f"UPDATE document_jobs SET {', '.join(set_clauses)} WHERE id = ?", - values, - ) - conn.commit() + values.append(str(job_id)) + with get_connection() as conn: + conn.execute(f"UPDATE workflows SET {', '.join(set_clauses)} WHERE id = %s::uuid", values) return get_document_job(job_id) -def get_document_job(job_id: int) -> Optional[dict]: +def get_document_job(job_id: str) -> Optional[dict]: + if not _is_uuid(str(job_id)): + return None with get_connection() as conn: - row = conn.execute( - "SELECT * FROM document_jobs WHERE id = ?", - (job_id,), - ).fetchone() - return dict(row) if row else None + return _job_from_row( + conn.execute(_JOB_SELECT + " WHERE w.id = %s::uuid", (str(job_id),)).fetchone() + ) def get_latest_document_job(workflow_id: str) -> Optional[dict]: with get_connection() as conn: - row = conn.execute(""" - SELECT * FROM document_jobs - WHERE workflow_id = ? - ORDER BY started_at DESC, id DESC - LIMIT 1 - """, (workflow_id,)).fetchone() - return dict(row) if row else None + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + return _job_from_row( + conn.execute( + _JOB_SELECT + " WHERE w.document_id = %s::uuid ORDER BY w.started_at DESC, w.id DESC LIMIT 1", + (doc_uuid,), + ).fetchone() + ) def list_document_jobs(workflow_id: str, limit: int = 50) -> list[dict]: with get_connection() as conn: - rows = conn.execute(""" - SELECT * FROM document_jobs - WHERE workflow_id = ? - ORDER BY started_at DESC, id DESC - LIMIT ? - """, (workflow_id, limit)).fetchall() - return [dict(row) for row in rows] + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return [] + rows = conn.execute( + _JOB_SELECT + " WHERE w.document_id = %s::uuid ORDER BY w.started_at DESC, w.id DESC LIMIT %s", + (doc_uuid, limit), + ).fetchall() + return [_job_from_row(r) for r in rows] def list_operations_queue( limit: int = 100, offset: int = 0, - include_demo: bool = False, include_disabled: bool = False, instances: Optional[list[str]] = None, ) -> tuple[list[dict], int]: """List documents needing action, optionally scoped to tenant instances. ``instances``: - - ``None`` — unrestricted (super admin / bypass) - - ``[]`` — restricted user with no states → empty result - - non-empty — only those document.instance values + - ``None`` - unrestricted (super admin / bypass) + - ``[]`` - restricted user with no states -> empty result + - non-empty - only those document.instance values """ - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" - with get_connection() as conn: - demo_filter = "" if include_demo else "AND (d.is_demo = 0 OR d.is_demo IS NULL)" - disabled_filter = "" if include_disabled else "AND (d.is_disabled = 0 OR d.is_disabled IS NULL)" - instance_filter = "" - instance_params: list = [] - if instances is not None: - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) - if not normalized: - return [], 0 - placeholders = ",".join("?" for _ in normalized) - instance_filter = ( - f"AND lower(COALESCE(NULLIF(trim(d.instance), ''), ?)) IN ({placeholders})" - ) - instance_params = [default_instance, *normalized] - where_clause = f""" - ( - d.stage IN ('ocr_review', 'translation_review', 'chunk_review', 'ready_for_ingestion', 'approval_for_prod', 'failed') - OR d.reindex_required = 1 - OR (j.status = 'running') - ) {demo_filter} {disabled_filter} {instance_filter} - """ - total_row = conn.execute( - f""" - SELECT COUNT(*) AS count - FROM documents d - LEFT JOIN document_jobs j ON j.id = d.latest_job_id - WHERE {where_clause} - """, - instance_params, - ).fetchone() + if instances is not None and not _normalize_instances(instances): + return [], 0 + clauses, params = _document_filters(include_disabled, instances) + clauses.append( + "(cw.current_stage IN ('ocr_review', 'translation_review', 'chunk_review', " + "'ready_for_ingestion', 'approval_for_prod', 'failed') " + "OR COALESCE((d.metadata->>'reindex_required')::boolean, FALSE) " + "OR cw.status = 'running')" + ) + where = f"WHERE {' AND '.join(clauses)}" + from_sql = """ + FROM documents d + JOIN catalogs c ON c.id = d.catalog_id + JOIN instances g ON g.id = c.instance_id + JOIN document_current_workflow cw ON cw.document_id = d.id + """ + with get_connection() as conn: + total_row = conn.execute(f"SELECT COUNT(*) AS count {from_sql} {where}", params).fetchone() rows = conn.execute( f""" SELECT - d.workflow_id, - d.filename, - d.stage, - d.instance, - d.error_message, - d.reindex_required, - d.reindex_reason, - j.id AS job_id, - j.job_type, - j.status AS job_status, - j.started_at - FROM documents d - LEFT JOIN document_jobs j ON j.id = d.latest_job_id - WHERE {where_clause} - ORDER BY COALESCE(j.started_at, d.updated_at, d.created_at) DESC - LIMIT ? OFFSET ? + d.id AS id, + (SELECT r.temporal_workflow_id FROM workflows r WHERE r.document_id = d.id + ORDER BY (r.workflow_type IN ('pipeline', 'ocr_only')) DESC, r.started_at ASC, r.id ASC + LIMIT 1) AS workflow_id, + d.source->>'original_filename' AS filename, + cw.current_stage AS stage, + lower(g.name) AS "instance", + cw.error_message, + COALESCE((d.metadata->>'reindex_required')::boolean, FALSE) AS reindex_required, + d.metadata->>'reindex_reason' AS reindex_reason, + cw.id AS job_id, + cw.workflow_type AS job_type, + cw.status AS job_status, + cw.started_at AS started_at + {from_sql} + {where} + ORDER BY cw.started_at DESC, d.id + LIMIT %s OFFSET %s """, - (*instance_params, limit, offset), + (*params, limit, offset), ).fetchall() - return [dict(row) for row in rows], (total_row["count"] if total_row else 0) + return [_clean_row(r) for r in rows], int(total_row["count"] if total_row else 0) def list_runs( @@ -1577,190 +2086,91 @@ def list_runs( status: Optional[str] = None, instances: Optional[list[str]] = None, ) -> list[dict]: - """List jobs with document title fields for the Runs UI. - - Joins ``documents`` so the UI can show filename / display_name instead of - falling back to "Untitled document". + """List workflows with document title fields for the Runs UI. Args: - instances: If provided, only return jobs whose document instance is in - this list (NULL/empty instance treated as DEFAULT_INSTANCE). - Empty list returns no rows. None = unrestricted (all tenants). + instances: If provided, only return runs whose document instance is in + this list. Empty list returns no rows. None = unrestricted. """ - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + where_parts: list[str] = [] + params: list = [] + if status: + where_parts.append("w.status = %s") + params.append(status) + if instances is not None: + normalized = _normalize_instances(instances) + if not normalized: + return [] + where_parts.append("lower(g.name) = ANY(%s)") + params.append(normalized) + where_sql = f"WHERE {' AND '.join(where_parts)}" if where_parts else "" with get_connection() as conn: - # Prefer document identity columns for list labels. - select_sql = """ - SELECT - j.*, - d.filename AS filename, - d.display_name AS display_name, - d.source_filename AS source_filename, - d.stage AS document_stage, - d.instance AS instance - FROM document_jobs j - LEFT JOIN documents d ON d.workflow_id = j.workflow_id - """ - where_parts: list[str] = [] - params: list = [] - - if status: - where_parts.append("j.status = ?") - params.append(status) - - if instances is not None: - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) - if not normalized: - return [] - placeholders = ",".join("?" for _ in normalized) - # Scope via owning document.instance (legacy NULL/empty → default). - where_parts.append( - f"lower(COALESCE(NULLIF(trim(d.instance), ''), ?)) IN ({placeholders})" - ) - params.append(default_instance) - params.extend(normalized) - - where_sql = f"WHERE {' AND '.join(where_parts)}" if where_parts else "" rows = conn.execute( - select_sql - + f""" + f""" + SELECT w.id, w.document_id, w.workflow_type, w.temporal_workflow_id, w.status, + w.current_stage, w.config, w.error_message, w.started_at, w.completed_at, + (SELECT r.temporal_workflow_id FROM workflows r WHERE r.document_id = w.document_id + ORDER BY (r.workflow_type IN ('pipeline', 'ocr_only')) DESC, r.started_at ASC, r.id ASC + LIMIT 1) AS doc_workflow_id, + d.source->>'original_filename' AS filename, + d.name AS name, + d.source->>'original_filename' AS source_filename, + cw.current_stage AS document_stage, + lower(g.name) AS "instance" + FROM workflows w + JOIN documents d ON d.id = w.document_id + JOIN catalogs c ON c.id = d.catalog_id + JOIN instances g ON g.id = c.instance_id + LEFT JOIN document_current_workflow cw ON cw.document_id = d.id {where_sql} - ORDER BY j.started_at DESC, j.id DESC - LIMIT ? OFFSET ? + ORDER BY w.started_at DESC, w.id DESC + LIMIT %s OFFSET %s """, (*params, limit, offset), ).fetchall() - return [dict(row) for row in rows] - - -def list_index_summaries(include_demo: bool = False, include_disabled: bool = False) -> list[dict]: - with get_connection() as conn: - demo_filter = "" if include_demo else "AND (d.is_demo = 0 OR d.is_demo IS NULL)" - disabled_filter = "" if include_disabled else "AND (d.is_disabled = 0 OR d.is_disabled IS NULL)" - rows = conn.execute(f""" - SELECT - s.index_name, - COUNT(*) AS documents, - SUM(COALESCE(s.chunk_count_indexed, 0)) AS indexed_chunks, - SUM(CASE WHEN d.reindex_required = 1 THEN 1 ELSE 0 END) AS stale_documents, - MAX(s.last_indexed_at) AS last_indexed_at, - GROUP_CONCAT(DISTINCT s.status) AS statuses - FROM document_index_status s - JOIN documents d ON d.workflow_id = s.workflow_id - WHERE 1=1 {demo_filter} {disabled_filter} - GROUP BY s.index_name - ORDER BY s.index_name - """).fetchall() - result: list[dict] = [] - for row in rows: - item = dict(row) - item["statuses"] = [part for part in (item.get("statuses") or "").split(",") if part] - result.append(item) - return result - - -def mark_document_reindex_required( - workflow_id: str, - required: bool = True, - reason: Optional[str] = None, -) -> Optional[dict]: - with _db_lock: - with get_connection() as conn: - conn.execute( - """ - UPDATE documents - SET reindex_required = ?, reindex_reason = ?, updated_at = ? - WHERE workflow_id = ? - """, - ( - 1 if required else 0, - reason if required else None, - datetime.utcnow().isoformat(), - workflow_id, - ), - ) - conn.commit() - return get_document(workflow_id) - + result = [] + for row in rows: + job = _job_from_row(row) + job["document_uuid"] = str(row["document_id"]) + for extra in ("filename", "name", "source_filename", "document_stage", "instance"): + job[extra] = row[extra] + result.append(job) + return result -def mark_prod_ready_requested( - workflow_id: str, - user_id: Optional[str] = None, - username: Optional[str] = None, -) -> Optional[dict]: - """Flag that a state_admin has asked a super_admin to review this document - for prod promotion. Purely informational — does not fire the approve_prod - signal itself; the super_admin still makes that call separately.""" - with _db_lock: - with get_connection() as conn: - conn.execute( - """ - UPDATE documents - SET prod_ready_requested_at = ?, - prod_ready_requested_by_user_id = ?, - prod_ready_requested_by_username = ?, - updated_at = ? - WHERE workflow_id = ? - """, - ( - datetime.utcnow().isoformat(), - user_id, - username, - datetime.utcnow().isoformat(), - workflow_id, - ), - ) - conn.commit() - return get_document(workflow_id) +# ============================================================================= +# Artifacts +# ============================================================================= -def set_document_validity( - workflow_id: str, - valid_from: str, - valid_to: str, -) -> Optional[dict]: - """Record the period this document's chunks are searchable in. Both ends - are `YYYY-MM-DD`; validation belongs to the caller - (`pipeline/document_validity.parse_period`), not to this SQL layer.""" - with _db_lock: - with get_connection() as conn: - conn.execute( - """ - UPDATE documents - SET valid_from = ?, - valid_to = ?, - updated_at = ? - WHERE workflow_id = ? - """, - ( - valid_from, - valid_to, - datetime.utcnow().isoformat(), - workflow_id, - ), - ) - conn.commit() - return get_document(workflow_id) +def _artifact_from_row(row: Optional[dict]) -> Optional[dict]: + if row is None: + return None + metadata = row["metadata"] or {} + artifact = { + "id": row["id"], + "workflow_id": row["doc_workflow_id"], + "job_id": row["workflow_pk"], + "artifact_type": row["artifact_type"], + "stage": row["stage"], + "storage_uri": row["storage_uri"], + "mime_type": row["mime_type"], + "filename": row["filename"], + "size_bytes": row["size_bytes"], + "metadata_json": json.dumps(metadata) if metadata else None, + "created_at": row["created_at"], + } + return {k: _clean(v) for k, v in artifact.items()} -def clear_prod_ready_requested(workflow_id: str) -> Optional[dict]: - """Reset the request flag once prod approval actually happens (or the - document leaves the approval_for_prod gate for any other reason).""" - with _db_lock: - with get_connection() as conn: - conn.execute( - """ - UPDATE documents - SET prod_ready_requested_at = NULL, - prod_ready_requested_by_user_id = NULL, - prod_ready_requested_by_username = NULL, - updated_at = ? - WHERE workflow_id = ? - """, - (datetime.utcnow().isoformat(), workflow_id), - ) - conn.commit() - return get_document(workflow_id) +_ARTIFACT_SELECT = """ + SELECT a.id, a.workflow_id AS workflow_pk, a.artifact_type, a.stage, a.storage_uri, + a.mime_type, a.filename, a.size_bytes, a.metadata, a.created_at, + (SELECT r.temporal_workflow_id FROM workflows r WHERE r.document_id = w.document_id + ORDER BY (r.workflow_type IN ('pipeline', 'ocr_only')) DESC, r.started_at ASC, r.id ASC + LIMIT 1) AS doc_workflow_id + FROM workflow_artifacts a + JOIN workflows w ON w.id = a.workflow_id +""" def add_document_artifact( @@ -1768,51 +2178,97 @@ def add_document_artifact( artifact_type: str, storage_uri: str, stage: Optional[str] = None, - job_id: Optional[int] = None, + job_id: Optional[str] = None, mime_type: Optional[str] = None, filename: Optional[str] = None, size_bytes: Optional[int] = None, metadata: Optional[dict] = None, -) -> int: - created_at = datetime.utcnow().isoformat() - metadata_json = json.dumps(metadata) if metadata else None - with _db_lock: - with get_connection() as conn: - cursor = conn.execute(""" - INSERT INTO document_artifacts ( - workflow_id, job_id, artifact_type, stage, storage_uri, - mime_type, filename, size_bytes, metadata_json, created_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, ( - workflow_id, job_id, artifact_type, stage, storage_uri, - mime_type, filename, size_bytes, metadata_json, created_at - )) - artifact_id = cursor.lastrowid - if artifact_type == "original_upload": - conn.execute("UPDATE documents SET original_artifact_id = ? WHERE workflow_id = ?", (artifact_id, workflow_id)) - elif artifact_type in {"normalized_pdf", "normalized_spreadsheet"}: - conn.execute("UPDATE documents SET normalized_artifact_id = ? WHERE workflow_id = ?", (artifact_id, workflow_id)) - conn.commit() - return artifact_id +) -> str: + """Record a file produced by a workflow; returns the artifact UUID. + + Belongs to ``job_id`` when given, else to the document's current workflow. + """ + with get_connection() as conn: + doc_uuid = _require_document_uuid(conn, workflow_id) + wf_uuid = None + if job_id and _is_uuid(str(job_id)): + row = conn.execute( + "SELECT id FROM workflows WHERE id = %s::uuid AND document_id = %s::uuid", + (str(job_id), doc_uuid), + ).fetchone() + wf_uuid = str(row["id"]) if row else None + if wf_uuid is None: + row = conn.execute( + "SELECT id FROM document_current_workflow WHERE document_id = %s::uuid", (doc_uuid,) + ).fetchone() + if row is None: + row = conn.execute( + """ + INSERT INTO workflows (document_id, workflow_type, temporal_workflow_id) + VALUES (%s::uuid, 'pipeline', %s) RETURNING id + """, + (doc_uuid, workflow_id), + ).fetchone() + wf_uuid = str(row["id"]) + created = conn.execute( + """ + INSERT INTO workflow_artifacts (workflow_id, artifact_type, stage, storage_uri, + mime_type, filename, size_bytes, metadata) + VALUES (%s::uuid, %s, %s, %s, %s, %s, %s, %s) + RETURNING id + """, + ( + wf_uuid, artifact_type, stage, storage_uri, mime_type, filename, size_bytes, + Jsonb(metadata or {}), + ), + ).fetchone() + return str(created["id"]) def list_document_artifacts(workflow_id: str) -> list[dict]: with get_connection() as conn: - rows = conn.execute(""" - SELECT * FROM document_artifacts - WHERE workflow_id = ? - ORDER BY created_at DESC, id DESC - """, (workflow_id,)).fetchall() - return [dict(row) for row in rows] + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return [] + rows = conn.execute( + _ARTIFACT_SELECT + " WHERE w.document_id = %s::uuid ORDER BY a.created_at DESC, a.id DESC", + (doc_uuid,), + ).fetchall() + return [_artifact_from_row(r) for r in rows] -def get_document_artifact(workflow_id: str, artifact_id: int) -> Optional[dict]: +def get_document_artifact(workflow_id: str, artifact_id: str) -> Optional[dict]: + if not _is_uuid(str(artifact_id)): + return None with get_connection() as conn: - row = conn.execute(""" - SELECT * FROM document_artifacts - WHERE workflow_id = ? AND id = ? - """, (workflow_id, artifact_id)).fetchone() - return dict(row) if row else None + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + return _artifact_from_row( + conn.execute( + _ARTIFACT_SELECT + " WHERE w.document_id = %s::uuid AND a.id = %s::uuid", + (doc_uuid, str(artifact_id)), + ).fetchone() + ) + + +# ============================================================================= +# Vector index state (documents.metadata.index_status) +# ============================================================================= + +def _index_entry_to_legacy(workflow_id: str, key: str, entry: dict) -> dict: + details = entry.get("details") or {} + return { + "workflow_id": workflow_id, + "index_name": entry.get("index_name") or key, + "doc_id": entry.get("doc_id"), + "chunk_count_indexed": entry.get("chunk_count_indexed") or 0, + "last_indexed_at": entry.get("last_indexed_at"), + "last_verified_at": entry.get("last_verified_at"), + "schema_version": entry.get("schema_version"), + "status": entry.get("status") or "unknown", + "details_json": json.dumps(details) if details else None, + } def upsert_document_index_status( @@ -1826,111 +2282,104 @@ def upsert_document_index_status( status: str = "unknown", details: Optional[dict] = None, ) -> dict: - details_json = json.dumps(details) if details else None - with _db_lock: - with get_connection() as conn: - conn.execute(""" - INSERT INTO document_index_status ( - workflow_id, index_name, doc_id, chunk_count_indexed, - last_indexed_at, last_verified_at, schema_version, status, details_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(workflow_id, index_name) DO UPDATE SET - doc_id = COALESCE(excluded.doc_id, document_index_status.doc_id), - chunk_count_indexed = COALESCE(excluded.chunk_count_indexed, document_index_status.chunk_count_indexed), - last_indexed_at = COALESCE(excluded.last_indexed_at, document_index_status.last_indexed_at), - last_verified_at = COALESCE(excluded.last_verified_at, document_index_status.last_verified_at), - schema_version = COALESCE(excluded.schema_version, document_index_status.schema_version), - status = excluded.status, - details_json = COALESCE(excluded.details_json, document_index_status.details_json) - """, ( - workflow_id, index_name, doc_id, chunk_count_indexed, - last_indexed_at, last_verified_at, schema_version, status, details_json - )) - conn.commit() - return get_document_index_status(workflow_id, index_name) or {} + """Merge one index's state into ``documents.metadata.index_status``. - -def get_document_index_status(workflow_id: str, index_name: str) -> Optional[dict]: + The key is ``prod`` for the PROD collection names and ``dev`` otherwise; + the collection name is kept inside the entry. ``None`` arguments leave the + stored value alone, except ``status`` which always replaces it. + """ + key = _index_key_for(index_name) with get_connection() as conn: - row = conn.execute(""" - SELECT * FROM document_index_status - WHERE workflow_id = ? AND index_name = ? - """, (workflow_id, index_name)).fetchone() - return dict(row) if row else None + doc_uuid = _require_document_uuid(conn, workflow_id) + row = conn.execute( + "SELECT metadata->'index_status'->%s::text AS entry FROM documents WHERE id = %s::uuid FOR UPDATE", + (key, doc_uuid), + ).fetchone() + existing = dict(row["entry"] or {}) + incoming = { + "index_name": index_name, + "doc_id": doc_id, + "chunk_count_indexed": chunk_count_indexed, + "last_indexed_at": last_indexed_at, + "last_verified_at": last_verified_at, + "schema_version": schema_version, + "details": details or None, + } + merged = {**existing, **{k: v for k, v in incoming.items() if v is not None}, "status": status} + validated = IndexState.model_validate(merged).model_dump() + conn.execute( + "UPDATE documents SET metadata = jsonb_set(" + "metadata || jsonb_build_object('index_status', COALESCE(metadata->'index_status', '{}'::jsonb)), " + "ARRAY['index_status', %s::text], %s::jsonb) " + "WHERE id = %s::uuid", + (key, Jsonb(validated), doc_uuid), + ) + root = _root_workflow_id(conn, doc_uuid) or workflow_id + return _index_entry_to_legacy(root, key, validated) def list_document_index_status(workflow_id: str) -> list[dict]: - with get_connection() as conn: - rows = conn.execute(""" - SELECT * FROM document_index_status - WHERE workflow_id = ? - ORDER BY last_verified_at DESC, last_indexed_at DESC - """, (workflow_id,)).fetchall() - return [dict(row) for row in rows] - - -def find_document_by_doc_identifier(identifier: str) -> Optional[dict]: - """Resolve a vector-store/chat doc identifier to a SQLite document row.""" - if not identifier: - return None - - with get_connection() as conn: - row = conn.execute( - "SELECT * FROM documents WHERE workflow_id = ?", - (identifier,), - ).fetchone() - if row: - return dict(row) + doc = get_document(workflow_id) + if not doc: + return [] + entries = [ + _index_entry_to_legacy(doc["workflow_id"], key, entry) + for key, entry in doc["index_status"].items() + ] + entries.sort( + key=lambda e: (e["last_verified_at"] or "", e["last_indexed_at"] or ""), reverse=True + ) + return entries - row = conn.execute( - "SELECT * FROM documents WHERE document_id = ?", - (identifier,), - ).fetchone() - if row: - return dict(row) - row = conn.execute( - "SELECT * FROM documents WHERE filename = ?", - (identifier,), - ).fetchone() - if row: - return dict(row) +def get_document_index_status(workflow_id: str, index_name: str) -> Optional[dict]: + for entry in list_document_index_status(workflow_id): + if entry["index_name"] == index_name: + return entry + return None - row = conn.execute( - """ - SELECT d.* - FROM document_index_status s - JOIN documents d ON d.workflow_id = s.workflow_id - WHERE s.doc_id = ? - LIMIT 1 + +def list_index_summaries(include_disabled: bool = False) -> list[dict]: + clauses, params = _document_filters(include_disabled, None) + where = f"AND {' AND '.join(clauses)}" if clauses else "" + with get_connection() as conn: + rows = conn.execute( + f""" + SELECT + COALESCE(e.value->>'index_name', e.key) AS index_name, + COUNT(*) AS documents, + SUM(COALESCE((e.value->>'chunk_count_indexed')::int, 0)) AS indexed_chunks, + SUM(CASE WHEN COALESCE((d.metadata->>'reindex_required')::boolean, FALSE) + THEN 1 ELSE 0 END) AS stale_documents, + MAX(e.value->>'last_indexed_at') AS last_indexed_at, + array_agg(DISTINCT COALESCE(e.value->>'status', 'unknown')) AS statuses + FROM documents d, jsonb_each(COALESCE(d.metadata->'index_status', '{{}}'::jsonb)) e + WHERE TRUE {where} + GROUP BY COALESCE(e.value->>'index_name', e.key) + ORDER BY index_name """, - (identifier,), - ).fetchone() - if row: - return dict(row) + params, + ).fetchall() + result = [] + for row in rows: + item = _clean_row(row) + item["indexed_chunks"] = int(item["indexed_chunks"] or 0) + item["stale_documents"] = int(item["stale_documents"] or 0) + item["statuses"] = sorted(s for s in row["statuses"] if s) + result.append(item) + return result - legacy_doc_id_hash = hashlib.md5(identifier.encode()).hexdigest() - if legacy_doc_id_hash != identifier: - row = conn.execute( - """ - SELECT d.* - FROM document_index_status s - JOIN documents d ON d.workflow_id = s.workflow_id - WHERE s.doc_id = ? - LIMIT 1 - """, - (legacy_doc_id_hash,), - ).fetchone() - if row: - return dict(row) - rows = conn.execute("SELECT * FROM documents").fetchall() - for candidate in rows: - document_id = candidate["document_id"] - if hashlib.md5(document_id.encode()).hexdigest() == identifier: - return dict(candidate) +def find_document_by_doc_identifier(identifier: str) -> Optional[dict]: + """Resolve a vector-store/chat doc identifier to a document. - return None + Matches the document UUID, a Temporal workflow id, the fingerprint, the + filename, an index ``doc_id``, or the legacy ``md5(fingerprint)`` doc id. + """ + if not identifier: + return None + with get_connection() as conn: + return _fetch_document(conn, _resolve_document_uuid(conn, identifier)) def resolve_chunk_provenance( @@ -1955,13 +2404,13 @@ def resolve_chunk_provenance( excerpt = text[:320] + ("..." if len(text) > 320 else "") page_start = int(chunk.get("page_start") or 1) page_end = int(chunk.get("page_end") or page_start) - display_name = doc.get("display_name") or doc.get("filename") or workflow_id + doc_name = doc.get("name") or doc.get("filename") or workflow_id return { "workflow_id": workflow_id, "doc_id": doc.get("document_id"), "filename": workflow_id, - "doc_name": display_name, + "doc_name": doc_name, "chunk_num": int(chunk_num), "page_start": page_start, "page_end": page_end, @@ -1975,6 +2424,9 @@ def resolve_chunk_provenance( # Audit Log Functions # ============================================================================= +_AUDIT_ENTITY_TYPES = {"document", "page", "chunk", "setting", "catalog"} + + def log_audit( workflow_id: str, document_id: str, @@ -1989,50 +2441,70 @@ def log_audit( actor_username: Optional[str] = None, actor_email: Optional[str] = None, actor_roles: Optional[str] = None, -) -> int: +) -> str: """ Log an audit entry. Args: - workflow_id: The Temporal workflow ID - document_id: The document hash ID + workflow_id: The Temporal workflow ID (or document identifier). Ids that + start with ``__`` mark system-level entries and are not stored. + document_id: Unused; kept for call-site compatibility. The document is + resolved from ``workflow_id``. action_type: Type of action (stage_change, page_edit, chunk_edit, approval, reset) - entity_type: Type of entity (page, chunk, document) - entity_id: ID of the entity (page_number or chunk_number) + entity_type: Type of entity (document, page, chunk, setting, catalog) + entity_id: Page number or chunk number the action applied to field_name: Name of the field that changed old_value: Previous value (as string or JSON) new_value: New value (as string or JSON) - metadata: Additional context as dict (will be JSON serialized) + metadata: Additional context as dict actor_*: Identity of the human/system that performed the action Returns: - The ID of the created audit log entry + The UUID of the created audit log entry """ - now = datetime.utcnow().isoformat() - metadata_json = json.dumps(metadata) if metadata else None - - with _db_lock: - with get_connection() as conn: - cursor = conn.execute(""" - INSERT INTO audit_logs ( - workflow_id, document_id, action_type, - entity_type, entity_id, field_name, - old_value, new_value, metadata, timestamp, - actor_user_id, actor_username, actor_email, actor_roles - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, ( - workflow_id, document_id, action_type, - entity_type, entity_id, field_name, - old_value, new_value, metadata_json, now, - actor_user_id, actor_username, actor_email, actor_roles, - )) - conn.commit() - return cursor.lastrowid + entity = entity_type if entity_type in _AUDIT_ENTITY_TYPES else None + system_entry = (workflow_id or "").startswith("__") + roles = _csv_roles(actor_roles) + with get_connection() as conn: + doc_uuid = None if system_entry else _resolve_document_uuid(conn, workflow_id) + instance_uuid = None + stored_workflow_id = None if system_entry else workflow_id + if doc_uuid: + doc_row = conn.execute( + "SELECT c.instance_id FROM documents d JOIN catalogs c ON c.id = d.catalog_id " + "WHERE d.id = %s::uuid", + (doc_uuid,), + ).fetchone() + instance_uuid = str(doc_row["instance_id"]) + if _is_uuid(workflow_id): + stored_workflow_id = _root_workflow_id(conn, doc_uuid) or workflow_id + if stored_workflow_id is None and entity not in ("setting", "catalog"): + stored_workflow_id = workflow_id + actor_uuid = ( + upsert_user(actor_user_id, actor_username, actor_email, roles, conn=conn) + if actor_user_id else None + ) + row = conn.execute( + """ + INSERT INTO audit_logs (document_id, workflow_id, instance_id, action_type, entity_type, + entity_number, field_name, old_value, new_value, metadata, + actor_user_id, actor) + VALUES (%s::uuid, %s, %s::uuid, %s, %s, %s, %s, %s, %s, %s, %s::uuid, %s) + RETURNING id + """, + ( + doc_uuid, stored_workflow_id, instance_uuid, action_type, entity, entity_id, + field_name, old_value, new_value, Jsonb(metadata or {}), actor_uuid, + Jsonb({"username": actor_username or None, "email": actor_email or None, "roles": roles}) + if (actor_user_id or actor_username or actor_email or roles) else Jsonb({}), + ), + ).fetchone() + return str(row["id"]) # Keycloak attaches these to every account regardless of what the user can # actually do here. Rendering them made the actor column a wall of noise -# ("…manage-account,manage-account-links,offline_access,uma_authorization…") +# ("...manage-account,manage-account-links,offline_access,uma_authorization...") # that buried the one role a reviewer cares about. _NOISE_ROLES = { "manage-account", @@ -2091,6 +2563,70 @@ def _normalize_audit_row(row) -> dict: return entry +_AUDIT_SELECT = """ + SELECT a.id, a.workflow_id, a.action_type, a.entity_type, a.entity_number, a.field_name, + a.old_value, a.new_value, a.metadata, a.created_at, a.actor, ag.name AS audit_instance, + au.keycloak_user_id AS actor_keycloak_id, + d.source->>'fingerprint' AS doc_fingerprint, + d.source->>'original_filename' AS filename, + d.name AS document_name, + dg.name AS doc_instance, + uu.username AS uploaded_by_username, uu.email AS uploaded_by_email + FROM audit_logs a + LEFT JOIN documents d ON d.id = a.document_id + LEFT JOIN instances ag ON ag.id = a.instance_id + LEFT JOIN catalogs dc ON dc.id = d.catalog_id + LEFT JOIN instances dg ON dg.id = dc.instance_id + LEFT JOIN users au ON au.id = a.actor_user_id + LEFT JOIN users uu ON uu.id = NULLIF(d.metadata->>'uploaded_by_user_id', '')::uuid +""" + + +def _audit_row_to_legacy(row: dict) -> dict: + actor = row["actor"] or {} + metadata = row["metadata"] or {} + entity_type = row["entity_type"] + if entity_type == "setting": + workflow_id, document_id = "__system__", "__settings__" + elif entity_type == "catalog" and not row["workflow_id"]: + workflow_id, document_id = "__catalog__", "__catalog__" + else: + workflow_id = row["workflow_id"] + document_id = row["doc_fingerprint"] or workflow_id + roles = actor.get("roles") or [] + entry = { + "id": row["id"], + "workflow_id": workflow_id, + "document_id": document_id, + "action_type": row["action_type"], + "entity_type": entity_type, + "entity_id": row["entity_number"], + "field_name": row["field_name"], + "old_value": row["old_value"], + "new_value": row["new_value"], + "metadata": json.dumps(metadata) if metadata else None, + "timestamp": row["created_at"], + "actor_user_id": row["actor_keycloak_id"], + "actor_username": actor.get("username"), + "actor_email": actor.get("email"), + "actor_roles": ",".join(roles) if roles else None, + "filename": row["filename"], + "name": row["document_name"], + "instance": row["doc_instance"] or row["audit_instance"], + "uploaded_by_username": row["uploaded_by_username"], + "uploaded_by_email": row["uploaded_by_email"], + } + return _normalize_audit_row({k: _clean(v) for k, v in entry.items()}) + + +def _audit_scope(workflow_id: str, conn) -> tuple[str, list]: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid: + root = _root_workflow_id(conn, doc_uuid) or workflow_id + return "(a.document_id = %s::uuid OR a.workflow_id = %s)", [doc_uuid, root] + return "a.workflow_id = %s", [workflow_id] + + def get_audit_logs( workflow_id: str, action_type: Optional[str] = None, @@ -2101,7 +2637,7 @@ def get_audit_logs( Get audit logs for a document. Args: - workflow_id: The Temporal workflow ID + workflow_id: The Temporal workflow ID (or document UUID) action_type: Optional filter by action type limit: Maximum number of entries to return offset: Offset for pagination @@ -2110,39 +2646,40 @@ def get_audit_logs( List of audit log entries as dicts """ with get_connection() as conn: + where, params = _audit_scope(workflow_id, conn) if action_type: - rows = conn.execute(""" - SELECT * FROM audit_logs - WHERE workflow_id = ? AND action_type = ? - ORDER BY timestamp DESC - LIMIT ? OFFSET ? - """, (workflow_id, action_type, limit, offset)).fetchall() - else: - rows = conn.execute(""" - SELECT * FROM audit_logs - WHERE workflow_id = ? - ORDER BY timestamp DESC - LIMIT ? OFFSET ? - """, (workflow_id, limit, offset)).fetchall() - - return [_normalize_audit_row(row) for row in rows] + where += " AND a.action_type = %s" + params.append(action_type) + rows = conn.execute( + f"{_AUDIT_SELECT} WHERE {where} ORDER BY a.created_at DESC, a.id LIMIT %s OFFSET %s", + (*params, limit, offset), + ).fetchall() + return [_audit_row_to_legacy(r) for r in rows] def get_audit_log_count(workflow_id: str, action_type: Optional[str] = None) -> int: """Get count of audit logs for a document.""" with get_connection() as conn: + where, params = _audit_scope(workflow_id, conn) if action_type: - row = conn.execute( - "SELECT COUNT(*) as cnt FROM audit_logs WHERE workflow_id = ? AND action_type = ?", - (workflow_id, action_type) - ).fetchone() - else: - row = conn.execute( - "SELECT COUNT(*) as cnt FROM audit_logs WHERE workflow_id = ?", - (workflow_id,) - ).fetchone() - - return row["cnt"] if row else 0 + where += " AND a.action_type = %s" + params.append(action_type) + row = conn.execute(f"SELECT COUNT(*) AS cnt FROM audit_logs a WHERE {where}", params).fetchone() + return int(row["cnt"]) if row else 0 + + +def _global_audit_filters( + action_type: Optional[str], instances: Optional[list[str]] +) -> tuple[list[str], list]: + where_parts: list[str] = ["a.action_type <> %s"] + params: list = [CATALOG_VERSION_ACTION] + if action_type: + where_parts.append("a.action_type = %s") + params.append(action_type) + if instances is not None: + where_parts.append("lower(COALESCE(ag.name, dg.name, %s)) = ANY(%s)") + params.extend([_default_instance(), _normalize_instances(instances)]) + return where_parts, params def get_all_audit_logs( @@ -2163,38 +2700,16 @@ def get_all_audit_logs( Returns: List of audit log entries as dicts, with document filename included """ - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + if instances is not None and not _normalize_instances(instances): + return [] + where_parts, params = _global_audit_filters(action_type, instances) with get_connection() as conn: - where_parts: list[str] = [] - params: list = [] - if action_type: - where_parts.append("a.action_type = ?") - params.append(action_type) - if instances is not None: - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) - if not normalized: - return [] - placeholders = ",".join("?" for _ in normalized) - where_parts.append( - f"lower(COALESCE(NULLIF(trim(d.instance), ''), ?)) IN ({placeholders})" - ) - params.append(default_instance) - params.extend(normalized) - where_sql = f"WHERE {' AND '.join(where_parts)}" if where_parts else "" rows = conn.execute( - f""" - SELECT a.*, d.filename, d.display_name, d.instance, - d.uploaded_by_username, d.uploaded_by_email - FROM audit_logs a - LEFT JOIN documents d ON a.workflow_id = d.workflow_id - {where_sql} - ORDER BY a.timestamp DESC - LIMIT ? OFFSET ? - """, + f"{_AUDIT_SELECT} WHERE {' AND '.join(where_parts)} " + "ORDER BY a.created_at DESC, a.id LIMIT %s OFFSET %s", (*params, limit, offset), ).fetchall() - - return [_normalize_audit_row(row) for row in rows] + return [_audit_row_to_legacy(r) for r in rows] def get_all_audit_log_count( @@ -2202,85 +2717,157 @@ def get_all_audit_log_count( instances: Optional[list[str]] = None, ) -> int: """Get total count of all audit logs (optionally tenant-scoped).""" - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + if instances is not None and not _normalize_instances(instances): + return 0 + where_parts, params = _global_audit_filters(action_type, instances) with get_connection() as conn: - where_parts: list[str] = [] - params: list = [] - if action_type: - where_parts.append("a.action_type = ?") - params.append(action_type) - if instances is not None: - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) - if not normalized: - return 0 - placeholders = ",".join("?" for _ in normalized) - where_parts.append( - f"lower(COALESCE(NULLIF(trim(d.instance), ''), ?)) IN ({placeholders})" - ) - params.append(default_instance) - params.extend(normalized) - where_sql = f"WHERE {' AND '.join(where_parts)}" if where_parts else "" - # Join documents when instance-scoped so tenant filter applies. - if instances is not None: - row = conn.execute( - f""" - SELECT COUNT(*) as cnt - FROM audit_logs a - LEFT JOIN documents d ON a.workflow_id = d.workflow_id - {where_sql} - """, - params, - ).fetchone() - elif action_type: - row = conn.execute( - "SELECT COUNT(*) as cnt FROM audit_logs a WHERE a.action_type = ?", - (action_type,), - ).fetchone() - else: - row = conn.execute("SELECT COUNT(*) as cnt FROM audit_logs").fetchone() - - return row["cnt"] if row else 0 + row = conn.execute( + "SELECT COUNT(*) AS cnt FROM audit_logs a " + "LEFT JOIN documents d ON d.id = a.document_id " + "LEFT JOIN instances ag ON ag.id = a.instance_id " + "LEFT JOIN catalogs dc ON dc.id = d.catalog_id " + "LEFT JOIN instances dg ON dg.id = dc.instance_id " + f"WHERE {' AND '.join(where_parts)}", + params, + ).fetchone() + return int(row["cnt"]) if row else 0 # ============================================================================= -# Page Functions (for persistence after workflow completion) +# Page Functions # ============================================================================= +_PAGE_SELECT = """ + SELECT p.id, p.document_id, p.page_number, p.original_markdown, p.edited_markdown, + p.translations, p.metadata, p.created_at, p.updated_at + FROM pages p +""" + + +def _primary_translation(translations: dict) -> tuple[Optional[str], dict]: + if not translations: + return None, {} + lang = max( + translations, + key=lambda k: (translations[k].get("translated_at") or "", k == "en"), + ) + return lang, translations[lang] + + +def _page_from_row(row: Optional[dict], workflow_id: str) -> Optional[dict]: + if row is None: + return None + meta = row["metadata"] or {} + translations = row["translations"] or {} + lang, tr = _primary_translation(translations) + page = { + "id": row["id"], + "workflow_id": workflow_id, + "page_number": row["page_number"], + "original_markdown": row["original_markdown"], + "edited_markdown": row["edited_markdown"], + "is_reviewed": bool(meta.get("is_reviewed")), + "reviewer_notes": meta.get("reviewer_notes"), + "detected_language": meta.get("detected_language"), + "translated_markdown": tr.get("translated_markdown"), + "edited_translation": tr.get("edited_translation"), + "translation_reviewed": bool(tr.get("is_reviewed")), + "translation_notes": tr.get("reviewer_notes"), + "translation_provider": tr.get("provider"), + "translation_model": tr.get("model"), + "translation_target_language": lang, + "translated_at": tr.get("translated_at"), + "translations": translations, + "metadata": meta, + } + return {k: _clean(v) for k, v in page.items()} + + +_TRANSLATION_FLAT_KEYS = ( + "translated_markdown", "edited_translation", "translation_notes", + "translation_provider", "translation_model", "translated_at", +) + + +def _translation_from_flat(page: dict) -> Optional[tuple[str, dict]]: + if not any(page.get(k) not in (None, "") for k in _TRANSLATION_FLAT_KEYS) and not page.get( + "translation_reviewed" + ): + return None + lang = page.get("translation_target_language") or "en" + entry = PageTranslation( + translated_markdown=page.get("translated_markdown"), + edited_translation=page.get("edited_translation"), + is_reviewed=bool(page.get("translation_reviewed")), + reviewer_notes=page.get("translation_notes"), + provider=page.get("translation_provider"), + model=page.get("translation_model"), + translated_at=page.get("translated_at"), + ).model_dump() + return lang, entry + + def save_pages(workflow_id: str, pages: list[dict]): """ Save all pages for a document (bulk upsert). - Called when workflow completes to persist data. + + Each call replaces the whole row, so the caller passes every field. Page + ``translations`` are kept as stored unless the page carries ``translations`` + (replaces them) or flat ``translated_*`` fields (merged in under + ``translation_target_language``); a page whose OCR text changed loses its + translations because they no longer describe it. """ - with _db_lock: - with get_connection() as conn: - for page in pages: - conn.execute(""" - INSERT OR REPLACE INTO pages ( - workflow_id, page_number, original_markdown, edited_markdown, - is_reviewed, reviewer_notes, detected_language, - translated_markdown, edited_translation, - translation_reviewed, translation_notes, - translation_provider, translation_model, - translation_target_language, translated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, ( - workflow_id, - page.get("page_number"), - page.get("original_markdown"), - page.get("edited_markdown"), - 1 if page.get("is_reviewed") else 0, - page.get("reviewer_notes"), - page.get("detected_language"), - page.get("translated_markdown"), - page.get("edited_translation"), - 1 if page.get("translation_reviewed") else 0, - page.get("translation_notes"), - page.get("translation_provider"), - page.get("translation_model"), - page.get("translation_target_language"), - page.get("translated_at"), - )) - conn.commit() + with get_connection() as conn: + doc_uuid = _require_document_uuid(conn, workflow_id) + existing = { + r["page_number"]: r + for r in conn.execute( + "SELECT page_number, original_markdown, translations FROM pages " + "WHERE document_id = %s::uuid", + (doc_uuid,), + ).fetchall() + } + for page in pages: + number = page.get("page_number") + current = existing.get(number) + meta = dict(page.get("metadata") or {}) + for flat_key in ("is_reviewed", "reviewer_notes", "detected_language"): + if flat_key in page: + meta[flat_key] = page[flat_key] + meta["is_reviewed"] = bool(meta.get("is_reviewed")) + page_meta = PageMetadata.model_validate( + {k: v for k, v in meta.items() if v is not None} + ).model_dump(exclude_none=True) + + translations = dict(current["translations"]) if current else {} + if current and current["original_markdown"] != page.get("original_markdown"): + translations = {} + if isinstance(page.get("translations"), dict): + translations = { + lang: PageTranslation.model_validate(entry).model_dump() + for lang, entry in page["translations"].items() + } + else: + flat = _translation_from_flat(page) + if flat: + translations[flat[0]] = flat[1] + + conn.execute( + """ + INSERT INTO pages (document_id, page_number, original_markdown, edited_markdown, + translations, metadata) + VALUES (%s::uuid, %s, %s, %s, %s, %s) + ON CONFLICT (document_id, page_number) DO UPDATE SET + original_markdown = EXCLUDED.original_markdown, + edited_markdown = EXCLUDED.edited_markdown, + translations = EXCLUDED.translations, + metadata = EXCLUDED.metadata + """, + ( + doc_uuid, number, page.get("original_markdown"), page.get("edited_markdown"), + Jsonb(translations), Jsonb(page_meta), + ), + ) def persist_document_content(workflow_id: str, pages: list[dict], chunks: list[dict]): @@ -2290,71 +2877,64 @@ def persist_document_content(workflow_id: str, pages: list[dict], chunks: list[d def get_pages(workflow_id: str) -> list[dict]: - """Get all pages for a document from SQLite.""" + """Get all pages for a document.""" with get_connection() as conn: - rows = conn.execute(""" - SELECT * FROM pages - WHERE workflow_id = ? - ORDER BY page_number - """, (workflow_id,)).fetchall() - - pages = [] - for row in rows: - page = dict(row) - # Convert SQLite integers back to booleans - page["is_reviewed"] = bool(page.get("is_reviewed")) - page["translation_reviewed"] = bool(page.get("translation_reviewed")) - pages.append(page) - return pages + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return [] + root = _root_workflow_id(conn, doc_uuid) or workflow_id + rows = conn.execute( + _PAGE_SELECT + " WHERE p.document_id = %s::uuid ORDER BY p.page_number", (doc_uuid,) + ).fetchall() + return [_page_from_row(r, root) for r in rows] def count_translated_pages(workflow_id: str) -> int: """Count pages with translation text without loading full page bodies.""" with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return 0 row = conn.execute( """ - SELECT COUNT(*) AS n - FROM pages - WHERE workflow_id = ? - AND ( - (translated_markdown IS NOT NULL AND TRIM(translated_markdown) != '') - OR (edited_translation IS NOT NULL AND TRIM(edited_translation) != '') + SELECT COUNT(*) AS n FROM pages p + WHERE p.document_id = %s::uuid + AND EXISTS ( + SELECT 1 FROM jsonb_each(p.translations) t + WHERE nullif(btrim(t.value->>'translated_markdown'), '') IS NOT NULL + OR nullif(btrim(t.value->>'edited_translation'), '') IS NOT NULL ) """, - (workflow_id,), + (doc_uuid,), ).fetchone() - return int(row["n"] if row else 0) + return int(row["n"] if row else 0) def get_saved_page_numbers(workflow_id: str) -> list[int]: """Get saved page numbers for a document without loading full page content.""" with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return [] rows = conn.execute( - """ - SELECT page_number - FROM pages - WHERE workflow_id = ? - ORDER BY page_number - """, - (workflow_id,), + "SELECT page_number FROM pages WHERE document_id = %s::uuid ORDER BY page_number", + (doc_uuid,), ).fetchall() - return [int(row["page_number"]) for row in rows] + return [int(row["page_number"]) for row in rows] def get_page(workflow_id: str, page_num: int) -> Optional[dict]: - """Get a specific page from SQLite.""" + """Get a specific page.""" with get_connection() as conn: - row = conn.execute(""" - SELECT * FROM pages - WHERE workflow_id = ? AND page_number = ? - """, (workflow_id, page_num)).fetchone() - - if row: - page = dict(row) - page["is_reviewed"] = bool(page.get("is_reviewed")) - page["translation_reviewed"] = bool(page.get("translation_reviewed")) - return page - return None + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + root = _root_workflow_id(conn, doc_uuid) or workflow_id + row = conn.execute( + _PAGE_SELECT + " WHERE p.document_id = %s::uuid AND p.page_number = %s", + (doc_uuid, page_num), + ).fetchone() + return _page_from_row(row, root) def update_page( @@ -2367,152 +2947,213 @@ def update_page( translation_reviewed: Optional[bool] = None, translation_notes: Optional[str] = None, ) -> Optional[dict]: - """Update a page in SQLite. Returns updated page or None if not found.""" - with _db_lock: - with get_connection() as conn: - # Check if page exists - existing = conn.execute( - "SELECT id FROM pages WHERE workflow_id = ? AND page_number = ?", - (workflow_id, page_num) - ).fetchone() - - if not existing: - return None - - # Build dynamic update - updates = [] - values = [] - - if edited_markdown is not None: - updates.append("edited_markdown = ?") - values.append(edited_markdown) - - if is_reviewed is not None: - updates.append("is_reviewed = ?") - values.append(1 if is_reviewed else 0) - - if reviewer_notes is not None: - updates.append("reviewer_notes = ?") - values.append(reviewer_notes) - - if edited_translation is not None: - updates.append("edited_translation = ?") - values.append(edited_translation) - - if translation_reviewed is not None: - updates.append("translation_reviewed = ?") - values.append(1 if translation_reviewed else 0) - - if translation_notes is not None: - updates.append("translation_notes = ?") - values.append(translation_notes) + """Update a page. Returns updated page or None if not found.""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + existing = conn.execute( + "SELECT translations FROM pages WHERE document_id = %s::uuid AND page_number = %s FOR UPDATE", + (doc_uuid, page_num), + ).fetchone() + if not existing: + return None - if updates: - values.extend([workflow_id, page_num]) - # Note: updates list contains only hardcoded field names, not user input - conn.execute( - f"UPDATE pages SET {', '.join(updates)} WHERE workflow_id = ? AND page_number = ?", - values - ) - conn.commit() + assignments: list[str] = [] + params: list = [] + if edited_markdown is not None: + assignments.append("edited_markdown = %s") + params.append(edited_markdown) + + meta_patch: dict[str, Any] = {} + if is_reviewed is not None: + meta_patch["is_reviewed"] = bool(is_reviewed) + if reviewer_notes is not None: + meta_patch["reviewer_notes"] = reviewer_notes + if meta_patch: + assignments.append("metadata = metadata || %s::jsonb") + params.append(Jsonb(meta_patch)) + + tr_patch: dict[str, Any] = {} + if edited_translation is not None: + tr_patch["edited_translation"] = edited_translation + if translation_reviewed is not None: + tr_patch["is_reviewed"] = bool(translation_reviewed) + if translation_notes is not None: + tr_patch["reviewer_notes"] = translation_notes + if tr_patch: + lang, _ = _primary_translation(existing["translations"] or {}) + lang = lang or "en" + assignments.append( + "translations = jsonb_set(translations, ARRAY[%s::text], " + "COALESCE(translations->%s::text, '{}'::jsonb) || %s::jsonb, TRUE)" + ) + params.extend([lang, lang, Jsonb(tr_patch)]) + if assignments: + conn.execute( + f"UPDATE pages SET {', '.join(assignments)} " + "WHERE document_id = %s::uuid AND page_number = %s", + (*params, doc_uuid, page_num), + ) return get_page(workflow_id, page_num) def reset_page(workflow_id: str, page_num: int) -> Optional[dict]: - """Reset a page to original markdown in SQLite.""" - with _db_lock: - with get_connection() as conn: - conn.execute(""" + """Reset a page to its original OCR markdown.""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is not None: + conn.execute( + """ UPDATE pages SET edited_markdown = NULL, - is_reviewed = 0, - reviewer_notes = NULL - WHERE workflow_id = ? AND page_number = ? - """, (workflow_id, page_num)) - conn.commit() - + metadata = (metadata - 'reviewer_notes') || '{"is_reviewed": false}'::jsonb + WHERE document_id = %s::uuid AND page_number = %s + """, + (doc_uuid, page_num), + ) return get_page(workflow_id, page_num) # ============================================================================= -# Chunk Functions (for persistence after workflow completion) +# Chunk Functions # ============================================================================= +_CHUNK_SELECT = """ + SELECT c.id, c.document_id, c.chunk_number, c.original_text, c.edited_text, c.token_count, + c.page_start, c.page_end, c.source_spans, c.section_title, c.content_type, + c.is_reference, c.is_reviewed, c.is_excluded, c.reviewer_notes, + c.created_at, c.updated_at, d.metadata->'chunking' AS chunking + FROM chunks c + JOIN documents d ON d.id = c.document_id +""" + + +def _chunk_from_row(row: Optional[dict], workflow_id: str) -> Optional[dict]: + if row is None: + return None + spans = row["source_spans"] or [] + page_numbers = sorted({int(s["page_number"]) for s in spans if "page_number" in s}) or list( + range(int(row["page_start"]), int(row["page_end"]) + 1) + ) + run = row.get("chunking") or {} + chunk = { + "id": row["id"], + "workflow_id": workflow_id, + "chunk_number": row["chunk_number"], + "original_text": row["original_text"], + "edited_text": row["edited_text"], + "token_count": row["token_count"], + "page_start": row["page_start"], + "page_end": row["page_end"], + "source_page_numbers_json": json.dumps(page_numbers), + "source_spans_json": json.dumps(spans), + "section_title": row["section_title"], + "content_type": row["content_type"], + "is_reference": bool(row["is_reference"]), + "chunking_provider": run.get("provider"), + "chunking_model": run.get("model"), + "chunking_config_json": json.dumps(run["config"]) if run.get("config") else None, + "chunking_run_id": f"run-{run['run_number']}" if run.get("run_number") else None, + "chunk_version": run.get("run_number") or 1, + "is_reviewed": bool(row["is_reviewed"]), + "is_excluded": bool(row["is_excluded"]), + "reviewer_notes": row["reviewer_notes"], + } + return {k: _clean(v) for k, v in chunk.items()} + + +def _load_json_value(raw: Any, default: Any) -> Any: + if raw is None or raw == "": + return default + if isinstance(raw, str): + return json.loads(raw) + return raw + + def save_chunks(workflow_id: str, chunks: list[dict]): """ - Save all chunks for a document (bulk upsert). - Called when workflow completes to persist data. + Save all chunks for a document. + + Re-chunking runs in one transaction: the document's chunks are deleted, the + new ones inserted, and ``metadata.chunking`` (with ``run_number`` + incremented) updated together. """ - with _db_lock: - with get_connection() as conn: - existing_version = conn.execute( - "SELECT COALESCE(MAX(chunk_version), 0) AS max_version FROM chunks WHERE workflow_id = ?", - (workflow_id,), - ).fetchone() - next_version = int(existing_version["max_version"] or 0) + 1 - conn.execute("DELETE FROM chunks WHERE workflow_id = ?", (workflow_id,)) - for chunk in chunks: - chunk_version = int(chunk.get("chunk_version") or next_version) - conn.execute(""" - INSERT INTO chunks ( - workflow_id, chunk_number, original_text, edited_text, - token_count, page_start, page_end, - source_page_numbers_json, source_spans_json, - section_title, content_type, is_reference, - chunking_provider, chunking_model, chunking_config_json, - chunking_run_id, chunk_version, - is_reviewed, is_excluded, reviewer_notes - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, ( - workflow_id, + with get_connection() as conn: + doc_uuid = _require_document_uuid(conn, workflow_id) + previous = conn.execute( + "SELECT metadata->'chunking'->>'run_number' AS run_number FROM documents " + "WHERE id = %s::uuid FOR UPDATE", + (doc_uuid,), + ).fetchone() + run_number = int(previous["run_number"] or 0) + 1 + conn.execute("DELETE FROM chunks WHERE document_id = %s::uuid", (doc_uuid,)) + for chunk in chunks: + spans = _load_json_value( + chunk.get("source_spans") if chunk.get("source_spans") is not None + else chunk.get("source_spans_json"), + [], + ) + page_start = int(chunk.get("page_start") or 1) + page_end = max(int(chunk.get("page_end") or page_start), page_start) + conn.execute( + """ + INSERT INTO chunks (document_id, chunk_number, original_text, edited_text, + token_count, page_start, page_end, source_spans, + section_title, content_type, is_reference, is_reviewed, + is_excluded, reviewer_notes) + VALUES (%s::uuid, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + """, + ( + doc_uuid, chunk.get("chunk_number"), - chunk.get("original_text"), + chunk.get("original_text") or "", chunk.get("edited_text"), - chunk.get("token_count", 0), - chunk.get("page_start", 1), - chunk.get("page_end", 1), - chunk.get("source_page_numbers_json"), - chunk.get("source_spans_json"), + chunk.get("token_count") or 0, + page_start, + page_end, + Jsonb(spans), chunk.get("section_title"), chunk.get("content_type"), - 1 if chunk.get("is_reference") else 0, - chunk.get("chunking_provider"), - chunk.get("chunking_model"), - chunk.get("chunking_config_json"), - chunk.get("chunking_run_id"), - chunk_version, - 1 if chunk.get("is_reviewed") else 0, - 1 if chunk.get("is_excluded") else 0, - chunk.get("reviewer_notes") - )) - conn.commit() + bool(chunk.get("is_reference")), + bool(chunk.get("is_reviewed")), + bool(chunk.get("is_excluded")), + chunk.get("reviewer_notes"), + ), + ) + meta_patch: dict[str, Any] = {} + if chunks: + first = chunks[0] + meta_patch["chunking"] = ChunkingRun( + run_number=run_number, + provider=first.get("chunking_provider"), + model=first.get("chunking_model"), + config=_load_json_value(first.get("chunking_config_json"), {}), + created_at=_iso_now(), + ).model_dump() + if meta_patch: + conn.execute( + "UPDATE documents SET metadata = metadata || %s::jsonb WHERE id = %s::uuid", + (Jsonb(meta_patch), doc_uuid), + ) + else: + conn.execute("UPDATE documents SET updated_at = now() WHERE id = %s::uuid", (doc_uuid,)) def get_chunks(workflow_id: str, include_excluded: bool = False) -> list[dict]: - """Get all chunks for a document from SQLite.""" - with get_connection() as conn: - if include_excluded: - rows = conn.execute(""" - SELECT * FROM chunks - WHERE workflow_id = ? - ORDER BY chunk_number - """, (workflow_id,)).fetchall() - else: - rows = conn.execute(""" - SELECT * FROM chunks - WHERE workflow_id = ? AND is_excluded = 0 - ORDER BY chunk_number - """, (workflow_id,)).fetchall() - - chunks = [] - for row in rows: - chunk = dict(row) - chunk["is_reviewed"] = bool(chunk.get("is_reviewed")) - chunk["is_excluded"] = bool(chunk.get("is_excluded")) - chunk["is_reference"] = bool(chunk.get("is_reference")) - chunks.append(chunk) - return chunks + """Get all chunks for a document.""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return [] + root = _root_workflow_id(conn, doc_uuid) or workflow_id + excluded = "" if include_excluded else "AND NOT c.is_excluded" + rows = conn.execute( + f"{_CHUNK_SELECT} WHERE c.document_id = %s::uuid {excluded} ORDER BY c.chunk_number", + (doc_uuid,), + ).fetchall() + return [_chunk_from_row(r, root) for r in rows] def search_chunks( @@ -2524,92 +3165,85 @@ def search_chunks( stage: Optional[str] = None, instances: Optional[list[str]] = None, ) -> tuple[list[dict], int]: - """Search chunks across documents (SQLite-first) for maintainer workflows. + """Search chunks across documents for maintainer workflows. instances: If provided, only return chunks whose owning document's instance is in this list (tenant scoping). ``None`` means no instance restriction. """ - where_clauses = ["1=1"] + where_clauses = ["TRUE"] params: list = [] if not include_excluded: - where_clauses.append("c.is_excluded = 0") + where_clauses.append("NOT c.is_excluded") if stage: - where_clauses.append("d.stage = ?") + where_clauses.append("cw.current_stage = %s") params.append(stage) if instances is not None: - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) + normalized = _normalize_instances(instances) if not normalized: - # Restricted caller with no instances: match nothing. - where_clauses.append("1 = 0") + where_clauses.append("FALSE") else: - placeholders = ",".join("?" for _ in normalized) - where_clauses.append( - f"LOWER(COALESCE(d.instance, ?)) IN ({placeholders})" - ) - params.append(default_instance) - params.extend(normalized) + where_clauses.append("lower(g.name) = ANY(%s)") + params.append(normalized) if query and query.strip(): - where_clauses.append("LOWER(COALESCE(c.edited_text, c.original_text, '')) LIKE ?") + where_clauses.append("lower(coalesce(c.edited_text, c.original_text)) LIKE %s") params.append(f"%{query.strip().lower()}%") - where_sql = " AND ".join(where_clauses) base_from_sql = f""" FROM chunks c - JOIN documents d ON d.workflow_id = c.workflow_id - WHERE {where_sql} + JOIN documents d ON d.id = c.document_id + JOIN catalogs cat ON cat.id = d.catalog_id + JOIN instances g ON g.id = cat.instance_id + LEFT JOIN document_current_workflow cw ON cw.document_id = d.id + WHERE {' AND '.join(where_clauses)} """ - with get_connection() as conn: - total_row = conn.execute( - f"SELECT COUNT(*) AS cnt {base_from_sql}", - params, - ).fetchone() + total_row = conn.execute(f"SELECT COUNT(*) AS cnt {base_from_sql}", params).fetchone() total = int(total_row["cnt"] if total_row and total_row["cnt"] is not None else 0) - rows = conn.execute( f""" SELECT - c.*, - d.document_id, - d.filename, - d.display_name, - d.stage, - d.updated_at AS document_updated_at + c.id, c.chunk_number, c.original_text, c.edited_text, c.token_count, + c.page_start, c.page_end, c.source_spans, c.section_title, c.content_type, + c.is_reference, c.is_reviewed, c.is_excluded, c.reviewer_notes, + d.metadata->'chunking' AS chunking, + d.source->>'fingerprint' AS document_id, + d.source->>'original_filename' AS filename, + d.name AS name, + cw.current_stage AS stage, + d.updated_at AS document_updated_at, + (SELECT r.temporal_workflow_id FROM workflows r WHERE r.document_id = d.id + ORDER BY (r.workflow_type IN ('pipeline', 'ocr_only')) DESC, r.started_at ASC, r.id ASC + LIMIT 1) AS root_workflow_id {base_from_sql} - ORDER BY d.updated_at DESC, c.workflow_id ASC, c.chunk_number ASC - LIMIT ? OFFSET ? + ORDER BY d.updated_at DESC, d.id, c.chunk_number + LIMIT %s OFFSET %s """, [*params, int(limit), int(offset)], ).fetchall() chunks: list[dict] = [] for row in rows: - chunk = dict(row) - chunk["is_reviewed"] = bool(chunk.get("is_reviewed")) - chunk["is_excluded"] = bool(chunk.get("is_excluded")) - chunk["is_reference"] = bool(chunk.get("is_reference")) + row = dict(row) + chunk = _chunk_from_row(row, row["root_workflow_id"]) + for extra in ("document_id", "filename", "name", "stage", "document_updated_at"): + chunk[extra] = _clean(row[extra]) chunks.append(chunk) - return chunks, total def get_chunk(workflow_id: str, chunk_num: int) -> Optional[dict]: - """Get a specific chunk from SQLite.""" + """Get a specific chunk.""" with get_connection() as conn: - row = conn.execute(""" - SELECT * FROM chunks - WHERE workflow_id = ? AND chunk_number = ? - """, (workflow_id, chunk_num)).fetchone() - - if row: - chunk = dict(row) - chunk["is_reviewed"] = bool(chunk.get("is_reviewed")) - chunk["is_excluded"] = bool(chunk.get("is_excluded")) - chunk["is_reference"] = bool(chunk.get("is_reference")) - return chunk - return None + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + root = _root_workflow_id(conn, doc_uuid) or workflow_id + row = conn.execute( + _CHUNK_SELECT + " WHERE c.document_id = %s::uuid AND c.chunk_number = %s", + (doc_uuid, chunk_num), + ).fetchone() + return _chunk_from_row(row, root) def update_chunk( @@ -2620,64 +3254,53 @@ def update_chunk( is_excluded: Optional[bool] = None, reviewer_notes: Optional[str] = None ) -> Optional[dict]: - """Update a chunk in SQLite. Returns updated chunk or None if not found.""" - with _db_lock: - with get_connection() as conn: - # Check if chunk exists - existing = conn.execute( - "SELECT id FROM chunks WHERE workflow_id = ? AND chunk_number = ?", - (workflow_id, chunk_num) - ).fetchone() - - if not existing: - return None - - # Build dynamic update - updates = [] - values = [] - - if edited_text is not None: - updates.append("edited_text = ?") - values.append(edited_text) - - if is_reviewed is not None: - updates.append("is_reviewed = ?") - values.append(1 if is_reviewed else 0) - - if is_excluded is not None: - updates.append("is_excluded = ?") - values.append(1 if is_excluded else 0) - - if reviewer_notes is not None: - updates.append("reviewer_notes = ?") - values.append(reviewer_notes) - - if updates: - values.extend([workflow_id, chunk_num]) - # Note: updates list contains only hardcoded field names, not user input - conn.execute( - f"UPDATE chunks SET {', '.join(updates)} WHERE workflow_id = ? AND chunk_number = ?", - values - ) - conn.commit() - + """Update a chunk. Returns updated chunk or None if not found.""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + existing = conn.execute( + "SELECT id FROM chunks WHERE document_id = %s::uuid AND chunk_number = %s", + (doc_uuid, chunk_num), + ).fetchone() + if not existing: + return None + + updates = [] + values: list = [] + if edited_text is not None: + updates.append("edited_text = %s") + values.append(edited_text) + if is_reviewed is not None: + updates.append("is_reviewed = %s") + values.append(bool(is_reviewed)) + if is_excluded is not None: + updates.append("is_excluded = %s") + values.append(bool(is_excluded)) + if reviewer_notes is not None: + updates.append("reviewer_notes = %s") + values.append(reviewer_notes) + if updates: + conn.execute( + f"UPDATE chunks SET {', '.join(updates)} WHERE document_id = %s::uuid AND chunk_number = %s", + (*values, doc_uuid, chunk_num), + ) return get_chunk(workflow_id, chunk_num) def reset_chunk(workflow_id: str, chunk_num: int) -> Optional[dict]: - """Reset a chunk to original text in SQLite.""" - with _db_lock: - with get_connection() as conn: - conn.execute(""" - UPDATE chunks SET - edited_text = NULL, - is_reviewed = 0, - is_excluded = 0, - reviewer_notes = NULL - WHERE workflow_id = ? AND chunk_number = ? - """, (workflow_id, chunk_num)) - conn.commit() - + """Reset a chunk to its original text.""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is not None: + conn.execute( + """ + UPDATE chunks SET edited_text = NULL, is_reviewed = FALSE, is_excluded = FALSE, + reviewer_notes = NULL + WHERE document_id = %s::uuid AND chunk_number = %s + """, + (doc_uuid, chunk_num), + ) return get_chunk(workflow_id, chunk_num) @@ -2685,88 +3308,57 @@ def reset_chunk(workflow_id: str, chunk_num: int) -> Optional[dict]: # Settings Functions # ============================================================================= -def get_setting(key: str) -> Optional[dict]: - """Get a single setting by key.""" - with get_connection() as conn: - row = conn.execute( - "SELECT * FROM settings WHERE key = ?", - (key,) - ).fetchone() - return dict(row) if row else None - - -def get_all_settings() -> dict: - """Get all settings as a dict of key -> value.""" - with get_connection() as conn: - rows = conn.execute("SELECT * FROM settings").fetchall() - return {row["key"]: dict(row) for row in rows} +_SETTING_GROUPS: dict[str, tuple[str, type]] = { + "searchMethod": ("retrieval", RetrievalSettings), + "alpha": ("retrieval", RetrievalSettings), + "rankingMethod": ("retrieval", RetrievalSettings), + "hybridRrfK": ("retrieval", RetrievalSettings), + "indexName": ("candidates", CandidateSettings), + "efSearch": ("candidates", CandidateSettings), + "candidateCap": ("candidates", CandidateSettings), + "candidateMultiplier": ("candidates", CandidateSettings), + "limit": ("results", ResultSettings), + "maxChunksPerDoc": ("results", ResultSettings), + "excludeReference": ("results", ResultSettings), + "showHighlights": ("results", ResultSettings), + "rerankMode": ("results", ResultSettings), + "useE5Prefix": ("query", QuerySettings), + "queryExpansionProfile": ("query", QuerySettings), +} +_GROUP_MODELS: dict[str, type] = {group: model for group, model in _SETTING_GROUPS.values()} -def get_search_settings() -> dict: - """Get all search-related settings as a simple dict.""" - all_settings = get_all_settings() - return { - "searchMethod": all_settings.get("search_method", {}).get("value", "HYBRID"), - "limit": int(all_settings.get("search_limit", {}).get("value", "12")), - "alpha": float(all_settings.get("search_alpha", {}).get("value", "0.6")), - "rankingMethod": all_settings.get("search_ranking_method", {}).get("value", "rrf"), - "showHighlights": all_settings.get("search_show_highlights", {}).get("value", "true") == "true", - "efSearch": int(all_settings.get("search_ef_search", {}).get("value", "256")), - "indexName": all_settings.get("search_index_name", {}).get("value", "documents-index"), - "candidateCap": int(all_settings.get("search_candidate_cap", {}).get("value", "120")), - "candidateMultiplier": int(all_settings.get("search_candidate_multiplier", {}).get("value", "10")), - "maxChunksPerDoc": int(all_settings.get("search_max_chunks_per_doc", {}).get("value", "2")), - "useE5Prefix": all_settings.get("search_use_e5_prefix", {}).get("value", "true") == "true", - "excludeReference": all_settings.get("search_exclude_reference", {}).get("value", "true") == "true", - "queryExpansionProfile": all_settings.get("search_query_expansion_profile", {}).get("value", "gu-v1"), - "rerankMode": all_settings.get("search_rerank_mode", {}).get("value", "none"), - "hybridRrfK": int(all_settings.get("search_hybrid_rrfk", {}).get("value", "60")), - } +_SETTINGS_ROW_SQL = ( + "SELECT retrieval, candidates, results, query FROM search_settings WHERE singleton" +) -def update_setting(key: str, value: str, log_change: bool = True) -> dict: - """ - Update a setting value. +def _settings_row(conn, lock: bool = False) -> dict: + sql = _SETTINGS_ROW_SQL + (" FOR UPDATE" if lock else "") + row = conn.execute(sql).fetchone() + if row is None: + conn.execute("INSERT INTO search_settings DEFAULT VALUES ON CONFLICT (singleton) DO NOTHING") + row = conn.execute(sql).fetchone() + return row - Args: - key: Setting key - value: New value (as string) - log_change: Whether to log to audit (default True) - Returns: - Updated setting dict - """ - now = datetime.utcnow().isoformat() - old_value = None +def _flatten_settings(row: dict) -> dict: + flat: dict[str, Any] = {} + for group, model in _GROUP_MODELS.items(): + flat.update(model.model_validate(row[group] or {}).model_dump()) + return flat - with _db_lock: - with get_connection() as conn: - # Get old value for audit - row = conn.execute( - "SELECT value FROM settings WHERE key = ?", - (key,) - ).fetchone() - old_value = row["value"] if row else None - conn.execute(""" - UPDATE settings SET value = ?, updated_at = ? - WHERE key = ? - """, (value, now, key)) - conn.commit() +def get_search_settings() -> dict: + """Get all search settings as a flat dict (stored values merged over defaults).""" + with get_connection() as conn: + return _flatten_settings(_settings_row(conn)) - # Log the change to audit - if log_change and old_value != value: - log_audit( - workflow_id="__system__", - document_id="__settings__", - action_type="settings_change", - entity_type="setting", - field_name=key, - old_value=old_value, - new_value=value - ) - return get_setting(key) +def _setting_text(value: Any) -> str: + if isinstance(value, bool): + return "true" if value else "false" + return str(value) def update_search_settings(settings: dict) -> dict: @@ -2778,55 +3370,61 @@ def update_search_settings(settings: dict) -> dict: Returns: Updated search settings - """ - # Map UI keys to DB keys - key_map = { - "searchMethod": "search_method", - "limit": "search_limit", - "alpha": "search_alpha", - "rankingMethod": "search_ranking_method", - "showHighlights": "search_show_highlights", - "efSearch": "search_ef_search", - "indexName": "search_index_name", - "candidateCap": "search_candidate_cap", - "candidateMultiplier": "search_candidate_multiplier", - "maxChunksPerDoc": "search_max_chunks_per_doc", - "useE5Prefix": "search_use_e5_prefix", - "excludeReference": "search_exclude_reference", - "queryExpansionProfile": "search_query_expansion_profile", - "rerankMode": "search_rerank_mode", - "hybridRrfK": "search_hybrid_rrfk", - } - - for ui_key, db_key in key_map.items(): - if ui_key in settings: - value = settings[ui_key] - # Convert to string for storage - if isinstance(value, bool): - value = "true" if value else "false" - else: - value = str(value) - update_setting(db_key, value) - return get_search_settings() + Raises: + ValueError: a value fails the setting's validation rules. + """ + unknown = sorted(set(settings) - set(_SETTING_GROUPS)) + if unknown: + raise ValueError(f"Unknown search settings: {', '.join(unknown)}") + changes: list[tuple[str, str, str]] = [] + with get_connection() as conn: + row = _settings_row(conn, lock=True) + before = _flatten_settings(row) + for group, model in _GROUP_MODELS.items(): + patch = {k: v for k, v in settings.items() if _SETTING_GROUPS[k][0] == group} + if not patch: + continue + try: + validated = model.model_validate({**model.model_validate(row[group] or {}).model_dump(), **patch}) + except Exception as exc: + raise ValueError(str(exc)) from exc + stored = {**(row[group] or {}), **{k: getattr(validated, k) for k in patch}} + conn.execute( + f"UPDATE search_settings SET {group} = {group} || %s::jsonb WHERE singleton", + (Jsonb({k: getattr(validated, k) for k in patch}),), + ) + for key in patch: + if before[key] != stored[key]: + changes.append((f"{group}.{key}", _setting_text(before[key]), _setting_text(stored[key]))) + for field_name, old_value, new_value in changes: + conn.execute( + """ + INSERT INTO audit_logs (action_type, entity_type, field_name, old_value, new_value) + VALUES ('settings_change', 'setting', %s, %s, %s) + """, + (field_name, old_value, new_value), + ) + return _flatten_settings( + conn.execute(_SETTINGS_ROW_SQL).fetchone() + ) def get_settings_audit_logs(limit: int = 50, offset: int = 0) -> list[dict]: """Get audit logs for settings changes.""" with get_connection() as conn: - rows = conn.execute(""" - SELECT * FROM audit_logs - WHERE document_id = '__settings__' - ORDER BY timestamp DESC - LIMIT ? OFFSET ? - """, (limit, offset)).fetchall() - return [dict(row) for row in rows] + rows = conn.execute( + f"{_AUDIT_SELECT} WHERE a.entity_type = 'setting' " + "ORDER BY a.created_at DESC, a.id LIMIT %s OFFSET %s", + (limit, offset), + ).fetchall() + return [_audit_row_to_legacy(r) for r in rows] def get_settings_audit_count() -> int: """Get count of settings audit logs.""" with get_connection() as conn: row = conn.execute( - "SELECT COUNT(*) as cnt FROM audit_logs WHERE document_id = '__settings__'" + "SELECT COUNT(*) AS cnt FROM audit_logs WHERE entity_type = 'setting'" ).fetchone() - return row["cnt"] if row else 0 + return int(row["cnt"]) if row else 0 diff --git a/pipeline/document_repository.py b/pipeline/document_repository.py index 83ff1ce..8c989e3 100644 --- a/pipeline/document_repository.py +++ b/pipeline/document_repository.py @@ -28,7 +28,7 @@ def get_document_kind(self, workflow_id: str) -> Optional[str]: doc = self._db.get_document(workflow_id) if not doc: return None - return doc.get("document_kind") + return doc.get("kind") def get_document_group(self, workflow_id: str) -> Optional[str]: doc = self._db.get_document(workflow_id) diff --git a/pipeline/instances.py b/pipeline/instances.py index 74fa154..0bc0777 100644 --- a/pipeline/instances.py +++ b/pipeline/instances.py @@ -1,11 +1,13 @@ """Instance (state / tenant) code → human-readable name. -``documents.instance`` stores a short lowercase code — ``bv`` for the Bharat +An instance code is a short lowercase ``instances.name`` — ``bv`` for the Bharat Vistaar portal, otherwise the state code matching the Keycloak group path -(``/states/MH/...`` → ``mh``). That code is what the pipeline filters on, but -it is not what a reader wants to see: consumers of the vector payload and the -master catalog get ``instance_name`` alongside it so they can show "Maharashtra" -without shipping their own lookup table. +(``/states/MH/...`` → ``mh``). A document's instance is derived through its +catalog (``documents.catalog_id`` → ``catalogs.instance_id``). That code is what +the pipeline filters on, but it is not what a reader wants to see: consumers of +the vector payload and the master catalog get ``instance_name`` (their field +name, unchanged) alongside the code so they can show "Maharashtra" without +shipping their own lookup table. Codes follow the standard two-letter Indian state/UT abbreviations. Unknown codes fall back to a title-cased version of the code itself rather than raising diff --git a/pipeline/master_catalog_pg.py b/pipeline/master_catalog_pg.py deleted file mode 100644 index beeccde..0000000 --- a/pipeline/master_catalog_pg.py +++ /dev/null @@ -1,418 +0,0 @@ -""" -Master Catalog (Postgres) — prompt/tool metadata for the AI layer. - -Generic across content types (scheme, advisory, or whatever else eventually -carries its own code/name/tool routing) — not scheme-specific by name. -Every document gets synced regardless of document_kind; code falls back to -doc_id/workflow_id when scheme_code isn't set. Separate from -`scheme_catalog.py` (SQLite `scheme_catalog_entries`, which tracks PROD -Qdrant publication state for search, and is legitimately scheme-only). - - - DEV ingest complete -> sync_catalog_entry(workflow_id, status="dev") - - PROD promote complete -> sync_catalog_entry(workflow_id, status="live") - - Document disabled -> remove_catalog_entry(workflow_id) - -Each row is keyed on `code` (upsert, never duplicated). `status` is a TEXT[] -that accumulates every tier a document has synced under ({dev} then -{dev,live}) rather than being overwritten — a later DEV re-sync (e.g. a -reingest after promotion) must not silently regress an already-live entry -back out of the live snapshot. Redis snapshot keys are -`master-catalog:dev:snapshot` (rows whose status overlaps {dev, live} — dev -testers should also see promoted entries) and `master-catalog:live:snapshot` -(rows whose status contains live, for prod). Both are refreshed on every -sync so they never drift relative to each other. - -Each snapshot also carries a pre-rendered `prompt` block (vector_schemes_ -bullets/identifiers, built from tool_name == "search_schemes" entries only) -so the AI layer can splice a live scheme list straight into its system -prompt instead of hand-maintaining one per language. Legacy schemes -(tool_name == "get_scheme_info") are intentionally excluded from that block -— they stay a separate, hardcoded list on the AI-layer side. -""" - -from __future__ import annotations - -import json -import logging -import os -from datetime import datetime, timezone -from typing import Optional - -import psycopg2 -import psycopg2.extras - -from . import db -from .instances import instance_display_name -from .scheme_catalog import resolve_scheme_aliases - -logger = logging.getLogger(__name__) - -_SCHEMA_SQL = """ -CREATE TABLE IF NOT EXISTS master_catalog ( - code TEXT PRIMARY KEY, - content_type TEXT NOT NULL DEFAULT 'scheme', - name TEXT NOT NULL, - tool_name TEXT NOT NULL, - doc_id TEXT, - prompt_snippet TEXT, - aliases TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[], - status TEXT[] NOT NULL DEFAULT ARRAY['dev']::TEXT[], - workflow_id TEXT, - instance TEXT, - updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), - created_at TIMESTAMPTZ NOT NULL DEFAULT now() -); - -CREATE TABLE IF NOT EXISTS master_catalog_meta ( - id INTEGER PRIMARY KEY DEFAULT 1, - version INTEGER NOT NULL DEFAULT 0, - updated_at TIMESTAMPTZ NOT NULL DEFAULT now() -); - -INSERT INTO master_catalog_meta (id, version) -VALUES (1, 0) -ON CONFLICT (id) DO NOTHING; - --- Migrate pre-existing installs where status was a scalar TEXT column: a --- promote-to-live sync used to overwrite it outright, so a later DEV --- re-sync (e.g. reingest) would silently regress a live entry back to dev --- and drop it from the live snapshot. Now it accumulates tiers instead. -DO $$ -BEGIN - IF (SELECT data_type FROM information_schema.columns - WHERE table_name = 'master_catalog' AND column_name = 'status') = 'text' THEN - ALTER TABLE master_catalog ALTER COLUMN status DROP DEFAULT; - ALTER TABLE master_catalog ALTER COLUMN status TYPE TEXT[] USING ARRAY[status]::TEXT[]; - ALTER TABLE master_catalog ALTER COLUMN status SET DEFAULT ARRAY['dev']::TEXT[]; - END IF; -END $$; - -ALTER TABLE master_catalog ADD COLUMN IF NOT EXISTS aliases TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[]; - --- Human-readable state/tenant name beside the `instance` code, so the AI layer --- can say "Maharashtra" without carrying its own code lookup table. -ALTER TABLE master_catalog ADD COLUMN IF NOT EXISTS instance_name TEXT; -""" - -_REDIS_KEY_PREFIX = "master-catalog" - -# How many aliases per scheme reach the AI system prompt. The catalog stores -# more than this for search; the prompt only needs enough to disambiguate. -_PROMPT_ALIAS_LIMIT = 4 - - -def _pg_dsn() -> str: - return ( - f"host={os.environ.get('MASTER_CATALOG_PG_HOST', 'localhost')} " - f"port={os.environ.get('MASTER_CATALOG_PG_PORT', '5432')} " - f"dbname={os.environ.get('MASTER_CATALOG_PG_DB', 'master_catalog')} " - f"user={os.environ.get('MASTER_CATALOG_PG_USER', 'master_catalog')} " - f"password={os.environ.get('MASTER_CATALOG_PG_PASSWORD', '')} " - f"sslmode={os.environ.get('MASTER_CATALOG_PG_SSLMODE', 'disable')}" - ) - - -def get_pg_connection(): - """New connection per call — sync calls are rare (ingest/promote events only), not per-request.""" - conn = psycopg2.connect(_pg_dsn()) - conn.autocommit = True - return conn - - -def get_redis_client(): - import redis # local import: keep this optional dep out of the module import path for tests - - host = os.environ.get("AI_LAYER_REDIS_HOST", "").strip() - if not host: - return None - return redis.Redis( - host=host, - port=int(os.environ.get("AI_LAYER_REDIS_PORT", "6379")), - db=int(os.environ.get("AI_LAYER_REDIS_DB", "0")), - password=os.environ.get("AI_LAYER_REDIS_PASSWORD") or None, - decode_responses=True, - socket_connect_timeout=3, - socket_timeout=3, - ) - - -def _redis_ttl_seconds() -> int: - return int(os.environ.get("MASTER_CATALOG_REDIS_TTL_SECONDS", "172800")) - - -def ensure_schema(conn) -> None: - with conn.cursor() as cur: - cur.execute(_SCHEMA_SQL) - - -def _utc_now_iso() -> str: - return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _resolve_tool_name(tool_routing: Optional[str]) -> str: - """Which existing AI-layer tool this entry routes to — a label, not code.""" - routing = (tool_routing or "qdrant").strip().lower() - return "get_scheme_info" if routing == "legacy" else "search_schemes" - - -def _generate_prompt_snippet(name: str, code: str, content_type: str) -> str: - """A single sentence meant to be dropped directly into an agent system - prompt as a reference entry — not a markdown bullet. There's no document - summary/description captured during ingestion to draw a real topic from, - so this stays generic (name + code + content type) rather than fabricate - one; `_build_vector_schemes_prompt_block` below is what actually renders - the searchable scheme list callers use, this field is a per-row preview.""" - kind = (content_type or "document").strip().lower() - label = "government scheme" if kind == "scheme" else kind - return f"Refer to '{name}' (code: {code}) for guidance on this {label}." - - -def sync_catalog_entry(workflow_id: str, status: str) -> Optional[int]: - """ - Upsert this document's catalog metadata into the Postgres master catalog - and refresh both Redis snapshot tiers. - - status: "dev" (DEV ingest complete) | "live" (PROD promote complete) - Returns the new catalog version, or None if the document doesn't exist - (nothing to sync). - """ - if status not in ("dev", "live"): - raise ValueError("status must be 'dev' or 'live'") - - doc = db.get_document(workflow_id) - if not doc: - logger.info("master_catalog_pg: %s not found, skipping sync", workflow_id) - return None - - content_type = (doc.get("document_kind") or "document").strip().lower() - - # code is the primary key — fall back to doc_id/workflow_id when this - # document has no scheme_code so every document still gets a row. - code = ( - (doc.get("scheme_code") or "").strip().lower() - or (doc.get("document_id") or "").strip().lower() - or workflow_id.strip().lower() - ) - - # Same fallback chain as the UI's getDocumentListLabel (ui/src/lib/pipelineUi.js) - # so a non-scheme document still gets a readable name instead of the bare code. - name = ( - (doc.get("scheme_name") or "").strip() - or (doc.get("display_name") or "").strip() - or (doc.get("filename") or "").strip() - or (doc.get("source_filename") or "").strip() - or code - ) - tool_name = _resolve_tool_name(doc.get("tool_routing")) - prompt_snippet = _generate_prompt_snippet(name, code, content_type) - # Same resolution as the vector payload, so the catalog and the indexed - # chunks never disagree about a scheme's aliases. - aliases = resolve_scheme_aliases( - doc.get("scheme_code"), name, doc.get("scheme_aliases_json") - ) - doc_id = doc.get("document_id") - instance = (doc.get("instance") or "default").strip().lower() - instance_name = instance_display_name(instance) - now = _utc_now_iso() - - conn = get_pg_connection() - try: - ensure_schema(conn) - with conn.cursor() as cur: - cur.execute( - """ - INSERT INTO master_catalog - (code, content_type, name, tool_name, doc_id, prompt_snippet, - aliases, status, workflow_id, instance, instance_name, updated_at) - VALUES (%s, %s, %s, %s, %s, %s, %s, ARRAY[%s]::TEXT[], %s, %s, %s, %s) - ON CONFLICT (code) DO UPDATE SET - content_type = EXCLUDED.content_type, - name = EXCLUDED.name, - tool_name = EXCLUDED.tool_name, - doc_id = EXCLUDED.doc_id, - prompt_snippet = EXCLUDED.prompt_snippet, - aliases = EXCLUDED.aliases, - status = ARRAY( - SELECT DISTINCT unnest(master_catalog.status || EXCLUDED.status) - ), - workflow_id = EXCLUDED.workflow_id, - instance = EXCLUDED.instance, - instance_name = EXCLUDED.instance_name, - updated_at = EXCLUDED.updated_at - """, - (code, content_type, name, tool_name, doc_id, prompt_snippet, - aliases, status, workflow_id, instance, instance_name, now), - ) - cur.execute( - """ - UPDATE master_catalog_meta - SET version = version + 1, updated_at = %s - WHERE id = 1 - RETURNING version - """, - (now,), - ) - row = cur.fetchone() - version = row[0] if row else None - - logger.info( - "master_catalog_pg: synced code=%s name=%s status=%s workflow_id=%s version=%s", - code, name, status, workflow_id, version, - ) - _push_snapshots_to_redis(conn, version, now) - return version - finally: - conn.close() - - -def remove_catalog_entry(workflow_id: str) -> Optional[int]: - """ - Remove this document's row from the Postgres master catalog and refresh - both Redis snapshot tiers, so a deleted document stops appearing to the - AI layer immediately instead of lingering in a stale snapshot until some - unrelated sync event happens to overwrite it. Counterpart to - sync_catalog_entry() — call this from the document-disable path. - - Deletes by the same `code` key sync_catalog_entry() would have derived, - with a workflow_id fallback in case the document row is already gone. - Returns the new catalog version, or None if there was no matching row. - """ - doc = db.get_document(workflow_id) - code = None - if doc: - code = ( - (doc.get("scheme_code") or "").strip().lower() - or (doc.get("document_id") or "").strip().lower() - or workflow_id.strip().lower() - ) - - conn = get_pg_connection() - try: - ensure_schema(conn) - with conn.cursor() as cur: - if code: - cur.execute("DELETE FROM master_catalog WHERE code = %s", (code,)) - else: - cur.execute("DELETE FROM master_catalog WHERE workflow_id = %s", (workflow_id,)) - deleted = cur.rowcount - - if not deleted: - logger.info( - "master_catalog_pg: no catalog row for workflow_id=%s (code=%s), nothing to remove", - workflow_id, code, - ) - return None - - now = _utc_now_iso() - cur.execute( - """ - UPDATE master_catalog_meta - SET version = version + 1, updated_at = %s - WHERE id = 1 - RETURNING version - """, - (now,), - ) - row = cur.fetchone() - version = row[0] if row else None - - logger.info( - "master_catalog_pg: removed code=%s workflow_id=%s version=%s", - code, workflow_id, version, - ) - _push_snapshots_to_redis(conn, version, now) - return version - finally: - conn.close() - - -def _fetch_rows(conn, tier: str) -> list[dict]: - """tier='dev' includes dev+live rows (dev testers should also see promoted entries); - tier='live' includes only live rows (prod). A row's `status` accumulates every - tier it has ever synced under, so this is an array-overlap check, not equality.""" - statuses = ("dev", "live") if tier == "dev" else ("live",) - with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: - cur.execute( - "SELECT * FROM master_catalog WHERE status && %s ORDER BY code", - (list(statuses),), - ) - return [dict(r) for r in cur.fetchall()] - - -def _row_to_json(row: dict) -> dict: - updated_at = row.get("updated_at") - return { - "code": row["code"], - "content_type": row.get("content_type"), - "name": row["name"], - "tool_name": row["tool_name"], - "doc_id": row.get("doc_id"), - "prompt_snippet": row.get("prompt_snippet"), - "aliases": row.get("aliases") or [], - "status": row.get("status"), - "workflow_id": row.get("workflow_id"), - "instance": row.get("instance"), - "instance_name": row.get("instance_name"), - "updated_at": updated_at.isoformat() if hasattr(updated_at, "isoformat") else updated_at, - } - - -def _build_vector_schemes_prompt_block(entries: list[dict]) -> dict: - """Pre-rendered system-prompt fragments for the AI layer's dynamic - "vector-indexed schemes" section (bharat-oan-api's search_schemes routing). - - Restricted to tool_name == "search_schemes": legacy get_scheme_info - schemes are a separate, still-hardcoded list in the AI layer and must - never be pulled into this block. - """ - vector_entries = sorted( - (e for e in entries if e.get("tool_name") == "search_schemes"), - key=lambda e: e["code"], - ) - bullets = [] - identifiers = [] - for e in vector_entries: - code = e["code"] - name = e["name"] - bullets.append(f"- **{name}** ({code})") - # The stored alias list is deliberately broad (curated + derived) because - # it costs nothing in Postgres and improves vector recall. The prompt is - # the opposite: every alias is tokens in every AI request, and the long - # derived tail adds noise rather than helping the model pick a scheme. - # Aliases are already in precedence order, so the head is the useful part. - aliases = (e.get("aliases") or [])[:_PROMPT_ALIAS_LIMIT] - alias_suffix = "".join(f" / {a}" for a in aliases) - identifiers.append(f"- `{code}`{alias_suffix}") - return { - "vector_schemes_bullets": "\n".join(bullets), - "vector_schemes_identifiers": "\n".join(identifiers), - "vector_scheme_count": len(vector_entries), - } - - -def _push_snapshots_to_redis(conn, version: Optional[int], updated_at: str) -> None: - """Best-effort: Postgres is the source of truth, Redis is a mirror. A failed - push here doesn't fail the sync — the next catalog event repairs it.""" - try: - client = get_redis_client() - if client is None: - logger.info("master_catalog_pg: AI_LAYER_REDIS_HOST unset, skipping Redis push") - return - ttl = _redis_ttl_seconds() - for tier in ("dev", "live"): - rows = _fetch_rows(conn, tier) - entries = [_row_to_json(r) for r in rows] - payload = { - "version": version, - "updated_at": updated_at, - "status": tier, - "entries": entries, - "prompt": _build_vector_schemes_prompt_block(entries), - } - client.set(f"{_REDIS_KEY_PREFIX}:{tier}:snapshot", json.dumps(payload), ex=ttl) - logger.info( - "master_catalog_pg: pushed %s:%s snapshot (%d entries, version=%s)", - _REDIS_KEY_PREFIX, tier, len(rows), version, - ) - except Exception as exc: - logger.warning("master_catalog_pg: Redis push failed: %s", exc) diff --git a/pipeline/migrate.py b/pipeline/migrate.py new file mode 100644 index 0000000..fe1a870 --- /dev/null +++ b/pipeline/migrate.py @@ -0,0 +1,76 @@ +"""Versioned SQL migration runner. + +Applies ``migrations/NNNN_.sql`` files in order, once each, recording +them in ``schema_migrations``. A session-level advisory lock serialises +concurrent api/worker startups so only one process applies a given file. +""" + +import logging +import re +from pathlib import Path + +import psycopg + +logger = logging.getLogger(__name__) + +MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations" +_MIGRATION_FILE_RE = re.compile(r"^(\d{4})_[a-z0-9_]+\.sql$") +_ADVISORY_LOCK_KEY = 7_204_911_338_105_221 + + +def list_migrations(directory: Path = MIGRATIONS_DIR) -> list[tuple[str, Path]]: + found = [] + for path in sorted(directory.glob("*.sql")): + match = _MIGRATION_FILE_RE.match(path.name) + if not match: + raise ValueError(f"Migration file name must be NNNN_name.sql: {path.name}") + found.append((match.group(1), path)) + return found + + +def apply_migrations(conninfo: str, directory: Path = MIGRATIONS_DIR) -> list[str]: + """Apply every pending migration. Returns the versions applied by this call.""" + applied: list[str] = [] + with psycopg.connect(conninfo, autocommit=True) as conn: + conn.execute("SELECT pg_advisory_lock(%s)", (_ADVISORY_LOCK_KEY,)) + try: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")} + for version, path in list_migrations(directory): + if version in done: + continue + with conn.transaction(): + conn.execute(path.read_text(encoding="utf-8")) + conn.execute( + "INSERT INTO schema_migrations (version, name) VALUES (%s, %s)", + (version, path.name), + ) + logger.info("applied migration %s", path.name) + applied.append(version) + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (_ADVISORY_LOCK_KEY,)) + return applied + + +def current_version(conninfo: str) -> str | None: + with psycopg.connect(conninfo, autocommit=True) as conn: + exists = conn.execute("SELECT to_regclass('schema_migrations')").fetchone()[0] + if not exists: + return None + row = conn.execute("SELECT max(version) FROM schema_migrations").fetchone() + return row[0] + + +def is_current(conninfo: str, directory: Path = MIGRATIONS_DIR) -> bool: + migrations = list_migrations(directory) + if not migrations: + return True + return current_version(conninfo) == migrations[-1][0] diff --git a/pipeline/models.py b/pipeline/models.py index c4d5522..011b101 100644 --- a/pipeline/models.py +++ b/pipeline/models.py @@ -7,7 +7,7 @@ from enum import Enum from typing import Any, Optional -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, Field, model_validator class DocumentStage(str, Enum): @@ -28,6 +28,34 @@ class DocumentStage(str, Enum): FAILED = "failed" +class WorkflowType(str, Enum): + """``workflows.workflow_type``: kind of processing run. Stored as TEXT.""" + PIPELINE = "pipeline" + OCR_ONLY = "ocr_only" + OCR_RETRY = "ocr_retry" + TRANSLATION_RETRY = "translation_retry" + CHUNKING_RETRY = "chunking_retry" + REINGEST = "reingest" + PROMOTE_PROD = "promote_prod" + + +class WorkflowStatus(str, Enum): + """``workflows.status``. Stored as TEXT.""" + RUNNING = "running" + WAITING_REVIEW = "waiting_review" + COMPLETED = "completed" + FAILED = "failed" + + +class AuditEntityType(str, Enum): + """``audit_logs.entity_type``. Stored as TEXT.""" + DOCUMENT = "document" + PAGE = "page" + CHUNK = "chunk" + SETTING = "setting" + CATALOG = "catalog" + + # Pipeline stage order for stepper UI PIPELINE_STAGES = [ ("registered", "Registered", "Document uploaded"), @@ -180,16 +208,17 @@ class ApprovalRequest(BaseModel): class DocumentSummary(BaseModel): + id: Optional[str] = None # documents.id (UUID); canonical identifier in API paths document_id: str canonical_document_id: Optional[str] = None workflow_id: str # The Temporal workflow ID (use this for API calls) filename: str - display_name: Optional[str] = None + name: Optional[str] = None # documents.name source_filename: Optional[str] = None source_manifest_name: Optional[str] = None source_file_fingerprint: Optional[str] = None authoritative: bool = False - instance: str = "default" + instance: str = "default" # tenant code of the document's catalog stage: DocumentStage page_count: int chunk_count: int @@ -203,11 +232,11 @@ class DocumentSummary(BaseModel): uploaded_by_username: Optional[str] = None uploaded_by_email: Optional[str] = None uploaded_by_roles: list[str] = [] - # Scheme catalog metadata (document_kind scheme → Master Catalog) - document_kind: str = "document" - scheme_code: Optional[str] = None - scheme_name: Optional[str] = None - scheme_aliases: list[str] = [] + # Catalog placement: kind comes from the catalog type, code/name/aliases are + # the document's own (a scheme's code feeds the Master Catalog). + kind: Optional[str] = None # scheme | advisory + code: Optional[str] = None + aliases: list[str] = [] tool_routing: Optional[str] = None catalog_visible: bool = True network_visible: bool = True @@ -228,12 +257,11 @@ class SchemeMetadataUpdate(BaseModel): period is part of classifying a document, not a separate errand. Omitting both leaves the stored period alone; the dates are validated in `document_validity.parse_period` so a bad period comes back as the same - 400 a bad kind does, not a Pydantic 422. + 400 other bad input does, not a Pydantic 422. """ - document_kind: Optional[str] = None # document | scheme - scheme_code: Optional[str] = None - scheme_name: Optional[str] = None - scheme_aliases: Optional[list[str]] = None + code: Optional[str] = None + name: Optional[str] = None + aliases: Optional[list[str]] = None tool_routing: Optional[str] = None # qdrant | legacy | both catalog_visible: Optional[bool] = None network_visible: Optional[bool] = None @@ -243,9 +271,9 @@ class SchemeMetadataUpdate(BaseModel): class DocumentArtifact(BaseModel): - id: int + id: str workflow_id: str - job_id: Optional[int] = None + job_id: Optional[str] = None artifact_type: str stage: Optional[str] = None storage_uri: str @@ -257,7 +285,7 @@ class DocumentArtifact(BaseModel): class DocumentJob(BaseModel): - id: int + id: str workflow_id: str job_type: str temporal_workflow_id: Optional[str] = None @@ -305,9 +333,9 @@ class DocumentDetail(DocumentSummary): source_type: Optional[str] = None canonical_input_type: Optional[str] = None stop_after_ocr: bool = False - original_artifact_id: Optional[int] = None - normalized_artifact_id: Optional[int] = None - latest_job_id: Optional[int] = None + original_artifact_id: Optional[str] = None + normalized_artifact_id: Optional[str] = None + latest_job_id: Optional[str] = None current_job: Optional[dict[str, Any]] = None artifacts: list[DocumentArtifact] = [] index_status: list[DocumentIndexStatus] = [] @@ -349,7 +377,7 @@ class OperationQueueEntry(BaseModel): workflow_id: str filename: str stage: str - job_id: Optional[int] = None + job_id: Optional[str] = None job_type: Optional[str] = None job_status: Optional[str] = None started_at: Optional[str] = None @@ -393,7 +421,7 @@ class ReindexStateRequest(BaseModel): class AuditLogEntry(BaseModel): """A single audit log entry.""" - id: int + id: str workflow_id: str document_id: str action_type: str # stage_change, page_edit, chunk_edit, approval, page_reset, chunk_reset @@ -416,7 +444,7 @@ class AuditLogEntry(BaseModel): # Document context — joined from documents. Without these declared, FastAPI # strips them from the response and the UI can only show a raw doc_id hash. filename: Optional[str] = None - display_name: Optional[str] = None + name: Optional[str] = None instance: Optional[str] = None uploaded_by_username: Optional[str] = None uploaded_by_email: Optional[str] = None @@ -438,7 +466,7 @@ class SearchSettings(BaseModel): """Search configuration settings.""" searchMethod: str = "HYBRID" # TENSOR, LEXICAL, HYBRID limit: int = 12 - alpha: float = 0.7 # 0=lexical, 1=semantic + alpha: float = 0.6 # 0=lexical, 1=semantic rankingMethod: str = "rrf" # rrf, normalize_linear showHighlights: bool = True efSearch: int = 256 @@ -486,3 +514,147 @@ class SettingsAuditResponse(BaseModel): total: int limit: int offset: int + + +# ============================================================================= +# JSONB column models +# +# The database only checks that these columns hold a JSON object; the value +# rules live here. ``db.py`` validates every value through these models before +# it writes one. +# ============================================================================= + +class DocumentSource(BaseModel): + """``documents.source``: facts about the uploaded file.""" + model_config = ConfigDict(extra="forbid") + + fingerprint: str = Field(pattern=r"^[0-9a-f]{32}$") + original_filename: Optional[str] = None + storage_uri: Optional[str] = None + source_type: Optional[str] = None # spreadsheet | document | unknown + canonical_input_type: Optional[str] = None # spreadsheet | pdf + + +class IndexState(BaseModel): + """One entry of ``documents.index_status`` (keys ``dev`` / ``prod``).""" + model_config = ConfigDict(extra="forbid") + + status: str = "unknown" # unknown | indexed | missing + index_name: Optional[str] = None + doc_id: Optional[str] = None + chunk_count_indexed: int = Field(default=0, ge=0) + last_indexed_at: Optional[str] = None + last_verified_at: Optional[str] = None + schema_version: Optional[str] = None + details: dict[str, Any] = Field(default_factory=dict) + + +class ChunkingRun(BaseModel): + model_config = ConfigDict(extra="forbid") + + run_number: int = Field(default=1, ge=1) + provider: Optional[str] = None + model: Optional[str] = None + config: dict[str, Any] = Field(default_factory=dict) + created_at: Optional[str] = None + + +class DocumentMetadata(BaseModel): + """``documents.metadata``: period, uploader, index state and flag fields.""" + model_config = ConfigDict(extra="forbid") + + valid_from: Optional[str] = None # YYYY-MM-DD + valid_to: Optional[str] = None # YYYY-MM-DD, None = no end + uploaded_by_user_id: Optional[str] = None # users.id (UUID string) + index_status: dict[str, IndexState] = Field(default_factory=dict) # keys dev / prod + catalog_visible: bool = True + network_visible: bool = True + reindex_required: bool = False + reindex_reason: Optional[str] = None + chunking: Optional[ChunkingRun] = None + # Vector-index keys (doc_id) of earlier file versions still present in an + # index, by env (``dev`` / ``prod``). A re-upload changes the fingerprint, so + # the old points would otherwise be orphaned; the next ingest/promote and + # disable/purge remove them. + superseded_doc_ids: dict[str, list[str]] = Field(default_factory=dict) + + @model_validator(mode="after") + def _period_is_ordered(self) -> "DocumentMetadata": + if self.valid_from and self.valid_to and self.valid_from > self.valid_to: + raise ValueError("valid_from must not be after valid_to") + return self + + +class CatalogNetworkState(BaseModel): + """``catalogs.metadata.network``: set once the catalog is published.""" + model_config = ConfigDict(extra="forbid") + + catalog_id: Optional[str] = None + last_published_at: Optional[str] = None + + +class CatalogMetadata(BaseModel): + """``catalogs.metadata``: Beckn publish information.""" + model_config = ConfigDict(extra="forbid") + + publish_directives: dict[str, Any] = Field(default_factory=dict) + visible_networks: list[str] = Field(default_factory=list) + criteria: dict[str, Any] = Field(default_factory=dict) + network: Optional[CatalogNetworkState] = None + + +class PageMetadata(BaseModel): + """``pages.metadata``: OCR review and language fields.""" + model_config = ConfigDict(extra="forbid") + + is_reviewed: bool = False + reviewer_notes: Optional[str] = None + detected_language: Optional[str] = None + + +class PageTranslation(BaseModel): + """One language entry of ``pages.translations``.""" + model_config = ConfigDict(extra="forbid") + + translated_markdown: Optional[str] = None + edited_translation: Optional[str] = None + is_reviewed: bool = False + reviewer_notes: Optional[str] = None + provider: Optional[str] = None + model: Optional[str] = None + translated_at: Optional[str] = None + + +class RetrievalSettings(BaseModel): + model_config = ConfigDict(extra="forbid") + + searchMethod: str = Field(default="HYBRID", pattern=r"^(TENSOR|LEXICAL|HYBRID)$") + alpha: float = Field(default=0.6, ge=0, le=1) + rankingMethod: str = Field(default="rrf", pattern=r"^(rrf|normalize_linear)$") + hybridRrfK: int = Field(default=60, ge=1) + + +class CandidateSettings(BaseModel): + model_config = ConfigDict(extra="forbid") + + indexName: str = "documents-index" + efSearch: int = Field(default=256, ge=1) + candidateCap: int = Field(default=120, ge=1) + candidateMultiplier: int = Field(default=10, ge=1) + + +class ResultSettings(BaseModel): + model_config = ConfigDict(extra="forbid") + + limit: int = Field(default=12, ge=1) + maxChunksPerDoc: int = Field(default=2, ge=1) + excludeReference: bool = True + showHighlights: bool = True + rerankMode: str = "none" + + +class QuerySettings(BaseModel): + model_config = ConfigDict(extra="forbid") + + useE5Prefix: bool = True + queryExpansionProfile: str = "gu-v1" diff --git a/pipeline/network_constants.py b/pipeline/network_constants.py index eb6a494..c4e6b39 100644 --- a/pipeline/network_constants.py +++ b/pipeline/network_constants.py @@ -35,3 +35,35 @@ PROVIDER_ID_TEMPLATE = "{instance}.{network_id}" GROUP_CATALOG_NAME_TEMPLATE = "{group_name} {catalog_name}" + +# Knowledge kinds the pipeline serves, and the network schema each maps to. +# ``catalogs.type`` stores the schema name; code asks this map what a type means +# instead of comparing ``openagrinet:*`` strings. The scheme mapping is a +# placeholder (ADR 0003 types schemes as KnowledgeAdvisory) - change it here only. +KIND_SCHEME = "scheme" +KIND_ADVISORY = "advisory" +KIND_VIDEO = "video" +KIND_TO_CATALOG_TYPE = { + KIND_SCHEME: "openagrinet:AgricultureResource", + KIND_ADVISORY: "openagrinet:KnowledgeAdvisory", + KIND_VIDEO: "openagrinet:VideoResource", +} +CATALOG_TYPE_TO_KIND = {schema: kind for kind, schema in KIND_TO_CATALOG_TYPE.items()} + + +def catalog_type_for_kind(kind: str) -> str: + """``catalogs.type`` for a knowledge kind; ValueError for an unknown kind.""" + try: + return KIND_TO_CATALOG_TYPE[(kind or "").strip().lower()] + except KeyError: + raise ValueError( + f"Unknown document kind {kind!r}. Use one of: {', '.join(KIND_TO_CATALOG_TYPE)}" + ) from None + + +def kind_for_catalog_type(catalog_type: str) -> str: + """Knowledge kind a ``catalogs.type`` stands for; ValueError for an unknown type.""" + try: + return CATALOG_TYPE_TO_KIND[catalog_type] + except KeyError: + raise ValueError(f"Unknown catalog type {catalog_type!r}") from None diff --git a/pipeline/scheme_catalog.py b/pipeline/scheme_catalog.py index c27d8a7..624c4bf 100644 --- a/pipeline/scheme_catalog.py +++ b/pipeline/scheme_catalog.py @@ -19,11 +19,6 @@ SCHEME_CODE_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") -# document_kind is a free-form slug (scheme, advisory, video, or a custom -# type entered in the UI) — same character rules as scheme_code but allows -# underscores too since kind names read more naturally that way (e.g. "how_to"). -DOCUMENT_KIND_RE = re.compile(r"^[a-z0-9]+(?:[_-][a-z0-9]+)*$") -SUGGESTED_DOCUMENT_KINDS = ("document", "scheme", "advisory", "video") _SCHEME_CODE_STOPWORDS = { "a", "an", "the", "of", "on", "for", "and", "or", "to", "in", "at", "by", @@ -198,8 +193,8 @@ def central_instances() -> set[str]: def default_network_visible(instance: str | None) -> bool: - inst = (instance or "default").strip().lower() or "default" - return inst in central_instances() + code = (instance or "default").strip().lower() or "default" + return code in central_instances() def scheme_collection_name() -> str: @@ -234,19 +229,21 @@ def _acronym_from_title(title: str) -> str: return letters -def derive_scheme_code(title: str, exclude_workflow_id: Optional[str] = None) -> str: +def derive_scheme_code( + title: str, exclude_workflow_id: Optional[str] = None, kind: str = "scheme" +) -> str: """Auto-generate a scheme_code from a document title: acronym of significant words, falling back to a truncated slug if the title has too few words to produce a useful acronym (e.g. a single-word title). Appends -2, -3, ... on - collision against every scheme_code already in use (documents table + - scheme_catalog_entries registry) so two schemes never share a code. + collision against every code already in use for the kind (``documents.code``) + so two documents of a kind never share a code across tenants. """ base = _acronym_from_title(title) if len(base) < 2: slug = re.sub(r"[^a-z0-9]+", "", (title or "").strip().lower()) base = slug[:8] or "scheme" - used = db.list_used_scheme_codes(exclude_workflow_id=exclude_workflow_id) + used = db.list_used_scheme_codes(exclude_workflow_id=exclude_workflow_id, kind=kind) if base not in used: return base n = 2 @@ -345,8 +342,8 @@ def _dedupe_aliases(aliases: list[str]) -> list[str]: def resolve_scheme_aliases( - scheme_code: str | None, - scheme_name: str | None, + code: str | None, + name: str | None, explicit: Any = None, ) -> list[str]: """Aliases to index for a scheme, in precedence order. @@ -358,254 +355,26 @@ def resolve_scheme_aliases( All three are merged rather than short-circuited, so a curator adding one alias does not silently discard the bootstrap and derived ones. """ - code = (scheme_code or "").strip().lower() + code = (code or "").strip().lower() merged: list[str] = list(parse_aliases(explicit)) bootstrap = BOOTSTRAP_VECTOR_SCHEMES.get(code) if bootstrap: merged.extend(bootstrap.get("scheme_aliases") or []) - if not (scheme_name or "").strip(): - scheme_name = bootstrap.get("scheme_name") + if not (name or "").strip(): + name = bootstrap.get("scheme_name") - merged.extend(_derive_aliases(code, scheme_name or "")) + merged.extend(_derive_aliases(code, name or "")) return _dedupe_aliases(merged) -def _content_hash(code: str, name: str, aliases: list[str], network_visible: bool, status: str) -> str: - payload = json.dumps( - { - "code": code, - "name": name, - "aliases": sorted(aliases), - "network_visible": network_visible, - "status": status, - }, - sort_keys=True, - ) - return hashlib.sha256(payload.encode()).hexdigest()[:16] - - -def ensure_catalog_schema() -> None: - """Create catalog tables and scheme metadata columns (idempotent).""" - db.init_scheme_catalog_schema() - - -def bootstrap_catalog_if_empty(*, force: bool = False) -> dict[str, Any]: - """Seed bootstrap vector + legacy schemes when catalog is empty (or force).""" - ensure_catalog_schema() - meta = db.get_catalog_meta() - entries = db.list_catalog_entries() - if entries and not force: - return {"bootstrapped": False, "version": meta.get("version", 0), "entry_count": len(entries)} - - now = utc_now_iso() - collection = scheme_collection_name() - for code, definition in BOOTSTRAP_VECTOR_SCHEMES.items(): - aliases = list(definition.get("scheme_aliases") or []) - name = definition["scheme_name"] - status = "live" - db.upsert_catalog_entry( - scheme_code=code, - scheme_name=name, - scheme_aliases=aliases, - instances=["default"], - workflow_ids=[], - collection_name=collection, - chunk_count=0, - status=status, - network_visible=True, - promoted_at=None, - content_hash=_content_hash(code, name, aliases, True, status), - source="bootstrap", - updated_at=now, - ) - version = db.bump_catalog_version("bootstrap_vector_schemes") - return { - "bootstrapped": True, - "version": version, - "vector_count": len(BOOTSTRAP_VECTOR_SCHEMES), - "legacy_count": len(BOOTSTRAP_LEGACY_SCHEMES), - } - - def rebuild_catalog_from_documents(*, reason: str = "rebuild") -> int: """ - Recompute scheme_catalog_entries from documents table, then bump version. + Bump the catalog version. - Pipeline-sourced live docs override bootstrap for the same scheme_code. - Bootstrap-only codes with no live docs remain live (migration phase A1). + Status, instances, chunk counts and the rest are computed by the + ``catalog_status`` view, so there is nothing to recompute here. """ - ensure_catalog_schema() - now = utc_now_iso() - collection = scheme_collection_name() - - docs = db.list_scheme_documents_for_catalog() - by_code: dict[str, list[dict]] = {} - for doc in docs: - code = (doc.get("scheme_code") or "").strip().lower() - if not code: - continue - by_code.setdefault(code, []).append(doc) - - # Track which codes are fully owned by pipeline live docs - live_pipeline_codes: set[str] = set() - - for code, group in by_code.items(): - live_docs = [ - d - for d in group - if (d.get("stage") == "completed") - and not int(d.get("is_disabled") or 0) - and int(d.get("catalog_visible") if d.get("catalog_visible") is not None else 1) - and (d.get("document_kind") or "document") == "scheme" - ] - pending_reindex = any( - (d.get("stage") == "completed") - and not int(d.get("is_disabled") or 0) - and (d.get("reindex_required") or d.get("_pending_reindex")) - for d in group - ) - # Also treat reindex_reason containing scheme_code_rename as pending_reindex - for d in group: - reason_txt = (d.get("reindex_reason") or "") - if "scheme_code_rename" in reason_txt and d.get("stage") == "completed" and not int(d.get("is_disabled") or 0): - pending_reindex = True - - if pending_reindex and live_docs: - # If any live doc is waiting reindex after rename, publish as pending_reindex - # unless we only have non-rename live docs — keep simple: any reindex_required on scheme - if any(int(d.get("reindex_required") or 0) for d in live_docs): - status = "pending_reindex" - else: - status = "live" - elif live_docs: - status = "live" - live_pipeline_codes.add(code) - elif any(d.get("stage") == "approval_for_prod" and not int(d.get("is_disabled") or 0) for d in group): - status = "pending_prod" - else: - status = "disabled" - - # Prefer live docs for aggregation; else all group - use = live_docs or group - names = [d.get("scheme_name") for d in use if d.get("scheme_name")] - scheme_name = names[0] if names else code - aliases: list[str] = [] - seen_alias: set[str] = set() - for d in use: - for a in parse_aliases(d.get("scheme_aliases_json")): - key = a.lower() - if key not in seen_alias: - seen_alias.add(key) - aliases.append(a) - instances = sorted( - { - (d.get("instance") or "default").strip().lower() or "default" - for d in use - } - ) - workflow_ids = [d["workflow_id"] for d in use if d.get("workflow_id")] - chunk_count = sum(int(d.get("chunk_count") or 0) for d in use) - network_visible = any(int(d.get("network_visible") or 0) for d in use) # OR - promoted_times = [d.get("ingested_at") or d.get("updated_at") for d in live_docs if d.get("ingested_at") or d.get("updated_at")] - promoted_at = max(promoted_times) if promoted_times else None - source = "pipeline" if any((d.get("document_kind") or "") == "scheme" for d in group) else "import" - - # pending_reindex entries must not appear as live for consumers - if status == "pending_reindex": - # still store entry but status excludes from vector_schemes - pass - - db.upsert_catalog_entry( - scheme_code=code, - scheme_name=scheme_name, - scheme_aliases=aliases, - instances=instances, - workflow_ids=workflow_ids, - collection_name=collection, - chunk_count=chunk_count, - status=status if status != "disabled" or live_docs else "disabled", - network_visible=1 if network_visible else 0, - promoted_at=promoted_at, - content_hash=_content_hash(code, scheme_name, aliases, network_visible, status), - source=source, - updated_at=now, - ) - - # Preserve bootstrap live entries for codes with no document rows; - # disable orphan pipeline / import codes that have no docs left. - existing = {e["scheme_code"]: e for e in db.list_catalog_entries()} - for code, definition in BOOTSTRAP_VECTOR_SCHEMES.items(): - if code in by_code: - continue - if code in existing and existing[code].get("source") == "pipeline": - # Was pipeline-owned but no longer eligible → mark disabled - if code not in live_pipeline_codes: - db.upsert_catalog_entry( - scheme_code=code, - scheme_name=existing[code].get("scheme_name") or definition["scheme_name"], - scheme_aliases=parse_aliases(existing[code].get("scheme_aliases_json")), - instances=parse_aliases(existing[code].get("instances_json")) or ["default"], - workflow_ids=parse_aliases(existing[code].get("workflow_ids_json")), - collection_name=collection, - chunk_count=int(existing[code].get("chunk_count") or 0), - status="disabled", - network_visible=int(existing[code].get("network_visible") or 0), - promoted_at=existing[code].get("promoted_at"), - content_hash=_content_hash( - code, - existing[code].get("scheme_name") or definition["scheme_name"], - parse_aliases(existing[code].get("scheme_aliases_json")), - bool(existing[code].get("network_visible")), - "disabled", - ), - source="pipeline", - updated_at=now, - ) - continue - # Keep or re-seed bootstrap - aliases = list(definition.get("scheme_aliases") or []) - name = definition["scheme_name"] - db.upsert_catalog_entry( - scheme_code=code, - scheme_name=name, - scheme_aliases=aliases, - instances=["default"], - workflow_ids=[], - collection_name=collection, - chunk_count=0, - status="live", - network_visible=1, - promoted_at=None, - content_hash=_content_hash(code, name, aliases, True, "live"), - source="bootstrap", - updated_at=now, - ) - - # Disable non-bootstrap entries that no longer appear in any document group - for code, entry in existing.items(): - if code in by_code or code in BOOTSTRAP_VECTOR_SCHEMES: - continue - if (entry.get("status") or "") == "disabled": - continue - name = entry.get("scheme_name") or code - aliases = parse_aliases(entry.get("scheme_aliases_json")) - db.upsert_catalog_entry( - scheme_code=code, - scheme_name=name, - scheme_aliases=aliases, - instances=parse_aliases(entry.get("instances_json")) or ["default"], - workflow_ids=[], - collection_name=collection, - chunk_count=int(entry.get("chunk_count") or 0), - status="disabled", - network_visible=int(entry.get("network_visible") or 0), - promoted_at=entry.get("promoted_at"), - content_hash=_content_hash(code, name, aliases, bool(entry.get("network_visible")), "disabled"), - source=entry.get("source") or "pipeline", - updated_at=now, - ) - return db.bump_catalog_version(reason) @@ -614,7 +383,7 @@ def on_scheme_document_promoted(workflow_id: str) -> Optional[int]: doc = db.get_document(workflow_id) if not doc: return None - if (doc.get("document_kind") or "document") != "scheme": + if doc.get("kind") != "scheme": return None # Clear reindex flag on successful promote (new code is live) if int(doc.get("reindex_required") or 0): @@ -626,7 +395,7 @@ def on_scheme_document_promoted(workflow_id: str) -> Optional[int]: return rebuild_catalog_from_documents(reason=f"promote_scheme:{workflow_id}") -def on_scheme_document_disabled(workflow_id: str, scheme_code: Optional[str] = None) -> Optional[int]: +def on_scheme_document_disabled(workflow_id: str, code: Optional[str] = None) -> Optional[int]: return rebuild_catalog_from_documents(reason=f"disable_scheme:{workflow_id}") @@ -668,7 +437,6 @@ def build_snapshot( include_pending: bool = False, network_visible: Optional[bool] = None, ) -> dict[str, Any]: - ensure_catalog_schema() meta = db.get_catalog_meta() version = int(meta.get("version") or 0) updated_at = meta.get("updated_at") or utc_now_iso() @@ -686,29 +454,27 @@ def build_snapshot( # Live snapshot for warmers: only status=live (pending excluded unless include_pending) if not include_pending and st != "live": continue - instances = parse_aliases(e.get("instances_json")) + instances = e["instances"] if instance: inst = instance.strip().lower() - if inst not in {i.lower() for i in instances} and instances: - # bootstrap may have ["default"] only - if inst not in {i.lower() for i in instances}: - continue - nv = bool(int(e.get("network_visible") or 0)) + if inst not in {g.lower() for g in instances} and instances: + continue + nv = bool(e.get("network_visible")) if network_visible is not None and nv != network_visible: continue vector_schemes.append( { - "scheme_code": e["scheme_code"], - "scheme_name": e.get("scheme_name") or e["scheme_code"], - "scheme_aliases": parse_aliases(e.get("scheme_aliases_json")), + "scheme_code": e["code"], + "scheme_name": e.get("name") or e["code"], + "scheme_aliases": e["aliases"], "tool": "search_schemes", "tool_routing": "qdrant", "instances": instances or ["default"], "network_visible": nv, "chunk_count": int(e.get("chunk_count") or 0), - "workflow_ids": parse_aliases(e.get("workflow_ids_json")), + "workflow_ids": e["workflow_ids"], "promoted_at": e.get("promoted_at"), - "source": e.get("source") or "bootstrap", + "source": e.get("source") or "pipeline", "status": st, } ) @@ -748,10 +514,9 @@ def build_snapshot( def apply_scheme_metadata( workflow_id: str, *, - document_kind: Optional[str] = None, - scheme_code: Optional[str] = None, - scheme_name: Optional[str] = None, - scheme_aliases: Optional[list[str]] = None, + code: Optional[str] = None, + name: Optional[str] = None, + aliases: Optional[list[str]] = None, tool_routing: Optional[str] = None, catalog_visible: Optional[bool] = None, network_visible: Optional[bool] = None, @@ -759,7 +524,7 @@ def apply_scheme_metadata( valid_to: Optional[str] = None, display_name: Optional[str] = None, ) -> dict[str, Any]: - """Update document scheme fields and rebuild catalog when needed. + """Update document catalog fields and rebuild catalog when needed. `valid_from`/`valid_to` set the document's validity period (see `pipeline/document_validity.py`). Either end alone is enough - the other is @@ -767,70 +532,60 @@ def apply_scheme_metadata( document has none - so a reviewer who only moves the end date does not have to restate the start. """ - ensure_catalog_schema() doc = db.get_document(workflow_id) if not doc: raise LookupError(f"Document not found: {workflow_id}") updates: dict[str, Any] = {} - old_code = (doc.get("scheme_code") or "").strip().lower() or None - old_kind = (doc.get("document_kind") or "document").strip().lower() + old_code = (doc.get("code") or "").strip().lower() or None + old_kind = (doc.get("kind") or "").strip().lower() code_renamed = False pending_reindex = False - stored_display_name = doc.get("display_name") + stored_display_name = doc.get("name") # v2: the display name is documents.name if display_name is not None and (stored_display_name is None or display_name.strip() != stored_display_name): display_name_rules.ensure_editable(doc) - updates["display_name"] = display_name_rules.normalize(display_name) - - if document_kind is not None: - kind = document_kind.strip().lower().replace(" ", "-") - if not kind or not DOCUMENT_KIND_RE.match(kind): - raise ValueError( - f"Invalid document_kind '{document_kind}'. Use a lowercase slug, " - f"e.g. one of {SUGGESTED_DOCUMENT_KINDS} or a custom type." - ) - updates["document_kind"] = kind - else: - kind = old_kind + updates["name"] = display_name_rules.normalize(display_name) + + new_kind = old_kind # the kind is fixed at upload; it is the document's catalog new_code = old_code - if scheme_code is not None: - new_code = validate_scheme_code(scheme_code) if scheme_code else None - updates["scheme_code"] = new_code + if code is not None: + new_code = validate_scheme_code(code) if code else None + updates["code"] = new_code if old_code and new_code and old_code != new_code: code_renamed = True - if scheme_name is not None: - updates["scheme_name"] = scheme_name.strip() if scheme_name else None + if name is not None: + updates["name"] = name.strip() if name else None - # Auto-derive scheme_code from the title when the caller set a name but no + # Auto-derive the code from the title when the caller set a name but no # explicit code (the UI form only collects a name and previews the code; # this is the authoritative, collision-checked source of truth for it). - final_name = (updates["scheme_name"] if "scheme_name" in updates else doc.get("scheme_name")) or "" - if scheme_code is None and not new_code and kind == "scheme" and final_name.strip(): - new_code = derive_scheme_code(final_name, exclude_workflow_id=workflow_id) - updates["scheme_code"] = new_code + final_name = (updates["name"] if "name" in updates else doc.get("name")) or "" + if code is None and not new_code and new_kind == "scheme" and final_name.strip(): + new_code = derive_scheme_code(final_name, exclude_workflow_id=workflow_id, kind=new_kind) + updates["code"] = new_code - if scheme_aliases is not None: - updates["scheme_aliases_json"] = json.dumps(parse_aliases(scheme_aliases)) + if aliases is not None: + updates["aliases"] = parse_aliases(aliases) if tool_routing is not None: tr = tool_routing.strip().lower() if tr not in ("qdrant", "legacy", "both"): raise ValueError("tool_routing must be qdrant|legacy|both") updates["tool_routing"] = tr - elif kind == "scheme" and not doc.get("tool_routing"): + elif new_kind == "scheme" and not doc.get("tool_routing"): updates["tool_routing"] = "qdrant" if catalog_visible is not None: updates["catalog_visible"] = 1 if catalog_visible else 0 - elif kind == "scheme" and doc.get("catalog_visible") is None: + elif new_kind == "scheme" and doc.get("catalog_visible") is None: updates["catalog_visible"] = 1 if network_visible is not None: updates["network_visible"] = 1 if network_visible else 0 - elif kind == "scheme" and doc.get("network_visible") is None: + elif new_kind == "scheme" and doc.get("network_visible") is None: updates["network_visible"] = 1 if default_network_visible(doc.get("instance")) else 0 if valid_from is not None or valid_to is not None: @@ -844,10 +599,6 @@ def apply_scheme_metadata( updates["valid_from"] = period.start_date updates["valid_to"] = period.end_date - if kind == "scheme" and not (new_code or updates.get("scheme_code") or old_code): - # Allow setting kind first without code only if not completing as scheme - pass - if code_renamed and (doc.get("stage") == "completed") and not int(doc.get("is_disabled") or 0): # Rule (b): hide new code until reindex updates["reindex_required"] = 1 @@ -859,7 +610,7 @@ def apply_scheme_metadata( # Rebuild catalog if scheme-related final = db.get_document(workflow_id) or doc - final_kind = (final.get("document_kind") or "document").strip().lower() + final_kind = (final.get("kind") or "").strip().lower() catalog_version = None if final_kind == "scheme" or old_kind == "scheme" or code_renamed: reason = "scheme_metadata_patch" diff --git a/pipeline/vector_store/embeddings.py b/pipeline/vector_store/embeddings.py index be9a30f..bc9feec 100644 --- a/pipeline/vector_store/embeddings.py +++ b/pipeline/vector_store/embeddings.py @@ -143,3 +143,15 @@ def embed_passages( batch = passages[i : i + batch_size] out.extend(embed_texts(batch, kind="passage", use_e5_prefix=use_e5_prefix)) return out + + +def count_tokens(text: str) -> int: + """Token count of ``text`` as the embedding model sees it (with the passage prefix). + + Uses the local model's tokenizer when embeddings run locally; for a remote + (OpenAI-compatible) provider there is no tokenizer here, so it estimates ~4 chars/token. + """ + prefixed = _prefix_passage(text) + if get_embedding_provider() == "openai_compatible": + return max(1, len(prefixed) // 4) + return len(_get_local_embedder().tokenizer.encode(prefixed)) diff --git a/pipeline/vector_store/qdrant_store.py b/pipeline/vector_store/qdrant_store.py index d2b7018..7e1596e 100644 --- a/pipeline/vector_store/qdrant_store.py +++ b/pipeline/vector_store/qdrant_store.py @@ -54,6 +54,21 @@ "scheme_aliases", "chunk_id", "chunk_index", + # Advisory payload (type=advisory; webhook records and PDF advisory documents) + "external_id", + "pop_id", + "slug", + "crop", + "domain", + "subdomain", + "institute", + "creator", + "problem_type", + "problem", + "season", + "start_period", + "end_period", + "updated_at", # Document validity - the period this chunk is searchable in. Absent on # points written before validity existed, which search treats as current. "start_date", diff --git a/pipeline/worker.py b/pipeline/worker.py index 438e36f..cfc7261 100644 --- a/pipeline/worker.py +++ b/pipeline/worker.py @@ -16,20 +16,25 @@ create_chunks_from_db, detect_and_translate_pages, detect_and_translate_pages_from_db, + index_webhook_advisory, ingest_document_from_db, ingest_to_vector_db, persist_document_content, prepare_for_ingestion, promote_document_to_prod_qdrant, + promote_webhook_advisory, publish_catalog_to_network, run_ocr, run_ocr_and_store, update_document_state, ) from .activity_logging import ActivityLogging +from .advisory_start import backfill_on_start from .network_group import validate_network_provider_config from .observability import configure_logs, reset_logs from .workflows import ( + AdvisoryIndexWorkflow, + AdvisoryPromoteWorkflow, ChunkingOnlyWorkflow, DocumentPipelineWorkflow, OcrOnlyWorkflow, @@ -77,8 +82,7 @@ async def _run(): logging.error("worker startup: TRANSLATION_VLLM_BASE_URL not set, exiting") return - # Initialize SQLite database - logging.info("worker startup: initializing SQLite database") + logging.info("worker startup: initializing PostgreSQL database") db.init_db() temporal_host = os.environ.get("TEMPORAL_HOST", "localhost:7233") @@ -99,6 +103,8 @@ async def _run(): TranslationOnlyWorkflow, OcrOnlyWorkflow, ChunkingOnlyWorkflow, + AdvisoryIndexWorkflow, + AdvisoryPromoteWorkflow, ], activities=[ run_ocr, @@ -110,6 +116,8 @@ async def _run(): ingest_to_vector_db, ingest_document_from_db, promote_document_to_prod_qdrant, + index_webhook_advisory, + promote_webhook_advisory, publish_catalog_to_network, update_document_state, detect_and_translate_pages, @@ -118,6 +126,10 @@ async def _run(): ], ) + backfilled = await backfill_on_start(client) + if backfilled: + logging.info("Advisory startup backfill queued %s workflows", backfilled) + logging.info( "worker started queue=%s temporal=%s activity_concurrency=%s", TASK_QUEUE, diff --git a/pipeline/workflows.py b/pipeline/workflows.py index 4765534..4ce63fb 100644 --- a/pipeline/workflows.py +++ b/pipeline/workflows.py @@ -17,8 +17,10 @@ auto_tag_chunks_from_db, create_chunks_from_db, detect_and_translate_pages_from_db, + index_webhook_advisory, ingest_document_from_db, promote_document_to_prod_qdrant, + promote_webhook_advisory, publish_catalog_to_network, run_ocr_and_store, update_document_state, @@ -96,7 +98,7 @@ async def _mirror_state( error_message: Optional[str] = None, ) -> dict: """ - Mirror workflow state into SQLite as a local activity. + Mirror workflow state into Postgres as a local activity. Local activities avoid starving tiny state updates behind long-running OCR, translation, or chunking activities on the shared task queue. @@ -887,3 +889,59 @@ def get_state(self) -> dict: "chunk_count": self.state.chunk_count, "chunks_completed_at": self.state.chunks_completed_at, } + + +@workflow.defn +class AdvisoryIndexWorkflow: + """Index one webhook advisory record into the dev Qdrant collection.""" + + def __init__(self): + self.state = None + + @workflow.run + async def run(self, record_id: str) -> dict: + self.state = {"record_id": record_id, "stage": "indexing"} + try: + result = await workflow.execute_activity( + index_webhook_advisory, + record_id, + start_to_close_timeout=timedelta(minutes=30), + retry_policy=INGEST_RETRY, + ) + except Exception as e: + self.state.update(stage="failed", error_message=str(e)) + raise + self.state.update(stage="completed", status=result.get("status"), chunks=result.get("chunks")) + return result + + @workflow.query + def get_state(self) -> dict: + return self.state or {} + + +@workflow.defn +class AdvisoryPromoteWorkflow: + """Promote one webhook advisory record into the PROD Qdrant collection.""" + + def __init__(self): + self.state = None + + @workflow.run + async def run(self, record_id: str) -> dict: + self.state = {"record_id": record_id, "stage": "promoting"} + try: + result = await workflow.execute_activity( + promote_webhook_advisory, + record_id, + start_to_close_timeout=timedelta(minutes=30), + retry_policy=INGEST_RETRY, + ) + except Exception as e: + self.state.update(stage="failed", error_message=str(e)) + raise + self.state.update(stage="completed", status=result.get("status"), chunks=result.get("chunks")) + return result + + @workflow.query + def get_state(self) -> dict: + return self.state or {} diff --git a/pyproject.toml b/pyproject.toml index 833e675..431b255 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,10 +35,6 @@ dependencies = [ # Object storage "minio>=7.0.0", - # Master Scheme Catalog: Postgres store + Redis push to AI layer - "psycopg2-binary>=2.9.0", - "redis>=5.0.0", - # HTTP client "httpx>=0.24.0", @@ -54,6 +50,9 @@ dependencies = [ "pypdf>=5.0.0", "PyMuPDF>=1.27.0", + # Postgres driver (sync) + connection pool + "psycopg[binary,pool]>=3.2", + # Rate limiting "slowapi>=0.1.9", @@ -71,6 +70,7 @@ dev = [ "pytest-asyncio>=0.21.0", "pytest-cov>=4.0.0", "ruff>=0.6.0", + "testcontainers[postgres]>=4.0.0", ] # sentence-transformers depends on torch; resolve it from the CPU-only wheel diff --git a/scripts/backup_volumes.sh b/scripts/backup_volumes.sh index 4dc91c0..6a7b08e 100644 --- a/scripts/backup_volumes.sh +++ b/scripts/backup_volumes.sh @@ -4,7 +4,8 @@ set -euo pipefail # Backup all persistent data volumes used by the docs pipeline: # - Temporal Postgres database # - MinIO object storage -# - SQLite documents/chunks DB +# - Application Postgres volume (documents, workflows, pages, chunks, audit) +# (a logical pg_dump is also written when the postgres container is running) # # Usage: # ./scripts/backup_volumes.sh # writes to ./backups @@ -52,8 +53,18 @@ backup_volume "temporal-db" "temporal-db-data" # 2) MinIO object storage (all PDFs / objects) backup_volume "minio" "minio-data" -# 3) SQLite documents/chunks DB -backup_volume "sqlite" "sqlite-data" +# 3) Application Postgres volume +backup_volume "app-db" "app-db-data" + +# 4) Logical dump of the application database, consistent while the stack runs +PG_CONTAINER="$(docker ps --format '{{.Names}}' | grep 'postgres' | grep -v 'temporal\|keycloak' | head -n 1 || true)" +if [[ -n "$PG_CONTAINER" ]]; then + echo "Dumping application database from ${PG_CONTAINER}" + docker exec "$PG_CONTAINER" sh -c 'pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc' \ + > "${BACKUP_DIR}/app-db-${TIMESTAMP}.dump" +else + echo "WARNING: application postgres container not running; skipped pg_dump" >&2 +fi echo echo "Backup complete. Files in ${BACKUP_DIR}:" diff --git a/scripts/drop_legacy_marqo_columns.py b/scripts/drop_legacy_marqo_columns.py deleted file mode 100644 index f178e3f..0000000 --- a/scripts/drop_legacy_marqo_columns.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python3 -""" -Drop the legacy marqo_doc_id column from document_index_status. - -Context: the Marqo backend was removed and document_index_status.marqo_doc_id -was superseded by a generically-named doc_id column, backfilled once at -startup (see pipeline/db.py init_db()). marqo_doc_id has been left in -place, unwritten, so the schema stayed usable by the pre-removal code during -rollout. This script is the followup: run it manually, once you've confirmed -the app has been stable on the new code for a while — it's a one-way door -(SQLite's DROP COLUMN can't be undone short of restoring a backup). - -No other legacy Marqo-only tables/columns were found in this codebase as of -this writing (only this one column) — if a future migration needs the same -treatment, extend this script rather than writing a new one. - -Requires SQLite 3.35+ for ALTER TABLE ... DROP COLUMN (bundled with Python -3.9+ on most platforms; the script checks and aborts with a clear message on -older versions rather than attempting a table-rebuild workaround). - -Usage: - uv run python scripts/drop_legacy_marqo_columns.py [--yes] -""" -import argparse -import sqlite3 -import sys -from pathlib import Path - -project_root = Path(__file__).parent.parent -sys.path.insert(0, str(project_root)) - -from pipeline import db # noqa: E402 - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--yes", action="store_true", help="Skip the interactive confirmation prompt.") - args = parser.parse_args() - - if sqlite3.sqlite_version_info < (3, 35, 0): - print( - f"ERROR: SQLite {sqlite3.sqlite_version} does not support " - "ALTER TABLE ... DROP COLUMN (needs 3.35+). Aborting — this " - "script does not implement a table-rebuild fallback.", - file=sys.stderr, - ) - sys.exit(1) - - with db.get_connection() as conn: - columns = {row[1] for row in conn.execute("PRAGMA table_info(document_index_status)")} - if "marqo_doc_id" not in columns: - print("document_index_status.marqo_doc_id doesn't exist — nothing to drop.") - sys.exit(0) - - stale = conn.execute( - """ - SELECT COUNT(*) FROM document_index_status - WHERE doc_id IS NULL AND marqo_doc_id IS NOT NULL - """ - ).fetchone()[0] - if stale: - print( - f"ERROR: {stale} row(s) in document_index_status have " - "marqo_doc_id set but doc_id NULL — the backfill in " - "init_db() hasn't run against this database yet. Start the " - "app once (so init_db() runs) before dropping the column.", - file=sys.stderr, - ) - sys.exit(1) - - total = conn.execute("SELECT COUNT(*) FROM document_index_status").fetchone()[0] - print(f"document_index_status: {total} row(s), all backfilled to doc_id.") - - if not args.yes: - answer = input( - "Drop the legacy marqo_doc_id column now? This cannot be undone " - "short of restoring a backup. [y/N] " - ).strip().lower() - if answer != "y": - print("Aborted.") - sys.exit(0) - - conn.execute("ALTER TABLE document_index_status DROP COLUMN marqo_doc_id") - conn.commit() - print("Dropped document_index_status.marqo_doc_id.") - - -if __name__ == "__main__": - main() diff --git a/scripts/ingest_test_docs.sh b/scripts/ingest_test_docs.sh index 03f7bf8..e6fa840 100644 --- a/scripts/ingest_test_docs.sh +++ b/scripts/ingest_test_docs.sh @@ -1,9 +1,15 @@ #!/usr/bin/env bash # Ingest all PDFs from ~/test-docs via H100 pipeline API. +# DOC_KIND: scheme (default) or advisory. PIPELINE_INSTANCE: tenant instance code (optional). set -euo pipefail DOCS_DIR="${1:-$HOME/test-docs}" API="${PIPELINE_API:-http://127.0.0.1:8001}" +DOC_KIND="${DOC_KIND:-scheme}" +QUERY="auto_approve=true&kind=${DOC_KIND}" +if [[ -n "${PIPELINE_INSTANCE:-}" ]]; then + QUERY="${QUERY}&instance=${PIPELINE_INSTANCE}" +fi if [[ ! -d "$DOCS_DIR" ]]; then echo "Missing directory: $DOCS_DIR" @@ -18,7 +24,7 @@ if [[ ${#pdfs[@]} -eq 0 ]]; then fi echo "API: $API" -echo "Files: ${#pdfs[@]}" +echo "Files: ${#pdfs[@]} kind: $DOC_KIND" curl -sf "$API/health" | python3 -m json.tool || { echo "API not reachable"; exit 1; } echo "" @@ -26,7 +32,7 @@ for f in "${pdfs[@]}"; do name="$(basename "$f")" echo "========================================" echo "UPLOAD: $name" - resp="$(curl -s -X POST "$API/upload?auto_approve=true" -F "file=@$f")" + resp="$(curl -s -X POST "$API/upload?$QUERY" -F "file=@$f")" echo "$resp" | python3 -m json.tool || echo "$resp" wf="$(echo "$resp" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('workflow_id',''))" 2>/dev/null || true)" if [[ -n "$wf" ]]; then diff --git a/scripts/list_failed_workflows.py b/scripts/list_failed_workflows.py index 4978822..75bbb12 100755 --- a/scripts/list_failed_workflows.py +++ b/scripts/list_failed_workflows.py @@ -3,7 +3,7 @@ Query failed workflows and write their IDs and error messages to a file. This script: -1. Queries SQLite database for all failed workflows +1. Queries the Postgres database (DATABASE_URL) for all failed workflows 2. Extracts workflow_id, filename, and error_message 3. Optionally queries Temporal for additional error details 4. Writes results to a CSV file @@ -12,7 +12,6 @@ import asyncio import csv import os -import sqlite3 import sys from pathlib import Path @@ -22,8 +21,7 @@ from temporalio.client import Client, WorkflowFailureError -# Database path - try environment variable first, then default -DB_PATH = os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db") +from pipeline import db # Configuration TEMPORAL_HOST = os.environ.get("TEMPORAL_HOST", "localhost:7233") @@ -94,38 +92,24 @@ async def get_temporal_error_details(workflow_id: str, client: Client) -> dict: } -def query_failed_workflows_from_db(db_path: str) -> list: - """Query SQLite database directly for failed workflows.""" - conn = sqlite3.connect(db_path) - conn.row_factory = sqlite3.Row - - try: - cursor = conn.execute(""" - SELECT workflow_id, document_id, filename, error_message, created_at - FROM documents - WHERE stage = 'failed' - ORDER BY created_at DESC - """) - rows = cursor.fetchall() - return [dict(row) for row in rows] - finally: - conn.close() +def query_failed_workflows_from_db() -> list: + """Query the database for failed workflows (including demo and disabled documents).""" + return db.list_documents( + stage="failed", limit=1_000_000, include_disabled=True + ) async def main(): """Main function to query failed workflows and write to file.""" - print(f"Querying failed workflows from SQLite database at {DB_PATH}...") - - # Check if database exists - if not os.path.exists(DB_PATH): - print(f"Error: Database not found at {DB_PATH}") - print("Make sure you're running this script inside the Docker container or") - print("that the database path is correct.") + print(f"Querying failed workflows from the database at {db.database_url().rsplit('@', 1)[-1]}...") + + try: + failed_docs = query_failed_workflows_from_db() + except Exception as e: + print(f"Error: could not query the database: {e}") + print("Check DATABASE_URL, and run inside the Docker network if the database is not published.") sys.exit(1) - # Query SQLite for failed workflows - failed_docs = query_failed_workflows_from_db(DB_PATH) - print(f"Found {len(failed_docs)} failed workflows") # Prepare data for CSV @@ -139,7 +123,7 @@ async def main(): print("Connected to Temporal") except Exception as e: print(f"Warning: Could not connect to Temporal: {e}") - print("Will only use SQLite error messages") + print("Will only use database error messages") temporal_client = None # Process each failed workflow @@ -154,7 +138,7 @@ async def main(): "workflow_id": workflow_id, "document_id": document_id, "filename": filename, - "sqlite_error_message": error_message, + "db_error_message": error_message, "created_at": created_at, } @@ -180,7 +164,7 @@ async def main(): "document_id", "filename", "created_at", - "sqlite_error_message", + "db_error_message", "temporal_status", "temporal_error_message", "temporal_error_type", @@ -196,7 +180,7 @@ async def main(): print(f"✓ Successfully wrote {len(rows)} failed workflows to {OUTPUT_FILE}") print("\nSummary:") print(f" - Total failed workflows: {len(rows)}") - print(f" - With SQLite error messages: {sum(1 for r in rows if r.get('sqlite_error_message'))}") + print(f" - With database error messages: {sum(1 for r in rows if r.get('db_error_message'))}") if temporal_client: print(f" - With Temporal error messages: {sum(1 for r in rows if r.get('temporal_error_message'))}") else: diff --git a/scripts/migrate_sqlite_to_postgres.py b/scripts/migrate_sqlite_to_postgres.py new file mode 100644 index 0000000..366e54b --- /dev/null +++ b/scripts/migrate_sqlite_to_postgres.py @@ -0,0 +1,926 @@ +#!/usr/bin/env python3 +"""One-off migration of the legacy SQLite store into Postgres. + +Run in the maintenance window, after the Postgres migrations are applied and +with the api and worker stopped. The SQLite file is opened read-only and is +never modified, so rolling back is redeploying the previous images on it. + + python scripts/migrate_sqlite_to_postgres.py --dry-run + python scripts/migrate_sqlite_to_postgres.py + +Every migrated row gets a deterministic UUID derived from its legacy key, so a +second run finds the rows already there and changes nothing. A document is +migrated in one transaction. ``--dry-run`` runs everything and rolls it back. + +Mapping (Schema v2): one legacy ``documents`` row becomes one document plus +its main workflow (``temporal_workflow_id`` = the legacy ``workflow_id``); each +``document_jobs`` row becomes a workflow. The legacy ``instance`` becomes a +``instances`` row (``bv`` is ``/global``, others ``/states/``, Keycloak id +filled in by the first login sync) and the instance's ``catalogs`` row for the +document's kind. ``scheme_code`` / ``scheme_name`` / ``scheme_aliases_json`` +(or the ``scheme_catalog_entries`` registry entry for the code) become the +document's ``code`` / ``name`` / ``aliases``; the validity period, uploader and +vector index state move into ``documents.metadata``. Not migrated, and listed in +the report: documents of kind ``document`` (no catalog), demo documents +(``is_demo``), registry entries no document uses (bootstrap-only), and +``document_manifest_entries``. +""" + +import argparse +import json +import os +import re +import sqlite3 +import sys +import uuid +from collections import Counter +from datetime import date, datetime, timezone +from pathlib import Path +from typing import Any, Optional + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import psycopg # noqa: E402 +from psycopg.rows import dict_row # noqa: E402 +from psycopg.types.json import Jsonb # noqa: E402 + +from pipeline import db, migrate # noqa: E402 +from pipeline.models import ( # noqa: E402 + ChunkingRun, + DocumentMetadata, + DocumentSource, + IndexState, + PageMetadata, + PageTranslation, +) +from pipeline.network_constants import KIND_TO_CATALOG_TYPE, catalog_type_for_kind # noqa: E402 + +_NAMESPACE = uuid.UUID("3f2b6c0e-5d1a-4a53-9a77-0c6a5c1d7e10") +_FINGERPRINT_RE = re.compile(r"^[0-9a-f]{32}$") +_CODE_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") +_STAGES = { + "registered", "ocr_processing", "ocr_review", "translation_processing", + "translation_review", "chunking", "chunk_review", "ready_for_ingestion", + "ingesting", "publishing_to_network", "approval_for_prod", "ingesting_prod", + "completed", "failed", +} +_WORKFLOW_TYPES = { + "pipeline", "ocr_only", "ocr_retry", "translation_retry", "chunking_retry", + "reingest", "promote_prod", +} +_WORKFLOW_STATUSES = {"running", "waiting_review", "completed", "failed"} +_REVIEW_STAGES = { + "ocr_review", "translation_review", "chunk_review", "ready_for_ingestion", "approval_for_prod", +} +_ENTITY_TYPES = {"document", "page", "chunk", "setting", "catalog"} + +_LEGACY_SETTING_KEYS = { + "search_method": "searchMethod", + "search_limit": "limit", + "search_alpha": "alpha", + "search_ranking_method": "rankingMethod", + "search_show_highlights": "showHighlights", + "search_ef_search": "efSearch", + "search_index_name": "indexName", + "search_candidate_cap": "candidateCap", + "search_candidate_multiplier": "candidateMultiplier", + "search_max_chunks_per_doc": "maxChunksPerDoc", + "search_use_e5_prefix": "useE5Prefix", + "search_exclude_reference": "excludeReference", + "search_query_expansion_profile": "queryExpansionProfile", + "search_rerank_mode": "rerankMode", + "search_hybrid_rrfk": "hybridRrfK", +} +_BOOL_SETTINGS = {"showHighlights", "useE5Prefix", "excludeReference"} +_INT_SETTINGS = { + "limit", "efSearch", "candidateCap", "candidateMultiplier", "maxChunksPerDoc", "hybridRrfK", +} +_FLOAT_SETTINGS = {"alpha"} + + +class Report: + """Per-table counts: rows read from SQLite and what became of them.""" + + def __init__(self) -> None: + self.tables: dict[str, Counter] = {} + self.skip_reasons: list[dict[str, Any]] = [] + self.notes: list[str] = [] + + def count(self, table: str, outcome: str, n: int = 1) -> None: + self.tables.setdefault(table, Counter())[outcome] += n + + def skip(self, table: str, key: str, reason: str) -> None: + self.count(table, "skipped") + self.skip_reasons.append({"table": table, "key": key, "reason": reason}) + + def mismatches(self) -> list[str]: + problems = [] + for table, counts in sorted(self.tables.items()): + accounted = counts["inserted"] + counts["existing"] + counts["skipped"] + counts["dropped"] + if counts["read"] != accounted: + problems.append(f"{table}: read {counts['read']} but accounted for {accounted}") + return problems + + def as_dict(self) -> dict[str, Any]: + return { + "tables": {t: dict(c) for t, c in sorted(self.tables.items())}, + "skipped": self.skip_reasons, + "notes": self.notes, + "mismatches": self.mismatches(), + } + + def render(self) -> str: + lines = [f"{'table':<28}{'read':>8}{'inserted':>10}{'existing':>10}{'skipped':>9}{'dropped':>9}"] + for table, c in sorted(self.tables.items()): + lines.append( + f"{table:<28}{c['read']:>8}{c['inserted']:>10}{c['existing']:>10}" + f"{c['skipped']:>9}{c['dropped']:>9}" + ) + for item in self.skip_reasons: + lines.append(f" skipped {item['table']} {item['key']}: {item['reason']}") + lines.extend(f" note: {n}" for n in self.notes) + problems = self.mismatches() + lines.append("verification: OK" if not problems else "verification: FAILED") + lines.extend(f" {p}" for p in problems) + return "\n".join(lines) + + +def det_id(kind: str, key: str) -> str: + return str(uuid.uuid5(_NAMESPACE, f"{kind}:{key}")) + + +def parse_ts(value: Any) -> Optional[datetime]: + if not value: + return None + try: + parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def parse_date(value: Any) -> Optional[date]: + if not value: + return None + try: + return date.fromisoformat(str(value)[:10]) + except ValueError: + return None + + +def load_json(text: Any, default: Any) -> Any: + if text in (None, ""): + return default + try: + return json.loads(text) + except (TypeError, ValueError): + return default + + +def truthy(value: Any, default: bool = False) -> bool: + if value is None: + return default + return bool(int(value)) + + +def _default_instance() -> str: + return (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + + +def _workflow_status_for_stage(stage: str) -> str: + if stage == "completed": + return "completed" + if stage == "failed": + return "failed" + return "waiting_review" if stage in _REVIEW_STAGES else "running" + + +def _index_key(index_name: str) -> str: + prod_names = { + n.strip() + for n in ( + os.environ.get("PROD_VECTOR_DB_COLLECTION_NAME"), + os.environ.get("PROD_SCHEME_VECTOR_DB_COLLECTION_NAME"), + ) + if n and n.strip() + } + return "prod" if index_name in prod_names else "dev" + + +def _has_table(sq: sqlite3.Connection, name: str) -> bool: + return sq.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (name,) + ).fetchone() is not None + + +def _rows(sq: sqlite3.Connection, table: str, where: str = "", params: tuple = ()) -> list[sqlite3.Row]: + if not _has_table(sq, table): + return [] + return sq.execute(f"SELECT * FROM {table} {where}", params).fetchall() + + +def _get(row: sqlite3.Row, key: str, default: Any = None) -> Any: + return row[key] if key in row.keys() else default + + +# ----------------------------------------------------------------------------- +# Catalog +# ----------------------------------------------------------------------------- + +def load_catalog_registry(sq, report: Report) -> dict[str, dict[str, Any]]: + """Legacy ``scheme_catalog_entries`` by code. Their name and aliases are merged + into the documents holding the code; see ``settle_catalog_registry``.""" + registry: dict[str, dict[str, Any]] = {} + for row in _rows(sq, "scheme_catalog_entries"): + report.count("scheme_catalog_entries", "read") + code = (row["scheme_code"] or "").strip().lower() + if not _CODE_RE.match(code): + report.skip("scheme_catalog_entries", str(row["scheme_code"]), "code is not a valid slug") + continue + aliases = [a for a in load_json(_get(row, "scheme_aliases_json"), []) if isinstance(a, str) and a] + registry[code] = {"name": row["scheme_name"] or code, "aliases": aliases} + return registry + + +def settle_catalog_registry(registry: dict[str, dict[str, Any]], consumed: set[str], report: Report) -> None: + """Registry entries a migrated document used count as inserted (merged into the + document); the rest have no document and are dropped.""" + unused = [] + for code in sorted(registry): + if code in consumed: + report.count("scheme_catalog_entries", "inserted") + else: + report.count("scheme_catalog_entries", "dropped") + unused.append(code) + if unused: + report.notes.append( + f"scheme_catalog_entries without a document were not migrated: {', '.join(unused)}" + ) + + +def migrate_catalog_version(sq, conn, report: Report) -> None: + rows = _rows(sq, "scheme_catalog_meta") + legacy_version = int(rows[0]["version"] or 0) if rows else 0 + if rows: + report.count("scheme_catalog_meta", "read") + report.count("scheme_catalog_meta", "inserted") + for n in range(1, legacy_version + 1): + conn.execute( + """ + INSERT INTO audit_logs (id, action_type, entity_type, metadata) + VALUES (%s::uuid, %s, 'catalog', %s) ON CONFLICT (id) DO NOTHING + """, + (det_id("catalog-bump", str(n)), db.CATALOG_VERSION_ACTION, + Jsonb({"reason": "sqlite_migration:carried_over"})), + ) + conn.execute( + """ + INSERT INTO audit_logs (id, action_type, entity_type, metadata) + VALUES (%s::uuid, %s, 'catalog', %s) ON CONFLICT (id) DO NOTHING + """, + (det_id("catalog-bump", "migration"), db.CATALOG_VERSION_ACTION, + Jsonb({"reason": "sqlite_migration"})), + ) + + +def _instance_of(doc: sqlite3.Row) -> str: + return (_get(doc, "instance") or _default_instance()).strip().lower() or _default_instance() + + +def _instance_for(conn, instance: str) -> str: + """The ``instances`` row of a tenant code, created with a deterministic id.""" + path = db._tenant_path(instance) + row = conn.execute( + """ + INSERT INTO instances (id, name, path) VALUES (%s::uuid, %s, %s) + ON CONFLICT (path) DO UPDATE SET name = instances.name RETURNING id + """, + (det_id("instance", path), db._tenant_code_from_path(path), path), + ).fetchone() + return str(row["id"]) + + +def _catalog_for(conn, instance_id: str, kind: str) -> str: + """The ``(instance, kind)`` catalog, created with a deterministic id.""" + catalog_type = catalog_type_for_kind(kind) + row = conn.execute( + """ + INSERT INTO catalogs (id, instance_id, type, name) + SELECT %s::uuid, g.id, %s, g.name || ' ' || %s FROM instances g WHERE g.id = %s::uuid + ON CONFLICT (instance_id, type) DO UPDATE SET type = EXCLUDED.type RETURNING id + """, + (det_id("catalog", f"{instance_id}:{catalog_type}"), catalog_type, kind, instance_id), + ).fetchone() + return str(row["id"]) + + +def _slug(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", (text or "").lower()).strip("-")[:40].strip("-") + + +def _unique_code(conn, catalog_id: str, base: str) -> str: + code, n = base, 1 + while conn.execute( + "SELECT 1 FROM documents WHERE catalog_id = %s::uuid AND code = %s", (catalog_id, code) + ).fetchone(): + n += 1 + code = f"{base}-{n}" + return code + + +# ----------------------------------------------------------------------------- +# Documents +# ----------------------------------------------------------------------------- + +def _user_for(conn, user_id, username, email, roles) -> Optional[str]: + if not (user_id or "").strip(): + return None + return db.upsert_user(user_id, username, email, roles, conn=conn) + + +def _build_index_status(sq, workflow_id: str) -> dict[str, dict]: + entries: dict[str, dict] = {} + for row in _rows(sq, "document_index_status", "WHERE workflow_id = ?", (workflow_id,)): + key = _index_key(row["index_name"]) + details = load_json(row["details_json"], {}) + state = IndexState( + status=row["status"] if row["status"] in ("unknown", "indexed", "missing") else "unknown", + index_name=row["index_name"], + doc_id=_get(row, "doc_id") or _get(row, "marqo_doc_id"), + chunk_count_indexed=max(int(row["chunk_count_indexed"] or 0), 0), + last_indexed_at=row["last_indexed_at"], + last_verified_at=row["last_verified_at"], + schema_version=row["schema_version"], + details=details if isinstance(details, dict) else {}, + ).model_dump() + current = entries.get(key) + if current is None or (state["last_indexed_at"] or "") >= (current["last_indexed_at"] or ""): + entries[key] = state + return entries + + +def _build_workflows( + sq, doc: sqlite3.Row, started_by: Optional[str], report: Report +) -> tuple[list[dict], dict[Any, str]]: + workflow_id = doc["workflow_id"] + stage = doc["stage"] if doc["stage"] in _STAGES else "registered" + created_at = parse_ts(doc["created_at"]) or datetime.now(timezone.utc) + stamps = { + key: value + for key, value in ( + ("ocr", _get(doc, "ocr_completed_at")), + ("translation", _get(doc, "translation_completed_at")), + ("chunks", _get(doc, "chunks_completed_at")), + ) + if value + } + + workflows: list[dict] = [] + job_ids: dict[Any, str] = {} + has_root = False + for job in _rows(sq, "document_jobs", "WHERE workflow_id = ? ORDER BY started_at, id", (workflow_id,)): + job_type = job["job_type"] + if job_type not in _WORKFLOW_TYPES: + report.skip("document_jobs", f"{workflow_id}#{job['id']}", f"unknown job_type {job_type!r}") + continue + temporal_id = job["temporal_workflow_id"] or workflow_id + is_root = job_type in ("pipeline", "ocr_only") and temporal_id == workflow_id and not has_root + has_root = has_root or is_root + wf_id = det_id("workflow", f"{workflow_id}:job:{job['id']}") + job_ids[job["id"]] = wf_id + status = job["status"] if job["status"] in _WORKFLOW_STATUSES else "failed" + job_stage = job["current_stage"] if job["current_stage"] in _STAGES else stage + config = load_json(job["config_json"], {}) + config = config if isinstance(config, dict) else {} + if job["temporal_run_id"]: + config["temporal_run_id"] = job["temporal_run_id"] + workflows.append({ + "id": wf_id, "type": job_type, "temporal_id": temporal_id, "status": status, + "stage": job_stage, "stamps": stamps if is_root else {}, + "stop_after_ocr": truthy(_get(doc, "stop_after_ocr")) and is_root, + "error_message": job["error_message"], "config": config, + "started_by": started_by if is_root else None, + "started_at": parse_ts(job["started_at"]) or created_at, + "completed_at": parse_ts(job["completed_at"]), "job_id": job["id"], + "inserted_as": "document_jobs", + }) + + if not has_root: + workflows.append({ + "id": det_id("workflow", f"{workflow_id}:root"), + "type": "ocr_only" if truthy(_get(doc, "stop_after_ocr")) else "pipeline", + "temporal_id": workflow_id, "status": _workflow_status_for_stage(stage), "stage": stage, + "stamps": stamps, "stop_after_ocr": truthy(_get(doc, "stop_after_ocr")), + "error_message": doc["error_message"], "config": {}, "started_by": started_by, + "started_at": created_at, + "completed_at": parse_ts(_get(doc, "ingested_at")) if stage == "completed" else None, + "job_id": None, "inserted_as": None, + }) + + requested_at = parse_ts(_get(doc, "prod_ready_requested_at")) + if requested_at and stage not in ("completed", "ingesting_prod"): + workflows.append({ + "id": det_id("workflow", f"{workflow_id}:prod-request"), "type": "promote_prod", + "temporal_id": f"{workflow_id}-prod-request", "status": "waiting_review", + "stage": "approval_for_prod", "stamps": {}, "stop_after_ocr": False, + "error_message": None, "config": {}, "started_by": None, "started_at": requested_at, + "completed_at": None, "job_id": None, "inserted_as": None, + "requested_by": ( + _get(doc, "prod_ready_requested_by_user_id"), + _get(doc, "prod_ready_requested_by_username"), + ), + }) + + workflows.sort(key=lambda w: w["started_at"]) + latest = workflows[-1] + latest["stage"] = stage + latest["error_message"] = doc["error_message"] + return workflows, job_ids + + +def _skip_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workflow: dict[str, list], reason: str) -> None: + """Leave a document behind. Its history rows are counted as skipped; its audit + entries are kept, tied to the tenant's instance but to no document.""" + workflow_id = doc["workflow_id"] + report.skip("documents", workflow_id, reason) + for table in ("document_jobs", "document_artifacts", "pages", "chunks"): + report.count(table, "skipped", len(_rows(sq, table, "WHERE workflow_id = ?", (workflow_id,)))) + for audit in audit_by_workflow.pop(workflow_id, []): + insert_audit(conn, audit, None, _instance_of(doc), report) + + +def migrate_document( + sq, conn, doc: sqlite3.Row, report: Report, audit_by_workflow: dict[str, list], + registry: Optional[dict[str, dict[str, Any]]] = None, consumed: Optional[set[str]] = None, +) -> bool: + registry = registry if registry is not None else {} + consumed = consumed if consumed is not None else set() + workflow_id = doc["workflow_id"] + doc_uuid = det_id("document", workflow_id) + exists = conn.execute("SELECT 1 FROM documents WHERE id = %s::uuid", (doc_uuid,)).fetchone() + if exists: + report.count("documents", "existing") + for table, label in ( + ("document_jobs", "document_jobs"), ("document_artifacts", "document_artifacts"), + ("pages", "pages"), ("chunks", "chunks"), + ): + report.count(label, "existing", len(_rows(sq, table, "WHERE workflow_id = ?", (workflow_id,)))) + return False + + kind = (_get(doc, "document_kind") or "document").strip().lower() + if truthy(_get(doc, "is_demo")): + _skip_document(sq, conn, doc, report, audit_by_workflow, "demo document (is_demo)") + return False + if kind not in KIND_TO_CATALOG_TYPE: + _skip_document(sq, conn, doc, report, audit_by_workflow, f"document_kind {kind!r} has no catalog") + return False + + fingerprint = next( + ( + c for c in ( + _get(doc, "source_file_fingerprint"), _get(doc, "canonical_document_id"), doc["document_id"] + ) if c and _FINGERPRINT_RE.match(c) + ), + None, + ) + if fingerprint is None: + import hashlib + + fingerprint = hashlib.md5((doc["document_id"] or workflow_id).encode()).hexdigest() + report.notes.append(f"{workflow_id}: no hex fingerprint, derived md5 of document_id") + + chunk_rows = _rows(sq, "chunks", "WHERE workflow_id = ? ORDER BY chunk_number", (workflow_id,)) + + chunking = None + if chunk_rows: + first = chunk_rows[0] + chunking = ChunkingRun( + run_number=max(max(int(r["chunk_version"] or 1) for r in chunk_rows), 1), + provider=_get(first, "chunking_provider"), + model=_get(first, "chunking_model"), + config=load_json(_get(first, "chunking_config_json"), {}), + ) + + valid_from, valid_to = parse_date(_get(doc, "valid_from")), parse_date(_get(doc, "valid_to")) + if valid_from and valid_to and valid_from > valid_to: + report.notes.append(f"{workflow_id}: valid_from after valid_to, valid_to cleared") + valid_to = None + + uploader = _user_for( + conn, _get(doc, "uploaded_by_user_id"), _get(doc, "uploaded_by_username"), + _get(doc, "uploaded_by_email"), _get(doc, "uploaded_by_roles"), + ) + + metadata = DocumentMetadata( + valid_from=valid_from.isoformat() if valid_from else None, + valid_to=valid_to.isoformat() if valid_to else None, + uploaded_by_user_id=uploader, + index_status=_build_index_status(sq, workflow_id), + catalog_visible=truthy(_get(doc, "catalog_visible"), True), + network_visible=truthy(_get(doc, "network_visible"), True), + reindex_required=truthy(_get(doc, "reindex_required")), + reindex_reason=_get(doc, "reindex_reason") or None, + chunking=chunking, + ).model_dump(exclude_none=True) + source = DocumentSource( + fingerprint=fingerprint, + original_filename=_get(doc, "source_filename") or doc["filename"], + storage_uri=doc["filepath"], + source_type=_get(doc, "source_type"), + canonical_input_type=_get(doc, "canonical_input_type"), + ).model_dump(exclude_none=True) + + catalog_id = _catalog_for(conn, _instance_for(conn, _instance_of(doc)), kind) + code = (_get(doc, "scheme_code") or "").strip().lower() + entry = registry.get(code) if kind == "scheme" and code else None + if entry is not None: + consumed.add(code) + name = (entry or {}).get("name") or _get(doc, "scheme_name") or _get(doc, "display_name") or None + aliases = (entry or {}).get("aliases") or [ + a for a in load_json(_get(doc, "scheme_aliases_json"), []) if isinstance(a, str) and a + ] + if code and not _CODE_RE.match(code): + report.notes.append(f"{workflow_id}: scheme_code {code!r} is not a valid slug, code derived") + code = "" + if not code: + code = _slug(name or "") or _slug(Path(doc["filename"] or "").stem) + if not _CODE_RE.match(code): + code = f"doc-{fingerprint[:8]}" + report.notes.append(f"{workflow_id}: no usable scheme_code, derived {code!r}") + unique = _unique_code(conn, catalog_id, code) + if unique != code: + report.notes.append(f"{workflow_id}: code {code!r} already used in its catalog, stored as {unique!r}") + code = unique + + created_at = parse_ts(doc["created_at"]) or datetime.now(timezone.utc) + updated_at = parse_ts(doc["updated_at"]) or created_at + conn.execute( + """ + INSERT INTO documents (id, catalog_id, code, name, aliases, source, is_disabled, metadata, + created_at, updated_at) + VALUES (%s::uuid, %s::uuid, %s, %s, %s, %s, %s, %s, %s, %s) + """, + ( + doc_uuid, catalog_id, code, name or code, aliases, Jsonb(source), + truthy(_get(doc, "is_disabled")), Jsonb(metadata), created_at, updated_at, + ), + ) + report.count("documents", "inserted") + + workflows, job_ids = _build_workflows(sq, doc, uploader, report) + root = next((w for w in workflows if w["type"] in ("pipeline", "ocr_only")), workflows[0]) + for w in workflows: + started_by = w["started_by"] + if w.get("requested_by"): + started_by = _user_for(conn, w["requested_by"][0], w["requested_by"][1], None, None) + conn.execute( + """ + INSERT INTO workflows (id, document_id, workflow_type, temporal_workflow_id, status, + current_stage, stage_timestamps, stop_after_ocr, error_message, + config, started_by_user_id, started_at, completed_at, updated_at) + VALUES (%s::uuid, %s::uuid, %s, %s, %s, + %s, %s, %s, %s, %s, %s::uuid, %s, %s, %s) + """, + ( + w["id"], doc_uuid, w["type"], w["temporal_id"], w["status"], w["stage"], + Jsonb(w["stamps"]), w["stop_after_ocr"], w["error_message"], Jsonb(w["config"]), + started_by, w["started_at"], w["completed_at"], w["completed_at"] or w["started_at"], + ), + ) + if w["inserted_as"]: + report.count("document_jobs", "inserted") + + for art in _rows(sq, "document_artifacts", "WHERE workflow_id = ? ORDER BY id", (workflow_id,)): + size = art["size_bytes"] + art_metadata = load_json(art["metadata_json"], {}) + conn.execute( + """ + INSERT INTO workflow_artifacts (id, workflow_id, artifact_type, stage, storage_uri, + mime_type, filename, size_bytes, metadata, created_at) + VALUES (%s::uuid, %s::uuid, %s, %s, %s, %s, %s, %s, %s, %s) + """, + ( + det_id("artifact", f"{workflow_id}:{art['id']}"), + job_ids.get(art["job_id"], root["id"]), + art["artifact_type"], art["stage"] if art["stage"] in _STAGES else None, + art["storage_uri"], art["mime_type"], art["filename"], + size if size is not None and size >= 0 else None, + Jsonb(art_metadata if isinstance(art_metadata, dict) else {}), + parse_ts(art["created_at"]) or created_at, + ), + ) + report.count("document_artifacts", "inserted") + + for page in _rows(sq, "pages", "WHERE workflow_id = ? ORDER BY page_number", (workflow_id,)): + if int(page["page_number"] or 0) < 1: + report.skip("pages", f"{workflow_id}#{page['page_number']}", "page_number < 1") + continue + translations = {} + if any( + _get(page, k) not in (None, "") + for k in ("translated_markdown", "edited_translation", "translation_notes") + ) or truthy(_get(page, "translation_reviewed")): + translations[_get(page, "translation_target_language") or "en"] = PageTranslation( + translated_markdown=_get(page, "translated_markdown"), + edited_translation=_get(page, "edited_translation"), + is_reviewed=truthy(_get(page, "translation_reviewed")), + reviewer_notes=_get(page, "translation_notes"), + provider=_get(page, "translation_provider"), + model=_get(page, "translation_model"), + translated_at=_get(page, "translated_at"), + ).model_dump() + page_meta = PageMetadata( + is_reviewed=truthy(_get(page, "is_reviewed")), + reviewer_notes=_get(page, "reviewer_notes"), + detected_language=_get(page, "detected_language"), + ).model_dump(exclude_none=True) + conn.execute( + """ + INSERT INTO pages (id, document_id, page_number, original_markdown, edited_markdown, + translations, metadata) + VALUES (%s::uuid, %s::uuid, %s, %s, %s, %s, %s) + """, + ( + det_id("page", f"{workflow_id}:{page['page_number']}"), doc_uuid, page["page_number"], + page["original_markdown"], page["edited_markdown"], Jsonb(translations), + Jsonb(page_meta), + ), + ) + report.count("pages", "inserted") + + for chunk in chunk_rows: + if int(chunk["chunk_number"] or 0) < 1: + report.skip("chunks", f"{workflow_id}#{chunk['chunk_number']}", "chunk_number < 1") + continue + page_start = max(int(chunk["page_start"] or 1), 1) + spans = load_json(_get(chunk, "source_spans_json"), []) + conn.execute( + """ + INSERT INTO chunks (id, document_id, chunk_number, original_text, edited_text, + token_count, page_start, page_end, source_spans, section_title, + content_type, is_reference, is_reviewed, is_excluded, reviewer_notes) + VALUES (%s::uuid, %s::uuid, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + """, + ( + det_id("chunk", f"{workflow_id}:{chunk['chunk_number']}"), doc_uuid, + chunk["chunk_number"], chunk["original_text"] or "", chunk["edited_text"], + max(int(chunk["token_count"] or 0), 0), page_start, + max(int(chunk["page_end"] or page_start), page_start), + Jsonb(spans if isinstance(spans, list) else []), chunk["section_title"], + chunk["content_type"], truthy(_get(chunk, "is_reference")), + truthy(_get(chunk, "is_reviewed")), truthy(_get(chunk, "is_excluded")), + chunk["reviewer_notes"], + ), + ) + report.count("chunks", "inserted") + + for audit in audit_by_workflow.pop(workflow_id, []): + insert_audit(conn, audit, doc_uuid, source_instance=_instance_of(doc), report=report) + return True + + +# ----------------------------------------------------------------------------- +# Audit, settings, OTPs, webhooks +# ----------------------------------------------------------------------------- + +def _legacy_audit_field(field_name: Optional[str], entity: Optional[str]) -> Optional[str]: + if entity == "setting" and field_name in _LEGACY_SETTING_KEYS: + flat = _LEGACY_SETTING_KEYS[field_name] + return f"{db._SETTING_GROUPS[flat][0]}.{flat}" + return field_name + + +def insert_audit(conn, audit: sqlite3.Row, doc_uuid: Optional[str], source_instance: Optional[str], report: Report) -> None: + entity = audit["entity_type"] if audit["entity_type"] in _ENTITY_TYPES else None + workflow_id = audit["workflow_id"] + if audit["document_id"] == "__settings__": + entity, workflow_id = "setting", None + elif workflow_id == "__catalog__": + entity, workflow_id = "catalog", None + metadata = load_json(audit["metadata"], None) + if not isinstance(metadata, dict): + metadata = {"raw": audit["metadata"]} if audit["metadata"] else {} + roles = [r.strip() for r in (_get(audit, "actor_roles") or "").split(",") if r.strip()] + actor_present = any((_get(audit, "actor_user_id"), _get(audit, "actor_username"), _get(audit, "actor_email"), roles)) + actor_uuid = _user_for( + conn, _get(audit, "actor_user_id"), _get(audit, "actor_username"), _get(audit, "actor_email"), roles + ) + inserted = conn.execute( + """ + INSERT INTO audit_logs (id, document_id, workflow_id, instance_id, action_type, entity_type, + entity_number, field_name, old_value, new_value, metadata, + actor_user_id, actor, created_at) + VALUES (%s::uuid, %s::uuid, %s, %s::uuid, %s, %s, %s, %s, %s, %s, %s, + %s::uuid, %s, %s) + ON CONFLICT (id) DO NOTHING RETURNING id + """, + ( + det_id("audit", str(audit["id"])), doc_uuid, workflow_id, + _instance_for(conn, source_instance.strip().lower()) if (source_instance or "").strip() else None, + audit["action_type"], entity, audit["entity_id"], + _legacy_audit_field(audit["field_name"], entity), audit["old_value"], audit["new_value"], + Jsonb(metadata), actor_uuid, + Jsonb({ + "username": _get(audit, "actor_username") or None, + "email": _get(audit, "actor_email") or None, + "roles": roles, + }) if actor_present else Jsonb({}), + parse_ts(audit["timestamp"]) or datetime.now(timezone.utc), + ), + ).fetchone() + report.count("audit_logs", "inserted" if inserted else "existing") + + +def migrate_settings(sq, conn, report: Report) -> None: + rows = _rows(sq, "settings") + for _ in rows: + report.count("settings", "read") + current = conn.execute( + "SELECT retrieval, candidates, results, query FROM search_settings WHERE singleton" + ).fetchone() + if current is None: + conn.execute("INSERT INTO search_settings DEFAULT VALUES") + current = {"retrieval": {}, "candidates": {}, "results": {}, "query": {}} + if any(current[g] for g in ("retrieval", "candidates", "results", "query")): + for _ in rows: + report.count("settings", "existing") + report.notes.append("search_settings already configured in Postgres, legacy settings not applied") + return + + groups: dict[str, dict[str, Any]] = {} + for row in rows: + flat = _LEGACY_SETTING_KEYS.get(row["key"]) + if flat is None: + report.skip("settings", row["key"], "unknown setting key") + continue + raw = row["value"] + try: + value: Any = ( + raw == "true" if flat in _BOOL_SETTINGS + else int(raw) if flat in _INT_SETTINGS + else float(raw) if flat in _FLOAT_SETTINGS + else raw + ) + except ValueError: + report.skip("settings", row["key"], f"unparseable value {raw!r}") + continue + group, model = db._SETTING_GROUPS[flat] + try: + model.model_validate({flat: value}) + except Exception as exc: + report.skip("settings", row["key"], f"invalid value: {exc}") + continue + groups.setdefault(group, {})[flat] = value + report.count("settings", "inserted") + for group, values in groups.items(): + conn.execute( + f"UPDATE search_settings SET {group} = {group} || %s::jsonb WHERE singleton", (Jsonb(values),) + ) + + +def migrate_email_otps(sq, conn, report: Report) -> None: + now = datetime.now(timezone.utc) + for row in _rows(sq, "email_otps"): + report.count("email_otps", "read") + expires = parse_ts(row["expires_at"]) + if expires is None or expires <= now: + report.count("email_otps", "dropped") + continue + inserted = conn.execute( + """ + INSERT INTO email_otps (email, code_hash, expires_at, attempts, last_sent_at) + VALUES (%s, %s, %s, %s, %s) ON CONFLICT (email) DO NOTHING RETURNING id + """, + (row["email"].strip().lower(), row["code_hash"], expires, max(int(row["attempts"] or 0), 0), + parse_ts(row["last_sent_at"]) or now), + ).fetchone() + report.count("email_otps", "inserted" if inserted else "existing") + + +def migrate_webhook_records(sq, conn, report: Report) -> None: + for row in _rows(sq, "webhook_records"): + report.count("webhook_records", "read") + payload = load_json(row["payload"], None) + if not isinstance(payload, dict): + report.skip("webhook_records", f"{row['source']}/{row['external_id']}", "payload is not a JSON object") + continue + try: + record_id = str(uuid.UUID(row["id"])) + except (ValueError, TypeError): + record_id = det_id("webhook", row["id"] or f"{row['source']}:{row['external_id']}") + published = row["is_published"] + inserted = conn.execute( + """ + INSERT INTO webhook_records (id, source, external_id, title, slug, status, is_published, + category, payload, payload_hash, created_at, updated_at) + VALUES (%s::uuid, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (source, external_id) DO NOTHING RETURNING id + """, + ( + record_id, row["source"], row["external_id"], row["title"], row["slug"], row["status"], + None if published is None else bool(published), row["category"], Jsonb(payload), + row["payload_hash"], parse_ts(row["created_at"]) or datetime.now(timezone.utc), + parse_ts(row["updated_at"]) or datetime.now(timezone.utc), + ), + ).fetchone() + report.count("webhook_records", "inserted" if inserted else "existing") + + +# ----------------------------------------------------------------------------- +# Driver +# ----------------------------------------------------------------------------- + +def run_migration(sqlite_path: str, database_url: str, dry_run: bool = False) -> Report: + report = Report() + sq = sqlite3.connect(f"file:{sqlite_path}?mode=ro", uri=True) + sq.row_factory = sqlite3.Row + try: + if not dry_run: + migrate.apply_migrations(database_url) + elif not migrate.is_current(database_url): + raise SystemExit("Postgres migrations are not applied; apply them before a dry run.") + + with psycopg.connect(database_url, autocommit=not dry_run, row_factory=dict_row) as conn: + conn.execute("SET timezone = 'UTC'") + documents = _rows(sq, "documents", "ORDER BY created_at, workflow_id") + known_workflows = {d["workflow_id"] for d in documents} + for table in ("document_jobs", "document_artifacts", "pages", "chunks"): + if _has_table(sq, table): + total = sq.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] + stray = sq.execute( + f"SELECT COUNT(*) FROM {table} WHERE workflow_id NOT IN " + "(SELECT workflow_id FROM documents)" + ).fetchone()[0] + report.count(table, "read", total) + if stray: + report.count(table, "skipped", stray) + report.notes.append(f"{table}: {stray} rows belong to no document and were not migrated") + + audit_by_workflow: dict[str, list] = {} + orphan_audit: list = [] + for audit in _rows(sq, "audit_logs", "ORDER BY id"): + report.count("audit_logs", "read") + if audit["workflow_id"] in known_workflows and audit["document_id"] != "__settings__": + audit_by_workflow.setdefault(audit["workflow_id"], []).append(audit) + else: + orphan_audit.append(audit) + + registry = load_catalog_registry(sq, report) + consumed: set[str] = set() + + for doc in documents: + report.count("documents", "read") + with conn.transaction(): + migrate_document(sq, conn, doc, report, audit_by_workflow, registry, consumed) + settle_catalog_registry(registry, consumed, report) + + skipped = Counter(i["reason"] for i in report.skip_reasons if i["table"] == "documents") + for reason, n in sorted(skipped.items()): + report.notes.append(f"documents not migrated: {n} x {reason}") + + with conn.transaction(): + for audits in audit_by_workflow.values(): + orphan_audit.extend(audits) + for audit in orphan_audit: + insert_audit(conn, audit, None, None, report) + migrate_settings(sq, conn, report) + migrate_email_otps(sq, conn, report) + migrate_webhook_records(sq, conn, report) + migrate_catalog_version(sq, conn, report) + + manifest = _rows(sq, "document_manifest_entries") + if manifest: + report.count("document_manifest_entries", "read", len(manifest)) + report.count("document_manifest_entries", "dropped", len(manifest)) + report.notes.append("document_manifest_entries is not part of the schema and was not migrated") + + if dry_run: + conn.rollback() + finally: + sq.close() + return report + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--sqlite-path", default=os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db")) + parser.add_argument("--database-url", default=os.environ.get("DATABASE_URL", db.DEFAULT_DATABASE_URL)) + parser.add_argument("--dry-run", action="store_true", help="run everything, then roll back") + parser.add_argument("--report", help="write the report as JSON to this path") + args = parser.parse_args() + + if not os.path.exists(args.sqlite_path): + print(f"SQLite database not found at {args.sqlite_path}", file=sys.stderr) + return 2 + + report = run_migration(args.sqlite_path, args.database_url, dry_run=args.dry_run) + print(("DRY RUN - nothing was written\n" if args.dry_run else "") + report.render()) + if args.report: + Path(args.report).write_text(json.dumps(report.as_dict(), indent=2, default=str)) + return 1 if report.mismatches() else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/start_manifest_translation_batch.py b/scripts/start_manifest_translation_batch.py deleted file mode 100644 index 2c9c95c..0000000 --- a/scripts/start_manifest_translation_batch.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python3 -"""Start translation-only workflows for manifest-backed Gujarati/mixed documents in OCR review.""" - -from __future__ import annotations - -import argparse -import json -import os -import sqlite3 -import sys -import time -from datetime import datetime -from pathlib import Path - -from temporalio.client import Client - -PROJECT_ROOT = Path(__file__).resolve().parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -from pipeline import db # noqa: E402 -from pipeline.workflows import TranslationOnlyWorkflow # noqa: E402 - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--temporal-host", default=os.environ.get("TEMPORAL_HOST", "localhost:7233")) - parser.add_argument("--task-queue", default=os.environ.get("TASK_QUEUE", "ocr-pipeline")) - parser.add_argument("--db-path", default=os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db")) - parser.add_argument("--limit", type=int, default=0) - parser.add_argument( - "--selection", - choices=["manifest_gujarati", "all_ocr_review"], - default="manifest_gujarati", - help="Choose whether to start only manifest rows marked Gujarati or any authoritative OCR-review docs.", - ) - parser.add_argument("--report-path", default="/tmp/start_manifest_translation_batch_report.json") - return parser.parse_args() - - -def eligible_documents(db_path: str, selection: str, limit: int = 0) -> list[dict]: - conn = sqlite3.connect(db_path) - conn.row_factory = sqlite3.Row - if selection == "all_ocr_review": - query = """ - SELECT d.workflow_id, d.document_id, d.filename, d.source_manifest_name, m.doc_language - FROM documents d - LEFT JOIN document_manifest_entries m - ON m.manifest_filename = d.source_manifest_name - WHERE d.source_manifest_name IS NOT NULL - AND d.stage = 'ocr_review' - AND d.translation_completed_at IS NULL - ORDER BY d.updated_at DESC, d.workflow_id - """ - else: - query = """ - SELECT d.workflow_id, d.document_id, d.filename, d.source_manifest_name, m.doc_language - FROM documents d - LEFT JOIN document_manifest_entries m - ON m.manifest_filename = d.source_manifest_name - WHERE d.source_manifest_name IS NOT NULL - AND d.stage = 'ocr_review' - AND d.translation_completed_at IS NULL - AND LOWER(COALESCE(m.doc_language, '')) LIKE '%gujarati%' - ORDER BY d.updated_at DESC, d.workflow_id - """ - rows = conn.execute(query).fetchall() - docs = [dict(r) for r in rows] - return docs[:limit] if limit > 0 else docs - - -async def main() -> int: - args = parse_args() - db.init_db() - client = await Client.connect(args.temporal_host) - - docs = eligible_documents(args.db_path, args.selection, args.limit) - report: dict[str, object] = { - "started_at": datetime.utcnow().isoformat(), - "temporal_host": args.temporal_host, - "task_queue": args.task_queue, - "selection": args.selection, - "translation_provider": os.environ.get("TRANSLATION_PROVIDER", "gemma_vllm"), - "translation_model": os.environ.get("TRANSLATION_MODEL", "gemma-4"), - "eligible_documents": len(docs), - "submitted": 0, - "skipped": 0, - "failed": [], - } - - for doc in docs: - temporal_workflow_id = f"{doc['workflow_id']}-translation-{int(time.time())}" - try: - await client.start_workflow( - TranslationOnlyWorkflow.run, - args=[doc["workflow_id"], doc["document_id"], doc["filename"]], - id=temporal_workflow_id, - task_queue=args.task_queue, - ) - job_id = db.create_document_job( - workflow_id=doc["workflow_id"], - job_type="translation_only", - temporal_workflow_id=temporal_workflow_id, - status="running", - current_stage="translation_processing", - config={ - "translation_provider": os.environ.get("TRANSLATION_PROVIDER", "gemma_vllm"), - "translation_model": os.environ.get("TRANSLATION_MODEL", "gemma-4"), - "source": "manifest_ocr_review", - }, - ) - db.update_document_fields(doc["workflow_id"], latest_job_id=job_id) - report["submitted"] = int(report["submitted"]) + 1 - except Exception as exc: - report["failed"].append( - { - "workflow_id": doc["workflow_id"], - "filename": doc["filename"], - "error": str(exc), - } - ) - - report["finished_at"] = datetime.utcnow().isoformat() - Path(args.report_path).write_text(json.dumps(report, indent=2), encoding="utf-8") - print(json.dumps(report, indent=2)) - return 1 if report["failed"] else 0 - - -if __name__ == "__main__": - raise SystemExit(__import__("asyncio").run(main())) diff --git a/scripts/start_ocr_only_batch.py b/scripts/start_ocr_only_batch.py index 70682c0..234fa61 100644 --- a/scripts/start_ocr_only_batch.py +++ b/scripts/start_ocr_only_batch.py @@ -4,14 +4,16 @@ import argparse import hashlib import json -import os -import sqlite3 import sys import urllib.error import urllib.parse import urllib.request from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from pipeline import db # noqa: E402 + ALLOWED_EXTENSIONS = { ".pdf", ".doc", ".docx", ".ppt", ".pptx", ".xls", ".xlsx", ".csv", @@ -31,35 +33,23 @@ def get_workflow_id(filepath: str) -> str: return f"doc-{hashlib.md5(filepath.encode()).hexdigest()[:12]}" -def already_has_ocr(conn: sqlite3.Connection, filepath: str, fingerprint: str) -> bool: - workflow_id = get_workflow_id(filepath) - row = conn.execute( - """ - SELECT workflow_id, canonical_document_id, ocr_completed_at, stage - FROM documents - WHERE workflow_id = ? - OR canonical_document_id = ? - ORDER BY updated_at DESC, created_at DESC - LIMIT 1 - """, - (workflow_id, fingerprint), - ).fetchone() - if not row: +def already_has_ocr(filepath: str, fingerprint: str) -> bool: + doc = db.get_document(get_workflow_id(filepath)) or db.find_document_by_doc_identifier(fingerprint) + if not doc: return False - if row["ocr_completed_at"]: + if doc["ocr_completed_at"]: return True - if row["stage"] in {"ocr_review", "translation_processing", "translation_review", "chunking", + if doc["stage"] in {"ocr_review", "translation_processing", "translation_review", "chunking", "chunk_review", "ready_for_ingestion", "ingesting", "completed"}: return True - page_row = conn.execute( - "SELECT COUNT(*) AS cnt FROM pages WHERE workflow_id = ?", - (row["workflow_id"],), - ).fetchone() - return bool(page_row and page_row["cnt"] > 0) + return bool(db.get_saved_page_numbers(doc["workflow_id"])) -def post_document(api_base: str, filepath: str) -> dict: - query = urllib.parse.urlencode({"stop_after_ocr": "true"}) +def post_document(api_base: str, filepath: str, kind: str, instance: str = "") -> dict: + params = {"stop_after_ocr": "true", "kind": kind} + if instance: + params["instance"] = instance + query = urllib.parse.urlencode(params) url = f"{api_base.rstrip('/')}/documents?{query}" payload = json.dumps({"filepath": filepath}).encode("utf-8") req = urllib.request.Request( @@ -76,8 +66,9 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("directory", help="Directory containing source docs") parser.add_argument("--api-base", default="http://localhost:8001", help="Docs-pipeline API base URL") - parser.add_argument("--db-path", default=os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db")) parser.add_argument("--limit", type=int, default=0, help="Optional max files to submit") + parser.add_argument("--kind", choices=["scheme", "advisory"], default="scheme", help="Document kind (catalog)") + parser.add_argument("--instance", default="", help="Tenant instance code (optional; defaults to your instance)") args = parser.parse_args() directory = Path(args.directory) @@ -85,9 +76,6 @@ def main() -> int: print(f"Directory not found: {directory}", file=sys.stderr) return 1 - conn = sqlite3.connect(args.db_path) - conn.row_factory = sqlite3.Row - files = sorted( p for p in directory.iterdir() if p.is_file() and p.suffix.lower() in ALLOWED_EXTENSIONS @@ -101,12 +89,12 @@ def main() -> int: for path in files: fingerprint = compute_fingerprint(path) - if already_has_ocr(conn, str(path), fingerprint): + if already_has_ocr(str(path), fingerprint): skipped += 1 print(f"SKIP {path.name}: OCR already present") continue try: - result = post_document(args.api_base, str(path)) + result = post_document(args.api_base, str(path), args.kind, args.instance) submitted += 1 print(f"SUBMIT {path.name}: {result.get('workflow_id')}") except urllib.error.HTTPError as exc: diff --git a/scripts/test_pipeline_e2e.py b/scripts/test_pipeline_e2e.py index 5c77f86..eda4ff7 100644 --- a/scripts/test_pipeline_e2e.py +++ b/scripts/test_pipeline_e2e.py @@ -37,10 +37,17 @@ def check_health(name: str, url: str) -> bool: return False -def upload_pdf(api: str, pdf_path: Path, *, auto_approve: bool, stop_after_ocr: bool) -> str: +def upload_pdf( + api: str, pdf_path: Path, *, auto_approve: bool, stop_after_ocr: bool, kind: str, instance: str = "" +) -> str: import subprocess - query = f"auto_approve={'true' if auto_approve else 'false'}&stop_after_ocr={'true' if stop_after_ocr else 'false'}" + query = ( + f"auto_approve={'true' if auto_approve else 'false'}" + f"&stop_after_ocr={'true' if stop_after_ocr else 'false'}&kind={kind}" + ) + if instance: + query += f"&instance={instance}" url = f"{api.rstrip('/')}/upload?{query}" result = subprocess.run( ["curl", "-sf", "-X", "POST", url, "-F", f"file=@{pdf_path}"], @@ -92,6 +99,8 @@ def main() -> int: parser.add_argument("--chandra-health", default=os.environ.get("CHANDRA_HEALTH_URL", "http://127.0.0.1:8010/health")) parser.add_argument("--gemma-health", default=os.environ.get("TRANSLATION_VLLM_BASE_URL", "http://localhost:8020/v1")) parser.add_argument("--mode", choices=["ocr", "full"], default="ocr", help="ocr=stop after OCR; full=entire pipeline") + parser.add_argument("--kind", choices=["scheme", "advisory"], default="scheme", help="Document kind (catalog)") + parser.add_argument("--instance", default="", help="Tenant instance code (optional; defaults to your instance)") parser.add_argument("--timeout", type=int, default=1800, help="Max seconds to wait for workflow") parser.add_argument("--poll-interval", type=float, default=5.0) args = parser.parse_args() @@ -121,7 +130,10 @@ def main() -> int: stop_after_ocr = args.mode == "ocr" print(f"==> Upload ({'OCR only' if stop_after_ocr else 'full pipeline'})") - workflow_id = upload_pdf(args.api, args.pdf, auto_approve=True, stop_after_ocr=stop_after_ocr) + workflow_id = upload_pdf( + args.api, args.pdf, auto_approve=True, stop_after_ocr=stop_after_ocr, + kind=args.kind, instance=args.instance, + ) print("==> Polling workflow") doc = poll_document(args.api, workflow_id, timeout_seconds=args.timeout, interval=args.poll_interval) diff --git a/scripts/upload_test_docs.ps1 b/scripts/upload_test_docs.ps1 index df3e934..788fb77 100644 --- a/scripts/upload_test_docs.ps1 +++ b/scripts/upload_test_docs.ps1 @@ -2,6 +2,10 @@ # Prerequisite: ssh -L 18001:127.0.0.1:8001 -N docs-pipeline-host $ErrorActionPreference = "Stop" $Api = if ($env:PIPELINE_API) { $env:PIPELINE_API } else { "http://127.0.0.1:18001" } +# DOC_KIND: scheme (default) or advisory. PIPELINE_INSTANCE: tenant instance code (optional). +$Kind = if ($env:DOC_KIND) { $env:DOC_KIND } else { "scheme" } +$Query = "auto_approve=true&kind=$Kind" +if ($env:PIPELINE_INSTANCE) { $Query = "$Query&instance=$($env:PIPELINE_INSTANCE)" } $DocsDir = Join-Path $PSScriptRoot ".." "test docs" | Resolve-Path Write-Host "API: $Api" @@ -31,7 +35,7 @@ Get-ChildItem -Path $DocsDir -Filter "*.pdf" | ForEach-Object { ) $body = $bodyLines -join $LF $bodyBytes = [System.Text.Encoding]::GetEncoding("iso-8859-1").GetBytes($body) - $resp = Invoke-RestMethod -Uri "$Api/upload?auto_approve=true" -Method Post ` + $resp = Invoke-RestMethod -Uri "$Api/upload?$Query" -Method Post ` -ContentType "multipart/form-data; boundary=$boundary" -Body $bodyBytes $resp | ConvertTo-Json -Depth 5 } diff --git a/scripts/verify_document_validity_e2e.py b/scripts/verify_document_validity_e2e.py index e2c1b66..79ab4d4 100644 --- a/scripts/verify_document_validity_e2e.py +++ b/scripts/verify_document_validity_e2e.py @@ -2,11 +2,12 @@ End-to-end check of document validity against a live Qdrant. Run against a local stack: uv run python scripts/verify_document_validity_e2e.py -Needs Qdrant on :6333 and the small embedding model cached; it creates and -deletes its own scratch collection and a throwaway SQLite file, and touches -neither the real index nor the real db. +Needs Qdrant on :6333, the small embedding model cached, and a scratch Postgres +(E2E_DATABASE_URL, migrations are applied to it). It creates and deletes its own +scratch collection and four documents named wf-*, and touches neither the real +index nor the real db. -Real path, no stubs on the parts that matter: real SQLite rows, the real +Real path, no stubs on the parts that matter: real Postgres rows, the real _prepare_records + QdrantVectorStore.upsert the ingest activity calls, real embeddings, a real Qdrant collection, and the real search filter. Only the embedding *model* is swapped for a small cached one so this does not download @@ -15,15 +16,15 @@ import os import sys -import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, REPO) -DB = os.path.join(tempfile.mkdtemp(), "e2e.db") COLLECTION = "e2e_validity_check" -os.environ["DOCUMENT_DB_PATH"] = DB +if not os.environ.get("E2E_DATABASE_URL"): + sys.exit("Set E2E_DATABASE_URL to a scratch Postgres database (it is never the real one).") +os.environ["DATABASE_URL"] = os.environ["E2E_DATABASE_URL"] os.environ["AUTH_DISABLED"] = "true" os.environ["VECTOR_DB_URL"] = "http://localhost:6333" os.environ["VECTOR_DB_COLLECTION_NAME"] = COLLECTION @@ -39,8 +40,10 @@ from pipeline.activities import _prepare_records, _validity_fields_from_doc # noqa: E402 from pipeline.vector_store.qdrant_store import QdrantVectorStore # noqa: E402 -db.DB_PATH = DB db.init_db() +SCRATCH_DOCUMENTS = ("wf-active", "wf-expired", "wf-future", "wf-legacy") +for scratch in SCRATCH_DOCUMENTS: + db.purge_document(scratch, keep_audit=False) PASS, FAIL = [], [] @@ -72,6 +75,9 @@ def make_document(workflow_id, text, valid_from=None, valid_to=None, legacy=Fals stage="chunk_review", valid_from=valid_from, valid_to=valid_to, + instance="bv", + kind="scheme", + code=workflow_id, ) if legacy: # A document that predates the columns: NULL, as the migration leaves it. @@ -232,5 +238,7 @@ def search_docs(**kwargs): check("the shortened period takes it out of today's results", search_docs(), ["wf-legacy"]) store.client.delete_collection(COLLECTION) +for scratch in SCRATCH_DOCUMENTS: + db.purge_document(scratch, keep_audit=False) print(f"\n{len(PASS)} passed, {len(FAIL)} failed") sys.exit(1 if FAIL else 0) diff --git a/tests/conftest.py b/tests/conftest.py index fc77ac7..4acf436 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,8 @@ Pytest configuration and fixtures for the document ingestion pipeline. Provides fixtures for: -- SQLite database (in-memory for isolation) +- Postgres database (a throwaway container, or the server named by + TEST_DATABASE_URL; migrations applied once, tables truncated per test) - FastAPI TestClient - Mock Temporal client - Mock MinIO client @@ -15,31 +16,58 @@ import pytest # Set test environment before imports -os.environ["DOCUMENT_DB_PATH"] = ":memory:" os.environ["MINIO_ACCESS_KEY"] = "test-access-key" os.environ["MINIO_SECRET_KEY"] = "test-secret-key" os.environ["TRANSLATION_VLLM_BASE_URL"] = "http://localhost:8000/v1" os.environ["AUTH_DISABLED"] = "true" -@pytest.fixture(scope="function") -def temp_db_path(tmp_path): - """Create a temporary database path for each test.""" - db_path = tmp_path / "test.db" - os.environ["DOCUMENT_DB_PATH"] = str(db_path) - yield str(db_path) - # Cleanup happens automatically with tmp_path +_TRUNCATE_SQL = """ + TRUNCATE documents, users, instances, catalogs, audit_logs, email_otps, + webhook_records, search_settings CASCADE +""" -@pytest.fixture(scope="function") -def db_connection(temp_db_path): - """Initialize database and provide connection for testing.""" +@pytest.fixture(scope="session") +def postgres_url(): + """URL of the Postgres the tests run against, with migrations applied. + + Every table is truncated before each test, so TEST_DATABASE_URL must name a + scratch database, never a real one. + """ + url = os.environ.get("TEST_DATABASE_URL") + container = None + if not url: + from testcontainers.postgres import PostgresContainer + + container = PostgresContainer("postgres:15-alpine", driver=None) + container.start() + url = container.get_connection_url() + os.environ["DATABASE_URL"] = url from pipeline import db - db.DB_PATH = temp_db_path + + db.close_pool() db.init_db() + yield url + db.close_pool() + if container is not None: + container.stop() + + +@pytest.fixture(scope="function") +def db_connection(postgres_url): + """The db module on an empty database (seed rows restored after each truncate).""" + from pipeline import db + + os.environ["DATABASE_URL"] = postgres_url + with db.get_connection() as conn: + conn.execute(_TRUNCATE_SQL) + conn.execute( + "INSERT INTO users (keycloak_user_id, username, roles) " + "VALUES ('local-admin', 'local-admin', '{superadmin}')" + ) + conn.execute("INSERT INTO search_settings DEFAULT VALUES") yield db - # Reset for next test - db.DB_PATH = os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db") @pytest.fixture @@ -206,7 +234,9 @@ def sample_document(db_connection): document_id=document_id, filename="test.pdf", filepath="/app/books/test.pdf", - stage="registered" + stage="registered", + kind="scheme", + code="test-scheme", ) return { diff --git a/tests/test_activities.py b/tests/test_activities.py index 8291c26..89a561c 100644 --- a/tests/test_activities.py +++ b/tests/test_activities.py @@ -10,6 +10,7 @@ """ import asyncio +import itertools import os from unittest.mock import MagicMock @@ -21,6 +22,14 @@ os.environ["TRANSLATION_VLLM_BASE_URL"] = "http://localhost:8000/v1" +_code_seq = itertools.count(1) + + +def _next_code() -> str: + """A unique document code for fixtures (documents need one in v2).""" + return f"doc-{next(_code_seq)}" + + class TestChunkingActivity: """Tests for the chunking activity.""" @@ -163,6 +172,36 @@ def test_prepare_records_carries_instance_name(self): assert records[0]["instance"] == code assert records[0]["instance_name"] == expected + @pytest.mark.unit + def test_prepare_records_type_carries_kind(self): + """Every record's payload type is its kind; document_kind is not written.""" + from pipeline.activities import _prepare_records + from pipeline.vector_store.qdrant_store import _record_payload + + chunks = [{"chunk_number": 1, "original_text": "Test chunk", "token_count": 5}] + + advisory = _prepare_records( + document_id="test-doc", filename="test.pdf", chunks=chunks, kind="advisory" + ) + assert advisory[0]["type"] == "advisory" + assert "document_kind" not in advisory[0] + assert _record_payload(advisory[0])["type"] == "advisory" + assert "document_kind" not in _record_payload(advisory[0]) + + default = _prepare_records(document_id="test-doc", filename="test.pdf", chunks=chunks) + assert default[0]["type"] == "document" + + scheme = _prepare_records( + document_id="test-doc", + filename="test.pdf", + chunks=chunks, + kind="scheme", + code="pm-kisan", + name="PM-KISAN", + ) + assert scheme[0]["type"] == "scheme" + assert "document_kind" not in scheme[0] + @pytest.mark.unit def test_prepare_records_fills_scheme_aliases(self): """A scheme must never index with scheme_aliases: [].""" @@ -175,10 +214,10 @@ def test_prepare_records_fills_scheme_aliases(self): filename="NMEO-OPGUIEDELINES.pdf", chunks=chunks, instance="bv", - document_kind="scheme", - scheme_code="nmeoop", - scheme_name="National Mission on Edible Oils - Oil Palm", - scheme_aliases=None, + kind="scheme", + code="nmeoop", + name="National Mission on Edible Oils - Oil Palm", + aliases=None, ) record = records[0] @@ -201,7 +240,7 @@ def test_prepare_records_non_scheme_has_no_aliases(self): filename="test.pdf", chunks=chunks, instance="mh", - document_kind="document", + kind="document", ) assert "scheme_aliases" not in records[0] @@ -395,11 +434,13 @@ class TestUpdateDocumentState: @pytest.mark.unit def test_update_state(self, db_connection): - """Test updating document state in SQLite.""" + """Test updating document state in the database.""" from pipeline.activities import update_document_state workflow_id = "state-update-test" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-state", filename="test.pdf", @@ -407,6 +448,12 @@ def test_update_state(self, db_connection): stage="registered" ) + # Page and chunk counts are derived from the pages table, not stored. + db_connection.save_pages( + workflow_id, + [{"page_number": n, "original_markdown": f"page {n}"} for n in range(1, 6)], + ) + asyncio.run(update_document_state( workflow_id=workflow_id, stage="ocr_processing", @@ -530,9 +577,9 @@ class FakeService: def __init__(self): pass - def publish(self, transaction_id, document_kind, workflow_id=None, group=None): + def publish(self, transaction_id, kind, workflow_id=None, group=None): assert transaction_id == "txn-1" - assert document_kind == "advisory" + assert kind == "advisory" assert workflow_id == "wf-1" return fake_result @@ -542,7 +589,7 @@ def publish(self, transaction_id, document_kind, workflow_id=None, group=None): "_upload_file_to_minio", lambda *a, **k: ("minio://documents/network_publish.json", 123, "application/json"), ) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "advisory"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory"}) monkeypatch.setattr(db, "get_latest_document_job", lambda workflow_id: {"id": 42}) recorded = {} @@ -579,11 +626,11 @@ async def test_publish_failure_propagates_and_records_nothing(self, monkeypatch) import pipeline.db as db class FakeService: - def publish(self, transaction_id, document_kind, workflow_id=None, group=None): + def publish(self, transaction_id, kind, workflow_id=None, group=None): raise RuntimeError("discovery service unreachable") monkeypatch.setattr(activities, "DiscoveryPublishService", FakeService) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "advisory"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory"}) add_artifact_calls = [] monkeypatch.setattr(db, "add_document_artifact", lambda **kwargs: add_artifact_calls.append(kwargs)) @@ -606,7 +653,7 @@ async def test_missing_publish_config_fails_without_retry(self, monkeypatch, uns monkeypatch.setenv("DISCOVERY_SERVICE_ENDPOINT", "https://discovery.example.com") monkeypatch.setenv("NETWORK_SENDER_ID", "provider.oan.dev") monkeypatch.delenv(unset) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "advisory"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory"}) with pytest.raises(ApplicationError, match=unset) as excinfo: await activities.publish_catalog_to_network("wf-1", "txn-1") @@ -620,7 +667,7 @@ async def test_unmapped_kind_records_a_skipped_artifact(self, monkeypatch): import pipeline.db as db class FakeService: - def publish(self, transaction_id, document_kind, workflow_id=None, group=None): + def publish(self, transaction_id, kind, workflow_id=None, group=None): return { "skipped": True, "envelope": None, @@ -636,7 +683,7 @@ def publish(self, transaction_id, document_kind, workflow_id=None, group=None): "_upload_file_to_minio", lambda *a, **k: ("minio://documents/network_publish_skipped.json", 45, "application/json"), ) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "document"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "document"}) monkeypatch.setattr(db, "get_latest_document_job", lambda workflow_id: None) recorded = {} @@ -679,7 +726,7 @@ async def test_disabled_publish_records_a_skip_without_calling_the_network(self, "_upload_file_to_minio", lambda *a, **k: ("minio://documents/network_publish_skipped.json", 45, "application/json"), ) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "advisory"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory"}) monkeypatch.setattr(db, "get_latest_document_job", lambda workflow_id: None) recorded = {} monkeypatch.setattr(db, "add_document_artifact", lambda **kwargs: recorded.update(kwargs)) @@ -700,7 +747,7 @@ async def test_non_rejected_results_complete_the_stage(self, monkeypatch, result import pipeline.db as db class FakeService: - def publish(self, transaction_id, document_kind, workflow_id=None, group=None): + def publish(self, transaction_id, kind, workflow_id=None, group=None): return { "skipped": False, "envelope": {"context": {"messageId": "msg-3"}}, @@ -716,7 +763,7 @@ def publish(self, transaction_id, document_kind, workflow_id=None, group=None): "_upload_file_to_minio", lambda *a, **k: ("minio://documents/network_publish.json", 12, "application/json"), ) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "advisory"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory"}) monkeypatch.setattr(db, "get_latest_document_job", lambda workflow_id: None) monkeypatch.setattr(db, "add_document_artifact", lambda **kwargs: 1) @@ -734,7 +781,7 @@ async def test_rejected_result_records_the_errors_then_fails(self, monkeypatch): errors = [{"code": "SCH_SCHEMA_VALIDATION_FAILED", "message": "topics is required"}] class FakeService: - def publish(self, transaction_id, document_kind, workflow_id=None, group=None): + def publish(self, transaction_id, kind, workflow_id=None, group=None): return { "skipped": False, "envelope": {"context": {"messageId": "msg-4"}}, @@ -750,7 +797,7 @@ def publish(self, transaction_id, document_kind, workflow_id=None, group=None): "_upload_file_to_minio", lambda *a, **k: ("minio://documents/network_publish.json", 12, "application/json"), ) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "advisory"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory"}) monkeypatch.setattr(db, "get_latest_document_job", lambda workflow_id: None) recorded = {} @@ -812,7 +859,7 @@ async def test_passes_the_documents_group(self, monkeypatch, instance, group): seen = {} self._wire( monkeypatch, - {"document_kind": "advisory", "instance": instance}, + {"kind": "advisory", "instance": instance}, self._PUBLISHED, seen, ) @@ -836,7 +883,7 @@ async def test_unmapped_kind_records_a_skip(self, monkeypatch): } recorded = self._wire( monkeypatch, - {"document_kind": "document", "instance": "mh"}, + {"kind": "document", "instance": "mh"}, skipped, {}, ) @@ -861,7 +908,7 @@ def publish(self, transaction_id, document_kind, workflow_id=None, group=None): raise InvalidProviderIdError("Provider id 'advisory.ladakh_ut.provider.oan.dev' is invalid") monkeypatch.setattr(activities, "DiscoveryPublishService", FakeService) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "advisory", "instance": "ladakh_ut"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory", "instance": "ladakh_ut"}) recorded = {} monkeypatch.setattr(db, "add_document_artifact", lambda **kwargs: recorded.update(kwargs)) @@ -900,3 +947,98 @@ def test_every_defined_activity_is_registered_on_the_worker(self): missing = {name for name in defined_activities if name not in worker_source} assert not missing, f"Activities defined but never registered on the worker: {missing}" + + +class TestSupersededPoints: + """After new content is indexed, points of earlier file versions are deleted.""" + + class _Store: + def __init__(self, failing=()): + self.calls = [] + self.failing = set(failing) + + def delete_by_doc_id(self, collection, doc_id): + self.calls.append((collection, doc_id)) + if doc_id in self.failing: + return {"deleted": 0, "doc_id": doc_id, "error": "boom"} + return {"deleted": -1, "doc_id": doc_id} + + @staticmethod + def _document(db): + doc_uuid = db.upsert_document( + workflow_id="wf-sup-act", + document_id="c" * 32, + filename="a.pdf", + filepath="minio://documents/a.pdf", + kind="scheme", + code="sup-act", + ) + db.upsert_document_index_status("wf-sup-act", "documents-index", doc_id="a" * 32, status="indexed") + db.reset_document_for_reupload( + doc_uuid, filename="b.pdf", filepath="minio://documents/b.pdf", fingerprint="c" * 32 + ) + return doc_uuid + + @pytest.mark.asyncio + @pytest.mark.db + @pytest.mark.unit + async def test_old_keys_are_deleted_and_forgotten(self, db_connection): + from pipeline import activities + + doc_uuid = self._document(db_connection) + store = self._Store() + + await activities._drop_superseded_points( + "dev", "wf-sup-act", db_connection.get_document(doc_uuid), "c" * 32, "documents-index", store=store + ) + + assert store.calls == [("documents-index", "a" * 32)] + assert db_connection.get_document(doc_uuid)["superseded_doc_ids"] == {} + + @pytest.mark.asyncio + @pytest.mark.db + @pytest.mark.unit + async def test_a_failed_delete_keeps_the_key_for_the_next_run(self, db_connection): + from pipeline import activities + + doc_uuid = self._document(db_connection) + store = self._Store(failing={"a" * 32}) + + await activities._drop_superseded_points( + "dev", "wf-sup-act", db_connection.get_document(doc_uuid), "c" * 32, "documents-index", store=store + ) + + assert db_connection.get_document(doc_uuid)["superseded_doc_ids"] == {"dev": ["a" * 32]} + + @pytest.mark.asyncio + @pytest.mark.db + @pytest.mark.unit + async def test_a_key_another_live_document_is_indexed_under_is_kept_and_remembered(self, db_connection): + from pipeline import activities + + doc_uuid = self._document(db_connection) + # A second document is indexed under the very key the first one replaced. + db_connection.upsert_document( + workflow_id="wf-twin", document_id="a" * 32, filename="a.pdf", + filepath="minio://documents/a.pdf", instance="gj", kind="scheme", code="twin", + ) + db_connection.upsert_document_index_status( + "wf-twin", "documents-index", doc_id="a" * 32, status="indexed" + ) + store = self._Store() + + await activities._drop_superseded_points( + "dev", "wf-sup-act", db_connection.get_document(doc_uuid), "c" * 32, "documents-index", store=store + ) + + # Points stay for the twin, and the key stays on the list for a later cleanup. + assert store.calls == [] + assert db_connection.get_document(doc_uuid)["superseded_doc_ids"] == {"dev": ["a" * 32]} + + # Once the twin is gone the same cleanup removes them. + db_connection.set_document_disabled("wf-twin", True) + await activities._drop_superseded_points( + "dev", "wf-sup-act", db_connection.get_document(doc_uuid), "c" * 32, "documents-index", store=store + ) + assert store.calls == [("documents-index", "a" * 32)] + assert db_connection.get_document(doc_uuid)["superseded_doc_ids"] == {} diff --git a/tests/test_advisory_activities.py b/tests/test_advisory_activities.py new file mode 100644 index 0000000..0ac0dd2 --- /dev/null +++ b/tests/test_advisory_activities.py @@ -0,0 +1,126 @@ +"""Tests for the advisory index/promote activities, the PROD target and the workflow exports.""" + +import asyncio +import copy +from pathlib import Path + +import pytest + +from pipeline import advisory_index +from tests.test_advisory_chunks import SORGHUM_PAYLOAD +from tests.test_advisory_index import FakeStore + + +def _seed(db): + db.upsert_webhook_records("webhook", [copy.deepcopy(SORGHUM_PAYLOAD)]) + with db.get_connection() as conn: + return str(conn.execute("SELECT id FROM webhook_records").fetchone()["id"]) + + +def _audit_actions(db): + with db.get_connection() as conn: + return [r["action_type"] for r in conn.execute("SELECT action_type FROM audit_logs")] + + +@pytest.fixture +def qdrant_env(monkeypatch): + monkeypatch.setenv("VECTOR_DB_COLLECTION_NAME", "document-data") + monkeypatch.delenv("PROD_VECTOR_DB_URL", raising=False) + monkeypatch.setattr("pipeline.vector_store.embeddings.count_tokens", lambda text: 250) + + +@pytest.mark.unit +@pytest.mark.db +class TestActivities: + def test_index_activity_writes_to_the_dev_collection_and_audits(self, db_connection, qdrant_env, monkeypatch): + from pipeline.activities import index_webhook_advisory + + rid = _seed(db_connection) + store = FakeStore() + monkeypatch.setattr("pipeline.vector_store.get_vector_store", lambda: store) + result = asyncio.run(index_webhook_advisory(rid)) + assert result["status"] == "indexed" + assert store.calls[0][1] == "document-data" + assert db_connection.get_unindexed_webhook_record_ids() == [] + assert "advisory_indexed" in _audit_actions(db_connection) + + def test_promote_activity_writes_to_prod_and_marks_promoted(self, db_connection, qdrant_env, monkeypatch): + from pipeline.activities import promote_webhook_advisory + + rid = _seed(db_connection) + prod = FakeStore() + monkeypatch.setattr(advisory_index, "prod_target", lambda: (prod, "prod-collection")) + result = asyncio.run(promote_webhook_advisory(rid)) + assert result["status"] == "indexed" + assert prod.calls[0][1] == "prod-collection" + assert db_connection.advisory_status_counts()["promoted"] == 1 + assert "advisory_promoted" in _audit_actions(db_connection) + + def test_promote_activity_requires_a_prod_target(self, db_connection, qdrant_env, monkeypatch): + from pipeline.activities import promote_webhook_advisory + + monkeypatch.setattr(advisory_index, "prod_target", lambda: None) + with pytest.raises(RuntimeError, match="PROD_VECTOR_DB_URL"): + asyncio.run(promote_webhook_advisory("any")) + + def test_dev_index_of_unpublished_record_also_clears_prod(self, db_connection, qdrant_env, monkeypatch): + from pipeline.activities import index_webhook_advisory + + payload = copy.deepcopy(SORGHUM_PAYLOAD) + payload["isPublished"] = 0 + db_connection.upsert_webhook_records("webhook", [payload]) + with db_connection.get_connection() as conn: + rid = str(conn.execute("SELECT id FROM webhook_records").fetchone()["id"]) + dev, prod = FakeStore(), FakeStore() + monkeypatch.setattr("pipeline.vector_store.get_vector_store", lambda: dev) + monkeypatch.setattr(advisory_index, "prod_target", lambda: (prod, "prod-collection")) + result = asyncio.run(index_webhook_advisory(rid)) + assert result["status"] == "deleted" + assert prod.calls == [("delete", "prod-collection", rid)] + + def test_dev_index_survives_a_broken_prod_config(self, db_connection, qdrant_env, monkeypatch): + from pipeline.activities import index_webhook_advisory + + rid = _seed(db_connection) + monkeypatch.setattr("pipeline.vector_store.get_vector_store", lambda: FakeStore()) + + def broken(): + raise ValueError("QDRANT_API_KEY is required for remote Qdrant") + + monkeypatch.setattr(advisory_index, "prod_target", broken) + assert asyncio.run(index_webhook_advisory(rid))["status"] == "indexed" + + +class TestProdTarget: + def test_none_without_prod_url(self, monkeypatch): + monkeypatch.delenv("PROD_VECTOR_DB_URL", raising=False) + assert advisory_index.prod_target() is None + + def test_uses_prod_env(self, monkeypatch): + monkeypatch.setenv("PROD_VECTOR_DB_URL", "http://prod:6333") + monkeypatch.setenv("PROD_VECTOR_DB_API_KEY", "k") + monkeypatch.setenv("PROD_VECTOR_DB_COLLECTION_NAME", "prod-docs") + seen = {} + + def fake_client(url=None, api_key=None): + seen.update(url=url, api_key=api_key) + return object() + + monkeypatch.setattr("pipeline.vector_store.qdrant_store.get_qdrant_client", fake_client) + target = advisory_index.prod_target() + assert target is not None and target[1] == "prod-docs" + assert seen == {"url": "http://prod:6333", "api_key": "k"} + + +def test_workflows_are_defined_and_registered(): + from pipeline.workflows import AdvisoryIndexWorkflow, AdvisoryPromoteWorkflow + + assert callable(AdvisoryIndexWorkflow.run) and callable(AdvisoryPromoteWorkflow.run) + source = (Path(__file__).resolve().parent.parent / "pipeline" / "worker.py").read_text() + for name in ( + "AdvisoryIndexWorkflow", + "AdvisoryPromoteWorkflow", + "index_webhook_advisory", + "promote_webhook_advisory", + ): + assert source.count(name) >= 2, name # imported and registered diff --git a/tests/test_advisory_api.py b/tests/test_advisory_api.py new file mode 100644 index 0000000..f045998 --- /dev/null +++ b/tests/test_advisory_api.py @@ -0,0 +1,175 @@ +"""API tests: /webhook starts advisory workflows; admin backfill / promote / status endpoints.""" + +import os +from unittest.mock import AsyncMock, patch + +import pytest + +TEST_KEY = "0123456789abcdef0123456789abcdef" +H = {"x-api-key": TEST_KEY} + + +def _rec(content_id, published=1, status="Published"): + return { + "contentID": content_id, + "title": f"Advisory {content_id}", + "status": status, + "isPublished": published, + "popList": [{"popID": 1, "status": "Published"}], + } + + +@pytest.fixture +def client(db_connection, monkeypatch): + from fastapi.testclient import TestClient + + from pipeline.api import api + + monkeypatch.setattr(api.limiter, "enabled", False) # many posts per test process + with patch.dict(os.environ, {"WEBHOOK_API_KEY": TEST_KEY, "AUTH_DISABLED": "true"}): + yield TestClient(api.app) + + +@pytest.fixture +def temporal(monkeypatch): + from pipeline.api import api + + mock = AsyncMock() + monkeypatch.setattr(api, "temporal_client", mock) + return mock + + +def _row_ids(db, *content_ids): + with db.get_connection() as conn: + return { + c: str(conn.execute( + "SELECT id FROM webhook_records WHERE external_id = %s", (str(c),) + ).fetchone()["id"]) + for c in content_ids + } + + +def _started(mock): + return sorted(call.kwargs["id"] for call in mock.start_workflow.await_args_list) + + +@pytest.mark.api +@pytest.mark.unit +class TestWebhookStartsIndexing: + def test_one_workflow_per_record_in_the_call(self, client, db_connection, temporal): + r = client.post("/webhook", json=[_rec(1), _rec(2)], headers=H) + assert r.status_code == 200 + assert r.json() == { + "success": True, + "message": "Webhook processed successfully.", + "recordCount": 2, + } + ids = _row_ids(db_connection, 1, 2) + assert _started(temporal) == sorted(f"advisory-{i}" for i in ids.values()) + + def test_only_records_in_the_call_are_started(self, client, db_connection, temporal): + db_connection.upsert_webhook_records("webhook", [_rec(1)]) # existing, unindexed + client.post("/webhook", json=[_rec(2)], headers=H) + ids = _row_ids(db_connection, 1, 2) + assert _started(temporal) == [f"advisory-{ids[2]}"] + + def test_existing_never_indexed_row_is_started_when_resent(self, client, db_connection, temporal): + db_connection.upsert_webhook_records("webhook", [_rec(1)]) + client.post("/webhook", json=[_rec(1)], headers=H) + ids = _row_ids(db_connection, 1) + assert _started(temporal) == [f"advisory-{ids[1]}"] + + def test_nothing_started_when_already_indexed(self, client, db_connection, temporal): + client.post("/webhook", json=[_rec(1)], headers=H) + rid = _row_ids(db_connection, 1)[1] + db_connection.mark_webhook_record_indexed(rid, db_connection.get_webhook_record(rid)["payload_hash"]) + temporal.start_workflow.reset_mock() + r = client.post("/webhook", json=[_rec(1)], headers=H) + assert r.status_code == 200 + temporal.start_workflow.assert_not_called() + + def test_still_200_when_temporal_fails(self, client, db_connection, temporal): + temporal.start_workflow.side_effect = RuntimeError("temporal down") + r = client.post("/webhook", json=[_rec(1)], headers=H) + assert r.status_code == 200 + assert len(db_connection.get_unindexed_webhook_record_ids()) == 1 + + def test_still_200_when_temporal_client_is_none(self, client, db_connection, monkeypatch): + from pipeline.api import api + + monkeypatch.setattr(api, "temporal_client", None) + assert client.post("/webhook", json=[_rec(1)], headers=H).status_code == 200 + + +@pytest.mark.api +@pytest.mark.unit +class TestAdminEndpoints: + def _seed(self, client, temporal, *records): + client.post("/webhook", json=list(records), headers=H) + temporal.start_workflow.reset_mock() + + def test_reindex_queues_all_unindexed_by_default(self, client, db_connection, temporal): + self._seed(client, temporal, _rec(1), _rec(2)) + r = client.post("/webhook/advisory/reindex", json={}) + assert r.status_code == 200 and r.json() == {"success": True, "queued": 2} + assert len(_started(temporal)) == 2 + + def test_reindex_force_queues_indexed_rows_too(self, client, db_connection, temporal): + self._seed(client, temporal, _rec(1)) + rid = _row_ids(db_connection, 1)[1] + db_connection.mark_webhook_record_indexed(rid, db_connection.get_webhook_record(rid)["payload_hash"]) + assert client.post("/webhook/advisory/reindex", json={}).json()["queued"] == 0 + assert client.post("/webhook/advisory/reindex", json={"force": True}).json()["queued"] == 1 + + def test_reindex_explicit_ids(self, client, db_connection, temporal): + self._seed(client, temporal, _rec(1), _rec(2)) + rid = _row_ids(db_connection, 2)[2] + r = client.post("/webhook/advisory/reindex", json={"record_ids": [rid]}) + assert r.json()["queued"] == 1 + assert _started(temporal) == [f"advisory-{rid}"] + + def test_promote_defaults_to_published_records_indexed_in_dev(self, client, db_connection, temporal): + self._seed(client, temporal, _rec(1), _rec(2, published=0, status="Draft")) + ids = _row_ids(db_connection, 1, 2) + for rid in ids.values(): + db_connection.mark_webhook_record_indexed(rid, db_connection.get_webhook_record(rid)["payload_hash"]) + temporal.start_workflow.reset_mock() + r = client.post("/webhook/advisory/promote", json={}) + assert r.json() == {"success": True, "queued": 1} + assert _started(temporal) == [f"advisory-promote-{ids[1]}"] + + def test_default_promote_skips_records_not_indexed_in_dev(self, client, db_connection, temporal): + self._seed(client, temporal, _rec(1)) + temporal.start_workflow.reset_mock() + r = client.post("/webhook/advisory/promote", json={}) + assert r.json() == {"success": True, "queued": 0} + assert _started(temporal) == [] + + def test_promote_explicit_ids(self, client, db_connection, temporal): + self._seed(client, temporal, _rec(1), _rec(2)) + rid = _row_ids(db_connection, 2)[2] + r = client.post("/webhook/advisory/promote", json={"record_ids": [rid]}) + assert r.json()["queued"] == 1 + assert _started(temporal) == [f"advisory-promote-{rid}"] + + def test_status_returns_counts(self, client, db_connection, temporal): + self._seed(client, temporal, _rec(1), _rec(2, published=0, status="Draft")) + r = client.get("/webhook/advisory/status") + assert r.status_code == 200 + assert r.json() == {"total": 2, "published": 1, "indexed": 0, "unindexed": 2, "promoted": 0} + + def test_endpoints_require_a_signed_in_admin(self, db_connection, temporal, monkeypatch): + from fastapi.testclient import TestClient + + from pipeline.api import api + + env = { + "AUTH_DISABLED": "false", + "KEYCLOAK_ISSUER": "https://example.com/realms/test", + "KEYCLOAK_JWKS_URL": "https://example.com/realms/test/protocol/openid-connect/certs", + } + with patch.dict(os.environ, env): + c = TestClient(api.app) + assert c.post("/webhook/advisory/reindex", json={}).status_code == 401 + assert c.post("/webhook/advisory/promote", json={}).status_code == 401 + assert c.get("/webhook/advisory/status").status_code == 401 diff --git a/tests/test_advisory_chunks.py b/tests/test_advisory_chunks.py new file mode 100644 index 0000000..016d2c6 --- /dev/null +++ b/tests/test_advisory_chunks.py @@ -0,0 +1,218 @@ +"""Tests for the webhook advisory chunk builder.""" + +import copy +import hashlib + +import pytest + +from pipeline.advisory_chunks import build_chunks, clean_value + +HEADER = ( + "Sorghum- Harvesting Management\n" + "Crop: Sorghum | Problem type: Harvesting Management | Season: Rabi" +) +OVERVIEW = ( + "Timely harvesting of AKJ-1 helps preserve grain appearance, processing quality and " + "suitability for flaking. Proper drying and handling after harvest are essential to " + "minimize grain damage and deterioration." +) +SYMPTOMS = ( + "Panicles become mature and dry, grains become hard and attain characteristic colour, " + "and most leaves begin to dry." +) +CONDITIONS = ( + "Crop has reached physiological maturity and panicles are fully developed; grains are " + "firm and suitable for flaking quality." +) +ADVISORY = ( + "Harvest AKJ-1 at physiological maturity when the grains are fully developed and firm. " + "Timely harvesting is important for maintaining grain quality required for flaking. " + "Avoid delayed harvesting, which may increase grain weathering, shattering and mould " + "risk. After harvesting, dry the panicles properly, thresh carefully, clean the grain " + "and reduce moisture to a safe level before storage or processing." +) + +SORGHUM_PAYLOAD = { + "contentID": 987, + "title": "Sorghum- Harvesting Management", + "commodityName": "Sorghum", + "creatorName": "jak", + "domainName": "Agriculture", + "subdomainName": "Cereals", + "instituteName": "ICAR–Directorate of Knowledge Management in Agriculture (DKMA)", + "isPublished": 1, + "status": "Published", + "popList": [ + { + "popID": 819, + "season": "Rabi", + "problemType": '"Harvesting Management"', + "problemDetails": '"Method Of Harvesting"', + "startPeriod": "105", + "endPeriod": "120", + "status": "Published", + "description": f'"{OVERVIEW}"', + "applicableSymptoms": [SYMPTOMS], + "applicableCondition": [CONDITIONS], + "advisory": f'"{ADVISORY}"', + } + ], +} + + +def make_row(payload=None, **overrides): + payload = copy.deepcopy(SORGHUM_PAYLOAD if payload is None else payload) + row = { + "id": "row-1", + "external_id": "987", + "title": payload.get("title", ""), + "slug": "987/sorghum--harvesting-management", + "status": payload.get("status"), + "is_published": payload.get("isPublished"), + "payload": payload, + "payload_hash": "hash-1", + "updated_at": "2026-10-16T09:15:10.000Z", + } + row.update(overrides) + return row + + +def long_when_combined(text: str) -> int: + """Over the limit only when the entry carries more than one question-side field.""" + return 600 if "Overview: " in text and "Symptoms: " in text else 300 + + +def pop_of(row): + return row["payload"]["popList"][0] + + +class TestCleanValue: + def test_strips_wrapping_quotes(self): + assert clean_value('"Aphids"') == "Aphids" + assert clean_value('""') == "" + + def test_handles_none_numbers_and_whitespace(self): + assert clean_value(None) == "" + assert clean_value(819) == "819" + assert clean_value(" x ") == "x" + + +class TestShortEntry: + def test_short_entry_is_one_full_chunk(self): + chunks = build_chunks(make_row(), lambda t: 250) + assert len(chunks) == 1 + assert chunks[0]["section"] == "full" + assert chunks[0]["text"] == "\n".join( + [ + HEADER, + f"Overview: {OVERVIEW}", + f"Symptoms: {SYMPTOMS}", + f"Conditions: {CONDITIONS}", + f"Advisory: {ADVISORY}", + ] + ) + assert chunks[0]["text_for_embedding"] == chunks[0]["text"] + + +class TestLongEntry: + def test_long_entry_splits_into_three_chunks_each_with_advice(self): + chunks = build_chunks(make_row(), long_when_combined) + assert [c["section"] for c in chunks] == ["overview", "symptoms", "conditions"] + for chunk in chunks: + assert chunk["text"].startswith(HEADER) + assert chunk["text"].endswith(f"Advisory: {ADVISORY}") + assert "Symptoms:" not in chunks[0]["text"] + assert f"Symptoms: {SYMPTOMS}" in chunks[1]["text"] + assert [c["chunk_num"] for c in chunks] == [0, 1, 2] + + def test_empty_conditions_gives_two_chunks(self): + row = make_row() + pop_of(row)["applicableCondition"] = [] + chunks = build_chunks(row, long_when_combined) + assert [c["section"] for c in chunks] == ["overview", "symptoms"] + + def test_no_advisory_gives_one_chunk_per_field(self): + row = make_row() + pop_of(row)["advisory"] = None + chunks = build_chunks(row, lambda t: 600) + assert [c["section"] for c in chunks] == ["overview", "symptoms", "conditions"] + for chunk in chunks: + assert "Advisory:" not in chunk["text"] + + def test_only_advisory_is_one_advisory_chunk(self): + row = make_row() + pop = pop_of(row) + pop["description"] = None + pop["applicableSymptoms"] = [] + pop["applicableCondition"] = [] + chunks = build_chunks(row, lambda t: 600) + assert [c["section"] for c in chunks] == ["advisory"] + assert chunks[0]["text"] == f"{HEADER}\nAdvisory: {ADVISORY}" + + +class TestPublishingAndShape: + @pytest.mark.parametrize( + "overrides", + [{"is_published": 0}, {"status": "Draft"}], + ) + def test_unpublished_record_returns_empty(self, overrides): + assert build_chunks(make_row(**overrides), lambda t: 250) == [] + + def test_unpublished_entry_is_skipped(self): + row = make_row() + row["payload"]["popList"].append( + {**copy.deepcopy(pop_of(row)), "popID": 820, "status": "Draft"} + ) + chunks = build_chunks(row, lambda t: 250) + assert [c["pop_id"] for c in chunks] == [819] + + @pytest.mark.parametrize("pop_list", [None, [], "nope"]) + def test_missing_or_empty_poplist_returns_empty(self, pop_list): + row = make_row() + row["payload"]["popList"] = pop_list + assert build_chunks(row, lambda t: 250) == [] + + def test_ids_are_stable_and_unique(self): + first = build_chunks(make_row(), long_when_combined) + second = build_chunks(make_row(), long_when_combined) + assert [c["_id"] for c in first] == [c["_id"] for c in second] + assert len({c["_id"] for c in first}) == 3 + full = build_chunks(make_row(), lambda t: 250)[0] + expected = hashlib.md5(b"row-1:819:full").hexdigest() + assert full["_id"] == expected + assert full["chunk_id"] == expected + + def test_metadata_fields(self): + chunk = build_chunks(make_row(), lambda t: 250)[0] + assert "document_kind" not in chunk + assert chunk["type"] == "advisory" + assert chunk["source"] == "webhook" + assert chunk["workflow_id"] == "" + assert chunk["doc_id"] == "row-1" + assert chunk["external_id"] == "987" + assert chunk["pop_id"] == 819 + assert chunk["title_en"] == "Sorghum- Harvesting Management" + assert chunk["slug"] == "987/sorghum--harvesting-management" + assert chunk["crop"] == "Sorghum" + assert chunk["domain"] == "Agriculture" + assert chunk["subdomain"] == "Cereals" + assert chunk["institute"].startswith("ICAR") + assert chunk["creator"] == "jak" + assert chunk["problem_type"] == "Harvesting Management" + assert chunk["problem"] == "Method Of Harvesting" + assert chunk["season"] == "Rabi" + assert chunk["start_period"] == "105" + assert chunk["end_period"] == "120" + assert chunk["updated_at"] == "2026-10-16T09:15:10.000Z" + + def test_null_values_never_appear_as_text(self): + row = make_row() + pop = pop_of(row) + pop["description"] = None + pop["season"] = None + pop["applicableSymptoms"] = [None, "x"] + chunks = build_chunks(row, lambda t: 250) + assert len(chunks) == 1 + assert "None" not in chunks[0]["text"] + assert "Season:" not in chunks[0]["text"] + assert "Symptoms: x" in chunks[0]["text"] diff --git a/tests/test_advisory_index.py b/tests/test_advisory_index.py new file mode 100644 index 0000000..440ee04 --- /dev/null +++ b/tests/test_advisory_index.py @@ -0,0 +1,153 @@ +"""Tests for indexing one webhook advisory record into a vector store.""" + +import copy + +import pytest + +from pipeline.advisory_chunks import build_chunks +from pipeline.advisory_index import index_advisory_record +from pipeline.vector_store.qdrant_store import _record_payload +from tests.test_advisory_chunks import SORGHUM_PAYLOAD, make_row + + +class FakeStore: + def __init__(self, delete_result=None, fail_upsert=False): + self.calls = [] + self.records = [] + self._delete_result = delete_result or {"deleted": -1} + self._fail_upsert = fail_upsert + + def delete_by_doc_id(self, name, doc_id): + self.calls.append(("delete", name, doc_id)) + return self._delete_result + + def upsert(self, name, records, batch_size=32): + self.calls.append(("upsert", name, len(records))) + if self._fail_upsert: + raise RuntimeError("qdrant down") + self.records = list(records) + return {"records_ingested": len(records)} + + +def _store_row(db, payload): + db.upsert_webhook_records("webhook", [payload]) + with db.get_connection() as conn: + return str(conn.execute( + "SELECT id FROM webhook_records WHERE external_id = %s", (str(payload["contentID"]),) + ).fetchone()["id"]) + + +def _count(_text): + return 250 + + +@pytest.mark.unit +@pytest.mark.db +class TestIndexAdvisoryRecord: + def test_published_record_is_deleted_then_upserted_then_marked(self, db_connection): + db = db_connection + rid = _store_row(db, copy.deepcopy(SORGHUM_PAYLOAD)) + store = FakeStore() + result = index_advisory_record(rid, store, "document-data", _count) + assert [c[0] for c in store.calls] == ["delete", "upsert"] + assert store.calls[0][1:] == ("document-data", rid) + assert result == {"record_id": rid, "status": "indexed", "chunks": 1} + assert db.get_unindexed_webhook_record_ids() == [] + + def test_unpublished_record_only_deletes(self, db_connection): + db = db_connection + payload = copy.deepcopy(SORGHUM_PAYLOAD) + payload["isPublished"] = 0 + rid = _store_row(db, payload) + store = FakeStore() + result = index_advisory_record(rid, store, "document-data", _count) + assert [c[0] for c in store.calls] == ["delete"] + assert result["status"] == "deleted" and result["chunks"] == 0 + assert db.get_unindexed_webhook_record_ids() == [] + + def test_unpublished_record_also_deletes_from_the_other_collection(self, db_connection): + db = db_connection + payload = copy.deepcopy(SORGHUM_PAYLOAD) + payload["isPublished"] = 0 + rid = _store_row(db, payload) + dev, prod = FakeStore(), FakeStore() + index_advisory_record( + rid, dev, "document-data", _count, also_delete_from=(prod, "prod-collection") + ) + assert prod.calls == [("delete", "prod-collection", rid)] + + def test_published_record_does_not_touch_the_other_collection(self, db_connection): + db = db_connection + rid = _store_row(db, copy.deepcopy(SORGHUM_PAYLOAD)) + dev, prod = FakeStore(), FakeStore() + index_advisory_record( + rid, dev, "document-data", _count, also_delete_from=(prod, "prod-collection") + ) + assert prod.calls == [] + + def test_missing_record_does_nothing(self, db_connection): + store = FakeStore() + result = index_advisory_record("nope", store, "document-data", _count) + assert store.calls == [] + assert result == {"record_id": "nope", "status": "missing", "chunks": 0} + + def test_record_without_poplist_is_marked_indexed(self, db_connection): + db = db_connection + payload = copy.deepcopy(SORGHUM_PAYLOAD) + payload["popList"] = [] + rid = _store_row(db, payload) + store = FakeStore() + result = index_advisory_record(rid, store, "document-data", _count) + assert [c[0] for c in store.calls] == ["delete"] + assert result["status"] == "deleted" + assert db.get_unindexed_webhook_record_ids() == [] + + def test_upsert_failure_leaves_record_unindexed(self, db_connection): + db = db_connection + rid = _store_row(db, copy.deepcopy(SORGHUM_PAYLOAD)) + with pytest.raises(RuntimeError, match="qdrant down"): + index_advisory_record(rid, FakeStore(fail_upsert=True), "document-data", _count) + assert db.get_unindexed_webhook_record_ids() == [rid] + + def test_delete_error_other_than_missing_collection_fails(self, db_connection): + db = db_connection + rid = _store_row(db, copy.deepcopy(SORGHUM_PAYLOAD)) + store = FakeStore(delete_result={"deleted": 0, "error": "connection refused"}) + with pytest.raises(RuntimeError, match="connection refused"): + index_advisory_record(rid, store, "document-data", _count) + assert db.get_unindexed_webhook_record_ids() == [rid] + + def test_missing_collection_on_delete_is_not_an_error(self, db_connection): + db = db_connection + rid = _store_row(db, copy.deepcopy(SORGHUM_PAYLOAD)) + store = FakeStore(delete_result={"deleted": 0, "error": "Collection `x` doesn't exist!"}) + result = index_advisory_record(rid, store, "x", _count) + assert result["status"] == "indexed" + + def test_promote_marks_promoted_not_indexed(self, db_connection): + db = db_connection + rid = _store_row(db, copy.deepcopy(SORGHUM_PAYLOAD)) + index_advisory_record(rid, FakeStore(), "prod", _count, mark="promoted") + assert db.get_unindexed_webhook_record_ids() == [rid] + assert db.advisory_status_counts()["promoted"] == 1 + + def test_reposting_unchanged_record_reuses_point_ids(self, db_connection): + db = db_connection + rid = _store_row(db, copy.deepcopy(SORGHUM_PAYLOAD)) + first, second = FakeStore(), FakeStore() + index_advisory_record(rid, first, "document-data", _count) + index_advisory_record(rid, second, "document-data", _count) + assert [r["_id"] for r in first.records] == [r["_id"] for r in second.records] + + +def test_payload_whitelist_keeps_advisory_fields(): + chunk = build_chunks(make_row(), _count)[0] + payload = _record_payload(chunk) + for key in ( + "type", "external_id", "pop_id", "slug", "crop", "domain", "subdomain", + "institute", "creator", "problem_type", "problem", "season", "start_period", + "end_period", "updated_at", "section", "chunk_id", "text", "doc_id", "title_en", + ): + assert key in payload, key + assert payload["type"] == "advisory" + assert "document_kind" not in payload diff --git a/tests/test_advisory_start.py b/tests/test_advisory_start.py new file mode 100644 index 0000000..0c7056e --- /dev/null +++ b/tests/test_advisory_start.py @@ -0,0 +1,98 @@ +"""Tests for starting advisory workflows (one per record).""" + +import asyncio +from unittest.mock import AsyncMock + +import pytest +from temporalio.exceptions import WorkflowAlreadyStartedError + +from pipeline.advisory_start import start_advisory_index, start_advisory_promote +from pipeline.workflows import AdvisoryIndexWorkflow, AdvisoryPromoteWorkflow + + +def _started_ids(client): + return [call.kwargs["id"] for call in client.start_workflow.await_args_list] + + +def test_index_starts_one_workflow_per_record(): + client = AsyncMock() + started = asyncio.run(start_advisory_index(client, ["a", "b"])) + assert started == 2 + assert _started_ids(client) == ["advisory-a", "advisory-b"] + first = client.start_workflow.await_args_list[0] + assert first.args[0] is AdvisoryIndexWorkflow.run + assert first.kwargs["args"] == ["a"] + assert first.kwargs["task_queue"] == "ocr-pipeline" + + +def test_promote_uses_separate_ids(): + client = AsyncMock() + assert asyncio.run(start_advisory_promote(client, ["a"])) == 1 + assert _started_ids(client) == ["advisory-promote-a"] + assert client.start_workflow.await_args_list[0].args[0] is AdvisoryPromoteWorkflow.run + + +def test_already_running_workflow_is_skipped_and_others_still_start(): + client = AsyncMock() + client.start_workflow.side_effect = [ + WorkflowAlreadyStartedError("advisory-a", "AdvisoryIndexWorkflow"), + None, + ] + assert asyncio.run(start_advisory_index(client, ["a", "b"])) == 1 + + +def test_other_errors_are_logged_and_do_not_stop_the_loop(): + client = AsyncMock() + client.start_workflow.side_effect = [RuntimeError("temporal down"), None] + assert asyncio.run(start_advisory_index(client, ["a", "b"])) == 1 + + +def test_no_ids_starts_nothing(): + client = AsyncMock() + assert asyncio.run(start_advisory_index(client, [])) == 0 + client.start_workflow.assert_not_called() + + +@pytest.mark.db +class TestBackfillOnStart: + def _seed(self, db): + db.upsert_webhook_records( + "webhook", + [{"contentID": 1, "status": "Published", "isPublished": 1, "popList": []}], + ) + + def test_off_by_default(self, db_connection, monkeypatch): + from pipeline.advisory_start import backfill_on_start + + monkeypatch.delenv("ADVISORY_BACKFILL_ON_START", raising=False) + self._seed(db_connection) + client = AsyncMock() + assert asyncio.run(backfill_on_start(client)) == 0 + client.start_workflow.assert_not_called() + + @pytest.mark.parametrize("value", ["true", "1", "yes", "on", "TRUE"]) + def test_on_queues_all_unindexed_rows(self, db_connection, monkeypatch, value): + from pipeline.advisory_start import backfill_on_start + + monkeypatch.setenv("ADVISORY_BACKFILL_ON_START", value) + self._seed(db_connection) + client = AsyncMock() + assert asyncio.run(backfill_on_start(client)) == 1 + assert _started_ids(client)[0].startswith("advisory-") + + def test_on_with_nothing_unindexed_starts_nothing(self, db_connection, monkeypatch): + from pipeline.advisory_start import backfill_on_start + + monkeypatch.setenv("ADVISORY_BACKFILL_ON_START", "true") + client = AsyncMock() + assert asyncio.run(backfill_on_start(client)) == 0 + client.start_workflow.assert_not_called() + + def test_off_values(self, db_connection, monkeypatch): + from pipeline.advisory_start import backfill_on_start + + self._seed(db_connection) + for value in ("false", "0", "", "no"): + monkeypatch.setenv("ADVISORY_BACKFILL_ON_START", value) + client = AsyncMock() + assert asyncio.run(backfill_on_start(client)) == 0 diff --git a/tests/test_api.py b/tests/test_api.py index 396d7a8..0cf8040 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -10,17 +10,26 @@ - Error handling """ +import hashlib +import itertools import os import pytest # Set test environment -os.environ["DOCUMENT_DB_PATH"] = ":memory:" os.environ["MINIO_ACCESS_KEY"] = "test-key" os.environ["MINIO_SECRET_KEY"] = "test-secret" os.environ["ALLOWED_FILE_PATHS"] = "/app/books,/tmp" +_code_seq = itertools.count(1) + + +def _next_code() -> str: + """A unique document code for fixtures (documents need one in v2).""" + return f"doc-{next(_code_seq)}" + + class TestHealthEndpoints: """Tests for health check endpoints.""" @@ -53,6 +62,8 @@ def test_list_documents(self, test_client, db_connection): """Test listing documents.""" # Create a test document db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id="api-test-001", document_id="doc-001", filename="test.pdf", @@ -70,6 +81,8 @@ def test_list_documents(self, test_client, db_connection): def test_list_documents_with_stage_filter(self, test_client, db_connection): """Test filtering documents by stage.""" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id="filter-api-001", document_id="doc-filter-001", filename="completed.pdf", @@ -86,9 +99,11 @@ def test_list_documents_with_stage_filter(self, test_client, db_connection): @pytest.mark.api @pytest.mark.unit def test_get_document_from_db(self, test_client, db_connection): - """Test getting document details from SQLite fallback.""" + """Test getting document details from the database.""" workflow_id = "get-test-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-get-001", filename="test.pdf", @@ -109,6 +124,150 @@ def test_get_document_not_found(self, test_client): assert response.status_code == 404 +class TestDocumentIdentifiers: + """`/documents/{id}` takes the UUID (canonical) or any legacy identifier.""" + + FINGERPRINT = hashlib.md5(b"identifiers").hexdigest() + WORKFLOW_ID = f"doc-{FINGERPRINT}" + + def _document(self, db_connection): + db_connection.upsert_document( + kind="scheme", + code=_next_code(), + workflow_id=self.WORKFLOW_ID, + document_id=self.FINGERPRINT, + filename="identifiers.pdf", + filepath="minio://documents/identifiers.pdf", + stage="ocr_review", + ) + db_connection.save_pages( + self.WORKFLOW_ID, [{"page_number": 1, "original_markdown": "page one"}] + ) + return db_connection.get_document(self.WORKFLOW_ID) + + @pytest.mark.api + @pytest.mark.unit + def test_the_response_carries_the_uuid_and_the_temporal_id(self, test_client, db_connection): + doc = self._document(db_connection) + body = test_client.get(f"/documents/{doc['id']}").json() + assert body["id"] == doc["id"] + assert body["workflow_id"] == self.WORKFLOW_ID + assert body["document_id"] == self.FINGERPRINT + + @pytest.mark.api + @pytest.mark.unit + @pytest.mark.parametrize("form", ["uuid", "workflow", "fingerprint", "filename"]) + def test_every_identifier_form_reaches_the_same_document(self, test_client, db_connection, form): + doc = self._document(db_connection) + identifier = { + "uuid": doc["id"], + "workflow": self.WORKFLOW_ID, + "fingerprint": self.FINGERPRINT, + "filename": "identifiers.pdf", + }[form] + detail = test_client.get(f"/documents/{identifier}") + pages = test_client.get(f"/documents/{identifier}/pages") + assert detail.status_code == 200 and pages.status_code == 200 + assert detail.json()["id"] == doc["id"] + assert [p["page_number"] for p in pages.json()] == [1] + + @pytest.mark.api + @pytest.mark.unit + def test_the_uuid_is_rewritten_to_the_temporal_id_before_handlers_run( + self, test_client, db_connection, mock_temporal_client + ): + doc = self._document(db_connection) + db_connection.update_document_stage(doc["id"], "approval_for_prod") + response = test_client.post(f"/documents/{doc['id']}/approve-prod") + assert response.status_code == 200 + queried = [call.args[0] for call in mock_temporal_client.get_workflow_handle.call_args_list] + assert queried and set(queried) == {self.WORKFLOW_ID} + + @pytest.mark.api + @pytest.mark.unit + def test_static_document_routes_are_not_mistaken_for_identifiers(self, test_client, db_connection): + db_connection.upsert_document( + kind="scheme", + code=_next_code(), + workflow_id="summary", document_id=hashlib.md5(b"summary").hexdigest(), + filename="summary", filepath="/tmp/s.pdf", + ) + response = test_client.get("/documents/summary") + assert response.status_code == 200 + assert "total_documents" in response.json() + + @pytest.mark.api + @pytest.mark.unit + def test_an_unknown_identifier_is_404(self, test_client, db_connection): + assert test_client.get("/documents/00000000-0000-0000-0000-000000000000").status_code == 404 + + +class TestArtifactAndJobIdentifiers: + """Artifact and job ids are UUIDs.""" + + def _document(self, db_connection): + db_connection.upsert_document( + kind="scheme", + code=_next_code(), + workflow_id="wf-art-api", document_id=hashlib.md5(b"art").hexdigest(), + filename="a.pdf", filepath="minio://documents/a.pdf", stage="ocr_review", + ) + artifact_id = db_connection.add_document_artifact( + "wf-art-api", "original_upload", "minio://documents/a.pdf", stage="registered", + filename="a.pdf", + ) + return artifact_id + + @pytest.mark.api + @pytest.mark.unit + def test_artifacts_are_addressed_by_uuid(self, test_client, db_connection): + artifact_id = self._document(db_connection) + listed = test_client.get("/documents/wf-art-api/artifacts").json() + assert [a["id"] for a in listed] == [artifact_id] + assert test_client.get(f"/documents/wf-art-api/artifacts/{artifact_id}").json()["id"] == artifact_id + assert test_client.get("/documents/wf-art-api/artifacts/7").status_code == 404 + + @pytest.mark.api + @pytest.mark.unit + def test_document_detail_derives_artifact_and_job_pointers(self, test_client, db_connection): + artifact_id = self._document(db_connection) + body = test_client.get("/documents/wf-art-api").json() + job = db_connection.get_latest_document_job("wf-art-api") + assert body["original_artifact_id"] == artifact_id + assert body["normalized_artifact_id"] is None + assert body["latest_job_id"] == job["id"] + assert body["artifacts"][0]["job_id"] == job["id"] + + @pytest.mark.api + @pytest.mark.unit + def test_runs_are_addressed_by_uuid(self, test_client, db_connection): + self._document(db_connection) + job = db_connection.get_latest_document_job("wf-art-api") + assert test_client.get(f"/runs/{job['id']}").json()["id"] == job["id"] + assert test_client.get("/runs/3").status_code == 404 + assert [r["id"] for r in test_client.get("/runs").json()] == [job["id"]] + + +class TestHealthAndSettings: + @pytest.mark.api + @pytest.mark.unit + def test_health_reports_the_database(self, test_client): + body = test_client.get("/health").json() + assert body["database"] == {"reachable": True, "migrations_current": True} + + @pytest.mark.api + @pytest.mark.unit + def test_settings_default_alpha_and_validation(self, test_client): + assert test_client.get("/settings/search").json()["alpha"] == 0.6 + assert test_client.put("/settings/search", json={"alpha": 2}).status_code == 400 + updated = test_client.put("/settings/search", json={"limit": 7}).json() + assert updated["limit"] == 7 + reset = test_client.post("/settings/search/reset").json() + assert reset["limit"] == 12 and reset["alpha"] == 0.6 + audit = test_client.get("/settings/search/audit").json() + assert audit["total"] == 2 and audit["logs"][0]["document_id"] == "__settings__" + + class TestProdApproval: """Promoting to PROD takes no body. @@ -118,6 +277,8 @@ class TestProdApproval: def _document_at_the_gate(self, db_connection, workflow_id): db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id=f"doc-{workflow_id}", filename="test.pdf", @@ -181,6 +342,8 @@ def test_the_document_carries_no_validity_window(self, test_client, db_connectio def test_the_wrong_stage_is_still_refused(self, test_client, db_connection): workflow_id = "prod-approval-005" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id=f"doc-{workflow_id}", filename="test.pdf", @@ -203,6 +366,8 @@ class TestDocumentValidityEndpoints: def _uploaded_document(self, db_connection, workflow_id): db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id=f"doc-{workflow_id}", filename="test.pdf", @@ -222,6 +387,7 @@ def test_upload_starts_today_and_never_expires(self, test_client, sample_pdf_con response = test_client.post( "/upload", + params={"kind": "scheme"}, files={"file": ("validity-upload.pdf", sample_pdf_content, "application/pdf")}, ) @@ -240,13 +406,13 @@ def test_an_end_date_can_be_set_and_then_cleared(self, test_client, db_connectio set_response = test_client.patch( f"/documents/{workflow_id}/scheme-metadata", - json={"document_kind": "advisory", "valid_from": "2026-01-01", "valid_to": "2026-06-30"}, + json={"kind": "advisory", "valid_from": "2026-01-01", "valid_to": "2026-06-30"}, ) assert set_response.json()["valid_to"] == "2026-06-30" cleared = test_client.patch( f"/documents/{workflow_id}/scheme-metadata", - json={"document_kind": "advisory", "valid_to": ""}, + json={"kind": "advisory", "valid_to": ""}, ) assert cleared.status_code == 200 @@ -276,7 +442,7 @@ def test_stores_the_reviewers_edit(self, test_client, db_connection): response = test_client.patch( f"/documents/{workflow_id}/scheme-metadata", json={ - "document_kind": "advisory", + "kind": "advisory", "valid_from": "2026-10-01", "valid_to": "2026-12-31", }, @@ -298,7 +464,7 @@ def test_moving_only_the_end_keeps_the_stored_start(self, test_client, db_connec response = test_client.patch( f"/documents/{workflow_id}/scheme-metadata", - json={"document_kind": "advisory", "valid_to": "2027-06-30"}, + json={"kind": "advisory", "valid_to": "2027-06-30"}, ) assert response.status_code == 200 @@ -314,7 +480,7 @@ def test_classifying_without_dates_leaves_the_period_alone(self, test_client, db test_client.patch( f"/documents/{workflow_id}/scheme-metadata", - json={"document_kind": "advisory"}, + json={"kind": "advisory"}, ) stored = db_connection.get_document(workflow_id) @@ -330,7 +496,7 @@ def test_rejects_an_end_before_the_start(self, test_client, db_connection): response = test_client.patch( f"/documents/{workflow_id}/scheme-metadata", json={ - "document_kind": "advisory", + "kind": "advisory", "valid_from": "2026-12-31", "valid_to": "2026-01-01", }, @@ -347,7 +513,7 @@ def test_rejects_a_malformed_date(self, test_client, db_connection): response = test_client.patch( f"/documents/{workflow_id}/scheme-metadata", - json={"document_kind": "advisory", "valid_to": "31-12-2026"}, + json={"kind": "advisory", "valid_to": "31-12-2026"}, ) assert response.status_code == 400 @@ -364,13 +530,13 @@ def test_a_rejected_period_stores_nothing(self, test_client, db_connection): test_client.patch( f"/documents/{workflow_id}/scheme-metadata", - json={"document_kind": "advisory", "valid_to": "not-a-date"}, + json={"kind": "advisory", "valid_to": "not-a-date"}, ) stored = db_connection.get_document(workflow_id) assert stored["valid_from"] == "2026-01-01" assert stored["valid_to"] == "2026-06-30" - assert (stored["document_kind"] or "document") == "document" + assert stored["kind"] == "scheme" @pytest.mark.api @pytest.mark.unit @@ -580,6 +746,8 @@ def test_list_pages(self, test_client, db_connection): """Test listing pages for a document.""" workflow_id = "pages-test-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-pages", filename="test.pdf", @@ -606,6 +774,8 @@ def test_get_page(self, test_client, db_connection): """Test getting a specific page.""" workflow_id = "page-get-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-page", filename="test.pdf", @@ -629,6 +799,8 @@ def test_get_page_invalid_number(self, test_client, db_connection): """Test validation rejects invalid page numbers.""" workflow_id = "page-invalid-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-invalid", filename="test.pdf", @@ -650,6 +822,8 @@ def test_update_page(self, test_client, db_connection): """Test updating a page.""" workflow_id = "page-update-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-update", filename="test.pdf", @@ -680,6 +854,8 @@ def test_list_chunks(self, test_client, db_connection): """Test listing chunks for a document.""" workflow_id = "chunks-test-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-chunks", filename="test.pdf", @@ -706,6 +882,8 @@ def test_list_chunks_include_excluded(self, test_client, db_connection): """Test including excluded chunks in listing.""" workflow_id = "chunks-excluded-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-excluded", filename="test.pdf", @@ -736,6 +914,8 @@ def test_get_chunk_invalid_number(self, test_client, db_connection): """Test validation rejects invalid chunk numbers.""" workflow_id = "chunk-invalid-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-chunk-invalid", filename="test.pdf", @@ -756,6 +936,8 @@ def test_get_document_audit(self, test_client, db_connection): """Test getting audit logs for a document.""" workflow_id = "audit-test-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-audit", filename="test.pdf", @@ -850,6 +1032,8 @@ def test_page_not_found(self, test_client, db_connection): """Test 404 for missing page.""" workflow_id = "page-404-test" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-404", filename="test.pdf", @@ -883,3 +1067,336 @@ def test_inline_content_disposition_strips_crlf(self): assert header.count(":") >= 1 assert "\r\nX-Injected" not in header assert header.startswith('inline; filename="') + + +class TestUploadCatalogIdentity: + """/upload picks the tenant's catalog from ``kind`` and names the document by ``code``.""" + + @pytest.fixture(autouse=True) + def _fresh_rate_limit(self): + # /upload is limited per IP; these tests post more often than one minute allows. + from pipeline.api import api + + api.limiter.reset() + yield + api.limiter.reset() + + @staticmethod + def _pdf(sample_pdf_content: bytes, tag: str) -> bytes: + # A different trailer gives a different fingerprint, hence a new workflow id. + return sample_pdf_content + f"\n% {tag}".encode() + + @pytest.mark.api + @pytest.mark.unit + def test_kind_is_required(self, test_client, sample_pdf_content): + response = test_client.post( + "/upload", files={"file": ("a.pdf", sample_pdf_content, "application/pdf")} + ) + + assert response.status_code == 422 + + @pytest.mark.api + @pytest.mark.unit + @pytest.mark.parametrize("kind", ["document", "podcast", "", "Scheme-X"]) + def test_an_unknown_kind_is_rejected(self, test_client, sample_pdf_content, kind): + response = test_client.post( + "/upload", + params={"kind": kind}, + files={"file": ("a.pdf", sample_pdf_content, "application/pdf")}, + ) + + assert response.status_code == 400 + assert "scheme" in response.json()["detail"] + + @pytest.mark.api + @pytest.mark.unit + def test_an_invalid_code_is_rejected(self, test_client, sample_pdf_content): + response = test_client.post( + "/upload", + params={"kind": "scheme", "code": "Not A Slug"}, + files={"file": ("a.pdf", sample_pdf_content, "application/pdf")}, + ) + + assert response.status_code == 400 + + @pytest.mark.api + @pytest.mark.unit + def test_upload_creates_the_catalog_and_derives_the_code_from_the_filename( + self, test_client, db_connection, sample_pdf_content + ): + response = test_client.post( + "/upload", + params={"kind": "scheme", "instance": "mh"}, + files={"file": ("National Mission on Natural Farming.pdf", sample_pdf_content, "application/pdf")}, + ) + + assert response.status_code == 200 + doc = test_client.get(f"/documents/{response.json()['workflow_id']}").json() + assert doc["kind"] == "scheme" + assert response.json()["instance"] == "mh" + assert doc["instance"] == "mh" + assert "group" not in doc + assert doc["code"] == "nmnf" + assert doc["name"] == "National Mission on Natural Farming" + with db_connection.get_connection() as conn: + catalog = conn.execute( + "SELECT c.type, g.name, g.path FROM documents d JOIN catalogs c ON c.id = d.catalog_id " + "JOIN instances g ON g.id = c.instance_id WHERE d.code = %s", + (doc["code"],), + ).fetchall() + assert [(r["type"], r["name"], r["path"]) for r in catalog] == [ + ("openagrinet:AgricultureResource", "mh", "/states/MH") + ] + + @pytest.mark.api + @pytest.mark.unit + def test_upload_rejects_an_unauthorized_instance( + self, test_client, db_connection, sample_pdf_content, monkeypatch + ): + from pipeline.api.api import app + from pipeline.auth.deps import get_current_user + + monkeypatch.setitem( + app.dependency_overrides, get_current_user, + lambda: self._user("/states/MH/contributor"), + ) + response = test_client.post( + "/upload", + params={"kind": "scheme", "instance": "gj"}, + files={"file": ("a.pdf", sample_pdf_content, "application/pdf")}, + ) + + assert response.status_code == 403 + with db_connection.get_connection() as conn: + assert conn.execute("SELECT count(*) AS n FROM documents").fetchone()["n"] == 0 + + @pytest.mark.api + @pytest.mark.unit + def test_a_second_upload_reuses_the_catalog_and_gets_a_fresh_code( + self, test_client, db_connection, sample_pdf_content + ): + first = test_client.post( + "/upload", + params={"kind": "advisory", "instance": "mh"}, + files={"file": ("Crop Advisory.pdf", self._pdf(sample_pdf_content, "one"), "application/pdf")}, + ) + second = test_client.post( + "/upload", + params={"kind": "advisory", "instance": "mh"}, + files={"file": ("Crop Advisory.pdf", self._pdf(sample_pdf_content, "two"), "application/pdf")}, + ) + + assert first.status_code == second.status_code == 200 + codes = { + test_client.get(f"/documents/{r.json()['workflow_id']}").json()["code"] + for r in (first, second) + } + assert codes == {"ca", "ca-2"} + with db_connection.get_connection() as conn: + assert conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] == 3 # seeded with the group + + @pytest.mark.api + @pytest.mark.unit + def test_an_existing_code_updates_the_same_document( + self, test_client, db_connection, sample_pdf_content + ): + first = test_client.post( + "/upload", + params={"kind": "scheme", "instance": "mh", "code": "pm-kisan"}, + files={"file": ("pmkisan.pdf", self._pdf(sample_pdf_content, "v1"), "application/pdf")}, + ) + assert first.status_code == 200 + before = test_client.get(f"/documents/{first.json()['workflow_id']}").json() + db_connection.save_pages( + before["id"], [{"page_number": 1, "original_markdown": "old text"}] + ) + db_connection.save_chunks(before["id"], [{"chunk_number": 1, "original_text": "old"}]) + # The first upload's run is still "running"; a replace waits for it to settle. + db_connection.update_document_job( + db_connection.get_latest_document_job(before["id"])["id"], status="waiting_review" + ) + + second = test_client.post( + "/upload", + params={"kind": "scheme", "instance": "mh", "code": "pm-kisan"}, + files={"file": ("pmkisan-2026.pdf", self._pdf(sample_pdf_content, "v2"), "application/pdf")}, + ) + + assert second.status_code == 200 + assert second.json()["workflow_id"] != first.json()["workflow_id"] + after = test_client.get(f"/documents/{before['id']}").json() + assert after["id"] == before["id"] + assert after["filename"] == "pmkisan-2026.pdf" + assert after["source_file_fingerprint"] != before["source_file_fingerprint"] + assert after["reindex_required"] is True + assert after["reindex_reason"] == "source_replaced" + assert (after["page_count"], after["chunk_count"]) == (0, 0) + with db_connection.get_connection() as conn: + assert conn.execute("SELECT count(*) AS n FROM documents").fetchone()["n"] == 1 + assert conn.execute("SELECT count(*) AS n FROM workflows").fetchone()["n"] == 2 + + @pytest.mark.api + @pytest.mark.unit + def test_the_same_code_in_another_tenant_is_a_different_document( + self, test_client, db_connection, sample_pdf_content + ): + for group, tag in (("mh", "a"), ("gj", "b")): + response = test_client.post( + "/upload", + params={"kind": "scheme", "instance": group, "code": "pm-kisan"}, + files={"file": ("p.pdf", self._pdf(sample_pdf_content, tag), "application/pdf")}, + ) + assert response.status_code == 200 + + with db_connection.get_connection() as conn: + assert conn.execute("SELECT count(*) AS n FROM documents").fetchone()["n"] == 2 + assert conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] == 6 # 3 per group + + @staticmethod + def _user(*group_paths): + from pipeline.auth.jwt import claims_to_user + + return claims_to_user({"sub": "u-" + "-".join(group_paths), "groups": list(group_paths)}) + + @staticmethod + def _mh_doc(db, **extra): + doc_uuid = db.upsert_document( + workflow_id="wf-mh-doc", + document_id="d-mh", + filename="a.pdf", + filepath="/tmp/a.pdf", + instance="mh", + kind="scheme", + code="pm-kisan", + **extra, + ) + # A new document starts with a "running" workflow; settle it so the + # document is at rest, as one waiting in review is. + db.update_document_job(db.get_latest_document_job(doc_uuid)["id"], status="waiting_review") + return doc_uuid + + @pytest.mark.api + @pytest.mark.unit + def test_replacing_a_document_needs_rights_in_its_instance(self, db_connection): + from fastapi import HTTPException + + from pipeline.api import api + + self._mh_doc(db_connection) + identity = lambda user: api._resolve_document_identity( # noqa: E731 + user, "mh", "scheme", "pm-kisan", "a.pdf" + ) + + # Another state, and a view-only member of this one, are both turned away + # before anything about the document is revealed. + for outsider in (self._user("/states/GJ/contributor"), self._user("/states/MH/view")): + with pytest.raises(HTTPException) as exc: + identity(outsider) + assert exc.value.status_code == 403 + + # The same state with a role that can upload and review may replace. + kind, code, existing = identity(self._user("/states/MH/contributor")) + assert (kind, code, existing["workflow_id"]) == ("scheme", "pm-kisan", "wf-mh-doc") + + @pytest.mark.api + @pytest.mark.unit + def test_upload_into_a_instance_needs_upload_rights_there(self, db_connection): + from fastapi import HTTPException + + from pipeline.api import api + + mixed = self._user("/states/MH/contributor", "/states/GJ/view") + # Allowed where the user can upload, refused where they only view. + api._resolve_document_identity(mixed, "mh", "scheme", "", "new scheme.pdf") + with pytest.raises(HTTPException) as exc: + api._resolve_document_identity(mixed, "gj", "scheme", "", "new scheme.pdf") + assert exc.value.status_code == 403 + + @pytest.mark.api + @pytest.mark.unit + def test_a_disabled_or_running_document_cannot_be_replaced(self, db_connection): + from fastapi import HTTPException + + from pipeline.api import api + + db = db_connection + contributor = self._user("/states/MH/contributor") + doc_uuid = self._mh_doc(db) + + db.create_document_job(doc_uuid, "pipeline", "wf-mh-doc", status="running") + with pytest.raises(HTTPException) as exc: + api._resolve_document_identity(contributor, "mh", "scheme", "pm-kisan", "a.pdf") + assert (exc.value.status_code, "in progress" in exc.value.detail) == (409, True) + + db.update_document_job(db.get_latest_document_job(doc_uuid)["id"], status="waiting_review") + db.set_document_disabled("wf-mh-doc", True) + with pytest.raises(HTTPException) as exc: + api._resolve_document_identity(contributor, "mh", "scheme", "pm-kisan", "a.pdf") + assert (exc.value.status_code, "disabled" in exc.value.detail) == (409, True) + + @pytest.mark.api + @pytest.mark.unit + def test_a_document_live_in_prod_needs_approval_rights_to_replace(self, db_connection): + from fastapi import HTTPException + + from pipeline.api import api + + db = db_connection + self._mh_doc(db) + db.upsert_document_index_status( + "wf-mh-doc", "documents-index", doc_id="d-mh", status="indexed" + ) + db.update_document_fields("wf-mh-doc") # no-op, keeps the dev entry only + with db.get_connection() as conn: + conn.execute( + "UPDATE documents SET metadata = jsonb_set(metadata, '{index_status,prod}', " + "'{\"status\": \"indexed\", \"doc_id\": \"d-mh\"}')" + ) + + with pytest.raises(HTTPException) as exc: + api._resolve_document_identity( + self._user("/states/MH/contributor"), "mh", "scheme", "pm-kisan", "a.pdf" + ) + assert exc.value.status_code == 403 + + _, _, existing = api._resolve_document_identity( + self._user("/states/MH/approver"), "mh", "scheme", "pm-kisan", "a.pdf" + ) + assert existing["workflow_id"] == "wf-mh-doc" + + @pytest.mark.api + @pytest.mark.unit + def test_the_demo_and_bootstrap_routes_are_gone(self, test_client): + assert test_client.post("/documents/anything/demo").status_code in (404, 405) + assert test_client.post("/catalog/v1/bootstrap").status_code in (404, 405) + + +class TestVectorKeysToDelete: + @pytest.mark.api + @pytest.mark.unit + def test_disable_never_deletes_a_key_another_live_document_is_indexed_under(self, db_connection): + from pipeline.api import api + + db = db_connection + uuid_a = db.upsert_document( + workflow_id="wf-k-a", document_id="a" * 32, filename="a.pdf", + filepath="minio://documents/a.pdf", instance="mh", kind="scheme", code="k-a", + ) + db.upsert_document( + workflow_id="wf-k-b", document_id="a" * 32, filename="a.pdf", + filepath="minio://documents/a.pdf", instance="gj", kind="scheme", code="k-b", + ) + db.upsert_document( + workflow_id="wf-k-c", document_id="b" * 32, filename="c.pdf", + filepath="minio://documents/c.pdf", instance="mh", kind="scheme", code="k-c", + ) + + # The twin was only uploaded, never indexed, so nothing of it is at risk. + assert api._vector_doc_ids(db.get_document(uuid_a), "dev") == ["a" * 32] + + db.upsert_document_index_status("wf-k-b", "documents-index", doc_id="a" * 32, status="indexed") + assert api._vector_doc_ids(db.get_document(uuid_a), "dev") == [] + assert api._vector_doc_ids(db.get_document("wf-k-c"), "dev") == ["b" * 32] + + db.set_document_disabled("wf-k-b", True) + assert api._vector_doc_ids(db.get_document(uuid_a), "dev") == ["a" * 32] diff --git a/tests/test_api_routes.py b/tests/test_api_routes.py index 351cc35..e8fc2a3 100644 --- a/tests/test_api_routes.py +++ b/tests/test_api_routes.py @@ -56,6 +56,7 @@ ("GET", "/runs/{job_id}"), ("GET", "/settings/search"), ("GET", "/settings/search/audit"), + ("GET", "/webhook/advisory/status"), ("PATCH", "/documents/{workflow_id}/chunks/{chunk_num}"), ("PATCH", "/documents/{workflow_id}/pages/{page_num}"), ("PATCH", "/documents/{workflow_id}/scheme-metadata"), @@ -66,7 +67,6 @@ ("POST", "/auth/otp/verify"), ("POST", "/auth/session/refresh"), ("POST", "/auth/sso/exchange"), - ("POST", "/catalog/v1/bootstrap"), ("POST", "/catalog/v1/rebuild"), ("POST", "/documents"), ("POST", "/documents/batch"), @@ -82,7 +82,6 @@ ("POST", "/documents/{workflow_id}/approve-translation"), ("POST", "/documents/{workflow_id}/chunks/{chunk_num}/reset"), ("POST", "/documents/{workflow_id}/clear-reindex-required"), - ("POST", "/documents/{workflow_id}/demo"), ("POST", "/documents/{workflow_id}/mark-reindex-required"), ("POST", "/documents/{workflow_id}/pages/{page_num}/reset"), ("POST", "/documents/{workflow_id}/reconcile"), @@ -96,6 +95,9 @@ ("POST", "/search"), ("POST", "/settings/search/reset"), ("POST", "/upload"), + ("POST", "/webhook"), + ("POST", "/webhook/advisory/promote"), + ("POST", "/webhook/advisory/reindex"), ("PUT", "/admin/users/{user_id}/access"), ("PUT", "/settings/search"), ] diff --git a/tests/test_auth.py b/tests/test_auth.py index 8623866..80483d2 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -200,6 +200,10 @@ def test_every_route_is_gated_or_explicitly_classified(): # credential, and the access token has usually expired by then. ("POST", "/auth/session/refresh"), ("POST", "/auth/otp/refresh"), + # Inbound webhook for external systems: gated by its own x-api-key + # (require_webhook_key, fails closed when WEBHOOK_API_KEY is unset), + # not by a user token. + ("POST", "/webhook"), } def dependency_calls(dependant): calls = {dependant.call} diff --git a/tests/test_db.py b/tests/test_db.py index b771611..01820a6 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -1,5 +1,5 @@ """ -Unit tests for pipeline/db.py - SQLite database operations. +Unit tests for pipeline/db.py against a real Postgres. Tests cover: - Document CRUD operations @@ -7,12 +7,45 @@ - Chunk operations - Audit logging - Settings management +- Schema constraints, JSONB merges, workflows, the catalog view, identifier + resolution and the migration runner """ +import hashlib +import re +import psycopg import pytest +def _md5(text): + return hashlib.md5(text.encode()).hexdigest() + + +def _code_for(workflow_id): + return re.sub(r"[^a-z0-9]+", "-", (workflow_id or "doc").lower()).strip("-") or "doc" + + +@pytest.fixture(autouse=True) +def _new_documents_default_to_a_scheme(monkeypatch): + """A new v2 document needs a kind and a code; these tests are about everything else. + + The requirement itself is pinned in test_schema_v2.py. Here a new document is a + scheme whose code is its workflow id, unless a test says otherwise. + """ + from pipeline import db + + real = db.upsert_document + + def with_defaults(*args, **kwargs): + kwargs.setdefault("kind", "scheme") + if not kwargs.get("code"): + kwargs["code"] = _code_for(kwargs.get("workflow_id")) + return real(*args, **kwargs) + + monkeypatch.setattr(db, "upsert_document", with_defaults) + + class TestDocumentOperations: """Tests for document CRUD operations.""" @@ -21,7 +54,7 @@ class TestDocumentOperations: def test_upsert_document_creates_new(self, db_connection): """Test creating a new document.""" workflow_id = "test-wf-001" - document_id = "test-doc-001" + document_id = _md5("test-doc-001") db_connection.upsert_document( workflow_id=workflow_id, @@ -125,34 +158,17 @@ def test_update_document_stage(self, db_connection): stage="registered" ) - db_connection.update_document_stage( - workflow_id=workflow_id, - stage="ocr_review", - page_count=5, - chunk_count=10 - ) + db_connection.update_document_stage(workflow_id=workflow_id, stage="ocr_review") doc = db_connection.get_document(workflow_id) assert doc["stage"] == "ocr_review" - assert doc["page_count"] == 5 - assert doc["chunk_count"] == 10 - - @pytest.mark.db - @pytest.mark.unit - def test_set_document_demo(self, db_connection): - """Test marking document as demo.""" - workflow_id = "demo-test" - db_connection.upsert_document( - workflow_id=workflow_id, - document_id="doc-demo", - filename="test.pdf", - filepath="/app/books/test.pdf", - stage="registered" - ) - - db_connection.set_document_demo(workflow_id, True) + # Counts are derived from pages and chunks, not stored. + assert (doc["page_count"], doc["chunk_count"]) == (0, 0) + db_connection.save_pages(workflow_id, [{"page_number": 1, "original_markdown": "a"}]) + db_connection.save_chunks(workflow_id, [{"chunk_number": 1, "original_text": "x"}, + {"chunk_number": 2, "original_text": "y"}]) doc = db_connection.get_document(workflow_id) - assert doc["is_demo"] == 1 + assert (doc["page_count"], doc["chunk_count"]) == (1, 2) @pytest.mark.db @pytest.mark.unit @@ -586,3 +602,678 @@ def test_settings_audit_log(self, db_connection): db_connection.update_search_settings({"limit": 50}) logs = db_connection.get_settings_audit_logs() assert len(logs) >= 1 + + +def _new_doc(db, workflow_id, **kwargs): + kwargs.setdefault("document_id", _md5(workflow_id)) + kwargs.setdefault("filename", f"{workflow_id}.pdf") + kwargs.setdefault("filepath", f"minio://documents/{workflow_id}.pdf") + return db.upsert_document(workflow_id=workflow_id, **kwargs) + + +class TestSchemaConstraints: + """Rules the database itself enforces.""" + + @staticmethod + def _catalog_id(db): + with db.get_connection() as conn: + return db.get_or_create_catalog(db.get_or_create_instance("mh", conn=conn), "scheme", conn=conn)["id"] + + @pytest.mark.db + @pytest.mark.unit + def test_document_code_must_be_a_slug(self, db_connection): + catalog_id = self._catalog_id(db_connection) + with db_connection.get_connection() as conn: + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute( + "INSERT INTO documents (catalog_id, code, name, source) VALUES (%s::uuid, 'Bad Code', 'N', %s)", + (catalog_id, psycopg.types.json.Jsonb({"fingerprint": _md5("k")})), + ) + + @pytest.mark.db + @pytest.mark.unit + def test_source_requires_a_hex_fingerprint(self, db_connection): + catalog_id = self._catalog_id(db_connection) + with db_connection.get_connection() as conn: + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute( + "INSERT INTO documents (catalog_id, code, name, source) VALUES (%s::uuid, 'c', 'C', %s)", + (catalog_id, psycopg.types.json.Jsonb({"fingerprint": "not-hex"})), + ) + + @pytest.mark.db + @pytest.mark.unit + def test_stage_timestamps_only_allow_known_keys(self, db_connection): + _new_doc(db_connection, "wf-ts") + with db_connection.get_connection() as conn: + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute( + "UPDATE workflows SET stage_timestamps = '{\"bogus\": \"x\"}'::jsonb " + "WHERE temporal_workflow_id = 'wf-ts'" + ) + + @pytest.mark.db + @pytest.mark.unit + def test_validity_period_must_not_end_before_it_starts(self, db_connection): + # The period lives in documents.metadata, so Pydantic enforces the order. + with pytest.raises(ValueError): + _new_doc(db_connection, "wf-bad-period", valid_from="2026-02-01", valid_to="2026-01-01") + + @pytest.mark.db + @pytest.mark.unit + def test_chunk_page_range_must_not_run_backwards(self, db_connection): + _new_doc(db_connection, "wf-range") + with pytest.raises(psycopg.errors.CheckViolation): + with db_connection.get_connection() as conn: + conn.execute( + "INSERT INTO chunks (document_id, chunk_number, original_text, page_start, page_end) " + "SELECT id, 1, 't', 3, 2 FROM documents" + ) + + @pytest.mark.db + @pytest.mark.unit + def test_audit_entry_needs_a_workflow_unless_system_level(self, db_connection): + with db_connection.get_connection() as conn: + conn.execute("INSERT INTO audit_logs (action_type, entity_type) VALUES ('x', 'setting')") + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute("INSERT INTO audit_logs (action_type, entity_type) VALUES ('x', 'page')") + + @pytest.mark.db + @pytest.mark.unit + def test_search_settings_is_a_single_row(self, db_connection): + with db_connection.get_connection() as conn: + with pytest.raises(psycopg.errors.UniqueViolation): + conn.execute("INSERT INTO search_settings DEFAULT VALUES") + + +class TestWorkflows: + @pytest.mark.db + @pytest.mark.unit + def test_upsert_creates_the_main_workflow(self, db_connection): + _new_doc(db_connection, "wf-main", stage="ocr_processing") + jobs = db_connection.list_document_jobs("wf-main") + assert [(j["job_type"], j["temporal_workflow_id"]) for j in jobs] == [("pipeline", "wf-main")] + + @pytest.mark.db + @pytest.mark.unit + def test_current_workflow_is_the_latest_by_start_time(self, db_connection): + _new_doc(db_connection, "wf-cur", stage="completed") + job_id = db_connection.create_document_job( + "wf-cur", "chunking_retry", temporal_workflow_id="wf-cur-retry-chunking-1", + current_stage="chunking", + ) + doc = db_connection.get_document("wf-cur") + assert doc["stage"] == "chunking" + assert doc["latest_job_id"] == job_id + assert doc["workflow_id"] == "wf-cur" + + @pytest.mark.db + @pytest.mark.unit + def test_stage_updates_follow_the_current_workflow(self, db_connection): + _new_doc(db_connection, "wf-follow", stage="completed") + db_connection.create_document_job("wf-follow", "ocr_retry", current_stage="ocr_processing") + db_connection.update_document_stage("wf-follow", "ocr_review") + by_type = {j["job_type"]: j["current_stage"] for j in db_connection.list_document_jobs("wf-follow")} + assert by_type == {"pipeline": "completed", "ocr_retry": "ocr_review"} + + @pytest.mark.db + @pytest.mark.unit + def test_stage_timestamps_are_set_once(self, db_connection): + _new_doc(db_connection, "wf-once") + db_connection.update_document_stage("wf-once", "ocr_review") + first = db_connection.get_document("wf-once")["ocr_completed_at"] + assert first + db_connection.update_document_stage("wf-once", "ocr_processing") + db_connection.update_document_stage("wf-once", "ocr_review") + assert db_connection.get_document("wf-once")["ocr_completed_at"] == first + + @pytest.mark.db + @pytest.mark.unit + def test_prod_request_is_a_waiting_promote_workflow_that_approval_adopts(self, db_connection): + _new_doc(db_connection, "wf-prod", stage="approval_for_prod") + doc = db_connection.mark_prod_ready_requested("wf-prod", user_id="kc-1", username="reviewer1") + assert doc["prod_ready_requested_by_username"] == "reviewer1" + latest = db_connection.get_latest_document_job("wf-prod") + assert (latest["job_type"], latest["status"]) == ("promote_prod", "waiting_review") + + db_connection.create_document_job( + "wf-prod", "promote_prod", temporal_workflow_id="wf-prod-promote-prod-1", + current_stage="ingesting_prod", + ) + jobs = [j for j in db_connection.list_document_jobs("wf-prod") if j["job_type"] == "promote_prod"] + assert len(jobs) == 1 + assert jobs[0]["temporal_workflow_id"] == "wf-prod-promote-prod-1" + assert db_connection.get_document("wf-prod")["prod_ready_requested_at"] is None + + @pytest.mark.db + @pytest.mark.unit + def test_job_config_merges(self, db_connection): + _new_doc(db_connection, "wf-cfg") + job = db_connection.get_latest_document_job("wf-cfg") + db_connection.update_document_job(job["id"], config_json={"a": 1}) + db_connection.update_document_job(job["id"], config_json={"chunking_progress": {"percent": 5}}) + config = db_connection.get_document_job(job["id"])["config"] + assert config == {"a": 1, "chunking_progress": {"percent": 5}} + + @pytest.mark.db + @pytest.mark.unit + def test_artifacts_belong_to_their_workflow_and_derive_pointers(self, db_connection): + _new_doc(db_connection, "wf-art") + job = db_connection.get_latest_document_job("wf-art") + art_id = db_connection.add_document_artifact( + "wf-art", "original_upload", "minio://documents/a.pdf", stage="registered", job_id=job["id"], + ) + norm_id = db_connection.add_document_artifact( + "wf-art", "normalized_pdf", "minio://documents/n.pdf", stage="ocr_processing", + ) + doc = db_connection.get_document("wf-art") + assert doc["original_artifact_id"] == art_id + assert doc["normalized_artifact_id"] == norm_id + artifact = db_connection.get_document_artifact(doc["id"], art_id) + assert artifact["job_id"] == job["id"] + assert artifact["workflow_id"] == "wf-art" + assert db_connection.get_document_artifact("wf-art", "123") is None + + +class TestPageTranslations: + @pytest.mark.db + @pytest.mark.unit + def test_an_ocr_save_keeps_existing_translations(self, db_connection): + _new_doc(db_connection, "wf-tr") + db_connection.save_pages("wf-tr", [{ + "page_number": 1, "original_markdown": "namaste", "detected_language": "hi", + "translated_markdown": "hello", "translation_target_language": "en", + "translation_provider": "p", "translation_model": "m", + }]) + db_connection.save_pages("wf-tr", [{"page_number": 1, "original_markdown": "namaste"}]) + page = db_connection.get_page("wf-tr", 1) + assert page["translated_markdown"] == "hello" + assert page["translations"]["en"]["provider"] == "p" + + @pytest.mark.db + @pytest.mark.unit + def test_changed_ocr_text_drops_stale_translations(self, db_connection): + _new_doc(db_connection, "wf-tr2") + db_connection.save_pages("wf-tr2", [{ + "page_number": 1, "original_markdown": "old", "translated_markdown": "t", + "translation_target_language": "en", + }]) + db_connection.save_pages("wf-tr2", [{"page_number": 1, "original_markdown": "new"}]) + assert db_connection.get_page("wf-tr2", 1)["translations"] == {} + + @pytest.mark.db + @pytest.mark.unit + def test_reading_and_saving_pages_back_is_lossless(self, db_connection): + _new_doc(db_connection, "wf-rt") + db_connection.save_pages("wf-rt", [{ + "page_number": 1, "original_markdown": "a", "detected_language": "gu", + "translated_markdown": "b", "translation_target_language": "en", "reviewer_notes": "n", + }]) + before = db_connection.get_pages("wf-rt") + db_connection.save_pages("wf-rt", before) + assert db_connection.get_pages("wf-rt") == before + + @pytest.mark.db + @pytest.mark.unit + def test_translation_edits_target_one_language_entry(self, db_connection): + _new_doc(db_connection, "wf-edit") + db_connection.save_pages("wf-edit", [{ + "page_number": 1, "original_markdown": "a", "translated_markdown": "b", + "translation_target_language": "en", + }]) + db_connection.update_page( + "wf-edit", 1, edited_translation="fixed", translation_reviewed=True, is_reviewed=True, + ) + page = db_connection.get_page("wf-edit", 1) + assert page["edited_translation"] == "fixed" + assert page["translated_markdown"] == "b" + assert page["translation_reviewed"] is True + assert page["is_reviewed"] is True + + @pytest.mark.db + @pytest.mark.unit + def test_page_metadata_merges_instead_of_replacing(self, db_connection): + _new_doc(db_connection, "wf-meta") + db_connection.save_pages("wf-meta", [ + {"page_number": 1, "original_markdown": "a", "detected_language": "hi"} + ]) + db_connection.update_page("wf-meta", 1, reviewer_notes="ok") + page = db_connection.get_page("wf-meta", 1) + assert page["detected_language"] == "hi" + assert page["reviewer_notes"] == "ok" + + @pytest.mark.db + @pytest.mark.unit + def test_count_translated_pages(self, db_connection): + _new_doc(db_connection, "wf-count") + db_connection.save_pages("wf-count", [ + {"page_number": 1, "original_markdown": "a", "translated_markdown": "x", + "translation_target_language": "en"}, + {"page_number": 2, "original_markdown": "b"}, + ]) + assert db_connection.count_translated_pages("wf-count") == 1 + + +class TestChunkRuns: + @pytest.mark.db + @pytest.mark.unit + def test_rechunking_replaces_chunks_and_bumps_the_run(self, db_connection): + _new_doc(db_connection, "wf-run") + first = [{"chunk_number": 1, "original_text": "a", "chunking_provider": "p1"}] + db_connection.save_chunks("wf-run", first) + old_id = db_connection.get_chunk("wf-run", 1)["id"] + db_connection.save_chunks("wf-run", [ + {"chunk_number": 1, "original_text": "b", "chunking_provider": "p2", + "chunking_config_json": '{"size": 450}'}, + {"chunk_number": 2, "original_text": "c", "chunking_provider": "p2"}, + ]) + doc = db_connection.get_document("wf-run") + chunk = db_connection.get_chunk("wf-run", 1) + assert doc["chunk_count"] == 2 + assert chunk["id"] != old_id + assert chunk["chunk_version"] == 2 + assert chunk["chunking_provider"] == "p2" + assert chunk["chunking_config_json"] == '{"size": 450}' + + @pytest.mark.db + @pytest.mark.unit + def test_a_failed_rechunk_leaves_the_previous_chunks(self, db_connection): + _new_doc(db_connection, "wf-atomic") + db_connection.save_chunks("wf-atomic", [{"chunk_number": 1, "original_text": "keep"}]) + with pytest.raises(psycopg.errors.UniqueViolation): + db_connection.save_chunks("wf-atomic", [ + {"chunk_number": 1, "original_text": "x"}, + {"chunk_number": 1, "original_text": "dup"}, + ]) + assert db_connection.get_chunk("wf-atomic", 1)["original_text"] == "keep" + assert db_connection.get_document("wf-atomic")["chunk_count"] == 1 + + @pytest.mark.db + @pytest.mark.unit + def test_source_spans_round_trip_and_derive_page_numbers(self, db_connection): + _new_doc(db_connection, "wf-spans") + db_connection.save_chunks("wf-spans", [{ + "chunk_number": 1, "original_text": "a", "page_start": 2, "page_end": 3, + "source_spans_json": '[{"page_number": 2, "start_char": 0, "end_char": 1}, ' + '{"page_number": 3, "start_char": 0, "end_char": 1}]', + }]) + chunk = db_connection.get_chunk("wf-spans", 1) + assert chunk["source_page_numbers_json"] == "[2, 3]" + + @pytest.mark.db + @pytest.mark.unit + def test_search_chunks_matches_edited_text_first(self, db_connection): + _new_doc(db_connection, "wf-search", instance="mh") + db_connection.save_chunks("wf-search", [ + {"chunk_number": 1, "original_text": "alpha beta", "edited_text": "gamma delta"}, + ]) + hits, total = db_connection.search_chunks(query="gamma") + assert total == 1 and hits[0]["workflow_id"] == "wf-search" + assert db_connection.search_chunks(query="alpha")[1] == 0 + assert db_connection.search_chunks(query="gamma", instances=["up"])[1] == 0 + + +class TestIndexStatus: + @pytest.mark.db + @pytest.mark.unit + def test_entries_merge_and_dev_prod_are_separate_keys(self, db_connection, monkeypatch): + monkeypatch.setenv("PROD_VECTOR_DB_COLLECTION_NAME", "prod-docs") + _new_doc(db_connection, "wf-idx") + db_connection.upsert_document_index_status( + "wf-idx", "documents-index", doc_id="abc", chunk_count_indexed=4, + last_indexed_at="2026-01-01T00:00:00", status="indexed", + ) + db_connection.upsert_document_index_status( + "wf-idx", "documents-index", last_verified_at="2026-01-02T00:00:00", status="indexed", + ) + db_connection.upsert_document_index_status("wf-idx", "prod-docs", status="missing") + doc = db_connection.get_document("wf-idx") + assert set(doc["index_status"]) == {"dev", "prod"} + dev = db_connection.get_document_index_status("wf-idx", "documents-index") + assert dev["chunk_count_indexed"] == 4 + assert dev["last_indexed_at"] == "2026-01-01T00:00:00" + assert dev["last_verified_at"] == "2026-01-02T00:00:00" + summaries = {s["index_name"]: s for s in db_connection.list_index_summaries()} + assert summaries["documents-index"]["indexed_chunks"] == 4 + + +class TestIdentifierResolution: + @pytest.mark.db + @pytest.mark.unit + def test_every_documented_form_resolves(self, db_connection): + fingerprint = _md5("content") + _new_doc(db_connection, "doc-" + fingerprint, document_id=fingerprint, filename="pm.pdf") + db_connection.create_document_job( + "doc-" + fingerprint, "ocr_retry", temporal_workflow_id=f"doc-{fingerprint}-retry-ocr-9", + ) + doc = db_connection.get_document("doc-" + fingerprint) + for identifier in ( + doc["id"], + "doc-" + fingerprint, + f"doc-{fingerprint}-retry-ocr-9", + fingerprint, + "pm.pdf", + _md5(fingerprint), + ): + found = db_connection.find_document_by_doc_identifier(identifier) + assert found and found["id"] == doc["id"], identifier + assert found["workflow_id"] == "doc-" + fingerprint + assert db_connection.find_document_by_doc_identifier("nope") is None + assert db_connection.get_document("00000000-0000-0000-0000-000000000000") is None + + +class TestAuditTrail: + @pytest.mark.db + @pytest.mark.unit + def test_audit_outlives_the_document_and_keeps_tenant_scope(self, db_connection): + _new_doc(db_connection, "wf-aud", instance="mh") + db_connection.log_audit( + workflow_id="wf-aud", document_id="x", action_type="purge_document", + entity_type="document", actor_user_id="kc-9", actor_username="boss", + actor_email="b@x.in", actor_roles="superadmin", + ) + db_connection.purge_document("wf-aud") + + logs = db_connection.get_audit_logs("wf-aud") + assert [entry["action_type"] for entry in logs] == ["purge_document"] + assert logs[0]["actor"] == "b@x.in (superadmin)" + assert logs[0]["actor_user_id"] == "kc-9" + assert db_connection.get_all_audit_log_count(instances=["mh"]) == 1 + assert db_connection.get_all_audit_log_count(instances=["up"]) == 0 + + @pytest.mark.db + @pytest.mark.unit + def test_system_entries_need_no_document(self, db_connection): + db_connection.log_audit( + workflow_id="__catalog__", document_id="__catalog__", action_type="catalog_rebuild", + entity_type="catalog", + ) + logs = db_connection.get_all_audit_logs(action_type="catalog_rebuild") + assert logs[0]["workflow_id"] == "__catalog__" + + @pytest.mark.db + @pytest.mark.unit + def test_settings_changes_are_logged_per_key_with_instance_names(self, db_connection): + db_connection.update_search_settings({"alpha": 0.2, "limit": 3}) + fields = {entry["field_name"] for entry in db_connection.get_settings_audit_logs()} + assert fields == {"retrieval.alpha", "results.limit"} + assert db_connection.get_settings_audit_count() == 2 + db_connection.update_search_settings({"alpha": 0.2}) + assert db_connection.get_settings_audit_count() == 2 + + +class TestSearchSettings: + @pytest.mark.db + @pytest.mark.unit + def test_defaults_come_from_the_models(self, db_connection): + settings = db_connection.get_search_settings() + assert settings["alpha"] == 0.6 + assert len(settings) == 15 + + @pytest.mark.db + @pytest.mark.unit + def test_values_are_validated(self, db_connection): + with pytest.raises(ValueError): + db_connection.update_search_settings({"alpha": 3}) + with pytest.raises(ValueError): + db_connection.update_search_settings({"searchMethod": "FUZZY"}) + with pytest.raises(ValueError): + db_connection.update_search_settings({"nonsense": 1}) + + @pytest.mark.db + @pytest.mark.unit + def test_a_partial_update_keeps_the_rest_of_its_instance(self, db_connection): + db_connection.update_search_settings({"alpha": 0.3}) + db_connection.update_search_settings({"hybridRrfK": 10}) + settings = db_connection.get_search_settings() + assert (settings["alpha"], settings["hybridRrfK"]) == (0.3, 10) + + +class TestCatalogView: + @staticmethod + def _scheme_doc(db, workflow_id, code, stage, kind="scheme", **kwargs): + _new_doc( + db, workflow_id, stage=stage, kind=kind, code=code, + name=code.upper(), **kwargs, + ) + + @pytest.mark.db + @pytest.mark.unit + def test_status_rules(self, db_connection): + db = db_connection + self._scheme_doc(db, "wf-live", "live-one", "completed") + self._scheme_doc(db, "wf-stale", "stale-one", "completed") + db.mark_document_reindex_required("wf-stale", True, "edit") + self._scheme_doc(db, "wf-gate", "gate-one", "approval_for_prod") + self._scheme_doc(db, "wf-wip", "wip-one", "chunk_review") + self._scheme_doc(db, "wf-off", "off-one", "completed") + db.set_document_disabled("wf-off", True) + + status = {e["code"]: e["status"] for e in db.list_catalog_entries()} + assert status == { + "live-one": "live", + "stale-one": "pending_reindex", + "gate-one": "pending_prod", + "wip-one": "disabled", + "off-one": "disabled", + } + + @pytest.mark.db + @pytest.mark.unit + def test_computed_columns(self, db_connection, monkeypatch): + monkeypatch.setenv("PROD_VECTOR_DB_COLLECTION_NAME", "prod-docs") + db = db_connection + self._scheme_doc(db, "wf-a", "scheme-a", "completed", instance="mh") + self._scheme_doc(db, "wf-b", "scheme-b", "completed", instance="up") + db.save_chunks("wf-a", [{"chunk_number": 1, "original_text": "x"}]) + db.save_chunks("wf-b", [{"chunk_number": 1, "original_text": "y"}, + {"chunk_number": 2, "original_text": "z"}]) + db.update_document_fields("wf-b", network_visible=False) + db.upsert_document_index_status("wf-a", "documents-index", last_indexed_at="2026-04-01T00:00:00") + db.upsert_document_index_status("wf-a", "prod-docs", last_indexed_at="2026-05-01T00:00:00") + + a = db.get_catalog_entry("scheme-a") + b = db.get_catalog_entry("scheme-b") + assert (a["instances"], a["workflow_ids"], a["chunk_count"]) == (["mh"], ["wf-a"], 1) + assert (b["instances"], b["workflow_ids"], b["chunk_count"]) == (["up"], ["wf-b"], 2) + assert a["network_visible"] is True and b["network_visible"] is False + assert a["promoted_at"].startswith("2026-05-01T00:00:00") + assert b["promoted_at"] is None + + @pytest.mark.db + @pytest.mark.unit + def test_catalogs_without_documents_are_not_listed(self, db_connection): + db = db_connection + with db.get_connection() as conn: + db.get_or_create_catalog(db.get_or_create_instance("mh", conn=conn), "scheme", conn=conn) + assert db.list_catalog_entries() == [] + assert db.get_catalog_entry("anything") is None + + @pytest.mark.db + @pytest.mark.unit + def test_other_kinds_stay_out_of_the_scheme_listing(self, db_connection): + self._scheme_doc(db_connection, "wf-adv", "kharif", "completed", kind="advisory") + assert db_connection.list_catalog_entries() == [] + assert [e["code"] for e in db_connection.list_catalog_entries(kind="advisory")] == ["kharif"] + assert db_connection.list_document_codes("advisory") == {"kharif"} + assert db_connection.list_document_codes("scheme") == set() + + @pytest.mark.db + @pytest.mark.unit + def test_the_kind_is_fixed_by_the_catalog_chosen_at_upload(self, db_connection): + _new_doc(db_connection, "wf-move", kind="scheme", code="movable", name="Movable") + before = db_connection.get_document("wf-move")["catalog_id"] + db_connection.update_document_fields("wf-move", kind="advisory", name="Renamed") + doc = db_connection.get_document("wf-move") + assert (doc["kind"], doc["code"], doc["name"]) == ("scheme", "movable", "Renamed") + assert doc["catalog_id"] == before + + @pytest.mark.db + @pytest.mark.unit + def test_used_codes_exclude_the_document_itself(self, db_connection): + self._scheme_doc(db_connection, "wf-u1", "taken", "completed") + self._scheme_doc(db_connection, "wf-u2", "other", "completed") + assert db_connection.list_used_scheme_codes() == {"taken", "other"} + assert db_connection.list_used_scheme_codes(exclude_workflow_id="wf-u1") == {"other"} + assert db_connection.list_used_scheme_codes(kind="advisory") == set() + + @pytest.mark.db + @pytest.mark.unit + def test_version_counts_bumps(self, db_connection): + assert db_connection.get_catalog_meta()["version"] == 0 + assert db_connection.bump_catalog_version("a") == 1 + assert db_connection.bump_catalog_version("b") == 2 + meta = db_connection.get_catalog_meta() + assert (meta["version"], meta["notes"]) == (2, "b") + + +class TestUsersAndOtps: + @pytest.mark.db + @pytest.mark.unit + def test_upsert_user_refreshes_and_survives_an_email_clash(self, db_connection): + first = db_connection.upsert_user("kc-a", "ann", "ann@x.in", ["reviewer"]) + again = db_connection.upsert_user("kc-a", "ann2", None, ["admin"]) + assert first == again + user = db_connection.get_user_by_keycloak_id("kc-a") + assert (user["username"], user["email"], user["roles"]) == ("ann2", "ann@x.in", ["admin"]) + other = db_connection.upsert_user("kc-b", "bob", "ANN@x.in", []) + assert other and other != first + assert db_connection.get_user_by_keycloak_id("kc-b")["email"] is None + + @pytest.mark.db + @pytest.mark.unit + def test_uploader_is_a_user_row(self, db_connection): + _new_doc(db_connection, "wf-up", uploaded_by_user_id="kc-u", uploaded_by_username="uma", + uploaded_by_email="u@x.in", uploaded_by_roles="contributor,reviewer") + doc = db_connection.get_document("wf-up") + assert (doc["uploaded_by_user_id"], doc["uploaded_by_username"]) == ("kc-u", "uma") + assert doc["uploaded_by_roles"] == "contributor,reviewer" + + @pytest.mark.db + @pytest.mark.unit + def test_otp_attempts_increment_atomically(self, db_connection): + db_connection.store_email_otp( + email="A@X.in", code_hash="h", expires_at="2099-01-01T00:00:00+00:00", + created_at="x", last_sent_at="2026-01-01T00:00:00+00:00", + ) + assert db_connection.increment_email_otp_attempts("a@x.in") == 1 + assert db_connection.increment_email_otp_attempts("a@x.in") == 2 + row = db_connection.get_email_otp("a@x.in") + assert row["attempts"] == 2 + assert row["expires_at"].endswith("+00:00") + db_connection.store_email_otp( + email="a@x.in", code_hash="h2", expires_at="2099-01-01T00:00:00+00:00", + created_at="x", last_sent_at="2026-01-02T00:00:00+00:00", + ) + assert db_connection.get_email_otp("a@x.in")["attempts"] == 0 + db_connection.delete_email_otp("a@x.in") + assert db_connection.get_email_otp("a@x.in") is None + + +class TestOperationsViews: + @pytest.mark.db + @pytest.mark.unit + def test_summary_counts_and_queue_use_the_current_workflow(self, db_connection): + _new_doc(db_connection, "wf-q1", stage="ocr_review", instance="mh") + _new_doc(db_connection, "wf-q2", stage="completed", instance="up") + db_connection.create_document_job("wf-q2", "reingest", current_stage="ingesting") + counts = db_connection.get_document_summary_counts() + assert counts["total_documents"] == 2 + assert counts["ocr_review_documents"] == 1 + assert counts["completed_documents"] == 0 + assert counts["running_jobs"] == 3 + assert db_connection.get_document_summary_counts(instances=[])["total_documents"] == 0 + by_instance = {r["instance"]: r for r in db_connection.get_document_counts_by_instance()} + assert by_instance["mh"]["dev_approval"] == 1 + + rows, total = db_connection.list_operations_queue() + assert total == 2 + assert {r["workflow_id"]: r["job_type"] for r in rows} == { + "wf-q1": "pipeline", "wf-q2": "reingest", + } + assert db_connection.list_operations_queue(instances=["up"])[1] == 1 + + @pytest.mark.db + @pytest.mark.unit + def test_disabled_documents_are_hidden_by_default(self, db_connection): + _new_doc(db_connection, "wf-off") + db_connection.set_document_disabled("wf-off", True) + assert db_connection.list_documents() == [] + assert len(db_connection.list_documents(include_disabled=True)) == 1 + + @pytest.mark.db + @pytest.mark.unit + def test_reconcile_moves_a_stalled_document_into_review(self, db_connection): + _new_doc(db_connection, "wf-rec", stage="ocr_processing") + db_connection.save_pages("wf-rec", [{"page_number": 1, "original_markdown": "a"}]) + result = db_connection.reconcile_materialized_state("wf-rec") + assert result["updated"] is True and result["stage"] == "ocr_review" + assert db_connection.get_latest_document_job("wf-rec")["status"] == "waiting_review" + + @pytest.mark.db + @pytest.mark.unit + def test_webhook_upsert_skips_unchanged_payloads(self, db_connection): + db = db_connection + assert db.upsert_webhook_records("webhook", [{"contentID": "c1", "title": "T", "isPublished": True}]) == 1 + assert db.upsert_webhook_records("webhook", [{"contentID": "c1", "title": "T", "isPublished": True}]) == 1 + with db.get_connection() as conn: + rows = conn.execute("SELECT title, is_published FROM webhook_records").fetchall() + assert rows == [{"title": "T", "is_published": True}] + + +class TestMigrationRunner: + @pytest.mark.db + @pytest.mark.unit + def test_applied_migrations_are_recorded_and_not_reapplied(self, db_connection, postgres_url): + from pipeline import migrate + + assert migrate.apply_migrations(postgres_url) == [] + assert migrate.is_current(postgres_url) + assert migrate.current_version(postgres_url) == migrate.list_migrations()[-1][0] + + @pytest.mark.db + @pytest.mark.unit + def test_migration_files_are_numbered_and_unversioned_by_schema_name(self): + from pipeline import migrate + + names = [path.name for _, path in migrate.list_migrations()] + assert names[0] == "0001_initial_schema.sql" + + @pytest.mark.db + @pytest.mark.unit + def test_a_new_migration_runs_once(self, db_connection, postgres_url, tmp_path): + from pipeline import migrate + + (tmp_path / "0001_initial_schema.sql").write_text("SELECT 1;") + (tmp_path / "0099_probe.sql").write_text("CREATE TABLE mig_probe (n int);") + try: + assert migrate.apply_migrations(postgres_url, tmp_path) == ["0099"] + assert migrate.apply_migrations(postgres_url, tmp_path) == [] + finally: + with db_connection.get_connection() as conn: + conn.execute("DROP TABLE IF EXISTS mig_probe") + conn.execute("DELETE FROM schema_migrations WHERE version = '0099'") + + @pytest.mark.db + @pytest.mark.unit + def test_a_failing_migration_applies_nothing_of_itself(self, db_connection, postgres_url, tmp_path): + from pipeline import migrate + + (tmp_path / "0001_initial_schema.sql").write_text("SELECT 1;") + (tmp_path / "0099_bad.sql").write_text("CREATE TABLE mig_half (n int); SELECT 1/0;") + with pytest.raises(psycopg.errors.DivisionByZero): + migrate.apply_migrations(postgres_url, tmp_path) + with db_connection.get_connection() as conn: + assert conn.execute("SELECT to_regclass('mig_half') AS t").fetchone()["t"] is None + assert conn.execute( + "SELECT count(*) AS n FROM schema_migrations WHERE version = '0099'" + ).fetchone()["n"] == 0 + + @pytest.mark.db + @pytest.mark.unit + def test_migration_file_names_are_validated(self, tmp_path): + from pipeline import migrate + + (tmp_path / "init.sql").write_text("SELECT 1;") + with pytest.raises(ValueError): + migrate.list_migrations(tmp_path) diff --git a/tests/test_display_name.py b/tests/test_display_name.py index 3e3308a..d32fddf 100644 --- a/tests/test_display_name.py +++ b/tests/test_display_name.py @@ -127,7 +127,7 @@ def capture(*args, **kwargs): monkeypatch.setattr(activities, "_prepare_records", capture) monkeypatch.setattr(db, "get_chunks", lambda *a, **k: [CHUNK]) monkeypatch.setattr( - db, "get_document", lambda workflow_id: {"display_name": "PM Kisan Guidelines", "instance": "mh"} + db, "get_document", lambda workflow_id: {"name": "PM Kisan Guidelines", "instance": "mh"} ) return seen @@ -174,24 +174,26 @@ def test_stores_the_trimmed_display_name(self, test_client, sample_pdf_content): response = test_client.post( "/upload", files={"file": ("scan_0042.pdf", sample_pdf_content, "application/pdf")}, + params={"kind": "scheme"}, data={"display_name": " PM Kisan Guidelines "}, ) assert response.status_code == 200 doc = test_client.get(f"/documents/{response.json()['workflow_id']}").json() - assert doc["display_name"] == "PM Kisan Guidelines" + assert doc["name"] == "PM Kisan Guidelines" @pytest.mark.api @pytest.mark.unit - def test_omitting_it_stores_nothing(self, test_client, sample_pdf_content): + def test_omitting_it_defaults_to_the_file_title(self, test_client, sample_pdf_content): response = test_client.post( "/upload", files={"file": ("no-name.pdf", sample_pdf_content + b" ", "application/pdf")}, + params={"kind": "scheme"}, ) assert response.status_code == 200 doc = test_client.get(f"/documents/{response.json()['workflow_id']}").json() - assert doc["display_name"] is None + assert doc["name"] == "no name" @pytest.mark.api @pytest.mark.unit @@ -200,6 +202,7 @@ def test_rejects_an_invalid_display_name(self, test_client, sample_pdf_content, response = test_client.post( "/upload", files={"file": ("bad-name.pdf", sample_pdf_content, "application/pdf")}, + params={"kind": "scheme"}, data={"display_name": value}, ) @@ -214,6 +217,8 @@ def _document(self, db_connection, workflow_id, stage="ready_for_ingestion"): filename="scan_0042.pdf", filepath="/app/books/scan_0042.pdf", stage=stage, + kind="scheme", + code=workflow_id, ) @pytest.mark.api @@ -227,8 +232,8 @@ def test_sets_the_display_name_before_ingest(self, test_client, db_connection): ) assert response.status_code == 200 - assert response.json()["display_name"] == "Cotton Pest Advisory" - assert db_connection.get_document("display-001")["display_name"] == "Cotton Pest Advisory" + assert response.json()["name"] == "Cotton Pest Advisory" + assert db_connection.get_document("display-001")["name"] == "Cotton Pest Advisory" @pytest.mark.api @pytest.mark.unit @@ -250,7 +255,7 @@ def test_refuses_once_ingest_has_started(self, test_client, db_connection): ) assert response.status_code == 409 - assert db_connection.get_document("display-003")["display_name"] is None + assert db_connection.get_document("display-003")["name"] != "Too Late" @pytest.mark.api @pytest.mark.unit @@ -259,7 +264,7 @@ def test_other_metadata_still_editable_after_ingest(self, test_client, db_connec response = test_client.patch( "/documents/display-004/scheme-metadata", - json={"document_kind": "advisory"}, + json={"aliases": ["cotton"]}, ) assert response.status_code == 200 @@ -268,15 +273,15 @@ def test_other_metadata_still_editable_after_ingest(self, test_client, db_connec @pytest.mark.unit def test_resending_the_unchanged_name_on_a_locked_document_is_accepted(self, test_client, db_connection): self._document(db_connection, "display-005", stage="ready_for_ingestion") - db_connection.update_document_fields("display-005", display_name="Cotton Pest Advisory", ingested_at="2026-11-01T10:00:00") + db_connection.update_document_fields("display-005", name="Cotton Pest Advisory", ingested_at="2026-11-01T10:00:00") response = test_client.patch( "/documents/display-005/scheme-metadata", - json={"document_kind": "advisory", "display_name": " Cotton Pest Advisory "}, + json={"aliases": ["cotton"], "display_name": " Cotton Pest Advisory "}, ) assert response.status_code == 200 - assert db_connection.get_document("display-005")["document_kind"] == "advisory" + assert db_connection.get_document("display-005")["name"] == "Cotton Pest Advisory" @pytest.mark.api @pytest.mark.unit diff --git a/tests/test_document_repository.py b/tests/test_document_repository.py index ae6dba4..3335332 100644 --- a/tests/test_document_repository.py +++ b/tests/test_document_repository.py @@ -1,7 +1,7 @@ """Unit tests for DocumentRepository. Two styles on purpose: the `db_connection` fixture exercises the repository -against real SQLite (so a drifted db.py signature fails here), and the fake-db +against real the database (so a drifted db.py signature fails here), and the fake-db tests pin the None guards that are the repository's own behaviour. """ @@ -24,7 +24,7 @@ def get_document(self, workflow_id): class TestGetDocumentKind: @pytest.mark.unit def test_returns_the_stored_kind(self): - repo = DocumentRepository(FakeDb(document={"document_kind": "advisory"})) + repo = DocumentRepository(FakeDb(document={"kind": "advisory"})) assert repo.get_document_kind("wf-1") == "advisory" @@ -35,42 +35,64 @@ def test_returns_none_for_a_missing_document(self): assert repo.get_document_kind("wf-nope") is None @pytest.mark.unit - def test_returns_none_when_the_column_is_null(self): - # Old rows predate the column default, so it can be NULL. - repo = DocumentRepository(FakeDb(document={"document_kind": None})) + def test_returns_none_when_the_kind_is_missing(self): + # A catalog type the pipeline has no kind for reads as None. + repo = DocumentRepository(FakeDb(document={"kind": None})) assert repo.get_document_kind("wf-1") is None @pytest.mark.unit def test_does_not_normalise_the_value(self): - repo = DocumentRepository(FakeDb(document={"document_kind": " Advisory "})) + repo = DocumentRepository(FakeDb(document={"kind": " Advisory "})) assert repo.get_document_kind("wf-1") == " Advisory " @pytest.mark.unit def test_passes_the_workflow_id_through(self): - fake = FakeDb(document={"document_kind": "scheme"}) + fake = FakeDb(document={"kind": "scheme"}) DocumentRepository(fake).get_document_kind("wf-42") assert fake.get_document_called_with == "wf-42" -class TestAgainstRealSqlite: +@pytest.fixture +def scheme_document(db_connection): + """A document registered the v2 way (it needs a kind and a code).""" + db_connection.upsert_document( + workflow_id="repo-wf-1", + document_id="repo-doc-1", + filename="repo.pdf", + filepath="/app/books/repo.pdf", + stage="registered", + kind="scheme", + code="repo-scheme", + ) + return "repo-wf-1" + + +class TestAgainstRealPostgres: """Guards against db.py drifting out from under the repository.""" @pytest.mark.unit @pytest.mark.db - def test_reads_a_kind_written_through_the_db_layer(self, db_connection, sample_document): - workflow_id = sample_document["workflow_id"] - db_connection.update_document_fields(workflow_id, document_kind="advisory") - - assert DocumentRepository().get_document_kind(workflow_id) == "advisory" + def test_reads_the_kind_of_the_documents_catalog(self, db_connection, scheme_document): + assert DocumentRepository().get_document_kind(scheme_document) == "scheme" @pytest.mark.unit @pytest.mark.db - def test_unclassified_document_reads_as_the_column_default(self, db_connection, sample_document): - assert DocumentRepository().get_document_kind(sample_document["workflow_id"]) == "document" + def test_reads_a_kind_written_through_the_db_layer(self, db_connection): + db_connection.upsert_document( + workflow_id="repo-wf-adv", + document_id="repo-doc-adv", + filename="adv.pdf", + filepath="/app/books/adv.pdf", + stage="registered", + kind="advisory", + code="repo-advisory", + ) + + assert DocumentRepository().get_document_kind("repo-wf-adv") == "advisory" @pytest.mark.unit @pytest.mark.db diff --git a/tests/test_embeddings_count_tokens.py b/tests/test_embeddings_count_tokens.py new file mode 100644 index 0000000..c469f9e --- /dev/null +++ b/tests/test_embeddings_count_tokens.py @@ -0,0 +1,24 @@ +"""Tests for the embedding token counter used to size advisory chunks.""" + +from types import SimpleNamespace + +from pipeline.vector_store import embeddings + + +def test_count_tokens_local_uses_tokenizer(monkeypatch): + monkeypatch.setenv("EMBEDDING_PROVIDER", "sentence_transformers") + fake = SimpleNamespace(tokenizer=SimpleNamespace(encode=lambda text: text.split())) + monkeypatch.setattr(embeddings, "_get_local_embedder", lambda: fake) + # "passage: a b c" -> passage:, a, b, c + assert embeddings.count_tokens("a b c") == 4 + + +def test_count_tokens_remote_estimates_from_length(monkeypatch): + monkeypatch.setenv("EMBEDDING_PROVIDER", "openai_compatible") + text = "x" * 400 + assert embeddings.count_tokens(text) == len("passage: " + text) // 4 + + +def test_count_tokens_remote_is_at_least_one(monkeypatch): + monkeypatch.setenv("EMBEDDING_PROVIDER", "openai_compatible") + assert embeddings.count_tokens("") >= 1 diff --git a/tests/test_instances.py b/tests/test_instances.py index 560a375..e37c3c7 100644 --- a/tests/test_instances.py +++ b/tests/test_instances.py @@ -1,9 +1,9 @@ -"""Tests for instance code → state name resolution.""" +"""Tests for instance (tenant) code → state name resolution.""" import pytest -class TestInstanceDisplayName: +class TestGroupDisplayName: @pytest.mark.unit @pytest.mark.parametrize( "code,expected", diff --git a/tests/test_master_catalog_pg.py b/tests/test_master_catalog_pg.py deleted file mode 100644 index cc7a747..0000000 --- a/tests/test_master_catalog_pg.py +++ /dev/null @@ -1,347 +0,0 @@ -"""Tests for the Postgres Master Catalog + Redis push (master_catalog_pg).""" - -from __future__ import annotations - -import json -import os - -import pytest - - -class FakeCursor: - def __init__(self, store: dict): - self.store = store - self._last_result = None - - def __enter__(self): - return self - - def __exit__(self, *exc): - return False - - def execute(self, sql: str, params=None): - sql_norm = " ".join(sql.split()) - if sql_norm.startswith("INSERT INTO master_catalog"): - (code, content_type, name, tool_name, doc_id, prompt_snippet, - aliases, status, workflow_id, instance, instance_name, - updated_at) = params - existing = self.store.setdefault("rows", {}).get(code) - # Mirrors the real ON CONFLICT: status accumulates tiers, never overwrites. - merged_status = sorted({*(existing["status"] if existing else []), status}) - self.store["rows"][code] = { - "code": code, - "content_type": content_type, - "name": name, - "tool_name": tool_name, - "doc_id": doc_id, - "prompt_snippet": prompt_snippet, - "aliases": aliases, - "status": merged_status, - "workflow_id": workflow_id, - "instance": instance, - "instance_name": instance_name, - "updated_at": updated_at, - } - self._last_result = None - elif sql_norm.startswith("UPDATE master_catalog_meta"): - self.store["version"] = self.store.get("version", 0) + 1 - self._last_result = (self.store["version"],) - elif sql_norm.startswith("SELECT * FROM master_catalog WHERE status"): - statuses = set(params[0]) - rows = [r for r in self.store.get("rows", {}).values() if statuses & set(r["status"])] - self._last_result = sorted(rows, key=lambda r: r["code"]) - elif sql_norm.startswith("CREATE TABLE") or "INSERT INTO master_catalog_meta" in sql_norm: - self._last_result = None - else: - raise AssertionError(f"Unexpected SQL in FakeCursor: {sql_norm}") - - def fetchone(self): - return self._last_result - - def fetchall(self): - return self._last_result or [] - - -class FakeConnection: - def __init__(self, store: dict): - self.store = store - self.closed = False - - def cursor(self, cursor_factory=None): - return FakeCursor(self.store) - - def close(self): - self.closed = True - - -class FakeRedis: - def __init__(self): - self.data: dict[str, str] = {} - self.ttls: dict[str, int] = {} - - def set(self, key, value, ex=None): - self.data[key] = value - self.ttls[key] = ex - - -@pytest.fixture -def catalog_pg(monkeypatch, temp_db_path): - from pipeline import db - from pipeline import master_catalog_pg as pg - - db.DB_PATH = temp_db_path - db.init_db() - - store: dict = {} - fake_conn = FakeConnection(store) - monkeypatch.setattr(pg, "get_pg_connection", lambda: fake_conn) - - fake_redis = FakeRedis() - monkeypatch.setattr(pg, "get_redis_client", lambda: fake_redis) - - yield db, pg, store, fake_redis - - db.DB_PATH = os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db") - - -def _make_scheme_doc(db, workflow_id="wf-1", scheme_code="pm-ddky", tool_routing="qdrant", aliases=None, instance="default"): - db.upsert_document( - workflow_id=workflow_id, - document_id=f"doc-{workflow_id}", - filename="scheme.pdf", - filepath="/tmp/scheme.pdf", - stage="approval_for_prod", - page_count=5, - chunk_count=3, - instance=instance, - ) - db.update_document_fields( - workflow_id, - document_kind="scheme", - scheme_code=scheme_code, - scheme_name="Prime Minister Dhan-Dhaanya Krishi Yojana", - tool_routing=tool_routing, - scheme_aliases_json=json.dumps(aliases) if aliases is not None else None, - ) - - -def test_resolve_tool_name(): - from pipeline.master_catalog_pg import _resolve_tool_name - - assert _resolve_tool_name("legacy") == "get_scheme_info" - assert _resolve_tool_name("qdrant") == "search_schemes" - assert _resolve_tool_name("both") == "search_schemes" - assert _resolve_tool_name(None) == "search_schemes" - - -def test_generate_prompt_snippet(): - from pipeline.master_catalog_pg import _generate_prompt_snippet - - assert ( - _generate_prompt_snippet("Micro Irrigation Fund", "mif", "scheme") - == "Refer to 'Micro Irrigation Fund' (code: mif) for guidance on this government scheme." - ) - assert ( - _generate_prompt_snippet("Weekly advisory", "wa", "advisory") - == "Refer to 'Weekly advisory' (code: wa) for guidance on this advisory." - ) - - -def test_sync_catalog_entry_rejects_bad_status(catalog_pg): - db, pg, store, fake_redis = catalog_pg - with pytest.raises(ValueError): - pg.sync_catalog_entry("wf-1", "prod") - - -def test_sync_catalog_entry_skips_missing_document(catalog_pg): - db, pg, store, fake_redis = catalog_pg - assert pg.sync_catalog_entry("does-not-exist", "dev") is None - assert store.get("rows") is None - - -def test_sync_catalog_entry_syncs_non_scheme_document_kind(catalog_pg): - db, pg, store, fake_redis = catalog_pg - db.upsert_document( - workflow_id="wf-doc-only", - document_id="doc-1", - filename="general.pdf", - filepath="/tmp/general.pdf", - stage="completed", - page_count=1, - chunk_count=1, - instance="default", - ) - # document_kind defaults to "document" — still synced, no eligibility gate - version = pg.sync_catalog_entry("wf-doc-only", "dev") - assert version == 1 - assert "doc-1" in store["rows"] - # No scheme_name — falls back to filename, not the bare code, so the - # prompt snippet is still readable. - assert store["rows"]["doc-1"]["name"] == "general.pdf" - - -def test_sync_catalog_entry_falls_back_to_doc_id_when_no_scheme_code(catalog_pg): - db, pg, store, fake_redis = catalog_pg - db.upsert_document( - workflow_id="wf-no-code", - document_id="doc-2", - filename="scheme.pdf", - filepath="/tmp/scheme.pdf", - stage="approval_for_prod", - page_count=1, - chunk_count=1, - instance="default", - ) - db.update_document_fields("wf-no-code", document_kind="scheme") - version = pg.sync_catalog_entry("wf-no-code", "dev") - assert version == 1 - assert "doc-2" in store["rows"] - - -def test_sync_catalog_entry_dev_upserts_and_pushes_redis(catalog_pg): - db, pg, store, fake_redis = catalog_pg - _make_scheme_doc(db, workflow_id="wf-1", scheme_code="pm-ddky", tool_routing="qdrant") - - version = pg.sync_catalog_entry("wf-1", "dev") - assert version == 1 - - row = store["rows"]["pm-ddky"] - assert row["content_type"] == "scheme" - assert row["name"] == "Prime Minister Dhan-Dhaanya Krishi Yojana" - assert row["tool_name"] == "search_schemes" - assert row["doc_id"] == "doc-wf-1" - assert row["prompt_snippet"] == ( - "Refer to 'Prime Minister Dhan-Dhaanya Krishi Yojana' (code: pm-ddky) " - "for guidance on this government scheme." - ) - assert row["status"] == ["dev"] - - dev_snapshot = json.loads(fake_redis.data["master-catalog:dev:snapshot"]) - assert dev_snapshot["version"] == 1 - codes = {e["code"] for e in dev_snapshot["entries"]} - assert codes == {"pm-ddky"} - - live_snapshot = json.loads(fake_redis.data["master-catalog:live:snapshot"]) - assert live_snapshot["entries"] == [] # not promoted yet - - assert fake_redis.ttls["master-catalog:dev:snapshot"] == 172800 - - -def test_sync_catalog_entry_live_appears_in_both_tiers(catalog_pg): - db, pg, store, fake_redis = catalog_pg - _make_scheme_doc(db, workflow_id="wf-2", scheme_code="mif", tool_routing="legacy") - - pg.sync_catalog_entry("wf-2", "dev") - pg.sync_catalog_entry("wf-2", "live") - - assert store["rows"]["mif"]["status"] == ["dev", "live"] - assert store["rows"]["mif"]["tool_name"] == "get_scheme_info" - - dev_snapshot = json.loads(fake_redis.data["master-catalog:dev:snapshot"]) - live_snapshot = json.loads(fake_redis.data["master-catalog:live:snapshot"]) - assert {e["code"] for e in dev_snapshot["entries"]} == {"mif"} - assert {e["code"] for e in live_snapshot["entries"]} == {"mif"} - - -def test_sync_catalog_entry_dev_resync_after_live_does_not_regress_status(catalog_pg): - """A reingest-to-dev after promotion must not silently drop the entry from - the live snapshot — status accumulates tiers, it doesn't get overwritten.""" - db, pg, store, fake_redis = catalog_pg - _make_scheme_doc(db, workflow_id="wf-5", scheme_code="nfsm") - - pg.sync_catalog_entry("wf-5", "dev") - pg.sync_catalog_entry("wf-5", "live") - pg.sync_catalog_entry("wf-5", "dev") # e.g. a later reingest to DEV - - assert store["rows"]["nfsm"]["status"] == ["dev", "live"] - live_snapshot = json.loads(fake_redis.data["master-catalog:live:snapshot"]) - assert {e["code"] for e in live_snapshot["entries"]} == {"nfsm"} - - -def test_sync_catalog_entry_upsert_no_duplicates(catalog_pg): - db, pg, store, fake_redis = catalog_pg - _make_scheme_doc(db, workflow_id="wf-3", scheme_code="cdp") - - pg.sync_catalog_entry("wf-3", "dev") - pg.sync_catalog_entry("wf-3", "dev") - - assert len(store["rows"]) == 1 - dev_snapshot = json.loads(fake_redis.data["master-catalog:dev:snapshot"]) - assert len(dev_snapshot["entries"]) == 1 - - -def test_sync_catalog_entry_stores_aliases(catalog_pg): - db, pg, store, fake_redis = catalog_pg - _make_scheme_doc( - db, workflow_id="wf-6", scheme_code="mif", - aliases=["MIF", "micro irrigation fund"], - ) - - pg.sync_catalog_entry("wf-6", "dev") - - # Curator aliases are preserved and lead, then bootstrap/derived ones are - # merged in so a scheme is never searchable by its exact name alone. - stored = store["rows"]["mif"]["aliases"] - assert stored[:2] == ["MIF", "micro irrigation fund"] - assert len(stored) > 2 - lowered = {a.lower() for a in stored} - assert "micro irrigation fund scheme" in lowered # from the bootstrap list - assert "mif" in lowered - - dev_snapshot = json.loads(fake_redis.data["master-catalog:dev:snapshot"]) - entry = next(e for e in dev_snapshot["entries"] if e["code"] == "mif") - assert entry["aliases"] == stored - - -def test_sync_catalog_entry_stores_instance_name(catalog_pg): - """The readable state name rides along with the instance code.""" - db, pg, store, fake_redis = catalog_pg - _make_scheme_doc(db, workflow_id="wf-inst", scheme_code="mif-2", instance="mh") - - pg.sync_catalog_entry("wf-inst", "dev") - - assert store["rows"]["mif-2"]["instance"] == "mh" - assert store["rows"]["mif-2"]["instance_name"] == "Maharashtra" - dev_snapshot = json.loads(fake_redis.data["master-catalog:dev:snapshot"]) - entry = next(e for e in dev_snapshot["entries"] if e["code"] == "mif-2") - assert entry["instance_name"] == "Maharashtra" - - -def test_vector_schemes_prompt_block_excludes_legacy_schemes(catalog_pg): - """The dynamic prompt block must only ever be built from search_schemes - (vector-indexed) entries — legacy get_scheme_info schemes stay a - separate, hardcoded list on the AI-layer side and must never leak in.""" - db, pg, store, fake_redis = catalog_pg - _make_scheme_doc( - db, workflow_id="wf-7", scheme_code="pm-ddky", tool_routing="qdrant", - aliases=["PM-DDKY", "dhan-dhaanya"], - ) - _make_scheme_doc( - db, workflow_id="wf-8", scheme_code="kcc", tool_routing="legacy", - ) - - pg.sync_catalog_entry("wf-7", "dev") - pg.sync_catalog_entry("wf-8", "dev") - - dev_snapshot = json.loads(fake_redis.data["master-catalog:dev:snapshot"]) - prompt = dev_snapshot["prompt"] - assert prompt["vector_scheme_count"] == 1 - assert "pm-ddky" in prompt["vector_schemes_bullets"] - assert "kcc" not in prompt["vector_schemes_bullets"] - assert prompt["vector_schemes_bullets"] == "- **Prime Minister Dhan-Dhaanya Krishi Yojana** (pm-ddky)" - # The stored alias list is broad for recall, but the prompt is capped so a - # long derived tail cannot bloat every AI request. - identifiers = prompt["vector_schemes_identifiers"] - assert identifiers.startswith("- `pm-ddky` / PM-DDKY / dhan-dhaanya") - assert identifiers.count(" / ") <= pg._PROMPT_ALIAS_LIMIT - assert len(store["rows"]["pm-ddky"]["aliases"]) > pg._PROMPT_ALIAS_LIMIT - - -def test_sync_catalog_entry_without_redis_host_skips_push_but_still_upserts(catalog_pg, monkeypatch): - db, pg, store, fake_redis = catalog_pg - monkeypatch.setattr(pg, "get_redis_client", lambda: None) - _make_scheme_doc(db, workflow_id="wf-4", scheme_code="pkvy") - - version = pg.sync_catalog_entry("wf-4", "dev") - assert version == 1 - assert "pkvy" in store["rows"] diff --git a/tests/test_migrate_sqlite_to_postgres.py b/tests/test_migrate_sqlite_to_postgres.py new file mode 100644 index 0000000..5af95ae --- /dev/null +++ b/tests/test_migrate_sqlite_to_postgres.py @@ -0,0 +1,442 @@ +"""scripts/migrate_sqlite_to_postgres.py: a legacy SQLite file lands in Postgres. + +The fixture file is built with the legacy schema DDL, the way a production +database looked at cutover. +""" + +import hashlib +import importlib.util +import json +import sqlite3 +from pathlib import Path + +import pytest + +_SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "migrate_sqlite_to_postgres.py" + +LEGACY_DDL = """ +CREATE TABLE documents ( + workflow_id TEXT PRIMARY KEY, document_id TEXT NOT NULL, canonical_document_id TEXT, + filename TEXT NOT NULL, display_name TEXT, source_filename TEXT, source_manifest_name TEXT, + source_file_fingerprint TEXT, filepath TEXT NOT NULL, stage TEXT NOT NULL DEFAULT 'registered', + page_count INTEGER DEFAULT 0, chunk_count INTEGER DEFAULT 0, error_message TEXT, + is_demo INTEGER DEFAULT 0, created_at TEXT, updated_at TEXT, ocr_completed_at TEXT, + translation_completed_at TEXT, chunks_completed_at TEXT, ingested_at TEXT, source_type TEXT, + canonical_input_type TEXT, stop_after_ocr INTEGER DEFAULT 0, reindex_required INTEGER DEFAULT 0, + reindex_reason TEXT, original_artifact_id INTEGER, normalized_artifact_id INTEGER, + latest_job_id INTEGER, is_disabled INTEGER DEFAULT 0, instance TEXT, uploaded_by_user_id TEXT, + uploaded_by_username TEXT, uploaded_by_email TEXT, uploaded_by_roles TEXT, + prod_ready_requested_at TEXT, prod_ready_requested_by_user_id TEXT, + prod_ready_requested_by_username TEXT, valid_from TEXT, valid_to TEXT, + document_kind TEXT DEFAULT 'document', scheme_code TEXT, scheme_name TEXT, + scheme_aliases_json TEXT, tool_routing TEXT, catalog_visible INTEGER DEFAULT 1, + network_visible INTEGER DEFAULT 1 +); +CREATE TABLE audit_logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, workflow_id TEXT NOT NULL, document_id TEXT NOT NULL, + action_type TEXT NOT NULL, entity_type TEXT, entity_id INTEGER, field_name TEXT, old_value TEXT, + new_value TEXT, metadata TEXT, timestamp TEXT NOT NULL, actor_user_id TEXT, actor_username TEXT, + actor_email TEXT, actor_roles TEXT +); +CREATE TABLE pages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, workflow_id TEXT NOT NULL, page_number INTEGER NOT NULL, + original_markdown TEXT, edited_markdown TEXT, is_reviewed INTEGER DEFAULT 0, reviewer_notes TEXT, + detected_language TEXT, translated_markdown TEXT, edited_translation TEXT, + translation_reviewed INTEGER DEFAULT 0, translation_notes TEXT, translation_provider TEXT, + translation_model TEXT, translation_target_language TEXT, translated_at TEXT, + UNIQUE(workflow_id, page_number) +); +CREATE TABLE chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, workflow_id TEXT NOT NULL, chunk_number INTEGER NOT NULL, + original_text TEXT, edited_text TEXT, token_count INTEGER DEFAULT 0, page_start INTEGER DEFAULT 1, + page_end INTEGER DEFAULT 1, source_page_numbers_json TEXT, source_spans_json TEXT, + section_title TEXT, content_type TEXT, is_reference INTEGER DEFAULT 0, chunking_provider TEXT, + chunking_model TEXT, chunking_config_json TEXT, chunking_run_id TEXT, chunk_version INTEGER DEFAULT 1, + is_reviewed INTEGER DEFAULT 0, is_excluded INTEGER DEFAULT 0, reviewer_notes TEXT, + UNIQUE(workflow_id, chunk_number) +); +CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT NOT NULL, description TEXT, updated_at TEXT NOT NULL); +CREATE TABLE document_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, workflow_id TEXT NOT NULL, job_type TEXT NOT NULL, + temporal_workflow_id TEXT, temporal_run_id TEXT, status TEXT NOT NULL DEFAULT 'running', + current_stage TEXT, started_at TEXT NOT NULL, completed_at TEXT, error_message TEXT, config_json TEXT +); +CREATE TABLE document_artifacts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, workflow_id TEXT NOT NULL, job_id INTEGER, + artifact_type TEXT NOT NULL, stage TEXT, storage_uri TEXT NOT NULL, mime_type TEXT, filename TEXT, + size_bytes INTEGER, metadata_json TEXT, created_at TEXT NOT NULL +); +CREATE TABLE document_index_status ( + workflow_id TEXT NOT NULL, index_name TEXT NOT NULL, doc_id TEXT, chunk_count_indexed INTEGER DEFAULT 0, + last_indexed_at TEXT, last_verified_at TEXT, schema_version TEXT, status TEXT NOT NULL DEFAULT 'unknown', + details_json TEXT, PRIMARY KEY (workflow_id, index_name) +); +CREATE TABLE document_manifest_entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, manifest_filename TEXT NOT NULL UNIQUE, title_en TEXT, + imported_at TEXT NOT NULL +); +CREATE TABLE email_otps ( + email TEXT PRIMARY KEY, code_hash TEXT NOT NULL, expires_at TEXT NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL, last_sent_at TEXT NOT NULL +); +CREATE TABLE webhook_records ( + id TEXT PRIMARY KEY, source TEXT NOT NULL DEFAULT 'webhook', external_id TEXT NOT NULL, title TEXT, + slug TEXT, status TEXT, is_published INTEGER, category TEXT, payload TEXT NOT NULL, payload_hash TEXT, + created_at TEXT NOT NULL, updated_at TEXT NOT NULL, UNIQUE (source, external_id) +); +CREATE TABLE scheme_catalog_meta ( + id INTEGER PRIMARY KEY CHECK (id = 1), version INTEGER NOT NULL DEFAULT 0, updated_at TEXT, notes TEXT +); +CREATE TABLE scheme_catalog_entries ( + scheme_code TEXT PRIMARY KEY, scheme_name TEXT, scheme_aliases_json TEXT, instances_json TEXT, + workflow_ids_json TEXT, collection_name TEXT, chunk_count INTEGER DEFAULT 0, + status TEXT NOT NULL DEFAULT 'disabled', network_visible INTEGER DEFAULT 0, promoted_at TEXT, + content_hash TEXT, source TEXT, updated_at TEXT +); +""" + +FP1 = hashlib.md5(b"scheme guideline").hexdigest() +FP2 = hashlib.md5(b"in flight").hexdigest() +WF1 = f"doc-{FP1}" +WF2 = f"doc-{FP2}" +WF3 = "doc-broken" +WF4 = "doc-plain" +WF5 = "doc-demo" +FUTURE = "2099-01-01T00:00:00" + + +def _load_script(): + spec = importlib.util.spec_from_file_location("migrate_sqlite_to_postgres", _SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _insert(conn, table, **values): + keys = ", ".join(values) + marks = ", ".join("?" for _ in values) + conn.execute(f"INSERT INTO {table} ({keys}) VALUES ({marks})", tuple(values.values())) + + +@pytest.fixture +def legacy_db(tmp_path): + path = tmp_path / "documents.db" + conn = sqlite3.connect(path) + conn.executescript(LEGACY_DDL) + + _insert( + conn, "documents", workflow_id=WF1, document_id=FP1, canonical_document_id=FP1, + filename="guideline.pdf", display_name="Guideline", source_filename="guideline.pdf", + source_file_fingerprint=FP1, filepath="minio://documents/guideline.pdf", stage="completed", + page_count=2, chunk_count=2, created_at="2026-03-01T10:00:00", updated_at="2026-03-05T10:00:00", + ocr_completed_at="2026-03-01T11:00:00", translation_completed_at="2026-03-02T11:00:00", + chunks_completed_at="2026-03-03T11:00:00", ingested_at="2026-03-04T11:00:00", + source_type="document", canonical_input_type="pdf", instance="mh", uploaded_by_user_id="kc-up", + uploaded_by_username="uma", uploaded_by_email="uma@x.in", uploaded_by_roles="contributor,reviewer", + valid_from="2026-03-01", valid_to="2027-03-01", document_kind="scheme", scheme_code="pm-kmy", + scheme_name="PM Kisan Maandhan", scheme_aliases_json=json.dumps(["PMKMY", "kmy"]), + catalog_visible=1, network_visible=0, reindex_required=1, reindex_reason="edit", + ) + _insert( + conn, "documents", workflow_id=WF2, document_id=FP2, filename="inflight.pdf", + filepath="/data/documents/inflight.pdf", stage="approval_for_prod", page_count=1, + created_at="2026-04-01T10:00:00", updated_at="2026-04-01T12:00:00", instance="up", + document_kind="advisory", prod_ready_requested_at="2026-04-02T09:00:00", + prod_ready_requested_by_user_id="kc-req", prod_ready_requested_by_username="rita", + ocr_completed_at="2026-04-01T11:00:00", + ) + _insert( + conn, "documents", workflow_id=WF3, document_id="not-a-hash", filename="broken.pdf", + filepath="/tmp/broken.pdf", stage="failed", error_message="OCR blew up", + created_at="2026-05-01T10:00:00", updated_at="2026-05-01T10:30:00", instance=" ", + document_kind="advisory", scheme_code="Bad Code", valid_from="2026-06-01", valid_to="2026-01-01", + ) + _insert( + conn, "documents", workflow_id=WF4, document_id="plain", filename="plain.pdf", + filepath="/tmp/plain.pdf", stage="registered", created_at="2026-05-02T10:00:00", + updated_at="2026-05-02T10:00:00", instance="mh", document_kind="document", + ) + _insert( + conn, "documents", workflow_id=WF5, document_id="demo", filename="demo.pdf", + filepath="/tmp/demo.pdf", stage="completed", created_at="2026-05-03T10:00:00", + updated_at="2026-05-03T10:00:00", instance="mh", document_kind="scheme", scheme_code="demo-s", + is_demo=1, + ) + + _insert(conn, "document_jobs", id=1, workflow_id=WF1, job_type="pipeline", temporal_workflow_id=WF1, + status="completed", current_stage="completed", started_at="2026-03-01T10:00:00", + completed_at="2026-03-04T11:00:00", config_json=json.dumps({"auto_approve": False}), + temporal_run_id="run-1") + _insert(conn, "document_jobs", id=2, workflow_id=WF1, job_type="reingest", + temporal_workflow_id=f"{WF1}-reingest-1", status="completed", current_stage="ingesting", + started_at="2026-03-10T10:00:00", completed_at="2026-03-10T10:05:00") + _insert(conn, "document_jobs", id=3, workflow_id=WF1, job_type="mystery", + started_at="2026-03-11T10:00:00") + + _insert(conn, "document_artifacts", workflow_id=WF1, job_id=1, artifact_type="original_upload", + stage="registered", storage_uri="minio://documents/guideline.pdf", size_bytes=100, + metadata_json=json.dumps({"uploaded_via": "upload_endpoint"}), created_at="2026-03-01T10:00:01") + _insert(conn, "document_artifacts", workflow_id=WF1, job_id=None, artifact_type="normalized_pdf", + stage="ocr_processing", storage_uri="minio://documents/n.pdf", size_bytes=-1, + created_at="2026-03-01T10:30:00") + _insert(conn, "document_artifacts", workflow_id=WF1, job_id=2, artifact_type="vector_payload_export", + stage="ingesting", storage_uri="minio://documents/v.json", created_at="2026-03-10T10:01:00") + + _insert(conn, "pages", workflow_id=WF1, page_number=1, original_markdown="namaste", is_reviewed=1, + detected_language="hi", translated_markdown="hello", translation_provider="azure", + translation_model="gpt", translation_target_language="en", translated_at="2026-03-02T10:00:00") + _insert(conn, "pages", workflow_id=WF1, page_number=2, original_markdown="english page", + detected_language="en", reviewer_notes="fine") + _insert(conn, "pages", workflow_id="doc-purged", page_number=1, original_markdown="stray") + _insert(conn, "pages", workflow_id=WF5, page_number=1, original_markdown="demo page") + + _insert(conn, "chunks", workflow_id=WF1, chunk_number=1, original_text="chunk one", token_count=5, + page_start=1, page_end=2, source_spans_json=json.dumps([{"page_number": 1, "start_char": 0, "end_char": 5}]), + chunking_provider="recursive", chunking_model="rs", chunking_config_json='{"size": 450}', + chunk_version=3, is_reviewed=1) + _insert(conn, "chunks", workflow_id=WF1, chunk_number=2, original_text="chunk two", edited_text="two", + page_start=2, page_end=1, is_excluded=1) + + _insert(conn, "document_index_status", workflow_id=WF1, index_name="documents-index", doc_id="abc", + chunk_count_indexed=1, last_indexed_at="2026-03-04T11:00:00", status="indexed", + details_json='{"backend": "qdrant"}') + _insert(conn, "document_index_status", workflow_id=WF1, index_name="schemes-index", doc_id="abc", + chunk_count_indexed=2, last_indexed_at="2026-03-10T10:05:00", status="indexed") + + _insert(conn, "audit_logs", workflow_id=WF1, document_id=FP1, action_type="stage_change", + entity_type="document", field_name="stage", old_value="chunk_review", new_value="completed", + metadata='{"page_count": 2}', timestamp="2026-03-04T11:00:00") + _insert(conn, "audit_logs", workflow_id=WF1, document_id=FP1, action_type="page_edit", + entity_type="page", entity_id=1, metadata="not json", timestamp="2026-03-02T12:00:00", + actor_user_id="kc-rev", actor_username="ravi", actor_email="ravi@x.in", actor_roles="reviewer") + _insert(conn, "audit_logs", workflow_id=WF5, document_id="demo", action_type="stage_change", + entity_type="document", timestamp="2026-05-03T11:00:00") + _insert(conn, "audit_logs", workflow_id="doc-purged", document_id="gone", action_type="purge_document", + entity_type="document", timestamp="2026-02-01T00:00:00") + _insert(conn, "audit_logs", workflow_id="__system__", document_id="__settings__", + action_type="settings_change", entity_type="setting", field_name="search_alpha", + old_value="0.7", new_value="0.6", timestamp="2026-01-01T00:00:00") + _insert(conn, "audit_logs", workflow_id="__catalog__", document_id="__catalog__", + action_type="catalog_rebuild", entity_type="catalog", timestamp="2026-01-02T00:00:00") + + for key, value in ( + ("search_alpha", "0.4"), ("search_limit", "20"), ("search_show_highlights", "false"), + ("search_method", "LEXICAL"), ("search_index_name", "documents-index"), ("search_mystery", "x"), + ("search_ef_search", "lots"), + ): + _insert(conn, "settings", key=key, value=value, updated_at="2026-01-01T00:00:00") + + _insert(conn, "email_otps", email="old@x.in", code_hash="h", expires_at="2020-01-01T00:00:00+00:00", + created_at="2020-01-01T00:00:00+00:00", last_sent_at="2020-01-01T00:00:00+00:00") + _insert(conn, "email_otps", email="LIVE@x.in", code_hash="h2", expires_at=FUTURE + "+00:00", + attempts=2, created_at="2026-01-01T00:00:00+00:00", last_sent_at="2026-01-01T00:00:00+00:00") + + _insert(conn, "webhook_records", id="3c8a2f0e-6b1d-4f6e-9d0a-5f8f0b2e7a11", external_id="c-1", + title="T", slug="t", status="published", is_published=1, category="Agri", + payload='{"contentID": "c-1"}', payload_hash="h", created_at="2026-01-01T00:00:00", + updated_at="2026-01-02T00:00:00") + + _insert(conn, "scheme_catalog_meta", id=1, version=5, updated_at="2026-02-01T00:00:00", notes="x") + _insert(conn, "scheme_catalog_entries", scheme_code="cdp", scheme_name="Crop Diversification Programme", + scheme_aliases_json='["CDP"]', status="live", source="bootstrap") + _insert(conn, "scheme_catalog_entries", scheme_code="pm-kmy", scheme_name="Registry name", + scheme_aliases_json='["registry"]', status="live", source="pipeline") + _insert(conn, "document_manifest_entries", manifest_filename="a.csv", imported_at="2026-01-01T00:00:00") + _insert(conn, "document_manifest_entries", manifest_filename="b.csv", imported_at="2026-01-01T00:00:00") + conn.commit() + conn.close() + return str(path) + + +def _counts(db): + with db.get_connection() as conn: + return { + table: conn.execute(f"SELECT count(*) AS n FROM {table}").fetchone()["n"] + for table in ( + "documents", "workflows", "workflow_artifacts", "pages", "chunks", "audit_logs", + "users", "instances", "catalogs", "email_otps", "webhook_records", + ) + } + + +@pytest.mark.db +@pytest.mark.unit +class TestSqliteToPostgresMigration: + def test_dry_run_writes_nothing_but_reports_everything(self, db_connection, postgres_url, legacy_db): + script = _load_script() + before = _counts(db_connection) + report = script.run_migration(legacy_db, postgres_url, dry_run=True) + assert _counts(db_connection) == before + assert report.tables["documents"]["inserted"] == 3 + assert report.tables["documents"]["skipped"] == 2 + assert report.mismatches() == [] + + def test_a_full_run_maps_every_table(self, db_connection, postgres_url, legacy_db): + db = db_connection + report = _load_script().run_migration(legacy_db, postgres_url) + + assert report.mismatches() == [] + tables = report.as_dict()["tables"] + assert tables["documents"] == {"read": 5, "inserted": 3, "skipped": 2} + assert tables["document_jobs"] == {"read": 3, "inserted": 2, "skipped": 1} + assert tables["pages"] == {"read": 4, "inserted": 2, "skipped": 2} + assert tables["chunks"]["inserted"] == 2 + assert tables["document_manifest_entries"] == {"read": 2, "dropped": 2} + assert tables["email_otps"] == {"read": 2, "inserted": 1, "dropped": 1} + assert tables["settings"]["skipped"] == 2 + assert tables["scheme_catalog_entries"] == {"read": 2, "inserted": 1, "dropped": 1} + skipped = {i["key"]: i["reason"] for i in report.skip_reasons if i["table"] == "documents"} + assert skipped == { + WF4: "document_kind 'document' has no catalog", + WF5: "demo document (is_demo)", + } + assert db.get_document(WF4) is None and db.get_document(WF5) is None + + doc = db.get_document(WF1) + assert doc["stage"] == "completed" + assert doc["instance"] == "mh" + assert doc["document_id"] == FP1 + assert doc["name"] == "Registry name" # one name per document in v2 + assert (doc["valid_from"], doc["valid_to"]) == ("2026-03-01", "2027-03-01") + assert (doc["code"], doc["kind"]) == ("pm-kmy", "scheme") + assert doc["network_visible"] is False and doc["reindex_required"] is True + assert doc["uploaded_by_username"] == "uma" + assert doc["uploaded_by_roles"] == "contributor,reviewer" + assert doc["created_at"] == "2026-03-01T10:00:00" + assert doc["ocr_completed_at"] == "2026-03-01T11:00:00" + assert doc["chunk_count"] == 2 + assert doc["page_count"] == 2 + assert doc["catalog_id"] + assert (doc["name"], doc["aliases"]) == ("Registry name", ["registry"]) + assert set(doc["index_status"]) == {"dev"} + assert doc["index_status"]["dev"]["index_name"] == "schemes-index" + + jobs = {j["job_type"]: j for j in db.list_document_jobs(WF1)} + assert set(jobs) == {"pipeline", "reingest"} + assert jobs["pipeline"]["temporal_workflow_id"] == WF1 + assert jobs["pipeline"]["temporal_run_id"] == "run-1" + assert jobs["reingest"]["temporal_workflow_id"] == f"{WF1}-reingest-1" + assert doc["workflow_id"] == WF1 + + artifacts = {a["artifact_type"]: a for a in db.list_document_artifacts(WF1)} + assert artifacts["original_upload"]["job_id"] == jobs["pipeline"]["id"] + assert artifacts["vector_payload_export"]["job_id"] == jobs["reingest"]["id"] + assert artifacts["normalized_pdf"]["job_id"] == jobs["pipeline"]["id"] + assert artifacts["normalized_pdf"]["size_bytes"] is None + assert doc["original_artifact_id"] == artifacts["original_upload"]["id"] + + pages = db.get_pages(WF1) + assert pages[0]["translated_markdown"] == "hello" + assert pages[0]["translation_target_language"] == "en" + assert pages[0]["is_reviewed"] is True and pages[0]["detected_language"] == "hi" + assert pages[1]["translations"] == {} and pages[1]["reviewer_notes"] == "fine" + + chunks = db.get_chunks(WF1, include_excluded=True) + assert [c["chunk_version"] for c in chunks] == [3, 3] + assert chunks[0]["chunking_config_json"] == '{"size": 450}' + assert chunks[1]["page_end"] == 2 and chunks[1]["is_excluded"] is True + + entries = {e["code"]: e for e in db.list_catalog_entries()} + assert set(entries) == {"pm-kmy"} # the bootstrap-only "cdp" entry has no document + assert entries["pm-kmy"]["name"] == "Registry name" + assert entries["pm-kmy"]["status"] == "pending_reindex" + assert entries["pm-kmy"]["workflow_ids"] == [WF1] + assert db.get_catalog_meta()["version"] == 6 + + with db.get_connection() as conn: + groups = {g["path"]: g for g in conn.execute("SELECT * FROM instances").fetchall()} + catalogs = conn.execute( + "SELECT g.path, c.type FROM catalogs c JOIN instances g ON g.id = c.instance_id ORDER BY 1, 2" + ).fetchall() + meta = conn.execute( + "SELECT metadata FROM documents WHERE id = %s::uuid", (doc["id"],) + ).fetchone()["metadata"] + assert set(groups) == {"/states/MH", "/states/UP", "/states/DEFAULT"} + assert all(g["keycloak_group_id"] is None for g in groups.values()) + assert [(c["path"], c["type"]) for c in catalogs] == [ + ("/states/DEFAULT", "openagrinet:KnowledgeAdvisory"), + ("/states/MH", "openagrinet:AgricultureResource"), + ("/states/UP", "openagrinet:KnowledgeAdvisory"), + ] + assert (meta["valid_from"], meta["valid_to"]) == ("2026-03-01", "2027-03-01") + assert meta["uploaded_by_user_id"] == db.get_user_by_keycloak_id("kc-up")["id"] + assert meta["index_status"]["dev"]["index_name"] == "schemes-index" + assert not {"is_demo", "chunk_count", "display_name"} & set(meta) + + users = {u: db.get_user_by_keycloak_id(u) for u in ("kc-up", "kc-rev", "kc-req")} + assert all(users.values()) + + logs = db.get_audit_logs(WF1) + assert {e["action_type"] for e in logs} == {"stage_change", "page_edit"} + edit = next(e for e in logs if e["action_type"] == "page_edit") + assert edit["actor"] == "ravi@x.in (reviewer)" + assert edit["metadata"] == json.dumps({"raw": "not json"}) + purged = db.get_audit_logs("doc-purged") + assert [e["action_type"] for e in purged] == ["purge_document"] + demo_trail = db.get_audit_logs(WF5) # kept for the skipped demo document, tied to its group + assert [e["action_type"] for e in demo_trail] == ["stage_change"] + assert demo_trail[0]["instance"] == "mh" + settings_log = db.get_settings_audit_logs() + assert settings_log[0]["field_name"] == "retrieval.alpha" + + settings = db.get_search_settings() + assert settings["alpha"] == 0.4 and settings["limit"] == 20 + assert settings["showHighlights"] is False and settings["searchMethod"] == "LEXICAL" + assert settings["efSearch"] == 256 + + assert db.get_email_otp("live@x.in")["attempts"] == 2 + assert db.get_email_otp("old@x.in") is None + with db.get_connection() as conn: + hook = conn.execute("SELECT * FROM webhook_records").fetchone() + assert str(hook["id"]) == "3c8a2f0e-6b1d-4f6e-9d0a-5f8f0b2e7a11" + assert hook["is_published"] is True and hook["payload"] == {"contentID": "c-1"} + + def test_in_flight_documents_keep_their_stage_and_prod_request(self, db_connection, postgres_url, legacy_db): + db = db_connection + _load_script().run_migration(legacy_db, postgres_url) + + doc = db.get_document(WF2) + assert doc["stage"] == "approval_for_prod" + assert doc["kind"] == "advisory" and doc["code"] == "inflight" + assert doc["prod_ready_requested_at"] == "2026-04-02T09:00:00" + assert doc["prod_ready_requested_by_username"] == "rita" + assert doc["instance"] == "up" + assert doc["ocr_completed_at"] == "2026-04-01T11:00:00" + types = sorted(j["job_type"] for j in db.list_document_jobs(WF2)) + assert types == ["pipeline", "promote_prod"] + assert db.get_latest_document_job(WF2)["status"] == "waiting_review" + assert db.get_document(doc["id"])["workflow_id"] == WF2 + assert db.get_document(FP2)["id"] == doc["id"] + + def test_unusable_values_are_repaired_and_noted(self, db_connection, postgres_url, legacy_db): + db = db_connection + report = _load_script().run_migration(legacy_db, postgres_url) + + doc = db.get_document(WF3) + assert doc["stage"] == "failed" and doc["error_message"] == "OCR blew up" + assert doc["document_id"] == hashlib.md5(b"not-a-hash").hexdigest() + assert doc["instance"] == "default" + assert doc["kind"] == "advisory" and doc["code"] == "broken" + assert doc["valid_from"] == "2026-06-01" and doc["valid_to"] is None + notes = " ".join(report.notes) + assert "valid_to cleared" in notes and "not a valid slug, code derived" in notes + + def test_running_it_twice_changes_nothing(self, db_connection, postgres_url, legacy_db): + script = _load_script() + script.run_migration(legacy_db, postgres_url) + first = _counts(db_connection) + + report = script.run_migration(legacy_db, postgres_url) + + assert _counts(db_connection) == first + assert report.tables["documents"] == {"read": 5, "existing": 3, "skipped": 2} + assert report.tables["audit_logs"]["inserted"] == 0 + assert report.tables["webhook_records"]["existing"] == 1 + assert report.mismatches() == [] + assert db_connection.get_catalog_meta()["version"] == 6 + + def test_legacy_file_is_opened_read_only(self, postgres_url, legacy_db): + script = _load_script() + before = Path(legacy_db).read_bytes() + script.run_migration(legacy_db, postgres_url) + assert Path(legacy_db).read_bytes() == before diff --git a/tests/test_network_constants.py b/tests/test_network_constants.py new file mode 100644 index 0000000..d4603c0 --- /dev/null +++ b/tests/test_network_constants.py @@ -0,0 +1,25 @@ +import pytest + +from pipeline.network_constants import ( + KIND_TO_CATALOG_TYPE, + catalog_type_for_kind, + kind_for_catalog_type, +) + + +@pytest.mark.parametrize("kind", ["scheme", "advisory", "video"]) +def test_every_kind_round_trips_through_its_catalog_type(kind): + assert kind_for_catalog_type(catalog_type_for_kind(kind)) == kind + + +def test_each_kind_has_its_own_catalog_type(): + assert len(set(KIND_TO_CATALOG_TYPE.values())) == len(KIND_TO_CATALOG_TYPE) + + +def test_video_is_a_known_kind(): + assert catalog_type_for_kind(" Video ") == "openagrinet:VideoResource" + + +def test_unknown_kind_is_rejected(): + with pytest.raises(ValueError): + catalog_type_for_kind("custom") diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 898f51c..6e1cd66 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -1,6 +1,7 @@ """Tests for vector-store provenance ingest fields and /provenance/chunk resolver.""" import hashlib +import itertools import pytest from fastapi.testclient import TestClient @@ -8,6 +9,13 @@ from pipeline.activities import prepare_ingestion_records from pipeline.api import app +_code_seq = itertools.count(1) + + +def _next_code() -> str: + """A unique document code for fixtures (documents need one in v2).""" + return f"doc-{next(_code_seq)}" + @pytest.mark.unit def test_prepare_records_includes_provenance_fields(): @@ -42,6 +50,7 @@ def test_prepare_records_includes_provenance_fields(): assert record["name_en"] == "Sample Circular" assert record["source"] == "docs-pipeline" assert record["type"] == "document" + assert "document_kind" not in record assert "text" in record assert record["is_reference"] is False @@ -52,12 +61,14 @@ def test_find_document_by_legacy_doc_id_hash(db_connection): document_id = "legacy-doc-id-001" workflow_id = "doc-legacy000001" db.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id=document_id, filename="legacy.pdf", filepath="/tmp/legacy.pdf", stage="completed", - display_name="Legacy PDF", + name="Legacy PDF", ) db.save_chunks( workflow_id, @@ -95,14 +106,16 @@ def test_find_document_by_legacy_doc_id_hash(db_connection): def test_provenance_chunk_endpoint(db_connection): db = db_connection workflow_id = "doc-provenance01" - document_id = "prov-doc-id-xyz" + document_id = hashlib.md5(b"prov-doc-id-xyz").hexdigest() db.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id=document_id, filename="prov.pdf", filepath="/tmp/prov.pdf", stage="completed", - display_name="Provenance PDF", + name="Provenance PDF", ) db.save_chunks( workflow_id, diff --git a/tests/test_provider_id_on_chunks.py b/tests/test_provider_id_on_chunks.py index 7a9ded0..f9e926c 100644 --- a/tests/test_provider_id_on_chunks.py +++ b/tests/test_provider_id_on_chunks.py @@ -44,14 +44,14 @@ class TestProviderIdFromDoc: def test_published_kinds_get_the_provider_id(self, kind, instance, expected): from pipeline.activities import _provider_id_from_doc - assert _provider_id_from_doc({"document_kind": kind, "instance": instance}) == expected + assert _provider_id_from_doc({"kind": kind, "instance": instance}) == expected @pytest.mark.unit @pytest.mark.parametrize("kind", ["document", "video", "how_to_faq", None]) def test_kinds_that_never_publish_get_none(self, kind): from pipeline.activities import _provider_id_from_doc - assert _provider_id_from_doc({"document_kind": kind, "instance": "mh"}) is None + assert _provider_id_from_doc({"kind": kind, "instance": "mh"}) is None @pytest.mark.unit def test_follows_the_overrides_file(self, monkeypatch, tmp_path): @@ -59,9 +59,9 @@ def test_follows_the_overrides_file(self, monkeypatch, tmp_path): _use_overrides(monkeypatch, tmp_path, '{"ka": "ka.example.com"}') - assert _provider_id_from_doc({"document_kind": "advisory", "instance": "ka"}) == "ka.example.com" - assert _provider_id_from_doc({"document_kind": "scheme", "instance": "ka"}) == "ka.example.com" - assert _provider_id_from_doc({"document_kind": "scheme", "instance": "mh"}) == "mh.provider.oan.dev" + assert _provider_id_from_doc({"kind": "advisory", "instance": "ka"}) == "ka.example.com" + assert _provider_id_from_doc({"kind": "scheme", "instance": "ka"}) == "ka.example.com" + assert _provider_id_from_doc({"kind": "scheme", "instance": "mh"}) == "mh.provider.oan.dev" @pytest.mark.unit def test_no_network_id_means_no_provider_id(self, monkeypatch): @@ -69,14 +69,14 @@ def test_no_network_id_means_no_provider_id(self, monkeypatch): monkeypatch.delenv("NETWORK_SENDER_ID", raising=False) - assert _provider_id_from_doc({"document_kind": "advisory", "instance": "mh"}) is None + assert _provider_id_from_doc({"kind": "advisory", "instance": "mh"}) is None @pytest.mark.unit def test_an_invalid_provider_id_fails_without_retry(self): from pipeline.activities import _provider_id_from_doc with pytest.raises(ApplicationError, match="ladakh_ut") as excinfo: - _provider_id_from_doc({"document_kind": "advisory", "instance": "ladakh_ut"}) + _provider_id_from_doc({"kind": "advisory", "instance": "ladakh_ut"}) assert excinfo.value.non_retryable is True @@ -101,7 +101,7 @@ def test_matches_the_provider_id_sent_in_publish(self, monkeypatch, tmp_path, ki DiscoveryPublishService().publish("txn", kind, "wf", group_for_instance(instance)) published = client.post.call_args.kwargs["json"]["message"]["catalogs"][0]["provider"]["id"] - assert _provider_id_from_doc({"document_kind": kind, "instance": instance}) == published + assert _provider_id_from_doc({"kind": kind, "instance": instance}) == published class TestProviderIdOnRecords: @@ -147,7 +147,7 @@ def capture(*args, **kwargs): monkeypatch.setattr(activities, "_prepare_records", capture) monkeypatch.setattr(db, "get_chunks", lambda *a, **k: [CHUNK]) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "scheme", "instance": "mh"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "scheme", "instance": "mh"}) return seen @pytest.mark.unit diff --git a/tests/test_schema_v2.py b/tests/test_schema_v2.py new file mode 100644 index 0000000..bfb8c77 --- /dev/null +++ b/tests/test_schema_v2.py @@ -0,0 +1,775 @@ +""" +Schema v2 rules: groups, catalogs, documents keyed by (catalog, code), the +group snapshot on audit_logs and the catalog_status view. + +Everything runs against the real Postgres from conftest.py. +""" + +import hashlib +import logging + +import psycopg +import pytest +from psycopg.types.json import Jsonb + + +def _md5(text): + return hashlib.md5(text.encode()).hexdigest() + + +def _doc(db, workflow_id, **kwargs): + kwargs.setdefault("document_id", _md5(workflow_id)) + kwargs.setdefault("filename", f"{workflow_id}.pdf") + kwargs.setdefault("filepath", f"minio://documents/{workflow_id}.pdf") + kwargs.setdefault("kind", "scheme") + kwargs.setdefault("code", workflow_id) + return db.upsert_document(workflow_id=workflow_id, **kwargs) + + +def _instance(db, instance="mh"): + return db.get_or_create_instance(instance) + + +class TestGroups: + @pytest.mark.db + @pytest.mark.unit + def test_get_or_create_instance_is_idempotent_and_maps_tenant_codes_to_paths(self, db_connection): + db = db_connection + mh = _instance(db, "mh") + assert _instance(db, "MH") == mh + assert _instance(db, "up") != mh + bv = _instance(db, "bv") + with db.get_connection() as conn: + rows = {r["name"]: r["path"] for r in conn.execute("SELECT name, path FROM instances").fetchall()} + assert rows == {"mh": "/states/MH", "up": "/states/UP", "bv": "/global"} + assert _instance(db, "bv") == bv + + @pytest.mark.db + @pytest.mark.unit + def test_upsert_instance_fills_the_keycloak_id_of_an_existing_path(self, db_connection): + db = db_connection + first = db.upsert_instance("/states/MH") # as created by the SQLite move: no Keycloak id yet + again = db.upsert_instance("/states/MH", keycloak_group_id="kc-group-1") + assert again == first + with db.get_connection() as conn: + row = conn.execute("SELECT keycloak_group_id, name FROM instances WHERE id = %s::uuid", (first,)).fetchone() + assert (row["keycloak_group_id"], row["name"]) == ("kc-group-1", "mh") + # A later sync that does not know the id must not erase it. + db.upsert_instance("/states/MH") + with db.get_connection() as conn: + row = conn.execute("SELECT keycloak_group_id FROM instances WHERE id = %s::uuid", (first,)).fetchone() + assert row["keycloak_group_id"] == "kc-group-1" + + @pytest.mark.db + @pytest.mark.unit + def test_keycloak_group_id_is_unique_but_nullable(self, db_connection): + with db_connection.get_connection() as conn: + conn.execute("INSERT INTO instances (name, path) VALUES ('a', '/states/A')") + conn.execute("INSERT INTO instances (name, path) VALUES ('b', '/states/B')") # two NULLs are fine + conn.execute("INSERT INTO instances (keycloak_group_id, name, path) VALUES ('k', 'c', '/states/C')") + with pytest.raises(psycopg.errors.UniqueViolation): + conn.execute("INSERT INTO instances (keycloak_group_id, name, path) VALUES ('k', 'd', '/states/D')") + + @pytest.mark.db + @pytest.mark.unit + def test_a_instance_with_catalogs_cannot_be_deleted(self, db_connection): + db = db_connection + instance_id = _instance(db) + db.get_or_create_catalog(instance_id, "scheme") + with db.get_connection() as conn: + with pytest.raises(psycopg.errors.ForeignKeyViolation): + conn.execute("DELETE FROM instances WHERE id = %s::uuid", (instance_id,)) + + @pytest.mark.db + @pytest.mark.unit + def test_deleting_a_instance_clears_users_and_audit_snapshots(self, db_connection): + db = db_connection + instance_id = _instance(db, "zz") + db.upsert_user("kc-zz", "zed", None, [], tenants=["zz"]) + with db.get_connection() as conn: + conn.execute( + "INSERT INTO audit_logs (action_type, entity_type, instance_id) VALUES ('x', 'setting', %s::uuid)", + (instance_id,), + ) + conn.execute("DELETE FROM catalogs WHERE instance_id = %s::uuid", (instance_id,)) + conn.execute("DELETE FROM instances WHERE id = %s::uuid", (instance_id,)) + assert conn.execute("SELECT instance_id FROM audit_logs WHERE action_type = 'x'").fetchone()["instance_id"] is None + assert db.get_user_by_keycloak_id("kc-zz")["instance_id"] is None + + +class TestCatalogs: + @pytest.mark.db + @pytest.mark.unit + def test_a_new_instance_gets_one_catalog_per_supported_type(self, db_connection): + from pipeline.network_constants import KIND_TO_CATALOG_TYPE + + db = db_connection + mh = _instance(db, "mh") + with db.get_connection() as conn: + rows = conn.execute( + "SELECT type, name FROM catalogs WHERE instance_id = %s::uuid ORDER BY type", (mh,) + ).fetchall() + assert [r["type"] for r in rows] == sorted(KIND_TO_CATALOG_TYPE.values()) + assert {r["name"] for r in rows} == { + "MH scheme catalog", "MH advisory catalog", "MH video catalog", + } + + @pytest.mark.db + @pytest.mark.unit + def test_upload_uses_the_seeded_catalog_and_creates_none(self, db_connection): + db = db_connection + mh = _instance(db, "mh") + with db.get_connection() as conn: + seeded = conn.execute( + "SELECT id FROM catalogs WHERE instance_id = %s::uuid AND type = 'openagrinet:AgricultureResource'", + (mh,), + ).fetchone()["id"] + before = conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] + _doc(db, "wf-seeded", instance="mh", code="seeded") + assert db.get_document("wf-seeded")["catalog_id"] == str(seeded) + with db.get_connection() as conn: + assert conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] == before + + @pytest.mark.db + @pytest.mark.unit + def test_a_missing_catalog_is_created_on_demand(self, db_connection): + db = db_connection + mh = _instance(db, "mh") + with db.get_connection() as conn: + conn.execute( + "DELETE FROM catalogs WHERE instance_id = %s::uuid AND type = 'openagrinet:KnowledgeAdvisory'", + (mh,), + ) + _doc(db, "wf-fallback", instance="mh", code="fallback", kind="advisory") + assert db.get_document("wf-fallback")["kind"] == "advisory" + + @pytest.mark.db + @pytest.mark.unit + def test_startup_sync_fills_missing_catalogs_for_every_instance_and_is_idempotent(self, db_connection): + db = db_connection + mh, up = _instance(db, "mh"), _instance(db, "up") + with db.get_connection() as conn: + conn.execute("DELETE FROM catalogs WHERE type = 'openagrinet:VideoResource'") + conn.execute( + "DELETE FROM catalogs WHERE instance_id = %s::uuid AND type = 'openagrinet:KnowledgeAdvisory'", (up,) + ) + assert db.sync_catalogs_for_all_instances() == 3 + assert db.sync_catalogs_for_all_instances() == 0 + with db.get_connection() as conn: + counts = { + str(r["instance_id"]): r["n"] + for r in conn.execute("SELECT instance_id, count(*) AS n FROM catalogs GROUP BY instance_id") + } + assert counts == {mh: 3, up: 3} + + @pytest.mark.db + @pytest.mark.unit + def test_get_or_create_catalog_is_idempotent_per_instance_and_kind(self, db_connection): + db = db_connection + mh = _instance(db, "mh") + scheme = db.get_or_create_catalog(mh, "scheme") + assert db.get_or_create_catalog(mh, "scheme")["id"] == scheme["id"] + assert scheme["type"] == "openagrinet:AgricultureResource" + assert scheme["name"] == "MH scheme catalog" + advisory = db.get_or_create_catalog(mh, "advisory") + assert advisory["id"] != scheme["id"] + assert advisory["type"] == "openagrinet:KnowledgeAdvisory" + assert db.get_or_create_catalog(_instance(db, "up"), "scheme")["id"] != scheme["id"] + + @pytest.mark.db + @pytest.mark.unit + def test_a_kind_without_a_catalog_type_is_rejected(self, db_connection): + with pytest.raises(ValueError): + db_connection.get_or_create_catalog(_instance(db_connection), "podcast") + + @pytest.mark.db + @pytest.mark.unit + def test_one_catalog_per_instance_and_type(self, db_connection): + db = db_connection + instance_id = _instance(db) + with db.get_connection() as conn: + conn.execute( + "INSERT INTO catalogs (instance_id, type, name) VALUES (%s::uuid, 'openagrinet:MandiPrice', 'A')", + (instance_id,), + ) + with pytest.raises(psycopg.errors.UniqueViolation): + conn.execute( + "INSERT INTO catalogs (instance_id, type, name) VALUES (%s::uuid, 'openagrinet:MandiPrice', 'B')", + (instance_id,), + ) + + @pytest.mark.db + @pytest.mark.unit + @pytest.mark.parametrize("bad_type", ["scheme", "openagrinet:", ":Thing", "open agrinet:X", "a:b:c", ""]) + def test_type_must_look_like_a_schema_name(self, db_connection, bad_type): + instance_id = _instance(db_connection) + with db_connection.get_connection() as conn: + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute( + "INSERT INTO catalogs (instance_id, type, name) VALUES (%s::uuid, %s, 'N')", + (instance_id, bad_type), + ) + + @pytest.mark.db + @pytest.mark.unit + def test_catalog_metadata_must_be_an_object(self, db_connection): + instance_id = _instance(db_connection) + with db_connection.get_connection() as conn: + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute( + "INSERT INTO catalogs (instance_id, type, name, metadata) " + "VALUES (%s::uuid, 'openagrinet:MandiPrice', 'N', '[]'::jsonb)", + (instance_id,), + ) + + @pytest.mark.db + @pytest.mark.unit + def test_a_catalog_with_documents_cannot_be_deleted(self, db_connection): + db = db_connection + _doc(db, "wf-held") + with db.get_connection() as conn: + with pytest.raises(psycopg.errors.ForeignKeyViolation): + conn.execute("DELETE FROM catalogs") + + +class TestDocumentIdentity: + @pytest.mark.db + @pytest.mark.unit + def test_a_new_document_needs_a_kind_and_a_code(self, db_connection): + db = db_connection + common = dict(workflow_id="wf-x", document_id=_md5("x"), filename="x.pdf", filepath="/x.pdf") + with pytest.raises(ValueError): + db.upsert_document(**common, code="x") + with pytest.raises(ValueError): + db.upsert_document(**common, kind="scheme") + assert db.get_document("wf-x") is None + + @pytest.mark.db + @pytest.mark.unit + def test_a_new_document_lands_in_its_tenants_catalog_of_that_kind(self, db_connection): + db = db_connection + _doc(db, "wf-land", instance="mh", kind="advisory", code="kharif", name="Kharif advice") + doc = db.get_document("wf-land") + assert (doc["instance"], doc["kind"], doc["code"], doc["name"]) == ( + "mh", "advisory", "kharif", "Kharif advice", + ) + expected = db.get_or_create_catalog(_instance(db, "mh"), "advisory")["id"] + assert doc["catalog_id"] == expected + + @pytest.mark.db + @pytest.mark.unit + def test_the_name_defaults_to_the_code(self, db_connection): + _doc(db_connection, "wf-name", code="pm-kisan") + assert db_connection.get_document("wf-name")["name"] == "pm-kisan" + + @pytest.mark.db + @pytest.mark.unit + def test_a_code_is_unique_within_its_catalog(self, db_connection): + db = db_connection + _doc(db, "wf-one", instance="mh", code="pm-kisan") + with pytest.raises(db.DuplicateDocumentCode): + _doc(db, "wf-two", instance="mh", code="pm-kisan") + assert db.get_document("wf-two") is None + + @pytest.mark.db + @pytest.mark.unit + def test_the_same_code_is_fine_in_another_instance_or_kind(self, db_connection): + db = db_connection + _doc(db, "wf-mh", instance="mh", code="pm-kisan") + _doc(db, "wf-up", instance="up", code="pm-kisan") + _doc(db, "wf-adv", instance="mh", code="pm-kisan", kind="advisory") + assert {d["workflow_id"] for d in db.list_documents()} == {"wf-mh", "wf-up", "wf-adv"} + + @pytest.mark.db + @pytest.mark.unit + def test_a_duplicate_code_leaves_no_half_written_rows(self, db_connection): + db = db_connection + _doc(db, "wf-first", code="dup") + with pytest.raises(db.DuplicateDocumentCode): + _doc(db, "wf-second", code="dup") + with db.get_connection() as conn: + assert conn.execute("SELECT count(*) AS n FROM documents").fetchone()["n"] == 1 + assert conn.execute("SELECT count(*) AS n FROM workflows").fetchone()["n"] == 1 + + @pytest.mark.db + @pytest.mark.unit + def test_find_document_by_code(self, db_connection): + db = db_connection + _doc(db, "wf-find", instance="mh", code="pm-kisan") + found = db.find_document_by_code("mh", "scheme", "pm-kisan") + assert found and found["workflow_id"] == "wf-find" + assert db.find_document_by_code("MH", "scheme", "pm-kisan")["id"] == found["id"] + assert db.find_document_by_code("up", "scheme", "pm-kisan") is None + assert db.find_document_by_code("mh", "advisory", "pm-kisan") is None + assert db.find_document_by_code("mh", "scheme", "other") is None + + @pytest.mark.db + @pytest.mark.unit + def test_the_kind_does_not_change_after_the_document_exists(self, db_connection): + db = db_connection + _doc(db, "wf-kind", instance="mh", code="fixed") + before = db.get_document("wf-kind")["catalog_id"] + db.update_document_fields("wf-kind", kind="advisory") + doc = db.get_document("wf-kind") + assert (doc["kind"], doc["catalog_id"]) == ("scheme", before) + + +class TestReupload: + @pytest.mark.db + @pytest.mark.unit + def test_reset_for_reupload_replaces_the_source_and_clears_the_content(self, db_connection): + db = db_connection + doc_uuid = _doc(db, "wf-re", filename="old.pdf", valid_from="2026-01-01", valid_to="2026-12-31") + db.save_pages("wf-re", [{"page_number": 1, "original_markdown": "old"}]) + db.save_chunks("wf-re", [{"chunk_number": 1, "original_text": "old", "chunking_provider": "p"}]) + db.upsert_document_index_status("wf-re", "documents-index", status="indexed") + artifact_id = db.add_document_artifact("wf-re", "original_upload", "minio://documents/old.pdf") + new_fingerprint = _md5("new content") + + db.reset_document_for_reupload( + doc_uuid, filename="new.pdf", filepath="minio://documents/new.pdf", fingerprint=new_fingerprint, + ) + + doc = db.get_document("wf-re") + assert doc["id"] == doc_uuid + assert doc["source_file_fingerprint"] == new_fingerprint + assert (doc["filename"], doc["filepath"]) == ("new.pdf", "minio://documents/new.pdf") + assert db.get_pages("wf-re") == [] + assert db.get_chunks("wf-re") == [] + assert (doc["page_count"], doc["chunk_count"]) == (0, 0) + assert doc["reindex_required"] is True + assert doc["reindex_reason"] == "source_replaced" + + # What is not content survives: workflows, artifacts, period, index state, code. + assert [j["job_type"] for j in db.list_document_jobs("wf-re")] == ["pipeline"] + assert db.get_document_artifact("wf-re", artifact_id) is not None + assert (doc["valid_from"], doc["valid_to"]) == ("2026-01-01", "2026-12-31") + assert "dev" in doc["index_status"] + assert doc["code"] == "wf-re" + with db.get_connection() as conn: + assert "chunking" not in conn.execute( + "SELECT metadata FROM documents WHERE id = %s::uuid", (doc_uuid,) + ).fetchone()["metadata"] + + @pytest.mark.db + @pytest.mark.unit + def test_a_rechunk_after_reupload_starts_a_fresh_run_number(self, db_connection): + db = db_connection + doc_uuid = _doc(db, "wf-run") + db.save_chunks("wf-run", [{"chunk_number": 1, "original_text": "a", "chunking_provider": "p"}]) + db.reset_document_for_reupload( + doc_uuid, filename="n.pdf", filepath="/n.pdf", fingerprint=_md5("n"), + ) + db.save_chunks("wf-run", [{"chunk_number": 1, "original_text": "b", "chunking_provider": "p"}]) + assert db.get_chunk("wf-run", 1)["chunk_version"] == 1 + + @pytest.mark.db + @pytest.mark.unit + def test_a_fingerprint_must_be_hex(self, db_connection): + doc_uuid = _doc(db_connection, "wf-bad") + with pytest.raises(ValueError): + db_connection.reset_document_for_reupload( + doc_uuid, filename="n.pdf", filepath="/n.pdf", fingerprint="not-hex", + ) + + +class TestUserGroup: + @pytest.mark.db + @pytest.mark.unit + def test_a_single_tenant_sets_the_instance_without_a_warning(self, db_connection, caplog): + db = db_connection + with caplog.at_level(logging.WARNING, logger="pipeline.db"): + db.upsert_user("kc-one", "one", None, [], tenants=["MH"]) + assert db.get_user_by_keycloak_id("kc-one")["instance_id"] == _instance(db, "mh") + assert not [r for r in caplog.records if "tenants" in r.getMessage()] + + @pytest.mark.db + @pytest.mark.unit + def test_several_tenants_use_the_first_sorted_one_and_warn(self, db_connection, caplog): + db = db_connection + with caplog.at_level(logging.WARNING, logger="pipeline.db"): + db.upsert_user("kc-many", "many", None, [], tenants=["up", "mh", "up"]) + assert db.get_user_by_keycloak_id("kc-many")["instance_id"] == _instance(db, "mh") + warnings = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] + assert any("kc-many" in w and "mh" in w for w in warnings) + + @pytest.mark.db + @pytest.mark.unit + def test_a_login_without_tenants_keeps_the_stored_instance(self, db_connection): + db = db_connection + db.upsert_user("kc-keep", "keep", None, [], tenants=["up"]) + db.upsert_user("kc-keep", "keep", None, []) + db.upsert_user("kc-keep", "keep", None, [], tenants=[]) + assert db.get_user_by_keycloak_id("kc-keep")["instance_id"] == _instance(db, "up") + + @pytest.mark.db + @pytest.mark.unit + def test_a_new_login_moves_the_user_to_the_new_first_tenant(self, db_connection): + db = db_connection + db.upsert_user("kc-move", "move", None, [], tenants=["up"]) + db.upsert_user("kc-move", "move", None, [], tenants=["mh"]) + assert db.get_user_by_keycloak_id("kc-move")["instance_id"] == _instance(db, "mh") + + @pytest.mark.db + @pytest.mark.unit + def test_users_without_tenants_have_no_instance(self, db_connection): + db_connection.upsert_user("kc-none", "none", None, []) + assert db_connection.get_user_by_keycloak_id("kc-none")["instance_id"] is None + + @pytest.mark.db + @pytest.mark.unit + def test_the_uploader_lands_in_the_documents_metadata_and_gets_the_tenant_instance(self, db_connection): + db = db_connection + _doc(db, "wf-up", instance="mh", uploaded_by_user_id="kc-up", uploaded_by_username="uma") + doc = db.get_document("wf-up") + user = db.get_user_by_keycloak_id("kc-up") + assert doc["uploaded_by_user_id"] == "kc-up" + assert user["instance_id"] == _instance(db, "mh") + with db.get_connection() as conn: + meta = conn.execute("SELECT metadata FROM documents").fetchone()["metadata"] + assert meta["uploaded_by_user_id"] == user["id"] + + +class TestAuditGroupSnapshot: + @pytest.mark.db + @pytest.mark.unit + def test_an_entry_snapshots_the_documents_instance(self, db_connection): + db = db_connection + _doc(db, "wf-aud", instance="mh") + db.log_audit(workflow_id="wf-aud", document_id="x", action_type="page_edit", entity_type="page") + with db.get_connection() as conn: + row = conn.execute("SELECT instance_id FROM audit_logs WHERE action_type = 'page_edit'").fetchone() + assert str(row["instance_id"]) == _instance(db, "mh") + + @pytest.mark.db + @pytest.mark.unit + def test_the_snapshot_survives_a_purge_and_keeps_tenant_scope(self, db_connection): + db = db_connection + _doc(db, "wf-gone", instance="mh") + _doc(db, "wf-other", instance="up", code="other") + for wf in ("wf-gone", "wf-other"): + db.log_audit(workflow_id=wf, document_id="x", action_type="purge_document", entity_type="document") + db.purge_document("wf-gone") + db.purge_document("wf-other") + + with db.get_connection() as conn: + rows = conn.execute( + "SELECT document_id, instance_id, workflow_id FROM audit_logs WHERE action_type = 'purge_document'" + ).fetchall() + assert {r["workflow_id"] for r in rows} == {"wf-gone", "wf-other"} + assert all(r["document_id"] is None for r in rows) + assert {str(r["instance_id"]) for r in rows} == {_instance(db, "mh"), _instance(db, "up")} + assert db.get_all_audit_log_count(instances=["mh"]) == 1 + assert db.get_all_audit_log_count(instances=["up"]) == 1 + assert db.get_all_audit_log_count(instances=["gj"]) == 0 + assert [e["workflow_id"] for e in db.get_all_audit_logs(instances=["mh"])] == ["wf-gone"] + assert db.get_all_audit_logs(instances=["mh"])[0]["instance"] == "mh" + + @pytest.mark.db + @pytest.mark.unit + def test_system_entries_have_no_instance(self, db_connection): + db = db_connection + db.log_audit(workflow_id="__catalog__", document_id="__catalog__", action_type="catalog_rebuild", + entity_type="catalog") + with db.get_connection() as conn: + row = conn.execute("SELECT instance_id, document_id FROM audit_logs WHERE action_type = 'catalog_rebuild'").fetchone() + assert row["instance_id"] is None and row["document_id"] is None + + +class TestCatalogStatusView: + @staticmethod + def _rows(db): + with db.get_connection() as conn: + return {r["code"]: r for r in conn.execute("SELECT * FROM catalog_status").fetchall()} + + @pytest.mark.db + @pytest.mark.unit + def test_one_row_per_document_with_catalog_instance_and_type(self, db_connection): + db = db_connection + _doc(db, "wf-1", instance="mh", code="alpha", stage="completed") + _doc(db, "wf-2", instance="mh", code="beta", stage="completed") + _doc(db, "wf-3", instance="up", code="gamma", stage="completed", kind="advisory") + rows = self._rows(db) + assert set(rows) == {"alpha", "beta", "gamma"} + assert rows["alpha"]["catalog_id"] == rows["beta"]["catalog_id"] != rows["gamma"]["catalog_id"] + assert str(rows["alpha"]["instance_id"]) == _instance(db, "mh") + assert (rows["alpha"]["instance_name"], rows["gamma"]["instance_name"]) == ("mh", "up") + assert rows["alpha"]["type"] == "openagrinet:AgricultureResource" + assert rows["gamma"]["type"] == "openagrinet:KnowledgeAdvisory" + assert rows["alpha"]["workflow_id"] == "wf-1" + assert rows["alpha"]["name"] == "alpha" + + @pytest.mark.db + @pytest.mark.unit + def test_status_follows_stage_flags_and_disabling(self, db_connection): + db = db_connection + _doc(db, "wf-live", code="live", stage="completed") + _doc(db, "wf-stale", code="stale", stage="completed") + db.mark_document_reindex_required("wf-stale", True, "edit") + _doc(db, "wf-gate", code="gate", stage="approval_for_prod") + _doc(db, "wf-wip", code="wip", stage="chunk_review") + _doc(db, "wf-off", code="off", stage="completed") + db.set_document_disabled("wf-off", True) + _doc(db, "wf-hidden", code="hidden", stage="completed") + db.update_document_fields("wf-hidden", catalog_visible=False) + _doc(db, "wf-gate-off", code="gate-off", stage="approval_for_prod") + db.set_document_disabled("wf-gate-off", True) + + status = {code: row["status"] for code, row in self._rows(db).items()} + assert status == { + "live": "live", + "stale": "pending_reindex", + "gate": "pending_prod", + "wip": "disabled", + "off": "disabled", + "hidden": "disabled", + "gate-off": "disabled", + } + + @pytest.mark.db + @pytest.mark.unit + def test_chunk_count_is_counted_from_chunks(self, db_connection): + db = db_connection + _doc(db, "wf-c", code="counted", stage="completed") + assert self._rows(db)["counted"]["chunk_count"] == 0 + db.save_chunks("wf-c", [{"chunk_number": n, "original_text": "t"} for n in (1, 2, 3)]) + assert self._rows(db)["counted"]["chunk_count"] == 3 + db.save_chunks("wf-c", [{"chunk_number": 1, "original_text": "t"}]) + assert self._rows(db)["counted"]["chunk_count"] == 1 + + @pytest.mark.db + @pytest.mark.unit + def test_network_visible_defaults_to_true_and_follows_the_flag(self, db_connection): + db = db_connection + _doc(db, "wf-v1", code="visible", stage="completed") + _doc(db, "wf-v2", code="private", stage="completed") + db.update_document_fields("wf-v2", network_visible=False) + rows = self._rows(db) + assert (rows["visible"]["network_visible"], rows["private"]["network_visible"]) == (True, False) + + @pytest.mark.db + @pytest.mark.unit + def test_promoted_at_comes_from_the_prod_index_of_live_documents_only(self, db_connection, monkeypatch): + monkeypatch.setenv("PROD_VECTOR_DB_COLLECTION_NAME", "prod-docs") + db = db_connection + _doc(db, "wf-p1", code="promoted", stage="completed") + _doc(db, "wf-p2", code="not-live", stage="chunk_review") + _doc(db, "wf-p3", code="dev-only", stage="completed") + for wf in ("wf-p1", "wf-p2"): + db.upsert_document_index_status(wf, "prod-docs", last_indexed_at="2026-05-01T10:00:00+00:00") + db.upsert_document_index_status("wf-p3", "documents-index", last_indexed_at="2026-05-02T10:00:00+00:00") + rows = self._rows(db) + assert rows["promoted"]["promoted_at"] is not None + assert rows["promoted"]["promoted_at"].isoformat().startswith("2026-05-01T10:00:00") + assert rows["not-live"]["promoted_at"] is None + assert rows["dev-only"]["promoted_at"] is None + + @pytest.mark.db + @pytest.mark.unit + def test_a_catalog_without_documents_has_no_rows(self, db_connection): + db = db_connection + db.get_or_create_catalog(_instance(db), "scheme") + assert self._rows(db) == {} + + +class TestDocumentJsonbShape: + @pytest.mark.db + @pytest.mark.unit + def test_a_new_document_carries_the_default_metadata_keys(self, db_connection): + db = db_connection + _doc(db, "wf-meta") + with db.get_connection() as conn: + meta = conn.execute("SELECT metadata FROM documents").fetchone()["metadata"] + assert meta["index_status"] == {} + assert (meta["catalog_visible"], meta["network_visible"], meta["reindex_required"]) == (True, True, False) + for dropped in ("is_demo", "chunk_count", "display_name"): + assert dropped not in meta + + @pytest.mark.db + @pytest.mark.unit + def test_the_source_has_no_page_count(self, db_connection): + db = db_connection + _doc(db, "wf-src") + db.save_pages("wf-src", [{"page_number": 1, "original_markdown": "a"}]) + with db.get_connection() as conn: + source = conn.execute("SELECT source FROM documents").fetchone()["source"] + assert "page_count" not in source + assert db.get_document("wf-src")["page_count"] == 1 + + @pytest.mark.db + @pytest.mark.unit + def test_there_are_no_instance_kind_or_demo_columns(self, db_connection): + with db_connection.get_connection() as conn: + cols = { + r["column_name"] + for r in conn.execute( + "SELECT column_name FROM information_schema.columns WHERE table_name = 'documents'" + ).fetchall() + } + assert {"instance", "document_kind", "is_demo", "catalog_entry_id", "valid_from", "valid_to", + "index_status", "uploaded_by_user_id"}.isdisjoint(cols) + assert {"catalog_id", "code", "name", "aliases", "source", "is_disabled", "metadata"} <= cols + + @pytest.mark.db + @pytest.mark.unit + def test_aliases_cannot_hold_an_empty_string(self, db_connection): + db = db_connection + _doc(db, "wf-al", aliases=["PM Kisan", "", "pmk"]) + with db.get_connection() as conn: + aliases = conn.execute("SELECT aliases FROM documents").fetchone()["aliases"] + assert aliases == ["PM Kisan", "pmk"] + with db.get_connection() as conn: + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute("UPDATE documents SET aliases = ARRAY['a', '']") + + @pytest.mark.db + @pytest.mark.unit + def test_document_metadata_must_be_an_object(self, db_connection): + db = db_connection + _doc(db, "wf-obj") + with db.get_connection() as conn: + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute("UPDATE documents SET metadata = %s", (Jsonb([1]),)) + + @pytest.mark.db + @pytest.mark.unit + def test_workflow_and_audit_columns_are_plain_text(self, db_connection): + with db_connection.get_connection() as conn: + types = { + (r["table_name"], r["column_name"]): r["data_type"] + for r in conn.execute( + "SELECT table_name, column_name, data_type FROM information_schema.columns " + "WHERE (table_name, column_name) IN (('workflows','workflow_type'), ('workflows','status'), " + "('workflows','current_stage'), ('workflow_artifacts','stage'), ('audit_logs','entity_type'))" + ).fetchall() + } + enums = conn.execute("SELECT count(*) AS n FROM pg_type WHERE typtype = 'e'").fetchone()["n"] + assert len(types) == 5 and set(types.values()) == {"text"} + assert enums == 0 + + +class TestSupersededIndexKeys: + """A replaced file changes the index key; the old key must be remembered for cleanup.""" + + @staticmethod + def _indexed(db, workflow_id, envs): + doc_uuid = _doc(db, workflow_id) + old_key = db.get_document(workflow_id)["document_id"] + for env in envs: + index = "prod-index" if env == "prod" else "documents-index" + db.upsert_document_index_status( + workflow_id, index, doc_id=old_key, chunk_count_indexed=1, status="indexed" + ) + return doc_uuid, old_key + + @pytest.mark.db + @pytest.mark.unit + def test_reupload_remembers_the_old_key_for_every_index_that_holds_it( + self, db_connection, monkeypatch + ): + db = db_connection + monkeypatch.setenv("PROD_VECTOR_DB_COLLECTION_NAME", "prod-index") + doc_uuid, old_key = self._indexed(db, "wf-sup", ["dev", "prod"]) + + db.reset_document_for_reupload( + doc_uuid, filename="n.pdf", filepath="minio://documents/n.pdf", fingerprint="b" * 32 + ) + + assert db.get_document(doc_uuid)["superseded_doc_ids"] == {"dev": [old_key], "prod": [old_key]} + + @pytest.mark.db + @pytest.mark.unit + def test_only_indexes_that_hold_the_old_key_are_tracked(self, db_connection): + db = db_connection + doc_uuid, old_key = self._indexed(db, "wf-sup-dev", ["dev"]) + + db.reset_document_for_reupload( + doc_uuid, filename="n.pdf", filepath="minio://documents/n.pdf", fingerprint="c" * 32 + ) + + assert db.get_document(doc_uuid)["superseded_doc_ids"] == {"dev": [old_key]} + + @pytest.mark.db + @pytest.mark.unit + def test_the_same_file_again_supersedes_nothing(self, db_connection): + db = db_connection + doc_uuid, old_key = self._indexed(db, "wf-sup-same", ["dev"]) + + db.reset_document_for_reupload( + doc_uuid, filename="n.pdf", filepath="minio://documents/n.pdf", fingerprint=old_key + ) + + assert db.get_document(doc_uuid)["superseded_doc_ids"] == {} + + @pytest.mark.db + @pytest.mark.unit + def test_two_replacements_keep_both_old_keys_until_cleared(self, db_connection): + db = db_connection + doc_uuid, first_key = self._indexed(db, "wf-sup-two", ["dev"]) + db.reset_document_for_reupload( + doc_uuid, filename="n.pdf", filepath="minio://documents/n.pdf", fingerprint="d" * 32 + ) + # The second version never reached the index, so its key is not tracked. + db.reset_document_for_reupload( + doc_uuid, filename="m.pdf", filepath="minio://documents/m.pdf", fingerprint="e" * 32 + ) + assert db.get_document(doc_uuid)["superseded_doc_ids"] == {"dev": [first_key]} + + db.clear_superseded_doc_ids(doc_uuid, "dev", [first_key]) + + assert db.get_document(doc_uuid)["superseded_doc_ids"] == {} + + +class TestSharedIndexKeys: + """Two documents holding the same file share an index key; neither may delete it for the other.""" + + @staticmethod + def _indexed_twin(db, workflow_id, key, env="dev", **kwargs): + _doc(db, workflow_id, document_id=key, instance="gj", **kwargs) + index = "prod-index" if env == "prod" else "documents-index" + db.upsert_document_index_status(workflow_id, index, doc_id=key, status="indexed") + + @pytest.mark.db + @pytest.mark.unit + def test_a_key_is_in_use_while_another_live_document_is_indexed_under_it( + self, db_connection, monkeypatch + ): + db = db_connection + monkeypatch.setenv("PROD_VECTOR_DB_COLLECTION_NAME", "prod-index") + first = _doc(db, "wf-twin-a", document_id="f" * 32) + self._indexed_twin(db, "wf-twin-b", "f" * 32, env="dev") + + assert db.doc_id_used_by_other_document("f" * 32, first, "dev") is True + # Another environment is checked on its own index entry. + assert db.doc_id_used_by_other_document("f" * 32, first, "prod") is False + assert db.doc_id_used_by_other_document("0" * 32, first, "dev") is False + + @pytest.mark.db + @pytest.mark.unit + def test_an_uploaded_but_unindexed_twin_does_not_block_deletion(self, db_connection): + db = db_connection + first = _doc(db, "wf-up-a", document_id="e" * 32) + _doc(db, "wf-up-b", document_id="e" * 32, instance="gj") # same file, never indexed + + assert db.doc_id_used_by_other_document("e" * 32, first, "dev") is False + + @pytest.mark.db + @pytest.mark.unit + def test_a_disabled_twin_does_not_block_deletion(self, db_connection): + db = db_connection + first = _doc(db, "wf-dis-a", document_id="d" * 32) + self._indexed_twin(db, "wf-dis-b", "d" * 32) + assert db.doc_id_used_by_other_document("d" * 32, first, "dev") is True + + db.set_document_disabled("wf-dis-b", True) + + assert db.doc_id_used_by_other_document("d" * 32, first, "dev") is False + + @pytest.mark.db + @pytest.mark.unit + def test_a_document_never_blocks_its_own_key(self, db_connection): + db = db_connection + only = _doc(db, "wf-own", document_id="c" * 32) + db.upsert_document_index_status("wf-own", "documents-index", doc_id="c" * 32, status="indexed") + + assert db.doc_id_used_by_other_document("c" * 32, only, "dev") is False diff --git a/tests/test_scheme_catalog.py b/tests/test_scheme_catalog.py index 2ffe77d..4e4cab7 100644 --- a/tests/test_scheme_catalog.py +++ b/tests/test_scheme_catalog.py @@ -2,69 +2,80 @@ from __future__ import annotations -import json -import os - import pytest @pytest.fixture -def catalog_db(temp_db_path): - from pipeline import db, scheme_catalog +def catalog_db(db_connection): + from pipeline import scheme_catalog - db.DB_PATH = temp_db_path - db.init_db() - scheme_catalog.ensure_catalog_schema() - yield db, scheme_catalog - db.DB_PATH = os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db") + yield db_connection, scheme_catalog + + +def _doc(db, workflow_id, *, code, kind="scheme", name=None, stage="completed", + instance="default", filename=None, **extra): + """Register a document the v2 way: it needs a kind (catalog) and a code.""" + return db.upsert_document( + workflow_id=workflow_id, + document_id=f"doc-{workflow_id}", + filename=filename or f"{workflow_id}.pdf", + filepath=f"/tmp/{workflow_id}.pdf", + stage=stage, + instance=instance, + kind=kind, + code=code, + name=name, + **extra, + ) -def test_bootstrap_seeds_13_vector_schemes(catalog_db): +def test_catalog_has_no_bootstrap_entries(catalog_db): + """The catalog is documents only; the code-defined lists stay as hints.""" db, scheme_catalog = catalog_db - result = scheme_catalog.bootstrap_catalog_if_empty() - assert result["bootstrapped"] is True - assert result["vector_count"] == 13 + assert not hasattr(scheme_catalog, "bootstrap_catalog_if_empty") snap = scheme_catalog.build_snapshot() - codes = {s["scheme_code"] for s in snap["vector_schemes"]} - assert len(codes) == 13 - assert "nbm" not in codes # legacy-only - assert "pkvy" in codes - assert "pm-ddky" in codes - assert snap["catalog_version"] >= 1 + assert snap["vector_schemes"] == [] + assert snap["catalog_version"] == 0 + # Static lists are untouched: curated alias source and legacy tool list. + assert len(scheme_catalog.BOOTSTRAP_VECTOR_SCHEMES) == 13 + assert "nbm" not in scheme_catalog.BOOTSTRAP_VECTOR_SCHEMES # legacy-only + assert any(s["scheme_code"] == "nbm" for s in snap["legacy_schemes"]) assert "search_schemes_doc" in snap["tool_prompts"] - assert "vector_schemes_bullets_en" in snap["tool_prompts"] assert snap["routing_exceptions"]["pkvy"] == "search_schemes" assert snap["routing_exceptions"]["nbm"] == "get_scheme_info" - # second call is no-op - again = scheme_catalog.bootstrap_catalog_if_empty() - assert again["bootstrapped"] is False + + +def test_snapshot_has_one_entry_per_live_scheme_document(catalog_db): + db, scheme_catalog = catalog_db + _doc(db, "wf-a", code="pkvy", name="Paramparagat Krishi Vikas Yojana", instance="mh") + _doc(db, "wf-b", code="mif", name="Micro Irrigation Fund", instance="gj") + _doc(db, "wf-adv", code="rain-advice", kind="advisory", name="Rain advisory") + _doc(db, "wf-draft", code="draft", stage="chunk_review") + snap = scheme_catalog.build_snapshot() + by_code = {s["scheme_code"]: s for s in snap["vector_schemes"]} + assert set(by_code) == {"pkvy", "mif"} # advisory kind and non-live docs stay out + assert by_code["pkvy"]["instances"] == ["mh"] + assert by_code["mif"]["instances"] == ["gj"] + assert by_code["pkvy"]["workflow_ids"] == ["wf-a"] + assert by_code["pkvy"]["source"] == "pipeline" + + only_mh = scheme_catalog.build_snapshot(instance="mh") + assert [s["scheme_code"] for s in only_mh["vector_schemes"]] == ["pkvy"] def test_scheme_metadata_and_promote_rebuild(catalog_db): db, scheme_catalog = catalog_db - scheme_catalog.bootstrap_catalog_if_empty() v0 = db.get_catalog_meta()["version"] - - db.upsert_document( - workflow_id="wf-pkvy-1", - document_id="doc-pkvy-1", - filename="pkvy.pdf", - filepath="/tmp/pkvy.pdf", - stage="completed", - page_count=10, - chunk_count=5, - instance="default", - ) + _doc(db, "wf-pkvy-1", code="pkvy") result = scheme_catalog.apply_scheme_metadata( "wf-pkvy-1", - document_kind="scheme", - scheme_code="pkvy", - scheme_name="Paramparagat Krishi Vikas Yojana", - scheme_aliases=["PKVY", "organic"], + code="pkvy", + name="Paramparagat Krishi Vikas Yojana", + aliases=["PKVY", "organic"], network_visible=True, catalog_visible=True, ) - assert result["document"]["scheme_code"] == "pkvy" + assert result["document"]["code"] == "pkvy" assert result["catalog_version"] is not None assert result["catalog_version"] > v0 @@ -72,37 +83,22 @@ def test_scheme_metadata_and_promote_rebuild(catalog_db): assert entry is not None assert entry["status"] == "live" assert entry["source"] == "pipeline" - aliases = json.loads(entry["scheme_aliases_json"]) - assert "organic" in aliases or "PKVY" in aliases + assert entry["name"] == "Paramparagat Krishi Vikas Yojana" + assert "organic" in entry["aliases"] and "PKVY" in entry["aliases"] def test_code_rename_pending_reindex(catalog_db): db, scheme_catalog = catalog_db - scheme_catalog.bootstrap_catalog_if_empty() - db.upsert_document( - workflow_id="wf-rename", - document_id="doc-rename", - filename="x.pdf", - filepath="/tmp/x.pdf", - stage="completed", - chunk_count=3, - instance="default", - ) - scheme_catalog.apply_scheme_metadata( - "wf-rename", - document_kind="scheme", - scheme_code="old-code", - scheme_name="Old", - ) + _doc(db, "wf-rename", code="old-code", name="Old") result = scheme_catalog.apply_scheme_metadata( "wf-rename", - scheme_code="new-code", - scheme_name="New", + code="new-code", + name="New", ) assert result["pending_reindex"] is True doc = db.get_document("wf-rename") - assert doc["scheme_code"] == "new-code" - assert int(doc["reindex_required"]) == 1 + assert doc["code"] == "new-code" + assert doc["reindex_required"] is True # New code must not appear in live vector_schemes snap = scheme_catalog.build_snapshot() live_codes = {s["scheme_code"] for s in snap["vector_schemes"]} @@ -110,6 +106,7 @@ def test_code_rename_pending_reindex(catalog_db): entry = db.get_catalog_entry("new-code") assert entry is not None assert entry["status"] == "pending_reindex" + assert db.get_catalog_entry("old-code") is None # the document holds one code def test_derive_scheme_code_acronym(catalog_db): @@ -121,59 +118,47 @@ def test_derive_scheme_code_acronym(catalog_db): def test_derive_scheme_code_collision_appends_suffix(catalog_db): db, scheme_catalog = catalog_db - db.upsert_document( - workflow_id="wf-nmnf-1", - document_id="doc-nmnf-1", - filename="a.pdf", - filepath="/tmp/a.pdf", - stage="completed", - instance="default", - ) - scheme_catalog.apply_scheme_metadata( - "wf-nmnf-1", document_kind="scheme", scheme_code="nmnf", scheme_name="National Mission on Natural Farming", - ) + _doc(db, "wf-nmnf-1", code="nmnf", name="National Mission on Natural Farming") # A second, unrelated document whose title happens to acronym to the same code. assert scheme_catalog.derive_scheme_code("New Model National Farming") == "nmnf-2" + # The collision domain is the kind: an advisory may reuse the code. + assert scheme_catalog.derive_scheme_code("New Model National Farming", kind="advisory") == "nmnf" + # A document's own code does not collide with itself. + assert scheme_catalog.derive_scheme_code( + "National Mission on Natural Farming", exclude_workflow_id="wf-nmnf-1" + ) == "nmnf" -def test_apply_scheme_metadata_auto_derives_code_from_name(catalog_db): +def test_collision_domain_spans_tenants(catalog_db): + """Codes are unique per catalog in SQL, but derivation avoids every tenant's codes.""" db, scheme_catalog = catalog_db - db.upsert_document( - workflow_id="wf-auto-code", - document_id="doc-auto-code", - filename="Guideline_of_NMNF_V2_Revised.pdf", - filepath="/tmp/x.pdf", - stage="ready_for_ingestion", - instance="default", - ) - result = scheme_catalog.apply_scheme_metadata( - "wf-auto-code", - document_kind="scheme", - scheme_name="National Mission on Natural Farming", - ) - # scheme_code was never passed explicitly -> derived from the title, not left blank/hash-based. - assert result["document"]["scheme_code"] == "nmnf" - assert result["document"]["scheme_name"] == "National Mission on Natural Farming" + _doc(db, "wf-mh", code="nmnf", instance="mh") + assert scheme_catalog.derive_scheme_code("National Mission on Natural Farming") == "nmnf-2" + # The same explicit code in another tenant's catalog is still allowed by the schema. + _doc(db, "wf-gj", code="nmnf", instance="gj") -def test_document_kind_widened_to_advisory_and_custom(catalog_db): +def test_apply_scheme_metadata_keeps_the_code_when_only_the_name_changes(catalog_db): + """Every document has a code from upload, so a name edit never re-derives it.""" db, scheme_catalog = catalog_db - db.upsert_document( - workflow_id="wf-kind", - document_id="doc-kind", - filename="advisory.pdf", - filepath="/tmp/advisory.pdf", - stage="ready_for_ingestion", - instance="default", + _doc(db, "wf-name-only", code="gn", stage="ready_for_ingestion") + result = scheme_catalog.apply_scheme_metadata( + "wf-name-only", + name="National Mission on Natural Farming", ) - result = scheme_catalog.apply_scheme_metadata("wf-kind", document_kind="advisory") - assert result["document"]["document_kind"] == "advisory" + assert result["document"]["code"] == "gn" + assert result["document"]["name"] == "National Mission on Natural Farming" + assert result["code_renamed"] is False - result = scheme_catalog.apply_scheme_metadata("wf-kind", document_kind="how_to_faq") - assert result["document"]["document_kind"] == "how_to_faq" - with pytest.raises(ValueError): - scheme_catalog.apply_scheme_metadata("wf-kind", document_kind="Not Valid!!") +def test_kind_is_fixed_after_upload(catalog_db): + db, scheme_catalog = catalog_db + _doc(db, "wf-kind", code="kind-doc", stage="ready_for_ingestion") + result = scheme_catalog.apply_scheme_metadata("wf-kind", name="Renamed") + assert result["document"]["kind"] == "scheme" + assert db.get_document("wf-kind")["kind"] == "scheme" + with pytest.raises(TypeError): + scheme_catalog.apply_scheme_metadata("wf-kind", kind="advisory") def test_prepare_records_scheme_payload(): @@ -195,10 +180,10 @@ def test_prepare_records_scheme_payload(): chunks, workflow_id="wf1", instance="default", - document_kind="scheme", - scheme_code="pkvy", - scheme_name="Paramparagat Krishi Vikas Yojana", - scheme_aliases=["PKVY"], + kind="scheme", + code="pkvy", + name="Paramparagat Krishi Vikas Yojana", + aliases=["PKVY"], ) assert len(records) == 1 r = records[0] @@ -230,27 +215,37 @@ def test_prepare_records_document_unchanged(): "doc2", "notes.pdf", chunks, - document_kind="document", + kind="document", ) assert records[0]["type"] == "document" + assert "document_kind" not in records[0] assert "scheme_code" not in records[0] +@pytest.mark.parametrize( + "kind,expected", + [("advisory", "advisory"), ("video", "video"), (" Advisory ", "advisory")], +) +def test_prepare_records_type_equals_kind(kind, expected): + from pipeline.activities import _prepare_records + + chunks = [{"chunk_number": 0, "original_text": "Some text.", "token_count": 3, "is_excluded": False}] + records = _prepare_records("doc3", "x.pdf", chunks, kind=kind) + assert records[0]["type"] == expected + assert "document_kind" not in records[0] + + @pytest.fixture -def asgi_client(temp_db_path, mock_temporal_client, mock_minio_client): +def asgi_client(db_connection, mock_temporal_client, mock_minio_client): """HTTP client that does not run real Temporal/MinIO lifespan.""" from contextlib import asynccontextmanager from fastapi.testclient import TestClient - from pipeline import db, scheme_catalog from pipeline.api import api - db.DB_PATH = temp_db_path - db.init_db() api.temporal_client = mock_temporal_client api.minio_client = mock_minio_client - scheme_catalog.ensure_catalog_schema() @asynccontextmanager async def _noop_lifespan(_app): @@ -263,13 +258,14 @@ async def _noop_lifespan(_app): yield client finally: api.app.router.lifespan_context = original - db.DB_PATH = os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db") def test_catalog_api_snapshot(asgi_client): - from pipeline import scheme_catalog + from pipeline import db, scheme_catalog - scheme_catalog.bootstrap_catalog_if_empty() + _doc(db, "wf-snap-1", code="pkvy", name="Paramparagat Krishi Vikas Yojana") + _doc(db, "wf-snap-2", code="pm-ddky", name="Deen Dayal Upadhyaya Krishi Kalyan Yojana") + scheme_catalog.rebuild_catalog_from_documents(reason="test") r = asgi_client.get("/catalog/v1/version") assert r.status_code == 200 @@ -280,7 +276,7 @@ def test_catalog_api_snapshot(asgi_client): assert r.status_code == 200 body = r.json() assert body["api_version"] == "1" - assert len(body["vector_schemes"]) == 13 + assert {s["scheme_code"] for s in body["vector_schemes"]} == {"pkvy", "pm-ddky"} assert "tool_prompts" in body assert "legacy_schemes" in body assert body["collections"]["scheme_qdrant"]["name"] == "schemes-index" @@ -291,13 +287,14 @@ def test_catalog_api_snapshot(asgi_client): r = asgi_client.get("/catalog/v1/schemes") assert r.status_code == 200 - assert len(r.json()["vector_schemes"]) == 13 + assert len(r.json()["vector_schemes"]) == 2 -def test_catalog_service_key_auth(asgi_client, monkeypatch): - from pipeline import scheme_catalog +def test_catalog_bootstrap_endpoint_is_gone(asgi_client): + assert asgi_client.post("/catalog/v1/bootstrap").status_code in (404, 405) - scheme_catalog.bootstrap_catalog_if_empty() + +def test_catalog_service_key_auth(asgi_client, monkeypatch): monkeypatch.setenv("CATALOG_SERVICE_API_KEYS", "test-secret-key") r = asgi_client.get( @@ -313,70 +310,47 @@ def test_catalog_service_key_auth(asgi_client, monkeypatch): def test_scheme_metadata_api(asgi_client): from pipeline import db - db.upsert_document( - workflow_id="wf-meta", - document_id="doc-meta", - filename="scheme.pdf", - filepath="/tmp/scheme.pdf", - stage="chunk_review", - instance="default", - ) + _doc(db, "wf-meta", code="meta-draft", stage="chunk_review", filename="scheme.pdf") r = asgi_client.patch( "/documents/wf-meta/scheme-metadata", json={ - "document_kind": "scheme", - "scheme_code": "mif", - "scheme_name": "Micro Irrigation Fund", - "scheme_aliases": ["MIF"], + "kind": "scheme", + "code": "mif", + "name": "Micro Irrigation Fund", + "aliases": ["MIF"], "network_visible": True, }, ) assert r.status_code == 200, r.text body = r.json() - assert body["scheme_code"] == "mif" - assert body["document_kind"] == "scheme" - assert "MIF" in body["scheme_aliases"] + assert body["code"] == "mif" + assert body["kind"] == "scheme" + assert "MIF" in body["aliases"] -def test_reingest_blocked_when_never_classified(asgi_client, mock_temporal_client): +def test_scheme_metadata_api_ignores_a_kind_in_the_body(asgi_client): from pipeline import db - db.upsert_document( - workflow_id="wf-unclassified", - document_id="doc-unclassified", - filename="x.pdf", - filepath="/tmp/x.pdf", - stage="completed", - instance="default", + _doc(db, "wf-meta-kind", code="meta-kind", stage="chunk_review") + r = asgi_client.patch( + "/documents/wf-meta-kind/scheme-metadata", json={"kind": "advisory", "name": "Still a scheme"} ) - db.save_chunks("wf-unclassified", [ - {"chunk_number": 1, "original_text": "some content", "token_count": 5}, - ]) - - r = asgi_client.post("/documents/wf-unclassified/reingest") - assert r.status_code == 400 - assert "never been classified" in r.json()["detail"] + assert r.status_code == 200 + assert r.json()["kind"] == "scheme" + assert r.json()["name"] == "Still a scheme" -def test_reingest_allowed_once_classified(asgi_client, mock_temporal_client): - from pipeline import db, scheme_catalog +@pytest.mark.parametrize("kind", ["scheme", "advisory", "video"]) +def test_reingest_is_allowed_for_every_kind(asgi_client, mock_temporal_client, kind): + """No 'never classified' guard any more: a document always has a kind and a code.""" + from pipeline import db - db.upsert_document( - workflow_id="wf-classified", - document_id="doc-classified", - filename="x.pdf", - filepath="/tmp/x.pdf", - stage="completed", - instance="default", - ) - db.save_chunks("wf-classified", [ + _doc(db, "wf-reingest", code="testcode", kind=kind) + db.save_chunks("wf-reingest", [ {"chunk_number": 1, "original_text": "some content", "token_count": 5}, ]) - scheme_catalog.apply_scheme_metadata( - "wf-classified", document_kind="scheme", scheme_code="testcode", scheme_name="Test Scheme", - ) - r = asgi_client.post("/documents/wf-classified/reingest") + r = asgi_client.post("/documents/wf-reingest/reingest") assert r.status_code == 200, r.text assert r.json()["status"] == "started" @@ -384,14 +358,7 @@ def test_reingest_allowed_once_classified(asgi_client, mock_temporal_client): def test_request_prod_ready_wrong_stage_rejected(asgi_client): from pipeline import db - db.upsert_document( - workflow_id="wf-prod-ready-wrong-stage", - document_id="doc-prod-ready-wrong-stage", - filename="x.pdf", - filepath="/tmp/x.pdf", - stage="chunk_review", - instance="default", - ) + _doc(db, "wf-prod-ready-wrong-stage", code="prw", stage="chunk_review") r = asgi_client.post("/documents/wf-prod-ready-wrong-stage/request-prod-ready") assert r.status_code == 400 @@ -399,14 +366,7 @@ def test_request_prod_ready_wrong_stage_rejected(asgi_client): def test_request_prod_ready_flags_document_and_clears_on_approve(asgi_client, mock_temporal_client): from pipeline import db - db.upsert_document( - workflow_id="wf-prod-ready", - document_id="doc-prod-ready", - filename="x.pdf", - filepath="/tmp/x.pdf", - stage="approval_for_prod", - instance="default", - ) + _doc(db, "wf-prod-ready", code="pr", stage="approval_for_prod") doc = db.get_document("wf-prod-ready") assert doc["prod_ready_requested_at"] is None @@ -508,7 +468,7 @@ def test_aliases_are_deduped_case_insensitively(self): @pytest.mark.unit def test_json_string_aliases_are_accepted(self): - """documents.scheme_aliases_json arrives as a JSON string.""" + """Aliases may arrive as a JSON string.""" from pipeline.scheme_catalog import resolve_scheme_aliases aliases = resolve_scheme_aliases("makhana", "Makhana Scheme", '["fox nut", "lotus seed"]') diff --git a/tests/test_search_provider_integration.py b/tests/test_search_provider_integration.py index 244aa9f..bd28f3f 100644 --- a/tests/test_search_provider_integration.py +++ b/tests/test_search_provider_integration.py @@ -64,4 +64,3 @@ def test_provider_composes_with_group(local_store, test_client): body = search(test_client, provider_id="bv.provider.oan.dev", group="mh") assert body["hits"] == [] - diff --git a/tests/test_tenancy.py b/tests/test_tenancy.py index 53a69c3..bf78d09 100644 --- a/tests/test_tenancy.py +++ b/tests/test_tenancy.py @@ -1,7 +1,10 @@ -"""Unit tests for multi-instance (tenant) scoping.""" +"""Unit tests for multi-group (tenant) scoping.""" from __future__ import annotations +import logging +import re + import pytest from fastapi import HTTPException @@ -105,7 +108,7 @@ def test_superadmin_token_is_instance_unrestricted_despite_scoped_claim(): assert user.is_superadmin is True assert user.is_admin is True # alias assert user.is_instance_unrestricted() is True - # Unrestricted -> allowed_instances is None (every instance visible). + # Unrestricted -> allowed_instances is None (every group visible). assert allowed_instances(user) is None assert user_can_access_instance(user, "tenant-b") doc = assert_document_instance_access( @@ -157,9 +160,20 @@ def test_content_curator_with_scoped_claim_cannot_cross_tenants(): assert exc.value.status_code == 404 +def _register(db, **kwargs): + """upsert_document the v2 way: a new document needs a kind and a code. + + The tenant (``group``) picks the group row; the kind plus that group pick the + catalog, which is created on first use. + """ + kwargs.setdefault("kind", "scheme") + kwargs.setdefault("code", re.sub(r"[^a-z0-9]+", "-", kwargs["workflow_id"].lower()).strip("-")) + return db.upsert_document(**kwargs) + + def test_list_documents_filters_by_instance(db_connection): db = db_connection - db.upsert_document( + _register(db, workflow_id="wf-tenant-a", document_id="d1", filename="a.pdf", @@ -167,7 +181,7 @@ def test_list_documents_filters_by_instance(db_connection): stage="completed", instance="tenant-a", ) - db.upsert_document( + _register(db, workflow_id="wf-tenant-b", document_id="d2", filename="b.pdf", @@ -188,7 +202,7 @@ def test_list_documents_filters_by_instance(db_connection): def test_summary_counts_honor_instance_filter(db_connection): db = db_connection - db.upsert_document( + _register(db, workflow_id="wf-tenant-a-2", document_id="d3", filename="c.pdf", @@ -196,7 +210,7 @@ def test_summary_counts_honor_instance_filter(db_connection): stage="completed", instance="tenant-a", ) - db.upsert_document( + _register(db, workflow_id="wf-tenant-b-2", document_id="d4", filename="d.pdf", @@ -213,7 +227,8 @@ def test_summary_counts_honor_instance_filter(db_connection): def test_list_runs_filters_by_instance(db_connection): """Runs must honor the same tenant scope as documents (role-based states).""" db = db_connection - db.upsert_document( + doc_a = _register(db, + with_workflow=False, workflow_id="wf-run-a", document_id="d-run-a", filename="a.pdf", @@ -221,7 +236,8 @@ def test_list_runs_filters_by_instance(db_connection): stage="completed", instance="tenant-a", ) - db.upsert_document( + doc_b = _register(db, + with_workflow=False, workflow_id="wf-run-b", document_id="d-run-b", filename="b.pdf", @@ -229,8 +245,8 @@ def test_list_runs_filters_by_instance(db_connection): stage="completed", instance="tenant-b", ) - job_a = db.create_document_job("wf-run-a", "ocr", status="completed") - job_b = db.create_document_job("wf-run-b", "ocr", status="running") + job_a = db.create_document_job(doc_a, "ocr_only", status="completed") + job_b = db.create_document_job(doc_b, "ocr_only", status="running") tenant_a_only = db.list_runs(instances=["tenant-a"]) assert {r["id"] for r in tenant_a_only} == {job_a} @@ -245,7 +261,7 @@ def test_list_runs_filters_by_instance(db_connection): unrestricted_rows = db.list_runs(instances=None) assert {r["id"] for r in unrestricted_rows} >= {job_a, job_b} - # Status filter still works under instance scope. + # Status filter still works under group scope. running_a = db.list_runs(status="running", instances=["tenant-a"]) assert running_a == [] running_b = db.list_runs(status="running", instances=["tenant-b"]) @@ -255,7 +271,7 @@ def test_list_runs_filters_by_instance(db_connection): def test_state_user_cannot_see_bv_portal_documents(db_connection): """State-scoped users must not see portal (BV) docs — only their state.""" db = db_connection - db.upsert_document( + _register(db, workflow_id="wf-bv", document_id="d-bv", filename="portal.pdf", @@ -263,7 +279,7 @@ def test_state_user_cannot_see_bv_portal_documents(db_connection): stage="ocr_review", instance=PORTAL_INSTANCE, ) - db.upsert_document( + _register(db, workflow_id="wf-mh", document_id="d-mh", filename="mh.pdf", @@ -271,8 +287,8 @@ def test_state_user_cannot_see_bv_portal_documents(db_connection): stage="ocr_review", instance="mh", ) - db.create_document_job("wf-bv", "ocr", status="running") - db.create_document_job("wf-mh", "ocr", status="running") + db.create_document_job("wf-bv", "ocr_only", status="running") + db.create_document_job("wf-mh", "ocr_only", status="running") # MH contributor from Keycloak group path mh_user = claims_to_user( @@ -316,7 +332,7 @@ def test_state_user_cannot_see_bv_portal_documents(db_connection): def test_upsert_does_not_reassign_instance(db_connection): db = db_connection - db.upsert_document( + _register(db, workflow_id="wf-owned", document_id="d1", filename="a.pdf", @@ -324,7 +340,7 @@ def test_upsert_does_not_reassign_instance(db_connection): stage="registered", instance="tenant-a", ) - db.upsert_document( + _register(db, workflow_id="wf-owned", document_id="d1", filename="a.pdf", @@ -343,7 +359,7 @@ def test_api_helpers_hide_cross_tenant_mutations(db_connection, monkeypatch): from pipeline.api import api monkeypatch.setattr(api, "db", db_mod) - db_mod.upsert_document( + _register(db_mod, workflow_id="wf-tenant-b-doc", document_id="d-tenant-b", filename="b.pdf", @@ -400,3 +416,111 @@ def test_follows_a_configured_default_instance(self, monkeypatch): assert group_for_instance("local") is None assert group_for_instance("mh") == "mh" + + +def test_instance_is_reached_through_the_documents_catalog(db_connection): + """No documents.instance column: a document's group is a groups row reached through its catalog.""" + db = db_connection + _register(db, workflow_id="wf-g-a", document_id="g1", filename="a.pdf", + filepath="/tmp/a.pdf", instance="tenant-a") + _register(db, workflow_id="wf-g-bv", document_id="g2", filename="b.pdf", + filepath="/tmp/b.pdf", instance=PORTAL_INSTANCE) + _register(db, workflow_id="wf-g-adv", document_id="g3", filename="c.pdf", + filepath="/tmp/c.pdf", instance="tenant-a", kind="advisory") + + with db.get_connection() as conn: + groups = {r["name"]: r["path"] for r in conn.execute("SELECT name, path FROM instances")} + catalogs = conn.execute( + "SELECT g.name AS instance_name, c.type FROM catalogs c JOIN instances g ON g.id = c.instance_id" + ).fetchall() + assert groups == { + "tenant-a": "/states/TENANT-A", + PORTAL_INSTANCE: "/global" if PORTAL_INSTANCE == "bv" else f"/states/{PORTAL_INSTANCE.upper()}", + } + # Every group is seeded with one catalog per supported type. + from pipeline.network_constants import KIND_TO_CATALOG_TYPE + + assert sorted((r["instance_name"], r["type"]) for r in catalogs) == sorted( + (group, catalog_type) + for group in ("tenant-a", PORTAL_INSTANCE) + for catalog_type in KIND_TO_CATALOG_TYPE.values() + ) + assert db.get_document("wf-g-a")["instance"] == "tenant-a" + assert db.get_document("wf-g-adv")["kind"] == "advisory" + assert {d["workflow_id"] for d in db.list_documents(instances=["tenant-a"])} == {"wf-g-a", "wf-g-adv"} + + +def test_same_code_is_allowed_in_another_tenants_catalog(db_connection): + db = db_connection + _register(db, workflow_id="wf-c-a", document_id="c1", filename="a.pdf", + filepath="/tmp/a.pdf", instance="tenant-a", code="pm-kisan") + _register(db, workflow_id="wf-c-b", document_id="c2", filename="b.pdf", + filepath="/tmp/b.pdf", instance="tenant-b", code="pm-kisan") + assert db.get_document("wf-c-a")["code"] == db.get_document("wf-c-b")["code"] == "pm-kisan" + # ...but not twice inside one tenant's catalog of that kind. + with pytest.raises(db.DuplicateDocumentCode): + _register(db, workflow_id="wf-c-a2", document_id="c3", filename="a2.pdf", + filepath="/tmp/a2.pdf", instance="tenant-a", code="pm-kisan") + + +def test_audit_keeps_the_tenant_scope_after_the_document_is_purged(db_connection): + """audit_logs snapshots the group, so a purged document's history stays tenant-scoped.""" + db = db_connection + _register(db, workflow_id="wf-aud-a", document_id="a1", filename="a.pdf", + filepath="/tmp/a.pdf", instance="tenant-a") + _register(db, workflow_id="wf-aud-b", document_id="a2", filename="b.pdf", + filepath="/tmp/b.pdf", instance="tenant-b") + db.log_audit(workflow_id="wf-aud-a", document_id="", action_type="page_edit", entity_type="document") + db.log_audit(workflow_id="wf-aud-b", document_id="", action_type="page_edit", entity_type="document") + db.purge_document("wf-aud-a", keep_audit=True) + + a_rows = db.get_all_audit_logs(instances=["tenant-a"]) + assert [r["workflow_id"] for r in a_rows if r["action_type"] == "page_edit"] == ["wf-aud-a"] + b_rows = db.get_all_audit_logs(instances=["tenant-b"]) + assert [r["workflow_id"] for r in b_rows if r["action_type"] == "page_edit"] == ["wf-aud-b"] + assert db.get_all_audit_log_count(instances=["tenant-a"]) == len(a_rows) + assert db.get_all_audit_logs(instances=[]) == [] + + +def test_user_instance_follows_the_first_tenant_in_the_token(db_connection, caplog): + db = db_connection + + def group_of(keycloak_id): + with db.get_connection() as conn: + row = conn.execute( + "SELECT g.name FROM users u LEFT JOIN instances g ON g.id = u.instance_id " + "WHERE u.keycloak_user_id = %s", (keycloak_id,)).fetchone() + return row["name"] + + db.upsert_user("kc-single", "single", "single@example.com", ["contributor"], tenants=["MH"]) + assert group_of("kc-single") == "mh" + + with caplog.at_level(logging.WARNING, logger="pipeline.db"): + db.upsert_user("kc-multi", "multi", "multi@example.com", ["contributor"], tenants=["up", "mh"]) + assert group_of("kc-multi") == "mh" # sorted, first wins + assert any("several tenants" in rec.message for rec in caplog.records) + + # No tenants in the token leaves an existing group alone. + db.upsert_user("kc-single", "single", "single@example.com", ["contributor"]) + assert group_of("kc-single") == "mh" + db.upsert_user("kc-none", "none", "none@example.com", ["contributor"]) + assert group_of("kc-none") is None + + +def test_instance_path_fills_the_keycloak_id_on_sync(db_connection): + """The SQLite move creates groups without a Keycloak id; a later sync matches by path.""" + db = db_connection + first = db.upsert_instance("/states/MH") + with db.get_connection() as conn: + assert conn.execute("SELECT keycloak_group_id FROM instances WHERE id = %s::uuid", + (first,)).fetchone()["keycloak_group_id"] is None + again = db.upsert_instance("/states/MH", keycloak_group_id="kc-group-123") + assert again == first + with db.get_connection() as conn: + row = conn.execute("SELECT name, keycloak_group_id FROM instances WHERE id = %s::uuid", (first,)).fetchone() + assert (row["name"], row["keycloak_group_id"]) == ("mh", "kc-group-123") + # A sync without an id never erases one already stored. + assert db.upsert_instance("/states/MH") == first + with db.get_connection() as conn: + assert conn.execute("SELECT keycloak_group_id FROM instances WHERE id = %s::uuid", + (first,)).fetchone()["keycloak_group_id"] == "kc-group-123" diff --git a/tests/test_webhook.py b/tests/test_webhook.py new file mode 100644 index 0000000..920e9f3 --- /dev/null +++ b/tests/test_webhook.py @@ -0,0 +1,129 @@ +"""Tests for POST /webhook (x-api-key protected).""" + +import os +from unittest.mock import patch + +import pytest + +TEST_KEY = "0123456789abcdef0123456789abcdef" + +SAMPLE = [ + { + "contentID": 101, + "title": "Bacterial Leaf Blight in Rice", + "domainName": "Agriculture", + "status": "Published", + "isPublished": 1, + "slug": "101/bacterial-leaf-blight-in-rice", + "createdAt": "2026-09-09T07:00:00.000Z", + "updatedAt": "2026-09-09T07:00:00.000Z", + "popCount": 1, + "popList": [{"popID": 1001, "contentID": 101, "keywords": ["rice"]}], + } +] + +OK_BODY = {"success": True, "message": "Webhook processed successfully.", "recordCount": 1} + + +@pytest.fixture +def test_client(db_connection): + """TestClient without lifespan (no Temporal/MinIO needed for /webhook).""" + from fastapi.testclient import TestClient + + from pipeline.api import api + + return TestClient(api.app) + + +@pytest.fixture +def key_env(): + with patch.dict(os.environ, {"WEBHOOK_API_KEY": TEST_KEY}): + yield + + +def _rows(db_connection): + from pipeline import db + + with db.get_connection() as conn: + return conn.execute("SELECT * FROM webhook_records").fetchall() + + +@pytest.mark.api +@pytest.mark.unit +class TestWebhookAuth: + def test_missing_header_401(self, test_client, db_connection, key_env): + assert test_client.post("/webhook", json=SAMPLE).status_code == 401 + + def test_empty_header_401(self, test_client, db_connection, key_env): + r = test_client.post("/webhook", json=SAMPLE, headers={"x-api-key": ""}) + assert r.status_code == 401 + + def test_wrong_key_401(self, test_client, db_connection, key_env): + r = test_client.post("/webhook", json=SAMPLE, headers={"x-api-key": "nope"}) + assert r.status_code == 401 + + @pytest.mark.parametrize("value", [None, ""]) + def test_unconfigured_key_503_even_if_auth_disabled(self, test_client, db_connection, value): + env = {"AUTH_DISABLED": "true"} + if value is not None: + env["WEBHOOK_API_KEY"] = value + with patch.dict(os.environ, env): + if value is None: + os.environ.pop("WEBHOOK_API_KEY", None) + r = test_client.post("/webhook", json=SAMPLE, headers={"x-api-key": "anything"}) + assert r.status_code == 503 + assert "WEBHOOK_API_KEY" in r.json()["detail"] + + +@pytest.mark.api +@pytest.mark.unit +class TestWebhookIngest: + H = {"x-api-key": TEST_KEY} + + def test_valid_payload(self, test_client, db_connection, key_env): + r = test_client.post("/webhook", json=SAMPLE, headers=self.H) + assert r.status_code == 200 + assert r.json() == OK_BODY + rows = _rows(db_connection) + assert len(rows) == 1 + assert rows[0]["external_id"] == "101" + assert rows[0]["id"].version == 4 + assert rows[0]["category"] == "Agriculture" + assert "popList" in rows[0]["payload"] + + def test_repost_is_idempotent(self, test_client, db_connection, key_env): + test_client.post("/webhook", json=SAMPLE, headers=self.H) + first_id = _rows(db_connection)[0]["id"] + changed = [{**SAMPLE[0], "title": "New title"}] + r = test_client.post("/webhook", json=changed, headers=self.H) + assert r.json()["recordCount"] == 1 + rows = _rows(db_connection) + assert len(rows) == 1 and rows[0]["title"] == "New title" + assert rows[0]["id"] == first_id + + def test_two_records(self, test_client, db_connection, key_env): + body = SAMPLE + [{**SAMPLE[0], "contentID": 102}] + r = test_client.post("/webhook", json=body, headers=self.H) + assert r.json()["recordCount"] == 2 + assert len(_rows(db_connection)) == 2 + + def test_empty_array(self, test_client, db_connection, key_env): + r = test_client.post("/webhook", json=[], headers=self.H) + assert r.status_code == 200 and r.json()["recordCount"] == 0 + + def test_non_array_422(self, test_client, db_connection, key_env): + r = test_client.post("/webhook", json={"contentID": 1}, headers=self.H) + assert r.status_code == 422 + + def test_non_object_item_422(self, test_client, db_connection, key_env): + r = test_client.post("/webhook", json=[1, 2], headers=self.H) + assert r.status_code == 422 + + def test_too_many_records_413(self, test_client, db_connection, key_env): + from pipeline.api import api + + with patch.object(api, "WEBHOOK_MAX_RECORDS", 1): + body = SAMPLE + [{**SAMPLE[0], "contentID": 102}] + r = test_client.post("/webhook", json=body, headers=self.H) + assert r.status_code == 413 + assert _rows(db_connection) == [] diff --git a/tests/test_webhook_index_db.py b/tests/test_webhook_index_db.py new file mode 100644 index 0000000..0c09b1a --- /dev/null +++ b/tests/test_webhook_index_db.py @@ -0,0 +1,107 @@ +"""Tests for the webhook_records indexing state (indexed_hash / promoted_hash) helpers.""" + + +import pytest + + +def _rec(content_id, title="Wheat Insect Management", published=1, status="Published"): + return { + "contentID": content_id, + "title": title, + "status": status, + "isPublished": published, + "domainName": "Agriculture", + "slug": f"{content_id}/x", + "popList": [{"popID": 1, "status": "Published"}], + } + + +def _row_id(db, content_id): + with db.get_connection() as conn: + return str(conn.execute( + "SELECT id FROM webhook_records WHERE external_id = %s", (str(content_id),) + ).fetchone()["id"]) + + +@pytest.mark.unit +@pytest.mark.db +class TestIndexState: + def test_new_rows_are_unindexed_until_marked(self, db_connection): + db = db_connection + db.upsert_webhook_records("webhook", [_rec(1)]) + rid = _row_id(db, 1) + assert db.get_unindexed_webhook_record_ids() == [rid] + row = db.get_webhook_record(rid) + db.mark_webhook_record_indexed(rid, row["payload_hash"]) + assert db.get_unindexed_webhook_record_ids() == [] + + def test_changed_payload_becomes_unindexed_again(self, db_connection): + db = db_connection + db.upsert_webhook_records("webhook", [_rec(1)]) + rid = _row_id(db, 1) + db.mark_webhook_record_indexed(rid, db.get_webhook_record(rid)["payload_hash"]) + + db.upsert_webhook_records("webhook", [_rec(1)]) # identical + assert db.get_unindexed_webhook_record_ids() == [] + + db.upsert_webhook_records("webhook", [_rec(1, title="Changed")]) + assert db.get_unindexed_webhook_record_ids() == [rid] + + def test_marking_a_stale_hash_leaves_the_row_unindexed(self, db_connection): + db = db_connection + db.upsert_webhook_records("webhook", [_rec(1)]) + rid = _row_id(db, 1) + stale = db.get_webhook_record(rid)["payload_hash"] + db.upsert_webhook_records("webhook", [_rec(1, title="Changed")]) + db.mark_webhook_record_indexed(rid, stale) + assert db.get_unindexed_webhook_record_ids() == [rid] + + def test_unindexed_ids_can_be_scoped_to_external_ids(self, db_connection): + db = db_connection + db.upsert_webhook_records("webhook", [_rec(1), _rec(2), _rec(3)]) + ids = db.get_unindexed_webhook_record_ids(external_ids=["1", "3"]) + assert sorted(ids) == sorted([_row_id(db, 1), _row_id(db, 3)]) + assert db.get_unindexed_webhook_record_ids(external_ids=[]) == [] + + def test_get_webhook_record_parses_payload(self, db_connection): + db = db_connection + db.upsert_webhook_records("webhook", [_rec(101)]) + row = db.get_webhook_record(_row_id(db, 101)) + assert row["payload"]["contentID"] == 101 + assert row["external_id"] == "101" + assert row["is_published"] == 1 + assert row["status"] == "Published" + assert db.get_webhook_record("missing") is None + + def test_webhook_external_id_matches_upsert_rule(self, db_connection): + db = db_connection + assert db.webhook_external_id({"contentID": 7}) == "7" + assert db.webhook_external_id({"id": "abc"}) == "abc" + assert db.webhook_external_id({"slug": "s"}) == "s" + assert len(db.webhook_external_id({"title": "x"})) == 64 # payload hash + + def test_record_id_lists(self, db_connection): + db = db_connection + db.upsert_webhook_records( + "webhook", [_rec(1), _rec(2, published=0, status="Draft")] + ) + assert len(db.get_webhook_record_ids()) == 2 + assert db.get_webhook_record_ids(published_only=True) == [_row_id(db, 1)] + + def test_status_counts(self, db_connection): + db = db_connection + db.upsert_webhook_records( + "webhook", [_rec(1), _rec(2), _rec(3, published=0, status="Draft")] + ) + a, b = _row_id(db, 1), _row_id(db, 2) + db.mark_webhook_record_indexed(a, db.get_webhook_record(a)["payload_hash"]) + db.mark_webhook_record_promoted(a, db.get_webhook_record(a)["payload_hash"]) + counts = db.advisory_status_counts() + assert counts == { + "total": 3, + "published": 2, + "indexed": 1, + "unindexed": 2, + "promoted": 1, + } + assert b # silence unused diff --git a/ui/src/lib/instanceLabels.js b/ui/src/lib/instanceLabels.js index ae0e843..b174b45 100644 --- a/ui/src/lib/instanceLabels.js +++ b/ui/src/lib/instanceLabels.js @@ -1,5 +1,5 @@ /** - * Plan 2: display labels for documents.instance (state / portal tag). + * Plan 2: display labels for the document instance (state / portal tag). * Authorization still comes from Keycloak JWT — this is UI only. */ diff --git a/ui/src/lib/pipelineUi.js b/ui/src/lib/pipelineUi.js index e4f833a..db7dbe2 100644 --- a/ui/src/lib/pipelineUi.js +++ b/ui/src/lib/pipelineUi.js @@ -206,7 +206,7 @@ export const DEFAULT_SEARCH_SETTINGS = { export function getDocumentListLabel(doc) { return ( - doc?.display_name || + doc?.name || doc?.name_en || doc?.name || doc?.filename || diff --git a/ui/src/views/ChunkExplorerView.jsx b/ui/src/views/ChunkExplorerView.jsx index d1ef3ea..e677724 100644 --- a/ui/src/views/ChunkExplorerView.jsx +++ b/ui/src/views/ChunkExplorerView.jsx @@ -166,7 +166,7 @@ export default function ChunkExplorerView() { const groups = new Map() chunks.forEach(chunk => { const key = chunk.workflow_id || '__unknown__' - const label = chunk.display_name || chunk.filename || key + const label = chunk.name || chunk.filename || key const existing = groups.get(key) || { key, label, chunks: [] } existing.chunks.push(chunk) groups.set(key, existing) @@ -401,7 +401,7 @@ export default function ChunkExplorerView() { openDocumentChunks(chunk.workflow_id) }} > - {chunk.display_name || chunk.filename || chunk.workflow_id} + {chunk.name || chunk.filename || chunk.workflow_id}

Chunk {chunk.chunk_number} · Pages {chunk.page_start}–{chunk.page_end} · {getStageLabel(chunk.stage)} diff --git a/ui/src/views/DocumentClassificationPanel.test.jsx b/ui/src/views/DocumentClassificationPanel.test.jsx index a670ec7..3dc6270 100644 --- a/ui/src/views/DocumentClassificationPanel.test.jsx +++ b/ui/src/views/DocumentClassificationPanel.test.jsx @@ -9,7 +9,7 @@ vi.mock('../lib/pipelineUi', async (importOriginal) => ({ fetchJson: vi.fn(), })) -const baseDoc = { document_kind: 'advisory', filename: 'scan_0042.pdf', display_name: 'Cotton Pest Advisory' } +const baseDoc = { kind: 'advisory', filename: 'scan_0042.pdf', name: 'Cotton Pest Advisory' } function renderPanel(doc = baseDoc, props = {}) { const onSaved = vi.fn().mockResolvedValue() @@ -35,7 +35,7 @@ describe('DocumentClassificationPanel display name', () => { }) it('falls back to the filename without its extension', () => { - renderPanel({ ...baseDoc, display_name: null }) + renderPanel({ ...baseDoc, name: null }) expect(screen.getByLabelText(/display name/i)).toHaveValue('scan_0042') }) @@ -53,7 +53,9 @@ describe('DocumentClassificationPanel display name', () => { await user.click(screen.getByRole('button', { name: /^save$/i })) await waitFor(() => expect(onSaved).toHaveBeenCalled()) - expect(patchBody()).toMatchObject({ document_kind: 'advisory', display_name: 'Bollworm Control' }) + expect(patchBody()).toMatchObject({ name: 'Bollworm Control' }) + // Document type is read-only after upload, so it is never sent back. + expect(patchBody()).not.toHaveProperty('kind') }) it('blocks saving a blank display name', async () => { @@ -80,18 +82,19 @@ describe('DocumentClassificationPanel display name', () => { expect(screen.getByLabelText(/display name/i)).toBeDisabled() }) - it('does not resend an unchanged display name', async () => { + it('does not offer to save a display name retyped unchanged', async () => { const user = userEvent.setup() - const { onSaved } = renderPanel({ ...baseDoc, document_kind: 'scheme', scheme_name: '' }) - await user.type(screen.getByPlaceholderText(/national mission/i), 'PM Kisan') - await user.click(screen.getByRole('button', { name: /^save$/i })) + renderPanel() + const field = screen.getByLabelText(/display name/i) + await user.clear(field) + await user.type(field, 'Cotton Pest Advisory') - await waitFor(() => expect(onSaved).toHaveBeenCalled()) - expect(patchBody()).not.toHaveProperty('display_name') + expect(screen.getByRole('button', { name: /saved/i })).toBeDisabled() + expect(fetchJson).not.toHaveBeenCalled() }) it('counts a document without a stored name as already saved', () => { - renderPanel({ ...baseDoc, display_name: null }) + renderPanel({ ...baseDoc, name: null }) expect(screen.getByRole('button', { name: /saved/i })).toBeDisabled() }) diff --git a/ui/src/views/DocumentOpsView.jsx b/ui/src/views/DocumentOpsView.jsx index 4543c1c..7b04f75 100644 --- a/ui/src/views/DocumentOpsView.jsx +++ b/ui/src/views/DocumentOpsView.jsx @@ -40,7 +40,6 @@ import { Badge } from '../components/ui/badge' import { Button } from '../components/ui/button' import { Checkbox } from '../components/ui/checkbox' import { Input } from '../components/ui/input' -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '../components/ui/select' import { Skeleton } from '../components/ui/skeleton' import { Tabs, TabsContent, TabsList, TabsTrigger } from '../components/ui/tabs' import { Textarea } from '../components/ui/textarea' @@ -95,22 +94,6 @@ function PanelNotice({ tone = 'error', title, message }) { ) } -// Client-side preview only — matches pipeline/scheme_catalog.py's -// derive_scheme_code() closely enough for instant feedback, but the server -// is the authoritative, collision-checked source of truth for the real code. -const SCHEME_CODE_STOPWORDS = new Set(['a', 'an', 'the', 'of', 'on', 'for', 'and', 'or', 'to', 'in', 'at', 'by', 'with', '&']) - -function previewSchemeCode(title) { - const words = (title || '').match(/[A-Za-z0-9]+/g) || [] - const letters = words - .filter(w => !SCHEME_CODE_STOPWORDS.has(w.toLowerCase())) - .map(w => w[0].toLowerCase()) - .join('') - if (letters.length >= 2) return letters - const slug = (title || '').toLowerCase().replace(/[^a-z0-9]+/g, '') - return slug.slice(0, 8) || 'scheme' -} - /** * Validity the document is searchable in. The start is prefilled from what the * server stamped at upload; the end is deliberately blank-able - an empty end @@ -123,29 +106,19 @@ function validateValidity(validFrom, validTo) { return '' } -const DOCUMENT_KIND_OPTIONS = [ - { value: 'scheme', label: 'Scheme' }, - { value: 'advisory', label: 'Advisory' }, - { value: 'video', label: 'Video' }, - { value: '__custom__', label: 'Custom…' }, -] +// The type is chosen at upload and fixed afterwards: it decides the catalog the +// document lives in, so this panel only displays it. +const DOCUMENT_KIND_LABELS = { + scheme: 'Scheme', + advisory: 'Advisory', + video: 'Video', +} export function DocumentClassificationPanel({ doc, workflowId, canClassify, onSaved }) { - const hasKind = doc.document_kind && doc.document_kind !== 'document' - // 'document' is the unclassified default, not a choice the panel offers, so - // it seeds the select as empty — otherwise the select renders blank while the - // save button reads "Saved", which tells a reviewer the opposite of the truth. - const [kind, setKind] = useState( - hasKind && !DOCUMENT_KIND_OPTIONS.some(o => o.value === doc.document_kind) - ? '__custom__' - : (hasKind ? doc.document_kind : '') - ) - const [customKind, setCustomKind] = useState(hasKind && !DOCUMENT_KIND_OPTIONS.some(o => o.value === doc.document_kind) ? doc.document_kind : '') - const [schemeName, setSchemeName] = useState(doc.scheme_name || '') - const savedDisplayName = doc.display_name || filenameStem(doc.filename) - const [displayName, setDisplayName] = useState(savedDisplayName) - const displayNameChanged = displayName.trim() !== savedDisplayName - const displayNameLocked = !canClassify || Boolean(doc.ingested_at) + // The display name is documents.name; main's lock rule still applies to it. + const savedName = doc.name || filenameStem(doc.filename) + const [name, setName] = useState(savedName) + const nameLocked = !canClassify || Boolean(doc.ingested_at) const [validFrom, setValidFrom] = useState(doc.valid_from || todayISODate()) // Blank when the document has no end date, which is the normal case - an // empty field here reads as "never expires", not as "not filled in yet". @@ -153,16 +126,13 @@ export function DocumentClassificationPanel({ doc, workflowId, canClassify, onSa const [saving, setSaving] = useState(false) const [error, setError] = useState('') - const effectiveKind = kind === '__custom__' ? customKind.trim().toLowerCase() : kind - const isScheme = effectiveKind === 'scheme' const validityError = DOCUMENT_VALIDITY_ENABLED ? validateValidity(validFrom, validTo) : '' - const nameError = displayNameError(displayName) + const nameError = displayNameError(name) + const nameChanged = name.trim() !== savedName const canSave = - Boolean(effectiveKind) && !nameError && - (!isScheme || schemeName.trim().length > 0) && !validityError async function save() { @@ -174,9 +144,7 @@ export function DocumentClassificationPanel({ doc, workflowId, canClassify, onSa method: 'PATCH', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ - document_kind: effectiveKind, - ...(displayNameChanged ? { display_name: displayName.trim() } : {}), - ...(isScheme ? { scheme_name: schemeName.trim() } : {}), + ...(nameChanged ? { name: name.trim() } : {}), // Omitted entirely when the feature is off, so classifying a // document leaves whatever period the upload stamped untouched. ...(DOCUMENT_VALIDITY_ENABLED @@ -193,9 +161,7 @@ export function DocumentClassificationPanel({ doc, workflowId, canClassify, onSa } const alreadySaved = - doc.document_kind === effectiveKind - && !displayNameChanged - && (!isScheme || doc.scheme_name === schemeName.trim()) + !nameChanged && (!DOCUMENT_VALIDITY_ENABLED || (doc.valid_from === validFrom && (doc.valid_to || '') === validTo)) @@ -214,24 +180,17 @@ export function DocumentClassificationPanel({ doc, workflowId, canClassify, onSa setDisplayName(e.target.value)} + onChange={e => setName(e.target.value)} />

Type - + + {DOCUMENT_KIND_LABELS[doc.kind] || doc.kind || '—'} +
{DOCUMENT_VALIDITY_ENABLED && ( <> @@ -258,38 +217,12 @@ export function DocumentClassificationPanel({ doc, workflowId, canClassify, onSa )} - {kind === '__custom__' && ( -
- Custom type - setCustomKind(e.target.value)} - placeholder="e.g. faq" - /> -
- )} - {isScheme && ( - <> -
- Scheme name - setSchemeName(e.target.value)} - placeholder="e.g. National Mission on Natural Farming" - /> -
-
- Scheme code (auto) - - {doc.scheme_code && doc.scheme_name === schemeName.trim() ? doc.scheme_code : (schemeName.trim() ? previewSchemeCode(schemeName) : '—')} - -
- - )} +
+ Code + + {doc.code || '—'} + +
{canClassify && ( - ) - })} + {visibleActions.slice(0, 4).map(action => ( + + ))} {canRemoveDocument && ( diff --git a/uv.lock b/uv.lock index 5e9aedd..2d43143 100644 --- a/uv.lock +++ b/uv.lock @@ -109,15 +109,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/34/32109943bace7729233cc4ee78530baa306d8cc3c6501a64ba8cb3b58129/argon2_cffi_bindings-26.1.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:0cc40f7b4050bb93eb67de95d2d759322fc7ce4930b9d645581ecf4913ec651e", size = 23584, upload-time = "2026-08-20T07:33:22.613Z" }, ] -[[package]] -name = "async-timeout" -version = "5.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a5/ae/136395dfbfe00dfc94da3f3e136d0b13f394cba8f4841120e34226265780/async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3", size = 9274, upload-time = "2024-11-06T16:41:39.6Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c", size = 6233, upload-time = "2024-11-06T16:41:37.9Z" }, -] - [[package]] name = "backports-asyncio-runner" version = "1.2.0" @@ -676,6 +667,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, ] +[[package]] +name = "docker" +version = "7.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "requests" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/88/7f/731ff914b0255d3d065f45fd4e626d4b8c95dbcbaada049f337a6ac16410/docker-7.2.0.tar.gz", hash = "sha256:cebb93773d334f778e023a7ee352a8d6e13ab1bd3b863a4d4a59dec897df43ac", size = 118731, upload-time = "2026-07-09T14:53:46.39Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/23/529140fe1aab80fc6992f93a706deec709140a6397439139a054e1515c45/docker-7.2.0-py3-none-any.whl", hash = "sha256:a3f45fdeb9165e2d25d9a1d02ddf3bc70fb572cf5ebbf9b58558c22caf29b71f", size = 148775, upload-time = "2026-07-09T14:53:45.224Z" }, +] + [[package]] name = "et-xmlfile" version = "2.0.0" @@ -690,7 +695,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1144,7 +1149,7 @@ dependencies = [ { name = "opentelemetry-sdk" }, { name = "pdf-inspector" }, { name = "pillow" }, - { name = "psycopg2-binary" }, + { name = "psycopg", extra = ["binary", "pool"] }, { name = "pydantic" }, { name = "pyfranc" }, { name = "pyjwt", extra = ["crypto"] }, @@ -1152,7 +1157,6 @@ dependencies = [ { name = "pypdf" }, { name = "python-multipart" }, { name = "qdrant-client" }, - { name = "redis" }, { name = "sentence-transformers" }, { name = "slowapi" }, { name = "temporalio" }, @@ -1169,6 +1173,7 @@ dev = [ { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "ruff" }, + { name = "testcontainers" }, ] [package.metadata] @@ -1185,7 +1190,7 @@ requires-dist = [ { name = "opentelemetry-sdk", specifier = ">=1.44.0" }, { name = "pdf-inspector", specifier = ">=1.14.0" }, { name = "pillow", specifier = ">=10.0.0" }, - { name = "psycopg2-binary", specifier = ">=2.9.0" }, + { name = "psycopg", extras = ["binary", "pool"], specifier = ">=3.2" }, { name = "pydantic", specifier = ">=2.0.0" }, { name = "pyfranc", specifier = ">=0.2.3" }, { name = "pyjwt", extras = ["crypto"], specifier = ">=2.8.0" }, @@ -1193,7 +1198,6 @@ requires-dist = [ { name = "pypdf", specifier = ">=5.0.0" }, { name = "python-multipart", specifier = ">=0.0.6" }, { name = "qdrant-client", specifier = ">=1.12.0" }, - { name = "redis", specifier = ">=5.0.0" }, { name = "sentence-transformers", specifier = "==5.1.0" }, { name = "slowapi", specifier = ">=0.1.9" }, { name = "temporalio", specifier = ">=1.0.0" }, @@ -1209,6 +1213,7 @@ dev = [ { name = "pytest-asyncio", specifier = ">=0.21.0" }, { name = "pytest-cov", specifier = ">=4.0.0" }, { name = "ruff", specifier = ">=0.6.0" }, + { name = "testcontainers", extras = ["postgres"], specifier = ">=4.0.0" }, ] [[package]] @@ -2097,66 +2102,109 @@ wheels = [ ] [[package]] -name = "psycopg2-binary" -version = "2.9.12" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2a/60/a3624f79acea344c16fbef3a94d28b89a8042ddfb8f3e4ca83f538671409/psycopg2_binary-2.9.12.tar.gz", hash = "sha256:5ac9444edc768c02a6b6a591f070b8aae28ff3a99be57560ac996001580f294c", size = 379686, upload-time = "2026-04-21T09:40:34.304Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/78/80/49bacf9e51617d8309f6f0123e29edc793f6f5f6700c7d1f1b20782fbb37/psycopg2_binary-2.9.12-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9b818ceff717f98851a64bffd4c5eb5b3059ae280276dcecc52ac658dcf006a4", size = 3712314, upload-time = "2026-04-20T23:33:31.363Z" }, - { url = "https://files.pythonhosted.org/packages/d3/f2/98eeac7d60c43df9338287834edf9b3e69be68a2db78a57b1b81d705e735/psycopg2_binary-2.9.12-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d2fa0d7caca8635c56e373055094eeda3208d901d55dd0ff5abc1d4e47f82b56", size = 3822389, upload-time = "2026-04-20T23:33:34.178Z" }, - { url = "https://files.pythonhosted.org/packages/9f/7c/30575e75f14d5351a56a1971bb43fe7f8bf7edf1b654fb1bec65c42a8812/psycopg2_binary-2.9.12-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:864c261b3690e1207d14bbfe0a61e27567981b80c47a778561e49f676f7ce433", size = 4578448, upload-time = "2026-04-20T23:33:37.073Z" }, - { url = "https://files.pythonhosted.org/packages/65/9b/4df366d89f28c527dc39d0b6c98a5ca74e30d37ac097b73f3352147568ae/psycopg2_binary-2.9.12-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c5ee5213445dd45312459029b8c4c0a695461eb517b753d2582315bd07995f5e", size = 4273705, upload-time = "2026-04-20T23:33:39.291Z" }, - { url = "https://files.pythonhosted.org/packages/4e/8a/c566803818eb03161ba869b6ba612bf7ad56816d98b9e5121e0a22ad6b0b/psycopg2_binary-2.9.12-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6f9cae1f848779b5b01f417e762c40d026ea93eb0648249a604728cda991dde3", size = 5893784, upload-time = "2026-04-20T23:33:41.658Z" }, - { url = "https://files.pythonhosted.org/packages/63/fe/0dfa5797e0b229e0567bc378695224caf14d547f73b05be0c80549089772/psycopg2_binary-2.9.12-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:63a3ebbd543d3d1eda088ac99164e8c5bac15293ee91f20281fd17d050aee1c4", size = 4109306, upload-time = "2026-04-20T23:33:43.953Z" }, - { url = "https://files.pythonhosted.org/packages/3c/89/28063adf17a4ba501eedd9890feab0c649ee4d8bd0a97df0ff1e9584feab/psycopg2_binary-2.9.12-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d6fcbba8c9fed08a73b8ac61ea79e4821e45b1e92bb466230c5e746bbf3d5256", size = 3654400, upload-time = "2026-04-20T23:33:46.115Z" }, - { url = "https://files.pythonhosted.org/packages/84/94/5a01de0aa4ead0b8d8d1aa4ec18cec0bd36d03fa714eaa5bb8a0b1b50020/psycopg2_binary-2.9.12-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:36512911ebb2b60a0c3e44d0bb5048c1980aced91235d133b7874f3d1d93487c", size = 3299215, upload-time = "2026-04-20T23:33:48.202Z" }, - { url = "https://files.pythonhosted.org/packages/7a/85/723bb085a61c6ac2dc0a0043f375f2fe7365363e27b073bad56ca5bda979/psycopg2_binary-2.9.12-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:8ffdb59fe88f99589e34354a130217aa1fd2d615612402d6edc8b3dbc7a44463", size = 3047724, upload-time = "2026-04-20T23:33:50.74Z" }, - { url = "https://files.pythonhosted.org/packages/b4/67/4d8b1e0d2fc4166677380eac0edf9cdff91013aca2546e8ef7bc04b56158/psycopg2_binary-2.9.12-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a46fe069b65255df410f856d842bc235f90e22ffdf532dda625fd4213d3fd9b1", size = 3349183, upload-time = "2026-04-20T23:33:59.635Z" }, - { url = "https://files.pythonhosted.org/packages/73/99/21af7a5498637ea4dc91a17c281a53bc1d632fbafe00f6689fbfb32a9fed/psycopg2_binary-2.9.12-cp310-cp310-win_amd64.whl", hash = "sha256:ab29414b25dcb698bf26bf213e3348abdcd07bbd5de032a5bec15bd75b298b03", size = 2757036, upload-time = "2026-04-20T23:34:01.606Z" }, - { url = "https://files.pythonhosted.org/packages/d5/19/d4ce60954f3bb9d8e3bc5e5c4d1f2487de2d3851bf2391d54954c9df12a6/psycopg2_binary-2.9.12-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5c8ce6c61bd1b1f6b9c24ee32211599f6166af2c55abb19456090a21fd16554b", size = 3712338, upload-time = "2026-04-20T23:34:03.961Z" }, - { url = "https://files.pythonhosted.org/packages/53/71/c85409ee0d78890f0660eff262e815e7dd2bb741a17611d82e9e8cd9dc5e/psycopg2_binary-2.9.12-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b4a9eaa6e7f4ff91bec10aa3fb296878e75187bced5cc4bafe17dc40915e1326", size = 3822407, upload-time = "2026-04-20T23:34:05.977Z" }, - { url = "https://files.pythonhosted.org/packages/3c/ed/60486c2c7f0d4d1ede2bfb1ed27e2498477ce646bc7f6b2759906303117e/psycopg2_binary-2.9.12-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c6528cefc8e50fcc6f4a107e27a672058b36cc5736d665476aeb413ba88dbb06", size = 4578425, upload-time = "2026-04-20T23:34:08.246Z" }, - { url = "https://files.pythonhosted.org/packages/0b/b9/656cb03fad9f4f49f2145c334b1126ee75189929ca4e6187d485a2d59951/psycopg2_binary-2.9.12-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e4e184b1fb6072bf05388aa41c697e1b2d01b3473f107e7ec44f186a32cfd0b8", size = 4273709, upload-time = "2026-04-20T23:34:10.974Z" }, - { url = "https://files.pythonhosted.org/packages/99/66/08cf0da0e25cc6fb142c89be45fc8418792858f0c4cbff5e24530ff02cd6/psycopg2_binary-2.9.12-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4766ab678563054d3f1d064a4db19cc4b5f9e3a8d9018592a8285cf200c248f3", size = 5893779, upload-time = "2026-04-20T23:34:13.905Z" }, - { url = "https://files.pythonhosted.org/packages/17/d7/eecd9ce8e146d3721115d82d3836efdbb712187e4590325df549989d18f4/psycopg2_binary-2.9.12-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5a0253224780c978746cb9be55a946bcdaf40fe3519c0f622924cdabdafe2c39", size = 4109308, upload-time = "2026-04-20T23:34:16.761Z" }, - { url = "https://files.pythonhosted.org/packages/b6/2e/b1dc289b362cc8d45697b57eefbd673186f49a4ea0906928988e3affcc98/psycopg2_binary-2.9.12-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0dc9228d47c46bda253d2ecd6bb93b56a9f2d7ad33b684a1fa3622bf74ffe30c", size = 3654405, upload-time = "2026-04-20T23:34:19.303Z" }, - { url = "https://files.pythonhosted.org/packages/eb/e4/4c4aea6473214dbdbd0fbba11aa4691e76dc01722c55724c5951719865ff/psycopg2_binary-2.9.12-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f921f3cd87035ef7df233383011d7a53ea1d346224752c1385f1edfd790ceb6a", size = 3299187, upload-time = "2026-04-20T23:34:21.206Z" }, - { url = "https://files.pythonhosted.org/packages/ba/5d/b03b99986446a4f57b170ed9a2579fb7ff9783ca0fa5226b19db99737fee/psycopg2_binary-2.9.12-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3d999bd982a723113c1a45b55a7a6a90d64d0ed2278020ed625c490ff7bef96c", size = 3047716, upload-time = "2026-04-20T23:34:23.077Z" }, - { url = "https://files.pythonhosted.org/packages/14/86/382ee4afbd1d97500c9d2862b20c2fdeddf4b7335e984df3fb4309f64108/psycopg2_binary-2.9.12-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29d4d134bd0ab46ffb04e94aa3c5fa3ef582e9026609165e2f758ff76fc3a3be", size = 3349237, upload-time = "2026-04-20T23:34:25.211Z" }, - { url = "https://files.pythonhosted.org/packages/a8/16/9a57c75ba1eda7165c017342f526810d5f5a12647dde749c99ae9a7141d7/psycopg2_binary-2.9.12-cp311-cp311-win_amd64.whl", hash = "sha256:cb4a1dacdd48077150dc762a9e5ddbf32c256d66cb46f80839391aa458774936", size = 2757036, upload-time = "2026-04-20T23:34:27.77Z" }, - { url = "https://files.pythonhosted.org/packages/e2/9f/ef4ef3c8e15083df90ca35265cfd1a081a2f0cc07bb229c6314c6af817f4/psycopg2_binary-2.9.12-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5cdc05117180c5fa9c40eea8ea559ce64d73824c39d928b7da9fb5f6a9392433", size = 3712459, upload-time = "2026-04-20T23:34:30.549Z" }, - { url = "https://files.pythonhosted.org/packages/b5/01/3dd14e46ba48c1e1a6ec58ee599fa1b5efa00c246d5046cd903d0eeb1af1/psycopg2_binary-2.9.12-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d3227a3bc228c10d21011a99245edca923e4e8bf461857e869a507d9a41fe9f6", size = 3822936, upload-time = "2026-04-20T23:34:32.77Z" }, - { url = "https://files.pythonhosted.org/packages/a6/f7/0640e4901119d8a9f7a1784b927f494e2198e213ceb593753d1f2c8b1b30/psycopg2_binary-2.9.12-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:995ce929eede89db6254b50827e2b7fd61e50d11f0b116b29fffe4a2e53c4580", size = 4578676, upload-time = "2026-04-20T23:34:35.18Z" }, - { url = "https://files.pythonhosted.org/packages/b0/55/44df3965b5f297c50cc0b1b594a31c67d6127a9d133045b8a66611b14dfb/psycopg2_binary-2.9.12-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9fe06d93e72f1c048e731a2e3e7854a5bfaa58fc736068df90b352cefe66f03f", size = 4274917, upload-time = "2026-04-20T23:34:37.982Z" }, - { url = "https://files.pythonhosted.org/packages/b0/4b/74535248b1eac0c9336862e8617c765ac94dac76f9e25d7c4a79588c8907/psycopg2_binary-2.9.12-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40e7b28b63aaf737cb3a1edc3a9bbc9a9f4ad3dcb7152e8c1130e4050eddcb7d", size = 5894843, upload-time = "2026-04-20T23:34:40.856Z" }, - { url = "https://files.pythonhosted.org/packages/f2/ba/f1bf8d2ae71868ad800b661099086ee52bc0f8d9f05be1acd8ebb06757cc/psycopg2_binary-2.9.12-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:89d19a9f7899e8eb0656a2b3a08e0da04c720a06db6e0033eab5928aabe60fa9", size = 4110556, upload-time = "2026-04-20T23:34:44.016Z" }, - { url = "https://files.pythonhosted.org/packages/45/46/c15706c338403b7c420bcc0c2905aad116cc064545686d8bf85f1999ea00/psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:612b965daee295ae2da8f8218ce1d274645dc76ef3f1abf6a0a94fd57eff876d", size = 3655714, upload-time = "2026-04-20T23:34:46.233Z" }, - { url = "https://files.pythonhosted.org/packages/b3/7c/a2d5dc09b64a4564db242a0fe418fde7d33f6f8259dd2c5b9d7def00fb5a/psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:b9a339b79d37c1b45f3235265f07cdeb0cb5ad7acd2ac7720a5920989c17c24e", size = 3301154, upload-time = "2026-04-20T23:34:49.528Z" }, - { url = "https://files.pythonhosted.org/packages/c0/e8/cc8c9a4ce71461f9ec548d38cadc41dc184b34c73e6455450775a9334ccd/psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:3471336e1acfd9c7fe507b8bad5af9317b6a89294f9eb37bd9a030bb7bebcdc6", size = 3048882, upload-time = "2026-04-20T23:34:51.86Z" }, - { url = "https://files.pythonhosted.org/packages/19/6a/31e2296bc0787c5ab75d3d118e40b239db8151b5192b90b77c72bc9256e9/psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7af18183109e23502c8b2ae7f6926c0882766f35b5175a4cd737ad825e4d7a1b", size = 3351298, upload-time = "2026-04-20T23:34:54.124Z" }, - { url = "https://files.pythonhosted.org/packages/5f/a8/75f4e3e11203b590150abed2cf7794b9c9c9f7eceddae955191138b44dde/psycopg2_binary-2.9.12-cp312-cp312-win_amd64.whl", hash = "sha256:398fcd4db988c7d7d3713e2b8e18939776fd3fb447052daae4f24fa39daede4c", size = 2757230, upload-time = "2026-04-20T23:34:56.242Z" }, - { url = "https://files.pythonhosted.org/packages/91/bb/4608c96f970f6e0c56572e87027ef4404f709382a3503e9934526d7ba051/psycopg2_binary-2.9.12-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7c729a73c7b1b84de3582f73cdd27d905121dc2c531f3d9a3c32a3011033b965", size = 3712419, upload-time = "2026-04-20T23:34:58.754Z" }, - { url = "https://files.pythonhosted.org/packages/5e/af/48f76af9d50d61cf390f8cd657b503168b089e2e9298e48465d029fcc713/psycopg2_binary-2.9.12-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4413d0caef93c5cf50b96863df4c2efe8c269bf2267df353225595e7e15e8df7", size = 3822990, upload-time = "2026-04-20T23:35:00.821Z" }, - { url = "https://files.pythonhosted.org/packages/7a/df/aba0f99397cd811d32e06fc0cc781f1f3ce98bc0e729cb423925085d781a/psycopg2_binary-2.9.12-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4dfcf8e45ebb0c663be34a3442f65e17311f3367089cd4e5e3a3e8e62c978777", size = 4578696, upload-time = "2026-04-20T23:35:03.409Z" }, - { url = "https://files.pythonhosted.org/packages/95/9c/eaa74021ac4e4d5c2f83d82fc6615a63f4fe6c94dc4e94c3990427053f67/psycopg2_binary-2.9.12-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c41321a14dd74aceb6a9a643b9253a334521babfa763fa873e33d89cfa122fb5", size = 4274982, upload-time = "2026-04-20T23:35:05.583Z" }, - { url = "https://files.pythonhosted.org/packages/35/ed/c25deff98bd26187ba48b3b250a3ffc3037c46c5b89362534a15d200e0db/psycopg2_binary-2.9.12-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:83946ba43979ebfdc99a3cd0ee775c89f221df026984ba19d46133d8d75d3cd9", size = 5894867, upload-time = "2026-04-20T23:35:07.902Z" }, - { url = "https://files.pythonhosted.org/packages/9a/81/8d0e21ca77373c6c9589e5c4528f6e8f0c08c62cafc76fb0bddb7a2cee22/psycopg2_binary-2.9.12-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:411e85815652d13560fbe731878daa5d92378c4995a22302071890ec3397d019", size = 4110578, upload-time = "2026-04-20T23:35:10.149Z" }, - { url = "https://files.pythonhosted.org/packages/00/fc/f481e2435bd8f742d0123309174aae4165160ad3ef17c1b99c3622c241d2/psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1c8ad4c08e00f7679559eaed7aff1edfffc60c086b976f93972f686384a95e2c", size = 3655816, upload-time = "2026-04-20T23:35:12.56Z" }, - { url = "https://files.pythonhosted.org/packages/53/79/b9f46466bdbe9f239c96cde8be33c1aace4842f06013b47b730dc9759187/psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:00814e40fa23c2b37ef0a1e3c749d89982c73a9cb5046137f0752a22d432e82f", size = 3301307, upload-time = "2026-04-20T23:35:15.029Z" }, - { url = "https://files.pythonhosted.org/packages/3f/19/7dc003b32fe35024df89b658104f7c8538a8b2dcbde7a4e746ce929742e7/psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:98062447aebc20ed20add1f547a364fd0ef8933640d5372ff1873f8deb9b61be", size = 3048968, upload-time = "2026-04-20T23:35:16.757Z" }, - { url = "https://files.pythonhosted.org/packages/91/58/2dbd7db5c604d45f4950d988506aae672a14126ec22998ced5021cbb76bb/psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:66a7685d7e548f10fb4ce32fb01a7b7f4aa702134de92a292c7bd9e0d3dbd290", size = 3351369, upload-time = "2026-04-20T23:35:18.933Z" }, - { url = "https://files.pythonhosted.org/packages/42/ee/dee8dcaad07f735824de3d6563bc67119fa6c28257b17977a8d624f02fab/psycopg2_binary-2.9.12-cp313-cp313-win_amd64.whl", hash = "sha256:b6937f5fe4e180aeee87de907a2fa982ded6f7f15d7218f78a083e4e1d68f2a0", size = 2757347, upload-time = "2026-04-20T23:35:21.283Z" }, - { url = "https://files.pythonhosted.org/packages/13/1b/708c0dca874acfad6d65314271859899a79007686f3a1f74e82a2ed4b645/psycopg2_binary-2.9.12-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:6f3b3de8a74ef8db215f22edffb19e32dc6fa41340456de7ec99efdc8a7b3ec2", size = 3712428, upload-time = "2026-04-20T23:35:23.453Z" }, - { url = "https://files.pythonhosted.org/packages/d6/39/ddbea9d4b4de6aca9431b6ed253f530f8a02d3b8f9bcfd0dbfe2b3de6fe4/psycopg2_binary-2.9.12-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1006fb62f0f0bc5ce256a832356c6262e91be43f5e4eb15b5eaf38079464caf2", size = 3823184, upload-time = "2026-04-20T23:35:25.92Z" }, - { url = "https://files.pythonhosted.org/packages/bf/a0/bc2fef74b106fa345567122a0659e6d94512ed7dc0131ec44c9e5aba3725/psycopg2_binary-2.9.12-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:840066105706cd2eb29b9a1c2329620056582a4bf3e8169dec5c447042d0869f", size = 4579157, upload-time = "2026-04-20T23:35:28.542Z" }, - { url = "https://files.pythonhosted.org/packages/57/d7/d4e3b2005d3de607ca4fbb0e8742e248056e52184a6b94ebda3c1c2c329b/psycopg2_binary-2.9.12-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:863f5d12241ebe1c76a72a04c2113b6dc905f90b9cef0e9be0efd994affd9354", size = 4274970, upload-time = "2026-04-20T23:35:30.418Z" }, - { url = "https://files.pythonhosted.org/packages/2e/42/c9853f8db3967fe08bcde11f53d53b85d351750cae726ce001cb68afa9c1/psycopg2_binary-2.9.12-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a99eaab34a9010f1a086b126de467466620a750634d114d20455f3a824aae033", size = 5895175, upload-time = "2026-04-20T23:35:33.584Z" }, - { url = "https://files.pythonhosted.org/packages/eb/fd/b82b5601a97630308bef079f545ffec481bbbc795c2ba5ec416a01d03f60/psycopg2_binary-2.9.12-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ffdd7dc5463ccd61845ac37b7012d0f35a1548df9febe14f8dd549be4a0bc81e", size = 4110658, upload-time = "2026-04-20T23:35:35.638Z" }, - { url = "https://files.pythonhosted.org/packages/62/8c/32ca69b0389ef25dd22937bf9e8fbe2ce27aea20b05ded48c4ce4cb42475/psycopg2_binary-2.9.12-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:54a0dfecab1b48731f934e06139dfe11e24219fb6d0ceb32177cf0375f14c7b5", size = 3656251, upload-time = "2026-04-20T23:35:37.854Z" }, - { url = "https://files.pythonhosted.org/packages/c4/29/96992a2b59e3b9d730fcf9612d0a387305025dc867a9fc490a9e496e074e/psycopg2_binary-2.9.12-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:96937c9c5d891f772430f418a7a8b4691a90c3e6b93cf72b5bd7cad8cbca32a5", size = 3301810, upload-time = "2026-04-20T23:35:39.927Z" }, - { url = "https://files.pythonhosted.org/packages/56/ad/44b06659949b243ae10112cd3b20a197f9bf3e81d5651379b9eb889bfaad/psycopg2_binary-2.9.12-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:77b348775efd4cdab410ec6609d81ccecd1139c90265fa583a7255c8064bc03d", size = 3048977, upload-time = "2026-04-20T23:35:41.806Z" }, - { url = "https://files.pythonhosted.org/packages/1d/f2/10a1bcebadb6aa55e280e1f58975c36a7b560ea525184c7aa4064c466633/psycopg2_binary-2.9.12-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:527e6342b3e44c2f0544f6b8e927d60de7f163f5723b8f1dfa7d2a84298738cd", size = 3351466, upload-time = "2026-04-20T23:35:43.993Z" }, - { url = "https://files.pythonhosted.org/packages/20/be/b732c8418ffa5bcfda002890f5dc4c869fc17db66ff11f53b17cfe44afc0/psycopg2_binary-2.9.12-cp314-cp314-win_amd64.whl", hash = "sha256:f12ae41fcafadb39b2785e64a40f9db05d6de2ac114077457e0e7c597f3af980", size = 2848762, upload-time = "2026-04-20T23:35:46.421Z" }, +name = "psycopg" +version = "3.3.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/26/3ea4ca5eaea1c0debcdf7ee7c1613fbe721dc27a03c461c0817ffd8a0601/psycopg-3.3.6.tar.gz", hash = "sha256:c081f2250df751a943036e42db6df4571c66cd0aabe8291a7a506512b12007d2", size = 168171, upload-time = "2026-09-18T13:22:55.152Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/de/748bd7609c71cae5d737f0ba9192f19329f70180ecda8fff3cac02c5abe3/psycopg-3.3.6-py3-none-any.whl", hash = "sha256:a1db9f7148b06a28606767efaca51fa6f9398c5c0a3810519be69d7000bdb631", size = 215490, upload-time = "2026-09-18T13:15:29.374Z" }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, +] +pool = [ + { name = "psycopg-pool" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.3.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/92/00350a66de0af05e41d01aa3134e3970045e816afed3f99d58ec1abe15b2/psycopg_binary-3.3.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7beb3e41c9a1e509f3ed85263386588cbe3e975aa67be21f79f44fd35ffaeefc", size = 4728682, upload-time = "2026-09-18T13:15:36.605Z" }, + { url = "https://files.pythonhosted.org/packages/91/fc/afa9c7fd316a469af7ede6ebb020eac482f5d827fae57d5310c9bc0c41ae/psycopg_binary-3.3.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:aa73160077345ec21b3f51e8e24b3de2e99586217e497629326eb9b2ea88c52e", size = 4777111, upload-time = "2026-09-18T13:15:46.566Z" }, + { url = "https://files.pythonhosted.org/packages/f2/44/7c1e015f1bc56b36ff1369f09e852b2d83ccefd5a669a42633a916cdedc4/psycopg_binary-3.3.6-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f87dbdc42e78ee0f7ea180c03f8c78e80a949e373066629bd90fefff10552dff", size = 5592755, upload-time = "2026-09-18T13:15:52.886Z" }, + { url = "https://files.pythonhosted.org/packages/3b/ae/314a251ca918cdac380bce1b87839ade9355382ea749e6ef3ba75ba0c09f/psycopg_binary-3.3.6-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a9348c5b43a3bb5ef8c2e89d5237c9c87eeafb01d338c84a7aebbc5cd0313299", size = 5267949, upload-time = "2026-09-18T13:16:00.53Z" }, + { url = "https://files.pythonhosted.org/packages/b6/9f/3bb0cfe9bb0f31ca57cf486ddc8c9ac51251aed8181bf88ff870b2623105/psycopg_binary-3.3.6-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a52991594ac4db888c7d39bccef331797e30cb31a95cae02cf2607f83a42dc2", size = 6862403, upload-time = "2026-09-18T13:16:10.385Z" }, + { url = "https://files.pythonhosted.org/packages/c4/d6/7032c10309c3155e9b24300fdcc9a1afa539cfd20ce52fdef74a46f10161/psycopg_binary-3.3.6-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5ea8beeb5541780b4b50b462eeacbc4f594ce3b911dc20c81c75f267876f71d2", size = 5107271, upload-time = "2026-09-18T13:16:16.843Z" }, + { url = "https://files.pythonhosted.org/packages/61/cc/79add2cf92684cf1a81da134b32caa662c25c72d0cc905d181ef4455f834/psycopg_binary-3.3.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:198a48e68cc99ccac03ba95ac857e73aa66f3bf6be77019fafb0832a05f7ad03", size = 4627984, upload-time = "2026-09-18T13:16:23.889Z" }, + { url = "https://files.pythonhosted.org/packages/c9/48/6dfb14f9350c14af6a2edb3c31262051b8cd94e2186e4b831e46dbbe8cd9/psycopg_binary-3.3.6-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:fa34eb47969297471db7b7f193622c7e3ee839ec05abd05f1fe104d5b1b1dcf4", size = 4322054, upload-time = "2026-09-18T13:16:29.33Z" }, + { url = "https://files.pythonhosted.org/packages/29/35/2982338716a91cbb4dfc866be015be4457ee8106a445aabf3d1fb6a270e0/psycopg_binary-3.3.6-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b979a42815410432420275412633960807178b1ce26591a16ce06e78a5bd4bb2", size = 4048471, upload-time = "2026-09-18T13:16:34.119Z" }, + { url = "https://files.pythonhosted.org/packages/24/e1/171b1db1542c5f76a678b7ee0a7800bebc9735a0a03417c76cf948bfd63c/psycopg_binary-3.3.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:889e42acec10450185e0cdfb396f375e2c1a8d7737c114830a7fde4654f59e30", size = 4353931, upload-time = "2026-09-18T13:16:38.692Z" }, + { url = "https://files.pythonhosted.org/packages/08/89/4424e62a944eef40bd9326ada4ae23802b28eab6502af91e84ef7bba74fb/psycopg_binary-3.3.6-cp310-cp310-win_amd64.whl", hash = "sha256:cbd5f73073ed19c378d4c35499db1e3e703a5b1a324e521204065967bfaa7a18", size = 3674362, upload-time = "2026-09-18T13:16:44.454Z" }, + { url = "https://files.pythonhosted.org/packages/70/86/b71166048974d49c6d136b2ed1c0e5bec0b974d8c4de5cbce7e86a9e412a/psycopg_binary-3.3.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:be4f9b3c9338ac5dd217c5847e21521b396c8117f78dc420d495a5c49bbef874", size = 4728002, upload-time = "2026-09-18T13:16:53.393Z" }, + { url = "https://files.pythonhosted.org/packages/12/1d/1e06c0de7ed5aed898acb87544eac6ef0bc7d752a67ec6e5d6b835e9b40c/psycopg_binary-3.3.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f0535693ce476a722b718b002d5d2c27d47e71ca945276ac194409c98e74c492", size = 4775403, upload-time = "2026-09-18T13:16:58.939Z" }, + { url = "https://files.pythonhosted.org/packages/84/02/2ffcbc43f8e4bbc38e5286a22013bcac01898d13cd38325f60dd5428a8af/psycopg_binary-3.3.6-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3c9e663b2e800e3218994cf948c11bcc2844e6491b34aa80d089baf6531827bf", size = 5594145, upload-time = "2026-09-18T13:17:08.515Z" }, + { url = "https://files.pythonhosted.org/packages/e1/25/031dae2c7d2e7e77dcf5b1962c1e0684fa548d7af0ff6707b6b5e6054ca7/psycopg_binary-3.3.6-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a2e44a342d2aee40508e28a563d8961c39d9bbd8cae36d8578f0a3c6658aab0f", size = 5267985, upload-time = "2026-09-18T13:17:16.24Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e5/94c89ada3c003a4d858178f3bba49a35e0297ef2aad659b80eb5e380e690/psycopg_binary-3.3.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f598f19fa9a91540b5cee17932ffd227b7b53a481605bcc4573c0eafa647300", size = 6860986, upload-time = "2026-09-18T13:17:23.348Z" }, + { url = "https://files.pythonhosted.org/packages/9d/a0/81bf499d095adee8413bd19822a6872fbfa21663ec78014a68d83a8db83c/psycopg_binary-3.3.6-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6ff05561e4a067d35507dc5c90f1deb2ec1c9703ac5cccc1bc26e08a197f9c5a", size = 5106702, upload-time = "2026-09-18T13:17:28.847Z" }, + { url = "https://files.pythonhosted.org/packages/00/75/99d56da64c27bd985fd82c6ecbf7976b724ac638fdd1654ef995323a1a26/psycopg_binary-3.3.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:566dd827f17728efdf7d88a5b066f815170f6fdad13967ae952842d90e6aaa9f", size = 4627090, upload-time = "2026-09-18T13:17:36.668Z" }, + { url = "https://files.pythonhosted.org/packages/3e/0c/0222171d11233332c6a24b1cef1578215f0ffddf3642eb8dd8c4448ad69f/psycopg_binary-3.3.6-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9b2f11794e017ce340934e35de46181c46ef71ec75ea3d85dd75cd836761c01e", size = 4320353, upload-time = "2026-09-18T13:17:42.526Z" }, + { url = "https://files.pythonhosted.org/packages/62/6f/e1cc2a28dd1228c67c969ba6fd37cd8726b312e2ff51380f847ddb38ccde/psycopg_binary-3.3.6-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:910ace140e3e7b7596898d083f37a8fe90c5c40684252ad4e682364b2cd3deba", size = 4047253, upload-time = "2026-09-18T13:17:47.068Z" }, + { url = "https://files.pythonhosted.org/packages/d8/fd/38b64790ce7a515b1dbd2bab3d119637a858aeb22c380cf4859bc4ce0e42/psycopg_binary-3.3.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37e517c146b185f9c0c6e8d0a0ebbdeeeb67896af28466e032bc810d0c7dc7a7", size = 4353208, upload-time = "2026-09-18T13:17:52.41Z" }, + { url = "https://files.pythonhosted.org/packages/f7/dc/45386530ceb2a8c789a226de9b9b34eca8fccf1feba2e4ef68a6aca50c56/psycopg_binary-3.3.6-cp311-cp311-win_amd64.whl", hash = "sha256:c7f92daa0d2a1c76f07264abddf8cbabd30152a2f09c3270e50f0c7efdf5dcac", size = 3675638, upload-time = "2026-09-18T13:17:58.112Z" }, + { url = "https://files.pythonhosted.org/packages/e6/01/2cdd1824e58b4467ee0b9498664cd28c42d8794db6b1e35b6bcb834f0044/psycopg_binary-3.3.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3f84dab25e0385692ee13274c68678377e0b1a70ab9d14e56264cbf61f60c62d", size = 4707086, upload-time = "2026-09-18T13:18:05.138Z" }, + { url = "https://files.pythonhosted.org/packages/f6/76/de9948ac06895261c84d5b9fbe283d8f3c5bc9f070691b8d9eaa1b51e322/psycopg_binary-3.3.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:612382ac3ed13651c7fa44b5fee9fbf7baaa2ddbc6f500391672682c5f1df9e0", size = 4769607, upload-time = "2026-09-18T13:18:12.83Z" }, + { url = "https://files.pythonhosted.org/packages/76/a9/72436c9915ee4905964689e7f0e182ce7767cc0a0390b3ce703be8177625/psycopg_binary-3.3.6-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:366db6e97e66b37211475f20c4c1324a2dc0dd825e46d4e87f9d599304d276f9", size = 5554134, upload-time = "2026-09-18T13:18:21.175Z" }, + { url = "https://files.pythonhosted.org/packages/0a/42/948bb3d2617795093512613fd96ba380e922992c7908fbc073858147d196/psycopg_binary-3.3.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1679a1cb93fbe5a6d1fd58d82cbddcc6fcb8c61446ba7cae6eb2a7b19bc585de", size = 5235723, upload-time = "2026-09-18T13:18:27.071Z" }, + { url = "https://files.pythonhosted.org/packages/99/47/93e823ff1b0088400703410939c9bda3e63ed9c850b3ee088e8769f4c10b/psycopg_binary-3.3.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:37d40450659401600e6d043ff586c89a71a69f33cbb8bcdba6cdb2569beecdbe", size = 6833587, upload-time = "2026-09-18T13:18:33.794Z" }, + { url = "https://files.pythonhosted.org/packages/5e/2d/ecc69c847795aa704041a9f5667a6b0938a088cf1853636d762a6938e493/psycopg_binary-3.3.6-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a5165300324efd5a772c48a88ab3a928513ab3979fca76553e62ee815f7b2b9c", size = 5070013, upload-time = "2026-09-18T13:18:39.628Z" }, + { url = "https://files.pythonhosted.org/packages/92/36/6126f0dac21713dcae91404f2a76da18598a6252339a8c669c46370d43b2/psycopg_binary-3.3.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d636338c8f21b0df2f84657b00bc34f9313f826ef93f1155bc743607e4a0c5eb", size = 4597367, upload-time = "2026-09-18T13:18:45.023Z" }, + { url = "https://files.pythonhosted.org/packages/4d/29/7ecfc04243b46c89ffd49924e9c5634ea904ef96c7d0f37e4073623584c1/psycopg_binary-3.3.6-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a4ee3bdd5468a725f2a4d9aab8a74b6d0279f768c8b5d3aeb102c5307ff3d59c", size = 4275419, upload-time = "2026-09-18T13:18:49.299Z" }, + { url = "https://files.pythonhosted.org/packages/6e/90/2f46d2e0de79706ac170df0a3637fe63c4498fc04f131f6049520b78b806/psycopg_binary-3.3.6-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:289aadd6a00e151203c081f708348ec89f1e483c9b510ef4ac3981f847f01f79", size = 4007358, upload-time = "2026-09-18T13:18:53.944Z" }, + { url = "https://files.pythonhosted.org/packages/03/48/6744e91291b751a8cf12d63d719977974bb94c84ceba913e7ddb2e478e51/psycopg_binary-3.3.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f21d057f3e5f5491067e5b292498073b73847d48799b099803fef100775fcc52", size = 4320156, upload-time = "2026-09-18T13:18:59.258Z" }, + { url = "https://files.pythonhosted.org/packages/1a/9b/94ff7fce53a64d5b286e2ec454e0a025cf3d6e6b4a9189bef16aa5de98b2/psycopg_binary-3.3.6-cp312-cp312-win_amd64.whl", hash = "sha256:e23a66a763fbe83fcc210bc77c27e5a5ea380ebf091c06f34d8561b695e5a40f", size = 3658864, upload-time = "2026-09-18T13:19:06.503Z" }, + { url = "https://files.pythonhosted.org/packages/b4/c3/c072584b69ad44a747b448cfc9766fecb8aae56e372a017e2ef668790057/psycopg_binary-3.3.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ad8f35e67cc16d1fad1fa8c88972dc9b3a3141ea67897399904edab96a301b6", size = 4712284, upload-time = "2026-09-18T13:19:13.451Z" }, + { url = "https://files.pythonhosted.org/packages/0a/b9/4283b785339e8e2318d03048994b093d650ea6289fabaa806b765dc0d449/psycopg_binary-3.3.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:373704aea331d3f3e3402c125a1543f5875e2986ebb54f97d1647942161f803f", size = 4772031, upload-time = "2026-09-18T13:19:18.524Z" }, + { url = "https://files.pythonhosted.org/packages/6f/72/7a1321d359246769fff1affffbd0132785a28f7f63c18524c15a502398f4/psycopg_binary-3.3.6-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b82491019b884d62318b5f30706c3d7e6d4e5a6cb7eabcb3edc0c1b0fdaceae9", size = 5556392, upload-time = "2026-09-18T13:19:24.418Z" }, + { url = "https://files.pythonhosted.org/packages/de/b0/c6f8a0585a5dacbea74e130bcfc66629390e8f5bbc79d2a8e806e8952150/psycopg_binary-3.3.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cec5ea900390897d0b46130f60bc2883bf19c314f9044235217c8be88b0ef269", size = 5237855, upload-time = "2026-09-18T13:19:31.257Z" }, + { url = "https://files.pythonhosted.org/packages/e2/fc/c3a7a8bbef7e945ec584ac61d460a612363ea398511cd0e220242b1d69f1/psycopg_binary-3.3.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98c02090d88f2ebc0ec1e8da538f77d225ce0fffecf372aa39262e62a1b054ef", size = 6833856, upload-time = "2026-09-18T13:19:43.622Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f2/8e80b921db728ebb68fc105bd7c4277f908210ad755bd6481d5ea7add740/psycopg_binary-3.3.6-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ee2c4728c691245e24501fcd7a97b5b381236b9985bc445bba88cdce7d1b5784", size = 5070730, upload-time = "2026-09-18T13:19:49.968Z" }, + { url = "https://files.pythonhosted.org/packages/54/6a/5b313e0c5348244f0e973aff3258bf86766656256d5ece8d541a53e35b4a/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f19cc87343eaa55255e76b31259a570072ac95d6ae82c92dd34b97691f5e49dc", size = 4598089, upload-time = "2026-09-18T13:19:56.426Z" }, + { url = "https://files.pythonhosted.org/packages/32/e9/db7f76ec24bf6699e92bf604e5c4bae10664a681a8999ef42aa0faf0f2c6/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fdccb3a0e184b03e9baa673b15a809cf36c339c85dbda0ebc25a698846dfbee8", size = 4278481, upload-time = "2026-09-18T13:20:04.681Z" }, + { url = "https://files.pythonhosted.org/packages/61/83/72c67013656f4d6b547caabffb193e91d57e63f90eefdcc6d045c400e97d/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:9892188bb15e5803beb51afe8a25add6b56be391a53058e8bca03b74e1e6bf22", size = 4009229, upload-time = "2026-09-18T13:20:11.905Z" }, + { url = "https://files.pythonhosted.org/packages/82/35/5e4500df2c999eb0faed8b184e6958b834172128274f06167a5deef4c19c/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3af90f92769d8cc10f94515ee7a0aef36ea85ca733a0ce22858f6e0953f41138", size = 4321467, upload-time = "2026-09-18T13:20:17.949Z" }, + { url = "https://files.pythonhosted.org/packages/55/7f/e350e1cf498ba2565c3f87b12f429d2012eb86b76c2b3845a19ee5fbb4d6/psycopg_binary-3.3.6-cp313-cp313-win_amd64.whl", hash = "sha256:0ebfad5d131de9f892ae9e70cc7616207768b6714b66a52d4612b8ceaf78b372", size = 3658179, upload-time = "2026-09-18T13:20:22.691Z" }, + { url = "https://files.pythonhosted.org/packages/6d/b9/60711317c284a442511644ea7185b56ebe627606d6741e732cd16108c47b/psycopg_binary-3.3.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:b3f75dee0f9afafabe4edc52c4842f1e1878ed2069bd05b22d6fe961e97e4dba", size = 4720512, upload-time = "2026-09-18T13:20:29.278Z" }, + { url = "https://files.pythonhosted.org/packages/63/da/28befc84454cbc6374550de7746f591f8fe1b6165c1fce249652cc8291c4/psycopg_binary-3.3.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5927b7ba63153cd8e9862987290a2b783a5c590daf2a4ef981700cc3569166d4", size = 4782318, upload-time = "2026-09-18T13:20:35.401Z" }, + { url = "https://files.pythonhosted.org/packages/a4/8a/0d21c2c833cdc0d4244c77e858e0ed37fa2abec2623be4fd686f617109ce/psycopg_binary-3.3.6-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:0bf08b749cc144f33b44a91b78e3f71c60eb07963746a0df5a100b36ce3d7475", size = 5567460, upload-time = "2026-09-18T13:20:41.902Z" }, + { url = "https://files.pythonhosted.org/packages/49/6d/7692d0d4e656b6cc9868d8acc2e3b42f17a0db4a625400a6d093cb0533a1/psycopg_binary-3.3.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:31cd942c23f613276b81a6e6598cefa12960058b0f46e1e874b540c793f6aca5", size = 5246902, upload-time = "2026-09-18T13:20:47.661Z" }, + { url = "https://files.pythonhosted.org/packages/d4/c1/b8a1f18fb1b7558a17f57f7cb3fc8bc93189feea2958925950b3acb15743/psycopg_binary-3.3.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4690cf67738f0e0e49a32aeec99bf0e4595cc2b4f1af984a4345394b1dcff91a", size = 6847192, upload-time = "2026-09-18T13:20:56.874Z" }, + { url = "https://files.pythonhosted.org/packages/a5/76/404f33519167c65cca88ec4998776f1dbebccc301ee977f0e62c47fb0826/psycopg_binary-3.3.6-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ad1c785e784cfd87e8436c6b7702f2d321fc39601bbaf29bc63a41a867091638", size = 5079573, upload-time = "2026-09-18T13:21:04.155Z" }, + { url = "https://files.pythonhosted.org/packages/f0/d9/79e8fbc8f37262a415f3550f0bcc5f98037442bf3d12ef6cbae2056655ae/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:79a2a1c3449f6c3409427078ed1cec10de79f3023cb5f2504f0597d350ad46c7", size = 4613633, upload-time = "2026-09-18T13:21:10.664Z" }, + { url = "https://files.pythonhosted.org/packages/d4/47/96225db74be7d2ce04b3a58678b53cda610225055edf5faa775c9f501d8b/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:86147cb5d140341c3363fb5bacce31f8d5543902a46699d3c536b101bbceaf9e", size = 4293375, upload-time = "2026-09-18T13:21:16.027Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d2/18e9c779a5efd565250329adaf529ecc2b8b2ed5be5cb0f6ccee208cbfd9/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:7308c93cf0b19bbaf8e6ff0a6ad50d3c442385739245fe15a8d593bf841734a6", size = 4019883, upload-time = "2026-09-18T13:21:21.587Z" }, + { url = "https://files.pythonhosted.org/packages/ef/28/0cc654afc6c2cda982767f5679d3646b30b1ec86545bdaa9402202d6776c/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:05a83ac9fd52b9bca7cb5ab04b3691163170bd16f53defa27216ea3aa07ee781", size = 4332607, upload-time = "2026-09-18T13:21:27.63Z" }, + { url = "https://files.pythonhosted.org/packages/f1/3e/0a753a74fbd7aef120f286c016e09d3cc3f1daf7688f4a145d27281260b2/psycopg_binary-3.3.6-cp314-cp314-win_amd64.whl", hash = "sha256:1fbd30e537dab22cafdf080608f10148fe2a5f3a61294ddb5113caac8a623840", size = 3755671, upload-time = "2026-09-18T13:21:33.855Z" }, + { url = "https://files.pythonhosted.org/packages/0e/b1/a372b9c02aea50148e71c9853e19efca8fa5ae2010a8e27243b9b8f790c0/psycopg_binary-3.3.6-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:bf8c8481d026b85dd70c5fa7dde85b2333aed0b32a2602bcd38a900cbd78a49c", size = 4719571, upload-time = "2026-09-18T13:21:41.437Z" }, + { url = "https://files.pythonhosted.org/packages/65/7c/811e3828c6b82e2f10c6c9cdd963cfc66f3e024026e5a69ac18530bad984/psycopg_binary-3.3.6-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:b599defe9190b17e9907c8b4d114c181e702c87efcd1b8a0ad40971cdcc4634a", size = 4781230, upload-time = "2026-09-18T13:21:49.516Z" }, + { url = "https://files.pythonhosted.org/packages/3e/15/9a784eed813ea9e97c294af3ead63d02b7b203502c66380336c50065e441/psycopg_binary-3.3.6-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b8ece331509f7a975b90501f41e83ad905e4141753fedf3f2711b2bc70a8efbc", size = 5566111, upload-time = "2026-09-18T13:21:58.089Z" }, + { url = "https://files.pythonhosted.org/packages/68/16/47194e002007c27337b11e49bf459c4b19727463f9aff2e1a90917bcc806/psycopg_binary-3.3.6-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c61617eaae0112ca154da87ffb99b73af2c74067acac28dfb9a4455b019dff2e", size = 5249963, upload-time = "2026-09-18T13:22:06.695Z" }, + { url = "https://files.pythonhosted.org/packages/53/84/5dcf9f310b11f0675cd860c6b2c70f58ce61798a3ee3f6f962b53fa358ca/psycopg_binary-3.3.6-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c6d19cb4999d03231e8730a5f66c8f5068bc3b532677eb39dab0f600bff3e312", size = 6847925, upload-time = "2026-09-18T13:22:13.088Z" }, + { url = "https://files.pythonhosted.org/packages/f3/06/1957a06dc22963c418c27b284929579de84f29c37ad1abe6dc6ee9e8cf25/psycopg_binary-3.3.6-cp315-cp315-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8cbb54454dbf1bbf2ff08dd7693e8d94ac94b1a20f70f4b3b813d52ecb5cbc1", size = 5087720, upload-time = "2026-09-18T13:22:17.959Z" }, + { url = "https://files.pythonhosted.org/packages/21/43/ac07d042bae99b57bf123bb473632f29af544008094da0ffd285ab8011e2/psycopg_binary-3.3.6-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:dc75da5a20951049f7b773145f998f69d181adad9c58a0ff36e0cf1d73c10e10", size = 4613412, upload-time = "2026-09-18T13:22:26.719Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b1/019156fbeafcefb4cccc9d109de4699493bceb8313c7545c8349e089dfbc/psycopg_binary-3.3.6-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:955e3dd94da361e052d2e49acf591017158dc8f8ed2c8a42c2e3943403c39dc2", size = 4292618, upload-time = "2026-09-18T13:22:33.042Z" }, + { url = "https://files.pythonhosted.org/packages/5d/0f/62113dc6b1df65983a1f2fc816c04b1edfa22f2ae9d4abee74ed267f4a96/psycopg_binary-3.3.6-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:c7753871eb57e6a5f4646f6168590c6653073dea5e9e720b201c8875332df4c8", size = 4027121, upload-time = "2026-09-18T13:22:38.334Z" }, + { url = "https://files.pythonhosted.org/packages/5d/d5/cf0cbd1ea5a7d8167fe2c6953efde19101f7b193bd61a23e6d622ad6854c/psycopg_binary-3.3.6-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:303732e798fe6729f8e12021b9c96107df8e95ecec4dd487c67b98ec2a59435e", size = 4336388, upload-time = "2026-09-18T13:22:45.576Z" }, + { url = "https://files.pythonhosted.org/packages/98/33/e2a5b36edf8aa422f6fa4b894756eb33dc93b36df5f65121280bb8b929c4/psycopg_binary-3.3.6-cp315-cp315-win_amd64.whl", hash = "sha256:2f122603f36050937982abf9668d8bc4769a79f7c93a65013b1c49f1cab7b56b", size = 3756154, upload-time = "2026-09-18T13:22:51.283Z" }, +] + +[[package]] +name = "psycopg-pool" +version = "3.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/5e/c0664b968b102ff68b811d999c728546c48d5c1eec03e3bbaf88c0cb4472/psycopg_pool-3.3.3.tar.gz", hash = "sha256:df87b5d9d0ad7db37f6cdad4fa8ce113d250f5997f6db38e9a99192fb67f9e1d", size = 32006, upload-time = "2026-09-22T15:53:24.947Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/b4/452c6607a0f479465cd8a9b0d9956919fcb150050c1f83f9f11e6b8ee8dc/psycopg_pool-3.3.3-py3-none-any.whl", hash = "sha256:9b9cd6a4fcec47a410f7e82d408540e7f77b478509e91b44c1a5457a13e5ff37", size = 40304, upload-time = "2026-09-22T15:53:23.712Z" }, ] [[package]] @@ -2493,7 +2541,7 @@ name = "python-dateutil" version = "2.9.0.post0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "six" }, + { name = "six", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } wheels = [ @@ -2627,18 +2675,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fc/3c/480c61cc8d5a3e76bb44e86231f408c93643e5498beadbbeb381ab55d02b/qdrant_client-1.19.0-py3-none-any.whl", hash = "sha256:13602a2b3478a95ecdf42f97b93d7f703b63a3361cd912a04495a33a5ac14121", size = 396157, upload-time = "2026-08-04T14:32:55.734Z" }, ] -[[package]] -name = "redis" -version = "8.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "async-timeout", marker = "python_full_version < '3.11.3'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a8/99/604f0b666d4c616d891cf77ebb9db6bb21601344c051aebf1b72b9ff915f/redis-8.1.0.tar.gz", hash = "sha256:6e1a19beef9225c83efd689c7e6b7da2d5215b1f42cd13b7fc3714d0a09c7b25", size = 5254356, upload-time = "2026-07-30T08:51:00.269Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/66/9d/c5731f6e3608663d4d3656fd8d3aecee8b509c3082818f5a13eae925baea/redis-8.1.0-py3-none-any.whl", hash = "sha256:a4fe1aac3d3b3cc791d4b3d5931c5a956045dc951ee74d1c913ee3ac4d2ee9fb", size = 560618, upload-time = "2026-07-30T08:50:58.497Z" }, -] - [[package]] name = "regex" version = "2026.7.19" @@ -2845,10 +2881,10 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform == 'darwin'", ] dependencies = [ - { name = "joblib" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" } }, - { name = "threadpoolctl" }, + { name = "joblib", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "threadpoolctl", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/98/c2/a7855e41c9d285dfe86dc50b250978105dce513d6e459ea66a6aeb0e1e0c/scikit_learn-1.7.2.tar.gz", hash = "sha256:20e9e49ecd130598f1ca38a1d85090e1a600147b9c02fa6f15d69cb53d968fda", size = 7193136, upload-time = "2025-09-09T08:21:29.075Z" } wheels = [ @@ -2899,13 +2935,13 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform == 'darwin'", ] dependencies = [ - { name = "joblib" }, - { name = "narwhals" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "joblib", marker = "python_full_version >= '3.11'" }, + { name = "narwhals", marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "scipy", version = "1.18.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "threadpoolctl" }, + { name = "threadpoolctl", marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" } wheels = [ @@ -2950,7 +2986,7 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform == 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" } wheels = [ @@ -3010,7 +3046,7 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform == 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -3089,7 +3125,7 @@ resolution-markers = [ "python_full_version == '3.12.*' and sys_platform == 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7e/74/66de6258867beb2ef08f35f9f2ac017a52cacd5081714d239ff1a442d458/scipy-1.18.1.tar.gz", hash = "sha256:52c4b7422442aba924d03ad4019852b08a92e64ea187b933135687bfe2747307", size = 30781235, upload-time = "2026-08-21T23:28:50.599Z" } wheels = [ @@ -3282,6 +3318,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, ] +[[package]] +name = "testcontainers" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docker" }, + { name = "python-dotenv" }, + { name = "typing-extensions" }, + { name = "urllib3" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4b/13/2cc466bddf26d0085f30a2b2bd56b7f8708b54a54db833eec97c5c69129b/testcontainers-4.15.0.tar.gz", hash = "sha256:085cde086337632e19002719460b7b80bbab2bdd51bb3ea04f77d0de96504706", size = 95340, upload-time = "2026-07-24T23:08:01.731Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/7e/424aac8b355597835deb333e757a0e94b5ccf38ad00f07fe6ed1f4e17c88/testcontainers-4.15.0-py3-none-any.whl", hash = "sha256:8796c14e76604031ad39cf0ed3b8e9806283a1fbf5270965c2b1c594caa31b74", size = 160771, upload-time = "2026-07-24T23:08:00.13Z" }, +] + [[package]] name = "threadpoolctl" version = "3.6.0" @@ -3455,14 +3507,14 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform == 'darwin'", ] dependencies = [ - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "setuptools" }, - { name = "sympy" }, - { name = "typing-extensions" }, + { name = "filelock", marker = "sys_platform == 'darwin'" }, + { name = "fsspec", marker = "sys_platform == 'darwin'" }, + { name = "jinja2", marker = "sys_platform == 'darwin'" }, + { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform == 'darwin'" }, + { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and sys_platform == 'darwin'" }, + { name = "setuptools", marker = "sys_platform == 'darwin'" }, + { name = "sympy", marker = "sys_platform == 'darwin'" }, + { name = "typing-extensions", marker = "sys_platform == 'darwin'" }, ] wheels = [ { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.13.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:94f0de129916f77b8dc2c7a8eff644cfeddfe59e39c9f55e9f6e17543410281d", upload-time = "2026-07-08T12:26:07Z" }, @@ -3485,14 +3537,14 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "setuptools" }, - { name = "sympy" }, - { name = "typing-extensions" }, + { name = "filelock", marker = "sys_platform != 'darwin'" }, + { name = "fsspec", marker = "sys_platform != 'darwin'" }, + { name = "jinja2", marker = "sys_platform != 'darwin'" }, + { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform != 'darwin'" }, + { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and sys_platform != 'darwin'" }, + { name = "setuptools", marker = "sys_platform != 'darwin'" }, + { name = "sympy", marker = "sys_platform != 'darwin'" }, + { name = "typing-extensions", marker = "sys_platform != 'darwin'" }, ] wheels = [ { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.13.0%2Bcpu-cp310-cp310-linux_s390x.whl", hash = "sha256:0555fde6108ca90247ae33d4e1237cbae475c86a223bb2f0f91d9addf1f611bd", upload-time = "2026-07-08T19:27:11Z" }, @@ -3602,6 +3654,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, ] +[[package]] +name = "tzdata" +version = "2026.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/68/f1b440335057bfce71b6e50a9d09445aa2ecbd08359a337976627b8409e7/tzdata-2026.5.tar.gz", hash = "sha256:8cc73c0a0bfca7dbfa59235d60b2eff82231dee33f53d206db1acd9173cfc0a7", size = 200404, upload-time = "2026-10-03T09:23:14.143Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/21/1e5995a1c920cce14e4bffae20c665ec10e7ed03ab25e006cd741092b718/tzdata-2026.5-py2.py3-none-any.whl", hash = "sha256:b683bd1b6659ddcd810ff02ad09ba821d4bf1065072805063eb35c49617905ac", size = 347996, upload-time = "2026-10-03T09:23:12.535Z" }, +] + [[package]] name = "urllib3" version = "2.7.0"
{canMutateQueue ? ( - toggleSelect(item.workflow_id)} aria-label={`Select ${item.display_name || item.filename || item.workflow_id}`} /> + toggleSelect(item.workflow_id)} aria-label={`Select ${item.name || item.filename || item.workflow_id}`} /> ) : null} navigate(`/documents/${item.workflow_id}`)}> - {item.display_name || item.filename} + {item.name || item.filename}
{item.workflow_id}