From 87d00da2b8307d3a8cdedf20542b22893a82dc22 Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 14 Oct 2026 15:39:48 +0530 Subject: [PATCH 01/53] Remove master_catalog sync and add PUBLISH_TO_NETWORK toggle Stop syncing documents into the Postgres master_catalog table and pushing a scheme-list snapshot to Redis. bharat-oan-api no longer reads that snapshot, and nothing else read the table. Removed the module and its tests, the sync after DEV ingest and PROD promote, the cleanup on document disable (and the master_catalog_* fields in the disable response), the related settings, docs and the psycopg2-binary and redis dependencies. The scheme catalog and its /catalog/v1/* pull API are unchanged. Add PUBLISH_TO_NETWORK (default off) to control the Beckn Discovery Service publish after PROD promotion, matching docs-pipeline which has no network publish. The setting is checked inside the publish activity so Temporal replay is unaffected. When off, the activity records a skipped artifact without building the Discovery Service client, so DISCOVERY_SERVICE_ENDPOINT and NETWORK_SENDER_ID are required only when the toggle is on. /pipeline/stages and the UI stepper hide the publishing_to_network stage while it is off. Documented in ENV.md and ADR 0005. --- AGENTS.md | 4 +- CONTEXT.md | 10 +- ENV.md | 26 +- ...0004-publishing-to-network-is-prod-only.md | 2 + docs/ADR/0005-publish-to-network-is-opt-in.md | 15 + pipeline/activities.py | 37 +- pipeline/api.py | 25 +- pipeline/config.py | 34 -- pipeline/instances.py | 13 + pipeline/master_catalog_pg.py | 418 ------------------ pyproject.toml | 4 - tests/test_activities.py | 36 ++ tests/test_api.py | 15 + tests/test_instances.py | 16 + tests/test_master_catalog_pg.py | 347 --------------- ui/src/components/PipelineStepper.jsx | 18 +- ui/src/lib/usePipelineConfig.js | 11 +- 17 files changed, 156 insertions(+), 875 deletions(-) create mode 100644 docs/ADR/0005-publish-to-network-is-opt-in.md delete mode 100644 pipeline/master_catalog_pg.py delete mode 100644 tests/test_master_catalog_pg.py diff --git a/AGENTS.md b/AGENTS.md index ef17b18..764a09e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,7 +10,7 @@ Full narrative docs (read these before deep architectural changes, don't re-deri - `README.md` — what the system does, API surface, supported inputs, auth model - `docs/SYSTEM_DESIGN.md` — C4 diagrams, containers, DEV/PROD indexing, stage/activity table - `docs/architecture-pipeline-rbac.md` — login flow, roles, DEV vs Super-Admin-PROD gate -- `docs/MASTER_SCHEME_CATALOG_AND_AI_SYNC_DESIGN.md` — master catalog → Redis sync to AI layer (bharat-oan-api) +- `docs/MASTER_SCHEME_CATALOG_AND_AI_SYNC_DESIGN.md` — original scheme catalog design (historical: the Postgres master catalog and Redis push it describes were removed; the pull API `/catalog/v1/*` remains) - `ENV.md` — full annotated environment variable reference (`.env.example` for a working template) ## Commands @@ -67,7 +67,7 @@ Core modules (all under `pipeline/`): - `models.py` — Pydantic models, including `DocumentStage` enum and `PIPELINE_STAGES` (the stepper-UI stage list) - `config.py` — `Config` dataclass reading env vars, with defaults - `instances.py` — instance/tenant (state) code ↔ human name mapping; `bv` = Bharat Vistaar platform-wide, others are state codes matching Keycloak group paths (`/states/MH/...` → `mh`) -- `master_catalog_pg.py`, `scheme_catalog.py` — Postgres-backed master scheme catalog, synced to Redis for the AI layer (bharat-oan-api) to consume +- `scheme_catalog.py` — SQLite-backed scheme catalog behind the pull-based `/catalog/v1/*` API - `ocr/`, `translation/`, `chunking/`, `vector_store/` — pluggable provider packages, each with a `base.py` protocol/ABC and a `service.py` (or equivalent) that dispatches to the configured provider - `auth/` — Keycloak JWT validation, group-based tenancy/permissions, email-OTP flow, admin user management diff --git a/CONTEXT.md b/CONTEXT.md index 9cf0d9f..9da1a46 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -9,19 +9,15 @@ The operator action that approves a document out of `ready_for_ingestion` and le _Avoid_: "publish" alone — ambiguous with Publish to Network. **Publish to Network**: -The pipeline stage that runs only after a document's PROD promotion succeeds, and POSTs a Beckn-shaped `publish` envelope to an external Discovery Service. Skipped entirely when PROD is disabled (`DISABLE_PROD_SETTING=true`) — a document never reaches the network without first promoting to PROD. The envelope it sends is kind-level (see Knowledge Kind), not document-level: it does not make any single document searchable, only re-announces that this provider serves that kind at all. +The pipeline stage that runs only after a document's PROD promotion succeeds, and POSTs a Beckn-shaped `publish` envelope to an external Discovery Service. Skipped entirely when PROD is disabled (`DISABLE_PROD_SETTING=true`) or when `PUBLISH_TO_NETWORK` is off (the default) — a document never reaches the network without first promoting to PROD. The envelope it sends is kind-level (see Knowledge Kind), not document-level: it does not make any single document searchable, only re-announces that this provider serves that kind at all. _Avoid_: "publish" alone — ambiguous with Publish to Dev. -**Master Catalog**: -The global, Postgres-backed registry of prompt/tool metadata (any content type: scheme, advisory, etc.), synced to Redis so the AI layer can pick up entries without a redeploy. -_Avoid_: "catalog" alone — ambiguous with Scheme Catalog and the Network Catalog Envelope. - **Scheme Catalog**: The SQLite-backed, scheme-only registry that aggregates one or more documents sharing a `scheme_code`. Backs the pull-based `GET /catalog/v1/*` API that other BAPs poll. -_Avoid_: "catalog" alone — ambiguous with Master Catalog and the Network Catalog Envelope. +_Avoid_: "catalog" alone — ambiguous with the Network Catalog Envelope. **Network Catalog Envelope**: -The Beckn wire-protocol `message.catalogs` object sent by Publish to Network to the Discovery Service. Structurally and semantically unrelated to the Master Catalog and Scheme Catalog tables — do not conflate the three. +The Beckn wire-protocol `message.catalogs` object sent by Publish to Network to the Discovery Service. Structurally and semantically unrelated to the Scheme Catalog tables — do not conflate the two. _Avoid_: "catalog" alone. **Knowledge Kind**: diff --git a/ENV.md b/ENV.md index 3e517c3..40a3b36 100644 --- a/ENV.md +++ b/ENV.md @@ -228,31 +228,15 @@ Exposes `/catalog/v1/*` so **bharat-oan-api** and **bharat-provider-backend** ca **Endpoints:** `GET /catalog/v1/snapshot`, `/version`, `/schemes`, `/tool-prompt`; `POST /catalog/v1/rebuild`, `/bootstrap`; `PATCH /documents/{id}/scheme-metadata`. -### Master Catalog — Postgres + Redis push (dev preview channel) - -Separate from the SQLite/Qdrant catalog above (which tracks PROD vector publication and is scheme-specific). This one is a generic Postgres table (`master_catalog`: code, content_type, name, tool_name, doc_id, prompt_snippet, status) — not scheme-only, since ingested documents can be schemes, advisories, or other kinds that later grow their own code/name/tool metadata. docs-pipeline writes to it on **DEV ingest complete** (`status=dev`) and **PROD promote complete** (`status=live`), then pushes directly into the AI layer's Redis so bharat-oan-api can test an entry's prompt/tool routing in the dev chatbot without an AI-layer redeploy. - -| Variable | Default | Purpose | -|----------|---------|---------| -| `MASTER_CATALOG_PG_HOST` | *(empty — required)* | Host of your existing Postgres instance. No DB container is provisioned by docker-compose for this — bring your own server and create the database on it (e.g. `CREATE DATABASE master_catalog`). Tables are created automatically on first sync. | -| `MASTER_CATALOG_PG_PORT` | `5432` | Postgres port | -| `MASTER_CATALOG_PG_DB` | `master_catalog` | Database name | -| `MASTER_CATALOG_PG_USER` / `MASTER_CATALOG_PG_PASSWORD` | `master_catalog` / *(empty)* | Credentials | -| `MASTER_CATALOG_PG_SSLMODE` | `disable` | Use `require` for managed prod Postgres | -| `AI_LAYER_REDIS_HOST` | *(empty)* | bharat-oan-api's Redis host. Empty = Postgres sync still happens, Redis push is skipped | -| `AI_LAYER_REDIS_PORT` / `AI_LAYER_REDIS_DB` / `AI_LAYER_REDIS_PASSWORD` | `6379` / `0` / *(empty)* | AI layer Redis connection | -| `MASTER_CATALOG_REDIS_TTL_SECONDS` | `172800` (48h) | Snapshot key TTL — a dead-man's switch, not a freshness mechanism (writes are push-driven) | - -**Redis keys:** `master-catalog:dev:snapshot` (dev + live entries — what the dev chatbot reads), `master-catalog:live:snapshot` (live only — what prod reads). Plain JSON via raw `redis-py`, not routed through bharat-oan-api's `aiocache` layer, since it's an external write contract rather than an internal cache value. - ### Publish to Network -First piece of the network-discovery push architecture: once a document's PROD promotion succeeds, the `publishing_to_network` pipeline stage POSTs a Beckn-shaped `publish` envelope to an external Discovery Service. It's a PROD-only stage (see `docs/ADR/0004-publishing-to-network-is-prod-only.md`) — skipped entirely when `DISABLE_PROD_SETTING=true`, since a document never reaches the network without first promoting to PROD. This is going to be the only way the experience layer discovers schemas going forward — the Master Scheme Catalog pull API above stays in place for backward compatibility. The catalog announces the *knowledge kind* the document was classified as — one catalog with exactly one `OnDemand` resource for `advisory`, another for `scheme`; any other `document_kind` publishes nothing. `DISCOVERY_SERVICE_ENDPOINT` and `NETWORK_SENDER_ID` are **required** whenever PROD promotion is enabled — there is no separate feature flag for this stage beyond `DISABLE_PROD_SETTING`. +First piece of the network-discovery push architecture: once a document's PROD promotion succeeds, the `publishing_to_network` pipeline stage POSTs a Beckn-shaped `publish` envelope to an external Discovery Service. It's a PROD-only stage (see `docs/ADR/0004-publishing-to-network-is-prod-only.md`) — skipped entirely when `DISABLE_PROD_SETTING=true`, since a document never reaches the network without first promoting to PROD. This is going to be the only way the experience layer discovers schemas going forward — the Master Scheme Catalog pull API above stays in place for backward compatibility. The catalog announces the *knowledge kind* the document was classified as — one catalog with exactly one `OnDemand` resource for `advisory`, another for `scheme`; any other `document_kind` publishes nothing. Publishing is **off by default**: set `PUBLISH_TO_NETWORK=true` to turn it on (see `docs/ADR/0005-publish-to-network-is-opt-in.md`). While it is off the activity records a `network_publish_skipped` artifact and the stage is left out of `GET /pipeline/stages`, so the UI stepper does not show it. `DISCOVERY_SERVICE_ENDPOINT` and `NETWORK_SENDER_ID` are **required only when `PUBLISH_TO_NETWORK` is on**. | Variable | Default | Purpose | |----------|---------|---------| -| `DISCOVERY_SERVICE_ENDPOINT` | *(empty — required)* | Base URL of the external Discovery Service; the pipeline POSTs to `{DISCOVERY_SERVICE_ENDPOINT}/publish` | -| `NETWORK_SENDER_ID` | *(empty — required)* | This instance's Beckn `senderId`, also sent as `networkId` | +| `PUBLISH_TO_NETWORK` | `false` | Turns the Discovery Service publish on (`true`/`1`/`yes`). Read when the publish activity runs, not when a workflow starts, so changing it never affects the path of a workflow already in flight | +| `DISCOVERY_SERVICE_ENDPOINT` | *(empty — required when `PUBLISH_TO_NETWORK` is on)* | Base URL of the external Discovery Service; the pipeline POSTs to `{DISCOVERY_SERVICE_ENDPOINT}/publish` | +| `NETWORK_SENDER_ID` | *(empty — required when `PUBLISH_TO_NETWORK` is on)* | This instance's Beckn `senderId`, also sent as `networkId` | | `DISCOVERY_SERVICE_PUBLISH_INITIAL_INTERVAL_SECONDS` | `30` | Wait before the first retry of a failed publish | | `DISCOVERY_SERVICE_PUBLISH_BACKOFF_COEFFICIENT` | `2.0` | Multiplier applied to the wait after each retry | | `DISCOVERY_SERVICE_PUBLISH_MAX_INTERVAL_SECONDS` | `300` | Cap on the wait between retries | @@ -269,7 +253,7 @@ First piece of the network-discovery push architecture: once a document's PROD p | Temporal / MinIO / SQLite | — | ✅ | ✅ | | OCR / translation / chunking / domain tags | — | config / status | ✅ runs jobs | | Qdrant (`VECTOR_DB_*`) | — | ✅ | ✅ | -| Publish to Network (`DISCOVERY_SERVICE_ENDPOINT`, `NETWORK_SENDER_ID`) | — | — | ✅ runs the publish activity | +| Publish to Network (`PUBLISH_TO_NETWORK`, `DISCOVERY_SERVICE_ENDPOINT`, `NETWORK_SENDER_ID`) | — | ✅ `PUBLISH_TO_NETWORK` only (stage list) | ✅ runs the publish activity | --- diff --git a/docs/ADR/0004-publishing-to-network-is-prod-only.md b/docs/ADR/0004-publishing-to-network-is-prod-only.md index b11c3d8..a87db48 100644 --- a/docs/ADR/0004-publishing-to-network-is-prod-only.md +++ b/docs/ADR/0004-publishing-to-network-is-prod-only.md @@ -12,3 +12,5 @@ Alternatives considered: - **Leave it at DEV-ingest time (status quo).** Rejected: too weak a bar for a live network-facing capability claim. - **Gate per-document instead of per-provider-capability.** Not possible: the envelope is structurally kind-level (ADR 0003). - **Treat post-promotion publish failure as non-fatal.** Rejected for v1: avoids a new "prod-live but not announced" intermediate document state; revisit if failures become operationally frequent. + +Update: publishing is now off unless `PUBLISH_TO_NETWORK` is set; see ADR 0005. The "skipped entirely when `DISABLE_PROD_SETTING=true`" rule above still holds. diff --git a/docs/ADR/0005-publish-to-network-is-opt-in.md b/docs/ADR/0005-publish-to-network-is-opt-in.md new file mode 100644 index 0000000..0cf35b3 --- /dev/null +++ b/docs/ADR/0005-publish-to-network-is-opt-in.md @@ -0,0 +1,15 @@ +# Publishing to the network is opt-in (PUBLISH_TO_NETWORK) + +Context: ADR 0004 made `publishing_to_network` a PROD-only stage, but every deployment that promotes to PROD still had to configure a Discovery Service (`DISCOVERY_SERVICE_ENDPOINT`, `NETWORK_SENDER_ID`) or its documents failed at that stage. Deployments that do not take part in the Beckn network, and the upstream docs-pipeline this project forked from, have no network publish at all. + +Decision: +- A new setting `PUBLISH_TO_NETWORK` (default `false`) controls whether the stage publishes. Accepted truthy values are `true`, `1` and `yes`, matching `DISABLE_PROD_SETTING`. +- The setting is checked inside the `publish_catalog_to_network` activity, not in the workflow. Activities are not replayed, so changing the setting never changes the path a replaying workflow takes and needs no `workflow.patched` marker. A document in flight simply publishes, or skips, according to the value in force when its publish activity runs. +- When off, the activity does not construct `DiscoveryPublishService`, records a `network_publish_skipped` artifact with the reason `PUBLISH_TO_NETWORK is off`, and the stage completes. The Discovery Service settings are therefore required only when the setting is on. +- `GET /pipeline/stages` omits `publishing_to_network` when the setting is off, and the UI stepper follows that response, so operators do not see a step that does nothing. `PIPELINE_STAGES` and `PROD_ONLY_STAGES` are unchanged. +- Ordering, retry policy and the fatal-on-rejection behaviour from ADR 0003 and ADR 0004 are unchanged when the setting is on. + +Alternatives considered: +- **Branch in the workflow on a value passed at start (like `skip_prod`).** Rejected: it needs the same plumbing at four call sites and a patch marker, for no benefit over an activity-level check. +- **Show the stage as "skipped" in the UI.** Rejected for now: the backend tracks one current stage per document and has no per-stage status, so a skipped state would need new storage. Hiding the stage reuses the existing `/pipeline/stages` mechanism. +- **Remove the publish stage altogether.** Rejected: the network publish remains the way the experience layer discovers knowledge kinds for deployments that opt in. diff --git a/pipeline/activities.py b/pipeline/activities.py index b29c479..b2ec3e4 100644 --- a/pipeline/activities.py +++ b/pipeline/activities.py @@ -28,7 +28,7 @@ from .chunking import chunk_pages, load_chunking_config from .discovery_publish_service import DiscoveryPublishService from .document_repository import DocumentRepository -from .instances import instance_display_name +from .instances import instance_display_name, publish_to_network_enabled from .network_constants import RESULT_REJECTED from .ocr import ocr_pdf as run_ocr_pdf from .ocr import ocr_pdf_in_segments as run_ocr_pdf_in_segments @@ -1374,12 +1374,6 @@ async def promote_document_to_prod_qdrant( catalog_version = on_scheme_document_promoted(workflow_id) except Exception as exc: activity.logger.warning("Catalog update after promote failed: %s", exc) - try: - from .master_catalog_pg import sync_catalog_entry - - await asyncio.to_thread(sync_catalog_entry, workflow_id, "live") - except Exception as exc: - activity.logger.warning("Master catalog (Postgres/Redis) sync after promote failed: %s", exc) return { **result, "collection": prod_collection, @@ -1443,12 +1437,6 @@ async def ingest_document_from_db( status="indexed", details=result.get("index_stats"), ) - try: - from .master_catalog_pg import sync_catalog_entry - - await asyncio.to_thread(sync_catalog_entry, workflow_id, "dev") - except Exception as exc: - activity.logger.warning("Master catalog (Postgres/Redis) sync after DEV ingest failed: %s", exc) return result @@ -1465,10 +1453,23 @@ async def publish_catalog_to_network(workflow_id: str, transaction_id: str) -> d repository = DocumentRepository() document_kind = normalize_document_kind(repository.get_document_kind(workflow_id)) - service = DiscoveryPublishService() - result = await asyncio.to_thread( - service.publish, transaction_id, document_kind, workflow_id - ) + if publish_to_network_enabled(): + service = DiscoveryPublishService() + result = await asyncio.to_thread( + service.publish, transaction_id, document_kind, workflow_id + ) + skip_reason = "no network catalog is mapped to this document kind" + else: + # Checked here, not in the workflow, so a replaying workflow never + # changes path when the setting flips. No Discovery Service settings + # are needed while publishing is off. + result = { + "skipped": True, + "result_status": None, + "envelope": None, + "errors": [], + } + skip_reason = "PUBLISH_TO_NETWORK is off" if result["skipped"]: artifact_type = "network_publish_skipped" @@ -1476,7 +1477,7 @@ async def publish_catalog_to_network(workflow_id: str, transaction_id: str) -> d exchange = { "skipped": True, "document_kind": document_kind, - "reason": "no network catalog is mapped to this document kind", + "reason": skip_reason, } else: artifact_type = "network_publish_payload" diff --git a/pipeline/api.py b/pipeline/api.py index 747704f..570eb70 100644 --- a/pipeline/api.py +++ b/pipeline/api.py @@ -78,7 +78,7 @@ unrestricted, user_can_access_instance, ) -from .instances import prod_stage_disabled +from .instances import prod_stage_disabled, publish_to_network_enabled from .models import ( PIPELINE_STAGES, PROD_ONLY_STAGES, @@ -1977,8 +1977,6 @@ async def disable_document( - Remove all chunks from the DEV vector index - Delete PROD Qdrant points too, if this document was ever promoted (schemes-index for scheme documents, documents-index otherwise) - - Remove the row from the Postgres master catalog and refresh the - Redis snapshots the AI layer reads, so a re-ingest starts clean - For scheme documents, also rebuild the legacy SQLite scheme catalog - Stop the workflow if it is still running @@ -2030,7 +2028,6 @@ async def disable_document( "scheme_qdrant_deleted": None, "prod_qdrant_deleted": None, "catalog_version": None, - "master_catalog_version": None, "minio_artifacts_deleted": None, "sqlite_rows_deleted": None, } @@ -2076,16 +2073,6 @@ async def disable_document( else: result["prod_qdrant_deleted"] = _delete_document_points_from_prod(doc) - # Master catalog (Postgres) + Redis snapshot cleanup — mirrors the sync - # done on ingest/promote so a deleted document stops appearing to the - # AI layer immediately, and a later re-ingest starts from a clean slate. - try: - from .master_catalog_pg import remove_catalog_entry - - result["master_catalog_version"] = await asyncio.to_thread(remove_catalog_entry, workflow_id) - except Exception as exc: - result["master_catalog_error"] = str(exc) - # Hard delete: drop the artifacts and the SQLite rows themselves. Ordered # storage-then-database so a failure mid-way leaves rows pointing at # already-deleted objects (visible, fixable) rather than orphaned objects @@ -2124,7 +2111,6 @@ async def disable_document( "prod_qdrant_deleted": result.get("prod_qdrant_deleted"), "scheme_code": doc.get("scheme_code"), "catalog_version": result.get("catalog_version"), - "master_catalog_version": result.get("master_catalog_version"), }, user=user, ) @@ -2364,7 +2350,7 @@ async def reingest_document( # (e.g. after an embedding/index change), but a trap if it's used as a # substitute for the real approval flow on a never-classified document — # it would silently ingest with document_kind='document' and no scheme - # metadata, the exact raw-hash/filename-fallback master_catalog rows we've + # metadata, the exact raw-hash/filename-fallback catalog rows we've # had to clean up by hand. 'document' is never offered as a choice in the # classification panel, so seeing it here means classification was never done. if (doc.get("document_kind") or "document") == "document" and not doc.get("scheme_code") and not doc.get("scheme_name"): @@ -4179,14 +4165,17 @@ async def reconcile_document_states(user: RequirePipeline): async def get_pipeline_stages(user: RequireSearch): """Get the pipeline stages for UI stepper display. - The PROD stages are omitted when DISABLE_PROD_SETTING is on, so the stepper - shows the pipeline the documents will actually follow. + The PROD stages are omitted when DISABLE_PROD_SETTING is on, and the + network publish stage is omitted unless PUBLISH_TO_NETWORK is on, so the + stepper shows the pipeline the documents will actually follow. """ prod_disabled = prod_stage_disabled() + network_disabled = not publish_to_network_enabled() return [ {"id": stage[0], "label": stage[1], "description": stage[2]} for stage in PIPELINE_STAGES if not (prod_disabled and stage[0] in PROD_ONLY_STAGES) + and not (network_disabled and stage[0] == "publishing_to_network") ] diff --git a/pipeline/config.py b/pipeline/config.py index 7754851..7ec2a4b 100644 --- a/pipeline/config.py +++ b/pipeline/config.py @@ -52,22 +52,6 @@ class Config: # File access allowed_file_paths: list[str] = None - # Master Catalog (Postgres) — code/name/tool_name/prompt_snippet, generic - # across content types (scheme, advisory, ...), not scheme-specific - master_catalog_pg_host: str = "localhost" - master_catalog_pg_port: int = 5432 - master_catalog_pg_db: str = "master_catalog" - master_catalog_pg_user: str = "master_catalog" - master_catalog_pg_password: str = "" - master_catalog_pg_sslmode: str = "disable" - - # AI layer Redis (bharat-oan-api) — direct write of the catalog snapshot - ai_layer_redis_host: str = "" - ai_layer_redis_port: int = 6379 - ai_layer_redis_db: int = 0 - ai_layer_redis_password: str = "" - master_catalog_redis_ttl_seconds: int = 172800 - def __post_init__(self): if self.cors_origins is None: self.cors_origins = ["http://localhost:3000"] @@ -152,21 +136,6 @@ def load_config() -> Config: # File access allowed_file_paths=os.environ.get("ALLOWED_FILE_PATHS", "").split(",") if os.environ.get("ALLOWED_FILE_PATHS") else None, - - # Master Catalog (Postgres) - master_catalog_pg_host=os.environ.get("MASTER_CATALOG_PG_HOST", "localhost"), - master_catalog_pg_port=int(os.environ.get("MASTER_CATALOG_PG_PORT", "5432")), - master_catalog_pg_db=os.environ.get("MASTER_CATALOG_PG_DB", "master_catalog"), - master_catalog_pg_user=os.environ.get("MASTER_CATALOG_PG_USER", "master_catalog"), - master_catalog_pg_password=os.environ.get("MASTER_CATALOG_PG_PASSWORD", ""), - master_catalog_pg_sslmode=os.environ.get("MASTER_CATALOG_PG_SSLMODE", "disable"), - - # AI layer Redis - ai_layer_redis_host=os.environ.get("AI_LAYER_REDIS_HOST", ""), - ai_layer_redis_port=int(os.environ.get("AI_LAYER_REDIS_PORT", "6379")), - ai_layer_redis_db=int(os.environ.get("AI_LAYER_REDIS_DB", "0")), - ai_layer_redis_password=os.environ.get("AI_LAYER_REDIS_PASSWORD", ""), - master_catalog_redis_ttl_seconds=int(os.environ.get("MASTER_CATALOG_REDIS_TTL_SECONDS", "172800")), ) @@ -213,9 +182,6 @@ def print_config_status(): ("CORS_ORIGINS", "(default)"), ("RATE_LIMIT_DEFAULT", "100/minute"), ("RATE_LIMIT_UPLOAD", "10/minute"), - ("MASTER_CATALOG_PG_HOST", "localhost"), - ("MASTER_CATALOG_PG_DB", "master_catalog"), - ("AI_LAYER_REDIS_HOST", "(unset — Redis push disabled)"), ] for var_name, default in optional: diff --git a/pipeline/instances.py b/pipeline/instances.py index 07782b8..c5486a7 100644 --- a/pipeline/instances.py +++ b/pipeline/instances.py @@ -35,6 +35,19 @@ def prod_stage_disabled() -> bool: return (os.environ.get("DISABLE_PROD_SETTING", "false").strip().lower() in {"true", "1", "yes"}) +def publish_to_network_enabled() -> bool: + """True when PUBLISH_TO_NETWORK turns on the Beckn Discovery Service publish. + + Defaults to off, so a deployment publishes to the network only when it opts + in. Unlike ``prod_stage_disabled`` this is read at publish time, inside the + ``publish_catalog_to_network`` activity: activities are not replayed, so a + flip changes only publishes that have not run yet and never changes the + path a replaying workflow takes. + """ + return (os.environ.get("PUBLISH_TO_NETWORK", "false").strip().lower() + in {"true", "1", "yes"}) + + INSTANCE_NAMES: dict[str, str] = { # Platform "bv": "Bharat Vistaar", diff --git a/pipeline/master_catalog_pg.py b/pipeline/master_catalog_pg.py deleted file mode 100644 index beeccde..0000000 --- a/pipeline/master_catalog_pg.py +++ /dev/null @@ -1,418 +0,0 @@ -""" -Master Catalog (Postgres) — prompt/tool metadata for the AI layer. - -Generic across content types (scheme, advisory, or whatever else eventually -carries its own code/name/tool routing) — not scheme-specific by name. -Every document gets synced regardless of document_kind; code falls back to -doc_id/workflow_id when scheme_code isn't set. Separate from -`scheme_catalog.py` (SQLite `scheme_catalog_entries`, which tracks PROD -Qdrant publication state for search, and is legitimately scheme-only). - - - DEV ingest complete -> sync_catalog_entry(workflow_id, status="dev") - - PROD promote complete -> sync_catalog_entry(workflow_id, status="live") - - Document disabled -> remove_catalog_entry(workflow_id) - -Each row is keyed on `code` (upsert, never duplicated). `status` is a TEXT[] -that accumulates every tier a document has synced under ({dev} then -{dev,live}) rather than being overwritten — a later DEV re-sync (e.g. a -reingest after promotion) must not silently regress an already-live entry -back out of the live snapshot. Redis snapshot keys are -`master-catalog:dev:snapshot` (rows whose status overlaps {dev, live} — dev -testers should also see promoted entries) and `master-catalog:live:snapshot` -(rows whose status contains live, for prod). Both are refreshed on every -sync so they never drift relative to each other. - -Each snapshot also carries a pre-rendered `prompt` block (vector_schemes_ -bullets/identifiers, built from tool_name == "search_schemes" entries only) -so the AI layer can splice a live scheme list straight into its system -prompt instead of hand-maintaining one per language. Legacy schemes -(tool_name == "get_scheme_info") are intentionally excluded from that block -— they stay a separate, hardcoded list on the AI-layer side. -""" - -from __future__ import annotations - -import json -import logging -import os -from datetime import datetime, timezone -from typing import Optional - -import psycopg2 -import psycopg2.extras - -from . import db -from .instances import instance_display_name -from .scheme_catalog import resolve_scheme_aliases - -logger = logging.getLogger(__name__) - -_SCHEMA_SQL = """ -CREATE TABLE IF NOT EXISTS master_catalog ( - code TEXT PRIMARY KEY, - content_type TEXT NOT NULL DEFAULT 'scheme', - name TEXT NOT NULL, - tool_name TEXT NOT NULL, - doc_id TEXT, - prompt_snippet TEXT, - aliases TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[], - status TEXT[] NOT NULL DEFAULT ARRAY['dev']::TEXT[], - workflow_id TEXT, - instance TEXT, - updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), - created_at TIMESTAMPTZ NOT NULL DEFAULT now() -); - -CREATE TABLE IF NOT EXISTS master_catalog_meta ( - id INTEGER PRIMARY KEY DEFAULT 1, - version INTEGER NOT NULL DEFAULT 0, - updated_at TIMESTAMPTZ NOT NULL DEFAULT now() -); - -INSERT INTO master_catalog_meta (id, version) -VALUES (1, 0) -ON CONFLICT (id) DO NOTHING; - --- Migrate pre-existing installs where status was a scalar TEXT column: a --- promote-to-live sync used to overwrite it outright, so a later DEV --- re-sync (e.g. reingest) would silently regress a live entry back to dev --- and drop it from the live snapshot. Now it accumulates tiers instead. -DO $$ -BEGIN - IF (SELECT data_type FROM information_schema.columns - WHERE table_name = 'master_catalog' AND column_name = 'status') = 'text' THEN - ALTER TABLE master_catalog ALTER COLUMN status DROP DEFAULT; - ALTER TABLE master_catalog ALTER COLUMN status TYPE TEXT[] USING ARRAY[status]::TEXT[]; - ALTER TABLE master_catalog ALTER COLUMN status SET DEFAULT ARRAY['dev']::TEXT[]; - END IF; -END $$; - -ALTER TABLE master_catalog ADD COLUMN IF NOT EXISTS aliases TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[]; - --- Human-readable state/tenant name beside the `instance` code, so the AI layer --- can say "Maharashtra" without carrying its own code lookup table. -ALTER TABLE master_catalog ADD COLUMN IF NOT EXISTS instance_name TEXT; -""" - -_REDIS_KEY_PREFIX = "master-catalog" - -# How many aliases per scheme reach the AI system prompt. The catalog stores -# more than this for search; the prompt only needs enough to disambiguate. -_PROMPT_ALIAS_LIMIT = 4 - - -def _pg_dsn() -> str: - return ( - f"host={os.environ.get('MASTER_CATALOG_PG_HOST', 'localhost')} " - f"port={os.environ.get('MASTER_CATALOG_PG_PORT', '5432')} " - f"dbname={os.environ.get('MASTER_CATALOG_PG_DB', 'master_catalog')} " - f"user={os.environ.get('MASTER_CATALOG_PG_USER', 'master_catalog')} " - f"password={os.environ.get('MASTER_CATALOG_PG_PASSWORD', '')} " - f"sslmode={os.environ.get('MASTER_CATALOG_PG_SSLMODE', 'disable')}" - ) - - -def get_pg_connection(): - """New connection per call — sync calls are rare (ingest/promote events only), not per-request.""" - conn = psycopg2.connect(_pg_dsn()) - conn.autocommit = True - return conn - - -def get_redis_client(): - import redis # local import: keep this optional dep out of the module import path for tests - - host = os.environ.get("AI_LAYER_REDIS_HOST", "").strip() - if not host: - return None - return redis.Redis( - host=host, - port=int(os.environ.get("AI_LAYER_REDIS_PORT", "6379")), - db=int(os.environ.get("AI_LAYER_REDIS_DB", "0")), - password=os.environ.get("AI_LAYER_REDIS_PASSWORD") or None, - decode_responses=True, - socket_connect_timeout=3, - socket_timeout=3, - ) - - -def _redis_ttl_seconds() -> int: - return int(os.environ.get("MASTER_CATALOG_REDIS_TTL_SECONDS", "172800")) - - -def ensure_schema(conn) -> None: - with conn.cursor() as cur: - cur.execute(_SCHEMA_SQL) - - -def _utc_now_iso() -> str: - return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _resolve_tool_name(tool_routing: Optional[str]) -> str: - """Which existing AI-layer tool this entry routes to — a label, not code.""" - routing = (tool_routing or "qdrant").strip().lower() - return "get_scheme_info" if routing == "legacy" else "search_schemes" - - -def _generate_prompt_snippet(name: str, code: str, content_type: str) -> str: - """A single sentence meant to be dropped directly into an agent system - prompt as a reference entry — not a markdown bullet. There's no document - summary/description captured during ingestion to draw a real topic from, - so this stays generic (name + code + content type) rather than fabricate - one; `_build_vector_schemes_prompt_block` below is what actually renders - the searchable scheme list callers use, this field is a per-row preview.""" - kind = (content_type or "document").strip().lower() - label = "government scheme" if kind == "scheme" else kind - return f"Refer to '{name}' (code: {code}) for guidance on this {label}." - - -def sync_catalog_entry(workflow_id: str, status: str) -> Optional[int]: - """ - Upsert this document's catalog metadata into the Postgres master catalog - and refresh both Redis snapshot tiers. - - status: "dev" (DEV ingest complete) | "live" (PROD promote complete) - Returns the new catalog version, or None if the document doesn't exist - (nothing to sync). - """ - if status not in ("dev", "live"): - raise ValueError("status must be 'dev' or 'live'") - - doc = db.get_document(workflow_id) - if not doc: - logger.info("master_catalog_pg: %s not found, skipping sync", workflow_id) - return None - - content_type = (doc.get("document_kind") or "document").strip().lower() - - # code is the primary key — fall back to doc_id/workflow_id when this - # document has no scheme_code so every document still gets a row. - code = ( - (doc.get("scheme_code") or "").strip().lower() - or (doc.get("document_id") or "").strip().lower() - or workflow_id.strip().lower() - ) - - # Same fallback chain as the UI's getDocumentListLabel (ui/src/lib/pipelineUi.js) - # so a non-scheme document still gets a readable name instead of the bare code. - name = ( - (doc.get("scheme_name") or "").strip() - or (doc.get("display_name") or "").strip() - or (doc.get("filename") or "").strip() - or (doc.get("source_filename") or "").strip() - or code - ) - tool_name = _resolve_tool_name(doc.get("tool_routing")) - prompt_snippet = _generate_prompt_snippet(name, code, content_type) - # Same resolution as the vector payload, so the catalog and the indexed - # chunks never disagree about a scheme's aliases. - aliases = resolve_scheme_aliases( - doc.get("scheme_code"), name, doc.get("scheme_aliases_json") - ) - doc_id = doc.get("document_id") - instance = (doc.get("instance") or "default").strip().lower() - instance_name = instance_display_name(instance) - now = _utc_now_iso() - - conn = get_pg_connection() - try: - ensure_schema(conn) - with conn.cursor() as cur: - cur.execute( - """ - INSERT INTO master_catalog - (code, content_type, name, tool_name, doc_id, prompt_snippet, - aliases, status, workflow_id, instance, instance_name, updated_at) - VALUES (%s, %s, %s, %s, %s, %s, %s, ARRAY[%s]::TEXT[], %s, %s, %s, %s) - ON CONFLICT (code) DO UPDATE SET - content_type = EXCLUDED.content_type, - name = EXCLUDED.name, - tool_name = EXCLUDED.tool_name, - doc_id = EXCLUDED.doc_id, - prompt_snippet = EXCLUDED.prompt_snippet, - aliases = EXCLUDED.aliases, - status = ARRAY( - SELECT DISTINCT unnest(master_catalog.status || EXCLUDED.status) - ), - workflow_id = EXCLUDED.workflow_id, - instance = EXCLUDED.instance, - instance_name = EXCLUDED.instance_name, - updated_at = EXCLUDED.updated_at - """, - (code, content_type, name, tool_name, doc_id, prompt_snippet, - aliases, status, workflow_id, instance, instance_name, now), - ) - cur.execute( - """ - UPDATE master_catalog_meta - SET version = version + 1, updated_at = %s - WHERE id = 1 - RETURNING version - """, - (now,), - ) - row = cur.fetchone() - version = row[0] if row else None - - logger.info( - "master_catalog_pg: synced code=%s name=%s status=%s workflow_id=%s version=%s", - code, name, status, workflow_id, version, - ) - _push_snapshots_to_redis(conn, version, now) - return version - finally: - conn.close() - - -def remove_catalog_entry(workflow_id: str) -> Optional[int]: - """ - Remove this document's row from the Postgres master catalog and refresh - both Redis snapshot tiers, so a deleted document stops appearing to the - AI layer immediately instead of lingering in a stale snapshot until some - unrelated sync event happens to overwrite it. Counterpart to - sync_catalog_entry() — call this from the document-disable path. - - Deletes by the same `code` key sync_catalog_entry() would have derived, - with a workflow_id fallback in case the document row is already gone. - Returns the new catalog version, or None if there was no matching row. - """ - doc = db.get_document(workflow_id) - code = None - if doc: - code = ( - (doc.get("scheme_code") or "").strip().lower() - or (doc.get("document_id") or "").strip().lower() - or workflow_id.strip().lower() - ) - - conn = get_pg_connection() - try: - ensure_schema(conn) - with conn.cursor() as cur: - if code: - cur.execute("DELETE FROM master_catalog WHERE code = %s", (code,)) - else: - cur.execute("DELETE FROM master_catalog WHERE workflow_id = %s", (workflow_id,)) - deleted = cur.rowcount - - if not deleted: - logger.info( - "master_catalog_pg: no catalog row for workflow_id=%s (code=%s), nothing to remove", - workflow_id, code, - ) - return None - - now = _utc_now_iso() - cur.execute( - """ - UPDATE master_catalog_meta - SET version = version + 1, updated_at = %s - WHERE id = 1 - RETURNING version - """, - (now,), - ) - row = cur.fetchone() - version = row[0] if row else None - - logger.info( - "master_catalog_pg: removed code=%s workflow_id=%s version=%s", - code, workflow_id, version, - ) - _push_snapshots_to_redis(conn, version, now) - return version - finally: - conn.close() - - -def _fetch_rows(conn, tier: str) -> list[dict]: - """tier='dev' includes dev+live rows (dev testers should also see promoted entries); - tier='live' includes only live rows (prod). A row's `status` accumulates every - tier it has ever synced under, so this is an array-overlap check, not equality.""" - statuses = ("dev", "live") if tier == "dev" else ("live",) - with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: - cur.execute( - "SELECT * FROM master_catalog WHERE status && %s ORDER BY code", - (list(statuses),), - ) - return [dict(r) for r in cur.fetchall()] - - -def _row_to_json(row: dict) -> dict: - updated_at = row.get("updated_at") - return { - "code": row["code"], - "content_type": row.get("content_type"), - "name": row["name"], - "tool_name": row["tool_name"], - "doc_id": row.get("doc_id"), - "prompt_snippet": row.get("prompt_snippet"), - "aliases": row.get("aliases") or [], - "status": row.get("status"), - "workflow_id": row.get("workflow_id"), - "instance": row.get("instance"), - "instance_name": row.get("instance_name"), - "updated_at": updated_at.isoformat() if hasattr(updated_at, "isoformat") else updated_at, - } - - -def _build_vector_schemes_prompt_block(entries: list[dict]) -> dict: - """Pre-rendered system-prompt fragments for the AI layer's dynamic - "vector-indexed schemes" section (bharat-oan-api's search_schemes routing). - - Restricted to tool_name == "search_schemes": legacy get_scheme_info - schemes are a separate, still-hardcoded list in the AI layer and must - never be pulled into this block. - """ - vector_entries = sorted( - (e for e in entries if e.get("tool_name") == "search_schemes"), - key=lambda e: e["code"], - ) - bullets = [] - identifiers = [] - for e in vector_entries: - code = e["code"] - name = e["name"] - bullets.append(f"- **{name}** ({code})") - # The stored alias list is deliberately broad (curated + derived) because - # it costs nothing in Postgres and improves vector recall. The prompt is - # the opposite: every alias is tokens in every AI request, and the long - # derived tail adds noise rather than helping the model pick a scheme. - # Aliases are already in precedence order, so the head is the useful part. - aliases = (e.get("aliases") or [])[:_PROMPT_ALIAS_LIMIT] - alias_suffix = "".join(f" / {a}" for a in aliases) - identifiers.append(f"- `{code}`{alias_suffix}") - return { - "vector_schemes_bullets": "\n".join(bullets), - "vector_schemes_identifiers": "\n".join(identifiers), - "vector_scheme_count": len(vector_entries), - } - - -def _push_snapshots_to_redis(conn, version: Optional[int], updated_at: str) -> None: - """Best-effort: Postgres is the source of truth, Redis is a mirror. A failed - push here doesn't fail the sync — the next catalog event repairs it.""" - try: - client = get_redis_client() - if client is None: - logger.info("master_catalog_pg: AI_LAYER_REDIS_HOST unset, skipping Redis push") - return - ttl = _redis_ttl_seconds() - for tier in ("dev", "live"): - rows = _fetch_rows(conn, tier) - entries = [_row_to_json(r) for r in rows] - payload = { - "version": version, - "updated_at": updated_at, - "status": tier, - "entries": entries, - "prompt": _build_vector_schemes_prompt_block(entries), - } - client.set(f"{_REDIS_KEY_PREFIX}:{tier}:snapshot", json.dumps(payload), ex=ttl) - logger.info( - "master_catalog_pg: pushed %s:%s snapshot (%d entries, version=%s)", - _REDIS_KEY_PREFIX, tier, len(rows), version, - ) - except Exception as exc: - logger.warning("master_catalog_pg: Redis push failed: %s", exc) diff --git a/pyproject.toml b/pyproject.toml index 7f65c98..9792bb6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,10 +35,6 @@ dependencies = [ # Object storage "minio>=7.0.0", - # Master Scheme Catalog: Postgres store + Redis push to AI layer - "psycopg2-binary>=2.9.0", - "redis>=5.0.0", - # HTTP client "httpx>=0.24.0", diff --git a/tests/test_activities.py b/tests/test_activities.py index 79904c0..4f847f2 100644 --- a/tests/test_activities.py +++ b/tests/test_activities.py @@ -496,6 +496,42 @@ async def test_run_ocr_calls_provider(self, monkeypatch, tmp_path): class TestPublishCatalogToNetworkActivity: """Tests for the publish_catalog_to_network activity (thin wrapper over DiscoveryPublishService).""" + @pytest.fixture(autouse=True) + def _publishing_on(self, monkeypatch): + # Publishing is off by default; these tests exercise the publish path. + monkeypatch.setenv("PUBLISH_TO_NETWORK", "true") + + @pytest.mark.unit + @pytest.mark.asyncio + async def test_publish_off_skips_without_building_the_service(self, monkeypatch): + import pipeline.activities as activities + import pipeline.db as db + + monkeypatch.delenv("PUBLISH_TO_NETWORK", raising=False) + + class ExplodingService: + def __init__(self): + raise AssertionError("service must not be built while publishing is off") + + monkeypatch.setattr(activities, "DiscoveryPublishService", ExplodingService) + monkeypatch.setattr( + activities, + "_upload_file_to_minio", + lambda *a, **k: ("minio://documents/network_publish_skipped.json", 45, "application/json"), + ) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "advisory"}) + monkeypatch.setattr(db, "get_latest_document_job", lambda workflow_id: None) + + recorded = {} + monkeypatch.setattr(db, "add_document_artifact", lambda **kwargs: recorded.update(kwargs)) + + result = await activities.publish_catalog_to_network("wf-5", "txn-5") + + assert result["status"] == "skipped" + assert result["message_id"] is None + assert recorded["artifact_type"] == "network_publish_skipped" + assert recorded["metadata"]["reason"] == "PUBLISH_TO_NETWORK is off" + @pytest.mark.unit @pytest.mark.asyncio async def test_publish_records_artifact_on_success(self, monkeypatch): diff --git a/tests/test_api.py b/tests/test_api.py index e9acfb5..822ea84 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -43,6 +43,21 @@ def test_pipeline_stages_endpoint(self, test_client): assert len(stages) > 0 assert all("id" in s and "label" in s for s in stages) + @pytest.mark.api + @pytest.mark.unit + def test_pipeline_stages_network_publish_follows_toggle(self, test_client, monkeypatch): + """publishing_to_network is listed only when PUBLISH_TO_NETWORK is on.""" + monkeypatch.delenv("DISABLE_PROD_SETTING", raising=False) + + monkeypatch.delenv("PUBLISH_TO_NETWORK", raising=False) + off = {s["id"] for s in test_client.get("/pipeline/stages").json()} + assert "publishing_to_network" not in off + assert "ingesting_prod" in off + + monkeypatch.setenv("PUBLISH_TO_NETWORK", "true") + on = {s["id"] for s in test_client.get("/pipeline/stages").json()} + assert "publishing_to_network" in on + class TestDocumentEndpoints: """Tests for document CRUD endpoints.""" diff --git a/tests/test_instances.py b/tests/test_instances.py index 560a375..8e7c043 100644 --- a/tests/test_instances.py +++ b/tests/test_instances.py @@ -112,3 +112,19 @@ def test_publishing_to_network_is_prod_only(self): stage_ids = [s[0] for s in PIPELINE_STAGES] assert stage_ids.index("publishing_to_network") == stage_ids.index("ingesting_prod") + 1 assert stage_ids.index("publishing_to_network") == stage_ids.index("completed") - 1 + + +@pytest.mark.unit +@pytest.mark.parametrize( + "value,expected", + [(None, False), ("", False), ("false", False), ("0", False), + ("true", True), ("TRUE", True), (" 1 ", True), ("yes", True)], +) +def test_publish_to_network_enabled(monkeypatch, value, expected): + from pipeline.instances import publish_to_network_enabled + + if value is None: + monkeypatch.delenv("PUBLISH_TO_NETWORK", raising=False) + else: + monkeypatch.setenv("PUBLISH_TO_NETWORK", value) + assert publish_to_network_enabled() is expected diff --git a/tests/test_master_catalog_pg.py b/tests/test_master_catalog_pg.py deleted file mode 100644 index cc7a747..0000000 --- a/tests/test_master_catalog_pg.py +++ /dev/null @@ -1,347 +0,0 @@ -"""Tests for the Postgres Master Catalog + Redis push (master_catalog_pg).""" - -from __future__ import annotations - -import json -import os - -import pytest - - -class FakeCursor: - def __init__(self, store: dict): - self.store = store - self._last_result = None - - def __enter__(self): - return self - - def __exit__(self, *exc): - return False - - def execute(self, sql: str, params=None): - sql_norm = " ".join(sql.split()) - if sql_norm.startswith("INSERT INTO master_catalog"): - (code, content_type, name, tool_name, doc_id, prompt_snippet, - aliases, status, workflow_id, instance, instance_name, - updated_at) = params - existing = self.store.setdefault("rows", {}).get(code) - # Mirrors the real ON CONFLICT: status accumulates tiers, never overwrites. - merged_status = sorted({*(existing["status"] if existing else []), status}) - self.store["rows"][code] = { - "code": code, - "content_type": content_type, - "name": name, - "tool_name": tool_name, - "doc_id": doc_id, - "prompt_snippet": prompt_snippet, - "aliases": aliases, - "status": merged_status, - "workflow_id": workflow_id, - "instance": instance, - "instance_name": instance_name, - "updated_at": updated_at, - } - self._last_result = None - elif sql_norm.startswith("UPDATE master_catalog_meta"): - self.store["version"] = self.store.get("version", 0) + 1 - self._last_result = (self.store["version"],) - elif sql_norm.startswith("SELECT * FROM master_catalog WHERE status"): - statuses = set(params[0]) - rows = [r for r in self.store.get("rows", {}).values() if statuses & set(r["status"])] - self._last_result = sorted(rows, key=lambda r: r["code"]) - elif sql_norm.startswith("CREATE TABLE") or "INSERT INTO master_catalog_meta" in sql_norm: - self._last_result = None - else: - raise AssertionError(f"Unexpected SQL in FakeCursor: {sql_norm}") - - def fetchone(self): - return self._last_result - - def fetchall(self): - return self._last_result or [] - - -class FakeConnection: - def __init__(self, store: dict): - self.store = store - self.closed = False - - def cursor(self, cursor_factory=None): - return FakeCursor(self.store) - - def close(self): - self.closed = True - - -class FakeRedis: - def __init__(self): - self.data: dict[str, str] = {} - self.ttls: dict[str, int] = {} - - def set(self, key, value, ex=None): - self.data[key] = value - self.ttls[key] = ex - - -@pytest.fixture -def catalog_pg(monkeypatch, temp_db_path): - from pipeline import db - from pipeline import master_catalog_pg as pg - - db.DB_PATH = temp_db_path - db.init_db() - - store: dict = {} - fake_conn = FakeConnection(store) - monkeypatch.setattr(pg, "get_pg_connection", lambda: fake_conn) - - fake_redis = FakeRedis() - monkeypatch.setattr(pg, "get_redis_client", lambda: fake_redis) - - yield db, pg, store, fake_redis - - db.DB_PATH = os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db") - - -def _make_scheme_doc(db, workflow_id="wf-1", scheme_code="pm-ddky", tool_routing="qdrant", aliases=None, instance="default"): - db.upsert_document( - workflow_id=workflow_id, - document_id=f"doc-{workflow_id}", - filename="scheme.pdf", - filepath="/tmp/scheme.pdf", - stage="approval_for_prod", - page_count=5, - chunk_count=3, - instance=instance, - ) - db.update_document_fields( - workflow_id, - document_kind="scheme", - scheme_code=scheme_code, - scheme_name="Prime Minister Dhan-Dhaanya Krishi Yojana", - tool_routing=tool_routing, - scheme_aliases_json=json.dumps(aliases) if aliases is not None else None, - ) - - -def test_resolve_tool_name(): - from pipeline.master_catalog_pg import _resolve_tool_name - - assert _resolve_tool_name("legacy") == "get_scheme_info" - assert _resolve_tool_name("qdrant") == "search_schemes" - assert _resolve_tool_name("both") == "search_schemes" - assert _resolve_tool_name(None) == "search_schemes" - - -def test_generate_prompt_snippet(): - from pipeline.master_catalog_pg import _generate_prompt_snippet - - assert ( - _generate_prompt_snippet("Micro Irrigation Fund", "mif", "scheme") - == "Refer to 'Micro Irrigation Fund' (code: mif) for guidance on this government scheme." - ) - assert ( - _generate_prompt_snippet("Weekly advisory", "wa", "advisory") - == "Refer to 'Weekly advisory' (code: wa) for guidance on this advisory." - ) - - -def test_sync_catalog_entry_rejects_bad_status(catalog_pg): - db, pg, store, fake_redis = catalog_pg - with pytest.raises(ValueError): - pg.sync_catalog_entry("wf-1", "prod") - - -def test_sync_catalog_entry_skips_missing_document(catalog_pg): - db, pg, store, fake_redis = catalog_pg - assert pg.sync_catalog_entry("does-not-exist", "dev") is None - assert store.get("rows") is None - - -def test_sync_catalog_entry_syncs_non_scheme_document_kind(catalog_pg): - db, pg, store, fake_redis = catalog_pg - db.upsert_document( - workflow_id="wf-doc-only", - document_id="doc-1", - filename="general.pdf", - filepath="/tmp/general.pdf", - stage="completed", - page_count=1, - chunk_count=1, - instance="default", - ) - # document_kind defaults to "document" — still synced, no eligibility gate - version = pg.sync_catalog_entry("wf-doc-only", "dev") - assert version == 1 - assert "doc-1" in store["rows"] - # No scheme_name — falls back to filename, not the bare code, so the - # prompt snippet is still readable. - assert store["rows"]["doc-1"]["name"] == "general.pdf" - - -def test_sync_catalog_entry_falls_back_to_doc_id_when_no_scheme_code(catalog_pg): - db, pg, store, fake_redis = catalog_pg - db.upsert_document( - workflow_id="wf-no-code", - document_id="doc-2", - filename="scheme.pdf", - filepath="/tmp/scheme.pdf", - stage="approval_for_prod", - page_count=1, - chunk_count=1, - instance="default", - ) - db.update_document_fields("wf-no-code", document_kind="scheme") - version = pg.sync_catalog_entry("wf-no-code", "dev") - assert version == 1 - assert "doc-2" in store["rows"] - - -def test_sync_catalog_entry_dev_upserts_and_pushes_redis(catalog_pg): - db, pg, store, fake_redis = catalog_pg - _make_scheme_doc(db, workflow_id="wf-1", scheme_code="pm-ddky", tool_routing="qdrant") - - version = pg.sync_catalog_entry("wf-1", "dev") - assert version == 1 - - row = store["rows"]["pm-ddky"] - assert row["content_type"] == "scheme" - assert row["name"] == "Prime Minister Dhan-Dhaanya Krishi Yojana" - assert row["tool_name"] == "search_schemes" - assert row["doc_id"] == "doc-wf-1" - assert row["prompt_snippet"] == ( - "Refer to 'Prime Minister Dhan-Dhaanya Krishi Yojana' (code: pm-ddky) " - "for guidance on this government scheme." - ) - assert row["status"] == ["dev"] - - dev_snapshot = json.loads(fake_redis.data["master-catalog:dev:snapshot"]) - assert dev_snapshot["version"] == 1 - codes = {e["code"] for e in dev_snapshot["entries"]} - assert codes == {"pm-ddky"} - - live_snapshot = json.loads(fake_redis.data["master-catalog:live:snapshot"]) - assert live_snapshot["entries"] == [] # not promoted yet - - assert fake_redis.ttls["master-catalog:dev:snapshot"] == 172800 - - -def test_sync_catalog_entry_live_appears_in_both_tiers(catalog_pg): - db, pg, store, fake_redis = catalog_pg - _make_scheme_doc(db, workflow_id="wf-2", scheme_code="mif", tool_routing="legacy") - - pg.sync_catalog_entry("wf-2", "dev") - pg.sync_catalog_entry("wf-2", "live") - - assert store["rows"]["mif"]["status"] == ["dev", "live"] - assert store["rows"]["mif"]["tool_name"] == "get_scheme_info" - - dev_snapshot = json.loads(fake_redis.data["master-catalog:dev:snapshot"]) - live_snapshot = json.loads(fake_redis.data["master-catalog:live:snapshot"]) - assert {e["code"] for e in dev_snapshot["entries"]} == {"mif"} - assert {e["code"] for e in live_snapshot["entries"]} == {"mif"} - - -def test_sync_catalog_entry_dev_resync_after_live_does_not_regress_status(catalog_pg): - """A reingest-to-dev after promotion must not silently drop the entry from - the live snapshot — status accumulates tiers, it doesn't get overwritten.""" - db, pg, store, fake_redis = catalog_pg - _make_scheme_doc(db, workflow_id="wf-5", scheme_code="nfsm") - - pg.sync_catalog_entry("wf-5", "dev") - pg.sync_catalog_entry("wf-5", "live") - pg.sync_catalog_entry("wf-5", "dev") # e.g. a later reingest to DEV - - assert store["rows"]["nfsm"]["status"] == ["dev", "live"] - live_snapshot = json.loads(fake_redis.data["master-catalog:live:snapshot"]) - assert {e["code"] for e in live_snapshot["entries"]} == {"nfsm"} - - -def test_sync_catalog_entry_upsert_no_duplicates(catalog_pg): - db, pg, store, fake_redis = catalog_pg - _make_scheme_doc(db, workflow_id="wf-3", scheme_code="cdp") - - pg.sync_catalog_entry("wf-3", "dev") - pg.sync_catalog_entry("wf-3", "dev") - - assert len(store["rows"]) == 1 - dev_snapshot = json.loads(fake_redis.data["master-catalog:dev:snapshot"]) - assert len(dev_snapshot["entries"]) == 1 - - -def test_sync_catalog_entry_stores_aliases(catalog_pg): - db, pg, store, fake_redis = catalog_pg - _make_scheme_doc( - db, workflow_id="wf-6", scheme_code="mif", - aliases=["MIF", "micro irrigation fund"], - ) - - pg.sync_catalog_entry("wf-6", "dev") - - # Curator aliases are preserved and lead, then bootstrap/derived ones are - # merged in so a scheme is never searchable by its exact name alone. - stored = store["rows"]["mif"]["aliases"] - assert stored[:2] == ["MIF", "micro irrigation fund"] - assert len(stored) > 2 - lowered = {a.lower() for a in stored} - assert "micro irrigation fund scheme" in lowered # from the bootstrap list - assert "mif" in lowered - - dev_snapshot = json.loads(fake_redis.data["master-catalog:dev:snapshot"]) - entry = next(e for e in dev_snapshot["entries"] if e["code"] == "mif") - assert entry["aliases"] == stored - - -def test_sync_catalog_entry_stores_instance_name(catalog_pg): - """The readable state name rides along with the instance code.""" - db, pg, store, fake_redis = catalog_pg - _make_scheme_doc(db, workflow_id="wf-inst", scheme_code="mif-2", instance="mh") - - pg.sync_catalog_entry("wf-inst", "dev") - - assert store["rows"]["mif-2"]["instance"] == "mh" - assert store["rows"]["mif-2"]["instance_name"] == "Maharashtra" - dev_snapshot = json.loads(fake_redis.data["master-catalog:dev:snapshot"]) - entry = next(e for e in dev_snapshot["entries"] if e["code"] == "mif-2") - assert entry["instance_name"] == "Maharashtra" - - -def test_vector_schemes_prompt_block_excludes_legacy_schemes(catalog_pg): - """The dynamic prompt block must only ever be built from search_schemes - (vector-indexed) entries — legacy get_scheme_info schemes stay a - separate, hardcoded list on the AI-layer side and must never leak in.""" - db, pg, store, fake_redis = catalog_pg - _make_scheme_doc( - db, workflow_id="wf-7", scheme_code="pm-ddky", tool_routing="qdrant", - aliases=["PM-DDKY", "dhan-dhaanya"], - ) - _make_scheme_doc( - db, workflow_id="wf-8", scheme_code="kcc", tool_routing="legacy", - ) - - pg.sync_catalog_entry("wf-7", "dev") - pg.sync_catalog_entry("wf-8", "dev") - - dev_snapshot = json.loads(fake_redis.data["master-catalog:dev:snapshot"]) - prompt = dev_snapshot["prompt"] - assert prompt["vector_scheme_count"] == 1 - assert "pm-ddky" in prompt["vector_schemes_bullets"] - assert "kcc" not in prompt["vector_schemes_bullets"] - assert prompt["vector_schemes_bullets"] == "- **Prime Minister Dhan-Dhaanya Krishi Yojana** (pm-ddky)" - # The stored alias list is broad for recall, but the prompt is capped so a - # long derived tail cannot bloat every AI request. - identifiers = prompt["vector_schemes_identifiers"] - assert identifiers.startswith("- `pm-ddky` / PM-DDKY / dhan-dhaanya") - assert identifiers.count(" / ") <= pg._PROMPT_ALIAS_LIMIT - assert len(store["rows"]["pm-ddky"]["aliases"]) > pg._PROMPT_ALIAS_LIMIT - - -def test_sync_catalog_entry_without_redis_host_skips_push_but_still_upserts(catalog_pg, monkeypatch): - db, pg, store, fake_redis = catalog_pg - monkeypatch.setattr(pg, "get_redis_client", lambda: None) - _make_scheme_doc(db, workflow_id="wf-4", scheme_code="pkvy") - - version = pg.sync_catalog_entry("wf-4", "dev") - assert version == 1 - assert "pkvy" in store["rows"] diff --git a/ui/src/components/PipelineStepper.jsx b/ui/src/components/PipelineStepper.jsx index d0da407..b4fe9bb 100644 --- a/ui/src/components/PipelineStepper.jsx +++ b/ui/src/components/PipelineStepper.jsx @@ -16,10 +16,19 @@ import { cn } from '../lib/utils' // would leave every finished document looking stuck. const PROD_ONLY_STEPS = new Set(['approval_for_prod', 'ingesting_prod', 'publishing_to_network']) +// Publishing to the network also needs PUBLISH_TO_NETWORK; without it the +// backend leaves the stage out of /pipeline/stages and so does the stepper. +const NETWORK_PUBLISH_STEP = 'publishing_to_network' + function resolveEffectiveIndex(stages, currentStage, hasPages, hasChunks) { const isFailed = currentStage === 'failed' const userStepId = mapStageToUserStep(currentStage) let effectiveIndex = stages.findIndex(stage => stage.id === userStepId) + if (effectiveIndex < 0 && userStepId === NETWORK_PUBLISH_STEP) { + // Stage is hidden but the document still passes through it briefly; keep + // showing the previous step as active instead of jumping back to the start. + effectiveIndex = stages.findIndex(stage => stage.id === 'ingesting_prod') + } if (isFailed) { // Approximate where failure happened so completed steps stay checked. @@ -48,10 +57,13 @@ function stageStatus(index, effectiveIndex, isFailed) { } export default function PipelineStepper({ currentStage, hasPages = false, hasChunks = false, className }) { - const { prodEnabled } = usePipelineConfig() + const { prodEnabled, networkPublishEnabled } = usePipelineConfig() const stages = React.useMemo( - () => (prodEnabled ? USER_PIPELINE_STAGES : USER_PIPELINE_STAGES.filter(s => !PROD_ONLY_STEPS.has(s.id))), - [prodEnabled], + () => + USER_PIPELINE_STAGES.filter( + s => (prodEnabled || !PROD_ONLY_STEPS.has(s.id)) && (networkPublishEnabled || s.id !== NETWORK_PUBLISH_STEP), + ), + [prodEnabled, networkPublishEnabled], ) const { isFailed, effectiveIndex } = resolveEffectiveIndex(stages, currentStage, hasPages, hasChunks) const total = stages.length diff --git a/ui/src/lib/usePipelineConfig.js b/ui/src/lib/usePipelineConfig.js index c0b5f32..b8a8db0 100644 --- a/ui/src/lib/usePipelineConfig.js +++ b/ui/src/lib/usePipelineConfig.js @@ -5,12 +5,13 @@ import { fetchJson } from './pipelineUi' * Runtime pipeline shape, read once from the backend. * * `/pipeline/stages` omits the PROD stages when DISABLE_PROD_SETTING is on, so - * their presence is what tells the UI whether promotion exists at all. Cached + * their presence is what tells the UI whether promotion exists at all. It also + * omits `publishing_to_network` unless PUBLISH_TO_NETWORK is on. Cached * at module level: the flag is deployment-wide and cannot change while the tab * is open, so every stepper on the page shares one request. */ -const DEFAULT_CONFIG = { prodEnabled: true, loaded: false } +const DEFAULT_CONFIG = { prodEnabled: true, networkPublishEnabled: true, loaded: false } let cache = null let inflight = null @@ -21,7 +22,11 @@ export function fetchPipelineConfig() { inflight = fetchJson('/pipeline/stages') .then(stages => { const ids = new Set((Array.isArray(stages) ? stages : []).map(s => s.id)) - cache = { prodEnabled: ids.has('approval_for_prod'), loaded: true } + cache = { + prodEnabled: ids.has('approval_for_prod'), + networkPublishEnabled: ids.has('publishing_to_network'), + loaded: true, + } return cache }) .catch(() => { From 5f9e03df9cfe155224ddd2c54ec9d012fb95e1ad Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 14 Oct 2026 15:42:08 +0530 Subject: [PATCH 02/53] feat: add benchmark docker-compose stack, Postman collection, and network publish config options --- .env.example | 38 +++++++++++++------------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/.env.example b/.env.example index 0e27f27..d6338fa 100644 --- a/.env.example +++ b/.env.example @@ -193,32 +193,15 @@ VITE_KEYCLOAK_IDP_HINT=google KEYCLOAK_ADMIN_USERNAME=admin KEYCLOAK_ADMIN_PASSWORD= -# --- Master Catalog (Postgres) --- -# code/name/tool_name/prompt_snippet, synced on DEV ingest (status=dev) -# and PROD promote (status=live). No DB container is provisioned in docker-compose — -# point these at your existing Postgres instance (any host/server is fine; this just -# needs its own database + user on it, e.g. `CREATE DATABASE master_catalog`). -MASTER_CATALOG_PG_HOST= -MASTER_CATALOG_PG_PORT=5432 -MASTER_CATALOG_PG_DB=master_catalog -MASTER_CATALOG_PG_USER=master_catalog -MASTER_CATALOG_PG_PASSWORD= -MASTER_CATALOG_PG_SSLMODE=disable - -# --- AI layer Redis (bharat-oan-api) --- -# docs-pipeline pushes the catalog snapshot directly into this Redis instance -# after every Postgres sync. Leave AI_LAYER_REDIS_HOST empty to disable the -# push (Postgres sync still happens; only the Redis mirror is skipped). -AI_LAYER_REDIS_HOST= -AI_LAYER_REDIS_PORT=6379 -AI_LAYER_REDIS_DB=0 -AI_LAYER_REDIS_PASSWORD= -MASTER_CATALOG_REDIS_TTL_SECONDS=172800 - # --- Publish to Network (Discovery Service) --- -# After DEV ingest, the publishing_to_network stage POSTs a Beckn catalog/publish -# envelope announcing the kind of knowledge this provider serves. Required — there -# is no feature flag to disable the stage. Worker-only; the API and UI never read these. +# After a document is promoted to PROD, the publishing_to_network stage POSTs a +# Beckn catalog/publish envelope announcing the kind of knowledge this provider +# serves. Publishing is off by default: set PUBLISH_TO_NETWORK=true to enable it. +# DISCOVERY_SERVICE_ENDPOINT and NETWORK_SENDER_ID are required only when it is on. +# The worker reads the Discovery Service settings. The API also reads +# PUBLISH_TO_NETWORK, to leave the stage out of the pipeline stage list; the UI +# never reads any of these. +PUBLISH_TO_NETWORK=false DISCOVERY_SERVICE_ENDPOINT= NETWORK_SENDER_ID= # Publicly reachable base URL of this provider, sent as each catalog's bppUri. @@ -259,3 +242,8 @@ KC_SMTP_STARTTLS=true KC_SMTP_SSL=false KC_SMTP_USERNAME= KC_SMTP_PASSWORD= + +# Publish the catalog to the Beckn Discovery Service after PROD promotion. +# Off by default. DISCOVERY_SERVICE_ENDPOINT and NETWORK_SENDER_ID are +# required only when this is true. +PUBLISH_TO_NETWORK=false \ No newline at end of file From 212207ee1b82dfd4a7ed6299ada3a291e9c1bd1f Mon Sep 17 00:00:00 2001 From: Naveen Date: Mon, 19 Oct 2026 14:30:04 +0530 Subject: [PATCH 03/53] feat: add x-api-key protected POST /webhook endpoint Accepts a JSON array of records and upserts them into a new SQLite webhook_records table keyed on (source, external_id). Auth via x-api-key checked against WEBHOOK_API_KEY (401 on mismatch, 503 when unset). Body capped by WEBHOOK_MAX_RECORDS (413). Store only; no ingestion. Ported from OpenAgriNet/docs-pipeline#165. --- .env.example | 11 +++- ENV.md | 9 +++ README.md | 1 + pipeline/api.py | 44 +++++++++++++- pipeline/db.py | 77 +++++++++++++++++++++++++ tests/test_webhook.py | 129 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 268 insertions(+), 3 deletions(-) create mode 100644 tests/test_webhook.py diff --git a/.env.example b/.env.example index d6338fa..5e26b69 100644 --- a/.env.example +++ b/.env.example @@ -246,4 +246,13 @@ KC_SMTP_PASSWORD= # Publish the catalog to the Beckn Discovery Service after PROD promotion. # Off by default. DISCOVERY_SERVICE_ENDPOINT and NETWORK_SENDER_ID are # required only when this is true. -PUBLISH_TO_NETWORK=false \ No newline at end of file +PUBLISH_TO_NETWORK=false + +# Shared secret for POST /webhook. Callers must send it in the `x-api-key` header. +# Required: if unset or empty, /webhook returns 503. Wrong or missing key returns 401. +# Generate a 32-char hex value with: openssl rand -hex 16 +# Never commit a real value. +WEBHOOK_API_KEY=your_32_character_hex_key_here + +# Max records accepted per POST /webhook request (more returns 413) +WEBHOOK_MAX_RECORDS=500 diff --git a/ENV.md b/ENV.md index 40a3b36..c4f7986 100644 --- a/ENV.md +++ b/ENV.md @@ -78,6 +78,15 @@ Root `.env` (see `.env.example`). Required by FastAPI (`pipeline/api.py`), Tempo | `DOCS_PIPELINE_UI_URL` | `http://localhost:3000` | Public UI base for links | | `DEFAULT_INSTANCE` | `default` | Default tenant / instance id | +### Webhook (`POST /webhook`) + +| Variable | Default | Purpose | +|----------|---------|---------| +| `WEBHOOK_MAX_RECORDS` | `500` | Max records per `/webhook` request; more returns 413 | +| `WEBHOOK_API_KEY` | *(empty — required)* | 32-char hex secret. Callers send it in the `x-api-key` header. Missing/invalid header → 401. Unset/empty → 503 (fails closed, ignores `AUTH_DISABLED`). Generate with `openssl rand -hex 16`. | + +Body is a JSON array of objects (max `WEBHOOK_MAX_RECORDS`); stored in the SQLite `webhook_records` table, upserted on `(source, external_id)` where `external_id` is `contentID` (else `id`, `slug`, payload hash). + ### Auth / Keycloak (backend JWT validation) | Variable | Default | Purpose | diff --git a/README.md b/README.md index daa705f..ab9db9c 100644 --- a/README.md +++ b/README.md @@ -205,6 +205,7 @@ It owns: - audit logs - search settings - document/index status +- inbound webhook records (`webhook_records`, written by `POST /webhook`, `x-api-key` = `WEBHOOK_API_KEY`) ### MinIO diff --git a/pipeline/api.py b/pipeline/api.py index 570eb70..1d86056 100644 --- a/pipeline/api.py +++ b/pipeline/api.py @@ -17,7 +17,7 @@ from datetime import datetime from io import BytesIO from pathlib import Path -from typing import Optional +from typing import Any, Dict, List, Optional # Load repo-root .env so KEYCLOAK_ADMIN_* etc. work without manual `source .env` try: @@ -31,7 +31,7 @@ from urllib.parse import quote -from fastapi import FastAPI, File, Header, HTTPException, Query, Request, UploadFile +from fastapi import Depends, FastAPI, File, Header, HTTPException, Query, Request, UploadFile from fastapi import Path as PathParam from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import StreamingResponse @@ -4020,6 +4020,46 @@ async def health(): } +# ============================================================================= +# Webhook (inbound pushes from external systems, x-api-key protected) +# ============================================================================= + +WEBHOOK_MAX_RECORDS = int(os.environ.get("WEBHOOK_MAX_RECORDS", "500")) + + +def require_webhook_key(request: Request) -> None: + """Dependency: x-api-key must equal WEBHOOK_API_KEY. Fails closed when unset.""" + import hmac + + expected = (os.environ.get("WEBHOOK_API_KEY") or "").strip() + if not expected: + raise HTTPException(503, "WEBHOOK_API_KEY is not configured") + provided = (request.headers.get("x-api-key") or "").strip() + if not provided or not hmac.compare_digest(provided, expected): + raise HTTPException(401, "Missing or invalid x-api-key") + + +@app.post("/webhook") +@limiter.limit(RATE_LIMIT_UPLOAD) +async def webhook( + request: Request, + records: List[Dict[str, Any]], + _: None = Depends(require_webhook_key), +): + """Accept a JSON array of records and upsert them into webhook_records.""" + if len(records) > WEBHOOK_MAX_RECORDS: + raise HTTPException(413, f"Too many records (max {WEBHOOK_MAX_RECORDS})") + try: + count = db.upsert_webhook_records("webhook", records) + except ValueError as e: + raise HTTPException(422, str(e)) + return { + "success": True, + "message": "Webhook processed successfully.", + "recordCount": count, + } + + # ============================================================================= # Vector index (passage schema) # ============================================================================= diff --git a/pipeline/db.py b/pipeline/db.py index 30b3ff0..22fd9bc 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -9,6 +9,7 @@ import json import os import sqlite3 +import uuid from contextlib import contextmanager from datetime import datetime from pathlib import Path @@ -385,7 +386,83 @@ def init_db(): last_sent_at TEXT NOT NULL ) """) + # Generic inbound webhook records (full body kept as JSON text in payload). + conn.execute(""" + CREATE TABLE IF NOT EXISTS webhook_records ( + id TEXT PRIMARY KEY, -- uuid4 + source TEXT NOT NULL DEFAULT 'webhook', + external_id TEXT NOT NULL, + title TEXT, + slug TEXT, + status TEXT, + is_published INTEGER, + category TEXT, + payload TEXT NOT NULL, + payload_hash TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + UNIQUE (source, external_id) + ) + """) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_webhook_records_source_status " + "ON webhook_records (source, status)" + ) + conn.commit() + + +def upsert_webhook_records(source: str, records: list) -> int: + """Upsert records by (source, external_id) in one transaction; returns len(records). + + external_id comes from contentID, else id, else slug, else a hash of the payload. + Unchanged payloads (same payload_hash) are not rewritten but still counted. + """ + now = datetime.utcnow().isoformat() + rows = [] + for rec in records: + if not isinstance(rec, dict): + raise ValueError("Each record must be a JSON object") + payload = json.dumps(rec, sort_keys=True, ensure_ascii=False, default=str) + payload_hash = hashlib.sha256(payload.encode("utf-8")).hexdigest() + ext = rec.get("contentID") + if ext in (None, ""): + ext = rec.get("id") + if ext in (None, ""): + ext = rec.get("slug") + if ext in (None, ""): + ext = payload_hash + pub = rec.get("isPublished") + rows.append(( + str(uuid.uuid4()), source, str(ext), rec.get("title"), rec.get("slug"), rec.get("status"), + int(pub) if isinstance(pub, (bool, int)) else None, + rec.get("domainName"), payload, payload_hash, + rec.get("createdAt") or now, rec.get("updatedAt") or now, + )) + with _db_lock: + with get_connection() as conn: + for r in rows: + conn.execute( + """ + INSERT INTO webhook_records + (id, source, external_id, title, slug, status, is_published, category, + payload, payload_hash, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(source, external_id) DO UPDATE SET + title = excluded.title, + slug = excluded.slug, + status = excluded.status, + is_published = excluded.is_published, + category = excluded.category, + payload = excluded.payload, + payload_hash = excluded.payload_hash, + created_at = excluded.created_at, + updated_at = excluded.updated_at + WHERE webhook_records.payload_hash IS NOT excluded.payload_hash + """, + r, + ) conn.commit() + return len(rows) def store_email_otp( diff --git a/tests/test_webhook.py b/tests/test_webhook.py new file mode 100644 index 0000000..d218885 --- /dev/null +++ b/tests/test_webhook.py @@ -0,0 +1,129 @@ +"""Tests for POST /webhook (x-api-key protected).""" + +import os +import uuid +from unittest.mock import patch + +import pytest + +TEST_KEY = "0123456789abcdef0123456789abcdef" + +SAMPLE = [ + { + "contentID": 101, + "title": "Bacterial Leaf Blight in Rice", + "domainName": "Agriculture", + "status": "Published", + "isPublished": 1, + "slug": "101/bacterial-leaf-blight-in-rice", + "createdAt": "2026-09-09T07:00:00.000Z", + "updatedAt": "2026-09-09T07:00:00.000Z", + "popCount": 1, + "popList": [{"popID": 1001, "contentID": 101, "keywords": ["rice"]}], + } +] + +OK_BODY = {"success": True, "message": "Webhook processed successfully.", "recordCount": 1} + + +@pytest.fixture +def test_client(db_connection): + """TestClient without lifespan (no Temporal/MinIO needed for /webhook).""" + from fastapi.testclient import TestClient + from pipeline import api + + return TestClient(api.app) + + +@pytest.fixture +def key_env(): + with patch.dict(os.environ, {"WEBHOOK_API_KEY": TEST_KEY}): + yield + + +def _rows(db_connection): + from pipeline import db + + with db.get_connection() as conn: + return conn.execute("SELECT * FROM webhook_records").fetchall() + + +@pytest.mark.api +@pytest.mark.unit +class TestWebhookAuth: + def test_missing_header_401(self, test_client, db_connection, key_env): + assert test_client.post("/webhook", json=SAMPLE).status_code == 401 + + def test_empty_header_401(self, test_client, db_connection, key_env): + r = test_client.post("/webhook", json=SAMPLE, headers={"x-api-key": ""}) + assert r.status_code == 401 + + def test_wrong_key_401(self, test_client, db_connection, key_env): + r = test_client.post("/webhook", json=SAMPLE, headers={"x-api-key": "nope"}) + assert r.status_code == 401 + + @pytest.mark.parametrize("value", [None, ""]) + def test_unconfigured_key_503_even_if_auth_disabled(self, test_client, db_connection, value): + env = {"AUTH_DISABLED": "true"} + if value is not None: + env["WEBHOOK_API_KEY"] = value + with patch.dict(os.environ, env): + if value is None: + os.environ.pop("WEBHOOK_API_KEY", None) + r = test_client.post("/webhook", json=SAMPLE, headers={"x-api-key": "anything"}) + assert r.status_code == 503 + assert "WEBHOOK_API_KEY" in r.json()["detail"] + + +@pytest.mark.api +@pytest.mark.unit +class TestWebhookIngest: + H = {"x-api-key": TEST_KEY} + + def test_valid_payload(self, test_client, db_connection, key_env): + r = test_client.post("/webhook", json=SAMPLE, headers=self.H) + assert r.status_code == 200 + assert r.json() == OK_BODY + rows = _rows(db_connection) + assert len(rows) == 1 + assert rows[0]["external_id"] == "101" + assert uuid.UUID(rows[0]["id"]).version == 4 + assert rows[0]["category"] == "Agriculture" + assert '"popList"' in rows[0]["payload"] + + def test_repost_is_idempotent(self, test_client, db_connection, key_env): + test_client.post("/webhook", json=SAMPLE, headers=self.H) + first_id = _rows(db_connection)[0]["id"] + changed = [{**SAMPLE[0], "title": "New title"}] + r = test_client.post("/webhook", json=changed, headers=self.H) + assert r.json()["recordCount"] == 1 + rows = _rows(db_connection) + assert len(rows) == 1 and rows[0]["title"] == "New title" + assert rows[0]["id"] == first_id + + def test_two_records(self, test_client, db_connection, key_env): + body = SAMPLE + [{**SAMPLE[0], "contentID": 102}] + r = test_client.post("/webhook", json=body, headers=self.H) + assert r.json()["recordCount"] == 2 + assert len(_rows(db_connection)) == 2 + + def test_empty_array(self, test_client, db_connection, key_env): + r = test_client.post("/webhook", json=[], headers=self.H) + assert r.status_code == 200 and r.json()["recordCount"] == 0 + + def test_non_array_422(self, test_client, db_connection, key_env): + r = test_client.post("/webhook", json={"contentID": 1}, headers=self.H) + assert r.status_code == 422 + + def test_non_object_item_422(self, test_client, db_connection, key_env): + r = test_client.post("/webhook", json=[1, 2], headers=self.H) + assert r.status_code == 422 + + def test_too_many_records_413(self, test_client, db_connection, key_env): + from pipeline import api + + with patch.object(api, "WEBHOOK_MAX_RECORDS", 1): + body = SAMPLE + [{**SAMPLE[0], "contentID": 102}] + r = test_client.post("/webhook", json=body, headers=self.H) + assert r.status_code == 413 + assert _rows(db_connection) == [] From ea76589ed01a006fcea4b80be4ce082891806cd1 Mon Sep 17 00:00:00 2001 From: Naveen Date: Mon, 26 Oct 2026 13:18:45 +0530 Subject: [PATCH 04/53] build: add Postgres service, psycopg dependencies and database config Add a postgres service to docker-compose, psycopg[binary,pool] and test dependencies, DATABASE_URL and DB_POOL_* settings, a Postgres service in the test workflow, and document the new variables in ENV.md. uv.lock also picks up unrelated lock refreshes that were already pending. --- .github/workflows/run-tests.yml | 16 ++ Dockerfile | 4 +- ENV.md | 12 +- docker-compose.yml | 63 ++++++- pipeline/config.py | 15 +- pyproject.toml | 4 + uv.lock | 291 +++++++++++++++++++------------- 7 files changed, 278 insertions(+), 127 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 6e01332..deaf75a 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -9,6 +9,22 @@ jobs: pipeline: name: pipeline (lint, test, coverage) runs-on: ubuntu-latest + services: + postgres: + image: postgres:15-alpine + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: knowledge_provider_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + TEST_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/knowledge_provider_test steps: - uses: actions/checkout@v4 diff --git a/Dockerfile b/Dockerfile index dcf1998..dd993cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,8 +29,10 @@ RUN --mount=type=cache,target=/root/.cache/uv \ ENV PATH="/app/.venv/bin:$PATH" -# Copy pipeline code +# Copy pipeline code and the SQL migrations the api/worker apply at startup COPY pipeline/ ./pipeline/ +COPY migrations/ ./migrations/ +COPY scripts/migrate_sqlite_to_postgres.py ./scripts/migrate_sqlite_to_postgres.py # Catch imports that work locally but fail on the shipped Python 3.10 image. RUN python -c "import pipeline.api" diff --git a/ENV.md b/ENV.md index c4f7986..df8b7b7 100644 --- a/ENV.md +++ b/ENV.md @@ -64,7 +64,11 @@ Root `.env` (see `.env.example`). Required by FastAPI (`pipeline/api.py`), Tempo | `LOG_LEVEL` | `INFO` | Worker log verbosity (`DEBUG`/`INFO`/`WARNING`/`ERROR`); the `temporalio` SDK logger stays at `INFO` regardless, to avoid its own noisy debug output | | `MINIO_ENDPOINT` | `localhost:9000` | MinIO API host:port | | `MINIO_BUCKET` | `documents` | Object storage bucket | -| `DOCUMENT_DB_PATH` | `/data/documents.db` | SQLite path (use `./data/documents.db` for local non-Docker) | +| `DATABASE_URL` | `postgresql://postgres:postgres@localhost:5432/knowledge_provider` | Application Postgres (documents, workflows, pages, chunks, audit, settings). The compose `postgres` service builds it from `DB_USERNAME` / `DB_PASSWORD` / `DB_NAME` (defaults `docs` / `docs` / `knowledge_provider`). Pending SQL migrations in `migrations/` are applied by the api and worker at startup | +| `DB_POOL_MIN_SIZE` | `1` | Connection pool minimum per process | +| `DB_POOL_MAX_SIZE` | `10` | Connection pool maximum per process (api and worker each hold their own pool) | +| `DB_POOL_TIMEOUT_SECONDS` | `30` | Seconds to wait for a free pooled connection | +| `DOCUMENT_DB_PATH` | `/data/documents.db` | Only read by `scripts/migrate_sqlite_to_postgres.py`: the legacy SQLite file to import | ### API HTTP surface @@ -85,7 +89,7 @@ Root `.env` (see `.env.example`). Required by FastAPI (`pipeline/api.py`), Tempo | `WEBHOOK_MAX_RECORDS` | `500` | Max records per `/webhook` request; more returns 413 | | `WEBHOOK_API_KEY` | *(empty — required)* | 32-char hex secret. Callers send it in the `x-api-key` header. Missing/invalid header → 401. Unset/empty → 503 (fails closed, ignores `AUTH_DISABLED`). Generate with `openssl rand -hex 16`. | -Body is a JSON array of objects (max `WEBHOOK_MAX_RECORDS`); stored in the SQLite `webhook_records` table, upserted on `(source, external_id)` where `external_id` is `contentID` (else `id`, `slug`, payload hash). +Body is a JSON array of objects (max `WEBHOOK_MAX_RECORDS`); stored in the `webhook_records` table, upserted on `(source, external_id)` where `external_id` is `contentID` (else `id`, `slug`, payload hash). ### Auth / Keycloak (backend JWT validation) @@ -259,7 +263,7 @@ First piece of the network-discovery push architecture: once a document's PROD p |---------|----|-----|--------| | `VITE_*` proxies & Keycloak browser login | ✅ | — | — | | JWT validation (`AUTH_*`, `KEYCLOAK_*`) | — | ✅ | — | -| Temporal / MinIO / SQLite | — | ✅ | ✅ | +| Temporal / MinIO / Postgres (`DATABASE_URL`) | — | ✅ | ✅ | | OCR / translation / chunking / domain tags | — | config / status | ✅ runs jobs | | Qdrant (`VECTOR_DB_*`) | — | ✅ | ✅ | | Publish to Network (`PUBLISH_TO_NETWORK`, `DISCOVERY_SERVICE_ENDPOINT`, `NETWORK_SENDER_ID`) | — | ✅ `PUBLISH_TO_NETWORK` only (stage list) | ✅ runs the publish activity | @@ -272,7 +276,7 @@ First piece of the network-discovery push architecture: once a document's PROD p ```bash set -a && source .env && set +a -export DOCUMENT_DB_PATH="$(pwd)/data/documents.db" +export DATABASE_URL="postgresql://docs:docs@localhost:5433/knowledge_provider" # the compose `postgres` service (`docker compose up -d postgres`) export ALLOWED_FILE_PATHS="$(pwd)/books,$(pwd)/data/documents" export CORS_ORIGINS="http://localhost:3000,http://localhost:3001" export PYTHONPATH="$(pwd)" diff --git a/docker-compose.yml b/docker-compose.yml index 1f6f11c..5a61620 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,7 @@ # API 127.0.0.1:${API_HOST_PORT:-8011}:8001 # UI 127.0.0.1:${UI_HOST_PORT:-3011}:80 # Keycloak 127.0.0.1:${KEYCLOAK_HOST_PORT:-8081}:8080 +# Postgres 127.0.0.1:${DB_HOST_PORT:-5433}:5432 # Temporal 127.0.0.1:7233:7233 # MinIO API 127.0.0.1:9000:9000 # MinIO UI 127.0.0.1:9001:9001 @@ -120,6 +121,30 @@ services: MINIO_BUCKET: ${MINIO_BUCKET:-documents} restart: "no" + # --------------------------------------------------------------------------- + # Application database — documents, workflows, pages, chunks, audit, settings. + # Schema is applied by api/worker at startup from migrations/ (see + # docs/POSTGRES_SCHEMA.md). A deployment that wants its own Postgres can set + # DATABASE_URL in .env and drop this service's depends_on entries. + # --------------------------------------------------------------------------- + postgres: + image: postgres:15-alpine + platform: linux/amd64 + environment: + POSTGRES_USER: ${DB_USERNAME:-docs} + POSTGRES_PASSWORD: ${DB_PASSWORD:-docs} + POSTGRES_DB: ${DB_NAME:-knowledge_provider} + ports: + - "127.0.0.1:${DB_HOST_PORT:-5433}:5432" + volumes: + - app-db-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USERNAME:-docs} -d ${DB_NAME:-knowledge_provider}"] + interval: 5s + timeout: 5s + retries: 10 + restart: unless-stopped + # --------------------------------------------------------------------------- # Qdrant — DEV vector index (see docs/SYSTEM_DESIGN.md's DEV vs PROD dual # indexing). api/worker's VECTOR_DB_URL is always pointed at this bundled @@ -249,7 +274,7 @@ services: TEMPORAL_HOST: temporal:7233 MINIO_ENDPOINT: minio:9000 VECTOR_DB_URL: http://qdrant:6333 - DOCUMENT_DB_PATH: /data/documents.db + DATABASE_URL: postgresql://${DB_USERNAME:-docs}:${DB_PASSWORD:-docs}@postgres:5432/${DB_NAME:-knowledge_provider} ALLOWED_FILE_PATHS: /app/books,/data/documents # HF cache path propagated into the container HF_HOME: ${HF_HOME:-/opt/hf-cache} @@ -257,6 +282,8 @@ services: ports: - "127.0.0.1:${API_HOST_PORT:-8011}:8001" depends_on: + postgres: + condition: service_healthy temporal: condition: service_healthy minio: @@ -268,7 +295,6 @@ services: command: uvicorn pipeline.api:app --host 0.0.0.0 --port 8001 volumes: - ./books:/app/books:ro - - sqlite-data:/data - documents-data:/data/documents - ${HF_HOME:-/opt/hf-cache}:${HF_HOME:-/opt/hf-cache} extra_hosts: @@ -291,11 +317,13 @@ services: TEMPORAL_HOST: temporal:7233 MINIO_ENDPOINT: minio:9000 VECTOR_DB_URL: http://qdrant:6333 - DOCUMENT_DB_PATH: /data/documents.db + DATABASE_URL: postgresql://${DB_USERNAME:-docs}:${DB_PASSWORD:-docs}@postgres:5432/${DB_NAME:-knowledge_provider} ALLOWED_FILE_PATHS: /app/books,/data/documents HF_HOME: ${HF_HOME:-/opt/hf-cache} HF_HUB_CACHE: ${HF_HOME:-/opt/hf-cache} depends_on: + postgres: + condition: service_healthy temporal: condition: service_healthy minio: @@ -307,13 +335,39 @@ services: command: python -m pipeline.worker volumes: - ./books:/app/books:ro - - sqlite-data:/data - documents-data:/data/documents - ${HF_HOME:-/opt/hf-cache}:${HF_HOME:-/opt/hf-cache} extra_hosts: - "host.docker.internal:host-gateway" restart: unless-stopped + # --------------------------------------------------------------------------- + # One-off SQLite -> Postgres data migration (maintenance window, api/worker + # stopped). The legacy volume is opened read-only by the script and is never + # modified, so rolling back is redeploying the previous images on it. + # docker compose --profile migrate run --rm migrate-sqlite # dry run + # MIGRATE_ARGS= docker compose --profile migrate run --rm migrate-sqlite # real run + # --------------------------------------------------------------------------- + migrate-sqlite: + image: ghcr.io/openagrinet/knowledge-provider:${IMAGE_TAG:-latest} + platform: linux/amd64 + profiles: + - migrate + build: + context: . + dockerfile: Dockerfile + environment: + DATABASE_URL: postgresql://${DB_USERNAME:-docs}:${DB_PASSWORD:-docs}@postgres:5432/${DB_NAME:-knowledge_provider} + DOCUMENT_DB_PATH: /legacy/documents.db + DEFAULT_INSTANCE: ${DEFAULT_INSTANCE:-default} + depends_on: + postgres: + condition: service_healthy + command: python scripts/migrate_sqlite_to_postgres.py ${MIGRATE_ARGS---dry-run} + volumes: + - sqlite-data:/legacy + restart: "no" + # --------------------------------------------------------------------------- # React UI # Dockerfile.prod builds a static nginx image; VITE_* args come from .env. @@ -349,5 +403,6 @@ volumes: keycloak-db-data: keycloak-data: minio-data: + app-db-data: sqlite-data: documents-data: diff --git a/pipeline/config.py b/pipeline/config.py index 7ec2a4b..426e41c 100644 --- a/pipeline/config.py +++ b/pipeline/config.py @@ -20,7 +20,10 @@ class Config: temporal_host: str = "localhost:7233" minio_endpoint: str = "localhost:9000" minio_bucket: str = "documents" - document_db_path: str = "/data/documents.db" + database_url: str = "postgresql://postgres:postgres@localhost:5432/knowledge_provider" + db_pool_min_size: int = 1 + db_pool_max_size: int = 10 + db_pool_timeout_seconds: float = 30.0 translation_provider: str = "gemma_vllm" translation_model: str = "google/gemma-4-31b-it" translation_vllm_base_url: str = "http://localhost:8020/v1" @@ -105,7 +108,10 @@ def load_config() -> Config: temporal_host=os.environ.get("TEMPORAL_HOST", "localhost:7233"), minio_endpoint=os.environ.get("MINIO_ENDPOINT", "localhost:9000"), minio_bucket=os.environ.get("MINIO_BUCKET", "documents"), - document_db_path=os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db"), + database_url=os.environ.get("DATABASE_URL", Config.database_url), + db_pool_min_size=int(os.environ.get("DB_POOL_MIN_SIZE", "1")), + db_pool_max_size=int(os.environ.get("DB_POOL_MAX_SIZE", "10")), + db_pool_timeout_seconds=float(os.environ.get("DB_POOL_TIMEOUT_SECONDS", "30")), translation_provider=os.environ.get("TRANSLATION_PROVIDER", "gemma_vllm"), translation_model=os.environ.get("TRANSLATION_MODEL", "google/gemma-4-31b-it"), translation_vllm_base_url=os.environ.get("TRANSLATION_VLLM_BASE_URL", "http://localhost:8020/v1"), @@ -158,7 +164,10 @@ def print_config_status(): ("TEMPORAL_HOST", "localhost:7233"), ("MINIO_ENDPOINT", "localhost:9000"), ("MINIO_BUCKET", "documents"), - ("DOCUMENT_DB_PATH", "/data/documents.db"), + ("DATABASE_URL", "postgresql://postgres:postgres@localhost:5432/knowledge_provider"), + ("DB_POOL_MIN_SIZE", "1"), + ("DB_POOL_MAX_SIZE", "10"), + ("DB_POOL_TIMEOUT_SECONDS", "30"), ("TRANSLATION_PROVIDER", "gemma_vllm"), ("TRANSLATION_MODEL", "gemma-4"), ("TRANSLATION_VLLM_BASE_URL", "http://localhost:8020/v1"), diff --git a/pyproject.toml b/pyproject.toml index 9792bb6..90d3e99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,9 @@ dependencies = [ "pypdf>=5.0.0", "PyMuPDF>=1.27.0", + # Postgres driver (sync) + connection pool + "psycopg[binary,pool]>=3.2", + # Rate limiting "slowapi>=0.1.9", @@ -63,6 +66,7 @@ dev = [ "pytest-asyncio>=0.21.0", "pytest-cov>=4.0.0", "ruff>=0.6.0", + "testcontainers[postgres]>=4.0.0", ] # sentence-transformers depends on torch; resolve it from the CPU-only wheel diff --git a/uv.lock b/uv.lock index 9292f9a..6a1bc0e 100644 --- a/uv.lock +++ b/uv.lock @@ -109,15 +109,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/34/32109943bace7729233cc4ee78530baa306d8cc3c6501a64ba8cb3b58129/argon2_cffi_bindings-26.1.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:0cc40f7b4050bb93eb67de95d2d759322fc7ce4930b9d645581ecf4913ec651e", size = 23584, upload-time = "2026-08-20T07:33:22.613Z" }, ] -[[package]] -name = "async-timeout" -version = "5.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a5/ae/136395dfbfe00dfc94da3f3e136d0b13f394cba8f4841120e34226265780/async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3", size = 9274, upload-time = "2024-11-06T16:41:39.6Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c", size = 6233, upload-time = "2024-11-06T16:41:37.9Z" }, -] - [[package]] name = "backports-asyncio-runner" version = "1.2.0" @@ -676,6 +667,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, ] +[[package]] +name = "docker" +version = "7.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "requests" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/88/7f/731ff914b0255d3d065f45fd4e626d4b8c95dbcbaada049f337a6ac16410/docker-7.2.0.tar.gz", hash = "sha256:cebb93773d334f778e023a7ee352a8d6e13ab1bd3b863a4d4a59dec897df43ac", size = 118731, upload-time = "2026-07-09T14:53:46.39Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/23/529140fe1aab80fc6992f93a706deec709140a6397439139a054e1515c45/docker-7.2.0-py3-none-any.whl", hash = "sha256:a3f45fdeb9165e2d25d9a1d02ddf3bc70fb572cf5ebbf9b58558c22caf29b71f", size = 148775, upload-time = "2026-07-09T14:53:45.224Z" }, +] + [[package]] name = "et-xmlfile" version = "2.0.0" @@ -690,7 +695,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1129,7 +1134,7 @@ dependencies = [ { name = "openpyxl" }, { name = "pdf-inspector" }, { name = "pillow" }, - { name = "psycopg2-binary" }, + { name = "psycopg", extra = ["binary", "pool"] }, { name = "pydantic" }, { name = "pyfranc" }, { name = "pyjwt", extra = ["crypto"] }, @@ -1137,7 +1142,6 @@ dependencies = [ { name = "pypdf" }, { name = "python-multipart" }, { name = "qdrant-client" }, - { name = "redis" }, { name = "sentence-transformers" }, { name = "slowapi" }, { name = "temporalio" }, @@ -1154,6 +1158,7 @@ dev = [ { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "ruff" }, + { name = "testcontainers" }, ] [package.metadata] @@ -1167,7 +1172,7 @@ requires-dist = [ { name = "openpyxl", specifier = ">=3.1.0" }, { name = "pdf-inspector", specifier = ">=1.14.0" }, { name = "pillow", specifier = ">=10.0.0" }, - { name = "psycopg2-binary", specifier = ">=2.9.0" }, + { name = "psycopg", extras = ["binary", "pool"], specifier = ">=3.2" }, { name = "pydantic", specifier = ">=2.0.0" }, { name = "pyfranc", specifier = ">=0.2.3" }, { name = "pyjwt", extras = ["crypto"], specifier = ">=2.8.0" }, @@ -1175,7 +1180,6 @@ requires-dist = [ { name = "pypdf", specifier = ">=5.0.0" }, { name = "python-multipart", specifier = ">=0.0.6" }, { name = "qdrant-client", specifier = ">=1.12.0" }, - { name = "redis", specifier = ">=5.0.0" }, { name = "sentence-transformers", specifier = "==5.1.0" }, { name = "slowapi", specifier = ">=0.1.9" }, { name = "temporalio", specifier = ">=1.0.0" }, @@ -1191,6 +1195,7 @@ dev = [ { name = "pytest-asyncio", specifier = ">=0.21.0" }, { name = "pytest-cov", specifier = ">=4.0.0" }, { name = "ruff", specifier = ">=0.6.0" }, + { name = "testcontainers", extras = ["postgres"], specifier = ">=4.0.0" }, ] [[package]] @@ -1938,66 +1943,109 @@ wheels = [ ] [[package]] -name = "psycopg2-binary" -version = "2.9.12" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2a/60/a3624f79acea344c16fbef3a94d28b89a8042ddfb8f3e4ca83f538671409/psycopg2_binary-2.9.12.tar.gz", hash = "sha256:5ac9444edc768c02a6b6a591f070b8aae28ff3a99be57560ac996001580f294c", size = 379686, upload-time = "2026-04-21T09:40:34.304Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/78/80/49bacf9e51617d8309f6f0123e29edc793f6f5f6700c7d1f1b20782fbb37/psycopg2_binary-2.9.12-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9b818ceff717f98851a64bffd4c5eb5b3059ae280276dcecc52ac658dcf006a4", size = 3712314, upload-time = "2026-04-20T23:33:31.363Z" }, - { url = "https://files.pythonhosted.org/packages/d3/f2/98eeac7d60c43df9338287834edf9b3e69be68a2db78a57b1b81d705e735/psycopg2_binary-2.9.12-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d2fa0d7caca8635c56e373055094eeda3208d901d55dd0ff5abc1d4e47f82b56", size = 3822389, upload-time = "2026-04-20T23:33:34.178Z" }, - { url = "https://files.pythonhosted.org/packages/9f/7c/30575e75f14d5351a56a1971bb43fe7f8bf7edf1b654fb1bec65c42a8812/psycopg2_binary-2.9.12-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:864c261b3690e1207d14bbfe0a61e27567981b80c47a778561e49f676f7ce433", size = 4578448, upload-time = "2026-04-20T23:33:37.073Z" }, - { url = "https://files.pythonhosted.org/packages/65/9b/4df366d89f28c527dc39d0b6c98a5ca74e30d37ac097b73f3352147568ae/psycopg2_binary-2.9.12-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c5ee5213445dd45312459029b8c4c0a695461eb517b753d2582315bd07995f5e", size = 4273705, upload-time = "2026-04-20T23:33:39.291Z" }, - { url = "https://files.pythonhosted.org/packages/4e/8a/c566803818eb03161ba869b6ba612bf7ad56816d98b9e5121e0a22ad6b0b/psycopg2_binary-2.9.12-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6f9cae1f848779b5b01f417e762c40d026ea93eb0648249a604728cda991dde3", size = 5893784, upload-time = "2026-04-20T23:33:41.658Z" }, - { url = "https://files.pythonhosted.org/packages/63/fe/0dfa5797e0b229e0567bc378695224caf14d547f73b05be0c80549089772/psycopg2_binary-2.9.12-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:63a3ebbd543d3d1eda088ac99164e8c5bac15293ee91f20281fd17d050aee1c4", size = 4109306, upload-time = "2026-04-20T23:33:43.953Z" }, - { url = "https://files.pythonhosted.org/packages/3c/89/28063adf17a4ba501eedd9890feab0c649ee4d8bd0a97df0ff1e9584feab/psycopg2_binary-2.9.12-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d6fcbba8c9fed08a73b8ac61ea79e4821e45b1e92bb466230c5e746bbf3d5256", size = 3654400, upload-time = "2026-04-20T23:33:46.115Z" }, - { url = "https://files.pythonhosted.org/packages/84/94/5a01de0aa4ead0b8d8d1aa4ec18cec0bd36d03fa714eaa5bb8a0b1b50020/psycopg2_binary-2.9.12-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:36512911ebb2b60a0c3e44d0bb5048c1980aced91235d133b7874f3d1d93487c", size = 3299215, upload-time = "2026-04-20T23:33:48.202Z" }, - { url = "https://files.pythonhosted.org/packages/7a/85/723bb085a61c6ac2dc0a0043f375f2fe7365363e27b073bad56ca5bda979/psycopg2_binary-2.9.12-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:8ffdb59fe88f99589e34354a130217aa1fd2d615612402d6edc8b3dbc7a44463", size = 3047724, upload-time = "2026-04-20T23:33:50.74Z" }, - { url = "https://files.pythonhosted.org/packages/b4/67/4d8b1e0d2fc4166677380eac0edf9cdff91013aca2546e8ef7bc04b56158/psycopg2_binary-2.9.12-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a46fe069b65255df410f856d842bc235f90e22ffdf532dda625fd4213d3fd9b1", size = 3349183, upload-time = "2026-04-20T23:33:59.635Z" }, - { url = "https://files.pythonhosted.org/packages/73/99/21af7a5498637ea4dc91a17c281a53bc1d632fbafe00f6689fbfb32a9fed/psycopg2_binary-2.9.12-cp310-cp310-win_amd64.whl", hash = "sha256:ab29414b25dcb698bf26bf213e3348abdcd07bbd5de032a5bec15bd75b298b03", size = 2757036, upload-time = "2026-04-20T23:34:01.606Z" }, - { url = "https://files.pythonhosted.org/packages/d5/19/d4ce60954f3bb9d8e3bc5e5c4d1f2487de2d3851bf2391d54954c9df12a6/psycopg2_binary-2.9.12-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5c8ce6c61bd1b1f6b9c24ee32211599f6166af2c55abb19456090a21fd16554b", size = 3712338, upload-time = "2026-04-20T23:34:03.961Z" }, - { url = "https://files.pythonhosted.org/packages/53/71/c85409ee0d78890f0660eff262e815e7dd2bb741a17611d82e9e8cd9dc5e/psycopg2_binary-2.9.12-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b4a9eaa6e7f4ff91bec10aa3fb296878e75187bced5cc4bafe17dc40915e1326", size = 3822407, upload-time = "2026-04-20T23:34:05.977Z" }, - { url = "https://files.pythonhosted.org/packages/3c/ed/60486c2c7f0d4d1ede2bfb1ed27e2498477ce646bc7f6b2759906303117e/psycopg2_binary-2.9.12-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c6528cefc8e50fcc6f4a107e27a672058b36cc5736d665476aeb413ba88dbb06", size = 4578425, upload-time = "2026-04-20T23:34:08.246Z" }, - { url = "https://files.pythonhosted.org/packages/0b/b9/656cb03fad9f4f49f2145c334b1126ee75189929ca4e6187d485a2d59951/psycopg2_binary-2.9.12-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e4e184b1fb6072bf05388aa41c697e1b2d01b3473f107e7ec44f186a32cfd0b8", size = 4273709, upload-time = "2026-04-20T23:34:10.974Z" }, - { url = "https://files.pythonhosted.org/packages/99/66/08cf0da0e25cc6fb142c89be45fc8418792858f0c4cbff5e24530ff02cd6/psycopg2_binary-2.9.12-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4766ab678563054d3f1d064a4db19cc4b5f9e3a8d9018592a8285cf200c248f3", size = 5893779, upload-time = "2026-04-20T23:34:13.905Z" }, - { url = "https://files.pythonhosted.org/packages/17/d7/eecd9ce8e146d3721115d82d3836efdbb712187e4590325df549989d18f4/psycopg2_binary-2.9.12-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5a0253224780c978746cb9be55a946bcdaf40fe3519c0f622924cdabdafe2c39", size = 4109308, upload-time = "2026-04-20T23:34:16.761Z" }, - { url = "https://files.pythonhosted.org/packages/b6/2e/b1dc289b362cc8d45697b57eefbd673186f49a4ea0906928988e3affcc98/psycopg2_binary-2.9.12-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0dc9228d47c46bda253d2ecd6bb93b56a9f2d7ad33b684a1fa3622bf74ffe30c", size = 3654405, upload-time = "2026-04-20T23:34:19.303Z" }, - { url = "https://files.pythonhosted.org/packages/eb/e4/4c4aea6473214dbdbd0fbba11aa4691e76dc01722c55724c5951719865ff/psycopg2_binary-2.9.12-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f921f3cd87035ef7df233383011d7a53ea1d346224752c1385f1edfd790ceb6a", size = 3299187, upload-time = "2026-04-20T23:34:21.206Z" }, - { url = "https://files.pythonhosted.org/packages/ba/5d/b03b99986446a4f57b170ed9a2579fb7ff9783ca0fa5226b19db99737fee/psycopg2_binary-2.9.12-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3d999bd982a723113c1a45b55a7a6a90d64d0ed2278020ed625c490ff7bef96c", size = 3047716, upload-time = "2026-04-20T23:34:23.077Z" }, - { url = "https://files.pythonhosted.org/packages/14/86/382ee4afbd1d97500c9d2862b20c2fdeddf4b7335e984df3fb4309f64108/psycopg2_binary-2.9.12-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29d4d134bd0ab46ffb04e94aa3c5fa3ef582e9026609165e2f758ff76fc3a3be", size = 3349237, upload-time = "2026-04-20T23:34:25.211Z" }, - { url = "https://files.pythonhosted.org/packages/a8/16/9a57c75ba1eda7165c017342f526810d5f5a12647dde749c99ae9a7141d7/psycopg2_binary-2.9.12-cp311-cp311-win_amd64.whl", hash = "sha256:cb4a1dacdd48077150dc762a9e5ddbf32c256d66cb46f80839391aa458774936", size = 2757036, upload-time = "2026-04-20T23:34:27.77Z" }, - { url = "https://files.pythonhosted.org/packages/e2/9f/ef4ef3c8e15083df90ca35265cfd1a081a2f0cc07bb229c6314c6af817f4/psycopg2_binary-2.9.12-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5cdc05117180c5fa9c40eea8ea559ce64d73824c39d928b7da9fb5f6a9392433", size = 3712459, upload-time = "2026-04-20T23:34:30.549Z" }, - { url = "https://files.pythonhosted.org/packages/b5/01/3dd14e46ba48c1e1a6ec58ee599fa1b5efa00c246d5046cd903d0eeb1af1/psycopg2_binary-2.9.12-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d3227a3bc228c10d21011a99245edca923e4e8bf461857e869a507d9a41fe9f6", size = 3822936, upload-time = "2026-04-20T23:34:32.77Z" }, - { url = "https://files.pythonhosted.org/packages/a6/f7/0640e4901119d8a9f7a1784b927f494e2198e213ceb593753d1f2c8b1b30/psycopg2_binary-2.9.12-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:995ce929eede89db6254b50827e2b7fd61e50d11f0b116b29fffe4a2e53c4580", size = 4578676, upload-time = "2026-04-20T23:34:35.18Z" }, - { url = "https://files.pythonhosted.org/packages/b0/55/44df3965b5f297c50cc0b1b594a31c67d6127a9d133045b8a66611b14dfb/psycopg2_binary-2.9.12-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9fe06d93e72f1c048e731a2e3e7854a5bfaa58fc736068df90b352cefe66f03f", size = 4274917, upload-time = "2026-04-20T23:34:37.982Z" }, - { url = "https://files.pythonhosted.org/packages/b0/4b/74535248b1eac0c9336862e8617c765ac94dac76f9e25d7c4a79588c8907/psycopg2_binary-2.9.12-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40e7b28b63aaf737cb3a1edc3a9bbc9a9f4ad3dcb7152e8c1130e4050eddcb7d", size = 5894843, upload-time = "2026-04-20T23:34:40.856Z" }, - { url = "https://files.pythonhosted.org/packages/f2/ba/f1bf8d2ae71868ad800b661099086ee52bc0f8d9f05be1acd8ebb06757cc/psycopg2_binary-2.9.12-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:89d19a9f7899e8eb0656a2b3a08e0da04c720a06db6e0033eab5928aabe60fa9", size = 4110556, upload-time = "2026-04-20T23:34:44.016Z" }, - { url = "https://files.pythonhosted.org/packages/45/46/c15706c338403b7c420bcc0c2905aad116cc064545686d8bf85f1999ea00/psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:612b965daee295ae2da8f8218ce1d274645dc76ef3f1abf6a0a94fd57eff876d", size = 3655714, upload-time = "2026-04-20T23:34:46.233Z" }, - { url = "https://files.pythonhosted.org/packages/b3/7c/a2d5dc09b64a4564db242a0fe418fde7d33f6f8259dd2c5b9d7def00fb5a/psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:b9a339b79d37c1b45f3235265f07cdeb0cb5ad7acd2ac7720a5920989c17c24e", size = 3301154, upload-time = "2026-04-20T23:34:49.528Z" }, - { url = "https://files.pythonhosted.org/packages/c0/e8/cc8c9a4ce71461f9ec548d38cadc41dc184b34c73e6455450775a9334ccd/psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:3471336e1acfd9c7fe507b8bad5af9317b6a89294f9eb37bd9a030bb7bebcdc6", size = 3048882, upload-time = "2026-04-20T23:34:51.86Z" }, - { url = "https://files.pythonhosted.org/packages/19/6a/31e2296bc0787c5ab75d3d118e40b239db8151b5192b90b77c72bc9256e9/psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7af18183109e23502c8b2ae7f6926c0882766f35b5175a4cd737ad825e4d7a1b", size = 3351298, upload-time = "2026-04-20T23:34:54.124Z" }, - { url = "https://files.pythonhosted.org/packages/5f/a8/75f4e3e11203b590150abed2cf7794b9c9c9f7eceddae955191138b44dde/psycopg2_binary-2.9.12-cp312-cp312-win_amd64.whl", hash = "sha256:398fcd4db988c7d7d3713e2b8e18939776fd3fb447052daae4f24fa39daede4c", size = 2757230, upload-time = "2026-04-20T23:34:56.242Z" }, - { url = "https://files.pythonhosted.org/packages/91/bb/4608c96f970f6e0c56572e87027ef4404f709382a3503e9934526d7ba051/psycopg2_binary-2.9.12-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7c729a73c7b1b84de3582f73cdd27d905121dc2c531f3d9a3c32a3011033b965", size = 3712419, upload-time = "2026-04-20T23:34:58.754Z" }, - { url = "https://files.pythonhosted.org/packages/5e/af/48f76af9d50d61cf390f8cd657b503168b089e2e9298e48465d029fcc713/psycopg2_binary-2.9.12-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4413d0caef93c5cf50b96863df4c2efe8c269bf2267df353225595e7e15e8df7", size = 3822990, upload-time = "2026-04-20T23:35:00.821Z" }, - { url = "https://files.pythonhosted.org/packages/7a/df/aba0f99397cd811d32e06fc0cc781f1f3ce98bc0e729cb423925085d781a/psycopg2_binary-2.9.12-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4dfcf8e45ebb0c663be34a3442f65e17311f3367089cd4e5e3a3e8e62c978777", size = 4578696, upload-time = "2026-04-20T23:35:03.409Z" }, - { url = "https://files.pythonhosted.org/packages/95/9c/eaa74021ac4e4d5c2f83d82fc6615a63f4fe6c94dc4e94c3990427053f67/psycopg2_binary-2.9.12-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c41321a14dd74aceb6a9a643b9253a334521babfa763fa873e33d89cfa122fb5", size = 4274982, upload-time = "2026-04-20T23:35:05.583Z" }, - { url = "https://files.pythonhosted.org/packages/35/ed/c25deff98bd26187ba48b3b250a3ffc3037c46c5b89362534a15d200e0db/psycopg2_binary-2.9.12-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:83946ba43979ebfdc99a3cd0ee775c89f221df026984ba19d46133d8d75d3cd9", size = 5894867, upload-time = "2026-04-20T23:35:07.902Z" }, - { url = "https://files.pythonhosted.org/packages/9a/81/8d0e21ca77373c6c9589e5c4528f6e8f0c08c62cafc76fb0bddb7a2cee22/psycopg2_binary-2.9.12-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:411e85815652d13560fbe731878daa5d92378c4995a22302071890ec3397d019", size = 4110578, upload-time = "2026-04-20T23:35:10.149Z" }, - { url = "https://files.pythonhosted.org/packages/00/fc/f481e2435bd8f742d0123309174aae4165160ad3ef17c1b99c3622c241d2/psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1c8ad4c08e00f7679559eaed7aff1edfffc60c086b976f93972f686384a95e2c", size = 3655816, upload-time = "2026-04-20T23:35:12.56Z" }, - { url = "https://files.pythonhosted.org/packages/53/79/b9f46466bdbe9f239c96cde8be33c1aace4842f06013b47b730dc9759187/psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:00814e40fa23c2b37ef0a1e3c749d89982c73a9cb5046137f0752a22d432e82f", size = 3301307, upload-time = "2026-04-20T23:35:15.029Z" }, - { url = "https://files.pythonhosted.org/packages/3f/19/7dc003b32fe35024df89b658104f7c8538a8b2dcbde7a4e746ce929742e7/psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:98062447aebc20ed20add1f547a364fd0ef8933640d5372ff1873f8deb9b61be", size = 3048968, upload-time = "2026-04-20T23:35:16.757Z" }, - { url = "https://files.pythonhosted.org/packages/91/58/2dbd7db5c604d45f4950d988506aae672a14126ec22998ced5021cbb76bb/psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:66a7685d7e548f10fb4ce32fb01a7b7f4aa702134de92a292c7bd9e0d3dbd290", size = 3351369, upload-time = "2026-04-20T23:35:18.933Z" }, - { url = "https://files.pythonhosted.org/packages/42/ee/dee8dcaad07f735824de3d6563bc67119fa6c28257b17977a8d624f02fab/psycopg2_binary-2.9.12-cp313-cp313-win_amd64.whl", hash = "sha256:b6937f5fe4e180aeee87de907a2fa982ded6f7f15d7218f78a083e4e1d68f2a0", size = 2757347, upload-time = "2026-04-20T23:35:21.283Z" }, - { url = "https://files.pythonhosted.org/packages/13/1b/708c0dca874acfad6d65314271859899a79007686f3a1f74e82a2ed4b645/psycopg2_binary-2.9.12-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:6f3b3de8a74ef8db215f22edffb19e32dc6fa41340456de7ec99efdc8a7b3ec2", size = 3712428, upload-time = "2026-04-20T23:35:23.453Z" }, - { url = "https://files.pythonhosted.org/packages/d6/39/ddbea9d4b4de6aca9431b6ed253f530f8a02d3b8f9bcfd0dbfe2b3de6fe4/psycopg2_binary-2.9.12-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1006fb62f0f0bc5ce256a832356c6262e91be43f5e4eb15b5eaf38079464caf2", size = 3823184, upload-time = "2026-04-20T23:35:25.92Z" }, - { url = "https://files.pythonhosted.org/packages/bf/a0/bc2fef74b106fa345567122a0659e6d94512ed7dc0131ec44c9e5aba3725/psycopg2_binary-2.9.12-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:840066105706cd2eb29b9a1c2329620056582a4bf3e8169dec5c447042d0869f", size = 4579157, upload-time = "2026-04-20T23:35:28.542Z" }, - { url = "https://files.pythonhosted.org/packages/57/d7/d4e3b2005d3de607ca4fbb0e8742e248056e52184a6b94ebda3c1c2c329b/psycopg2_binary-2.9.12-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:863f5d12241ebe1c76a72a04c2113b6dc905f90b9cef0e9be0efd994affd9354", size = 4274970, upload-time = "2026-04-20T23:35:30.418Z" }, - { url = "https://files.pythonhosted.org/packages/2e/42/c9853f8db3967fe08bcde11f53d53b85d351750cae726ce001cb68afa9c1/psycopg2_binary-2.9.12-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a99eaab34a9010f1a086b126de467466620a750634d114d20455f3a824aae033", size = 5895175, upload-time = "2026-04-20T23:35:33.584Z" }, - { url = "https://files.pythonhosted.org/packages/eb/fd/b82b5601a97630308bef079f545ffec481bbbc795c2ba5ec416a01d03f60/psycopg2_binary-2.9.12-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ffdd7dc5463ccd61845ac37b7012d0f35a1548df9febe14f8dd549be4a0bc81e", size = 4110658, upload-time = "2026-04-20T23:35:35.638Z" }, - { url = "https://files.pythonhosted.org/packages/62/8c/32ca69b0389ef25dd22937bf9e8fbe2ce27aea20b05ded48c4ce4cb42475/psycopg2_binary-2.9.12-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:54a0dfecab1b48731f934e06139dfe11e24219fb6d0ceb32177cf0375f14c7b5", size = 3656251, upload-time = "2026-04-20T23:35:37.854Z" }, - { url = "https://files.pythonhosted.org/packages/c4/29/96992a2b59e3b9d730fcf9612d0a387305025dc867a9fc490a9e496e074e/psycopg2_binary-2.9.12-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:96937c9c5d891f772430f418a7a8b4691a90c3e6b93cf72b5bd7cad8cbca32a5", size = 3301810, upload-time = "2026-04-20T23:35:39.927Z" }, - { url = "https://files.pythonhosted.org/packages/56/ad/44b06659949b243ae10112cd3b20a197f9bf3e81d5651379b9eb889bfaad/psycopg2_binary-2.9.12-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:77b348775efd4cdab410ec6609d81ccecd1139c90265fa583a7255c8064bc03d", size = 3048977, upload-time = "2026-04-20T23:35:41.806Z" }, - { url = "https://files.pythonhosted.org/packages/1d/f2/10a1bcebadb6aa55e280e1f58975c36a7b560ea525184c7aa4064c466633/psycopg2_binary-2.9.12-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:527e6342b3e44c2f0544f6b8e927d60de7f163f5723b8f1dfa7d2a84298738cd", size = 3351466, upload-time = "2026-04-20T23:35:43.993Z" }, - { url = "https://files.pythonhosted.org/packages/20/be/b732c8418ffa5bcfda002890f5dc4c869fc17db66ff11f53b17cfe44afc0/psycopg2_binary-2.9.12-cp314-cp314-win_amd64.whl", hash = "sha256:f12ae41fcafadb39b2785e64a40f9db05d6de2ac114077457e0e7c597f3af980", size = 2848762, upload-time = "2026-04-20T23:35:46.421Z" }, +name = "psycopg" +version = "3.3.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/26/3ea4ca5eaea1c0debcdf7ee7c1613fbe721dc27a03c461c0817ffd8a0601/psycopg-3.3.6.tar.gz", hash = "sha256:c081f2250df751a943036e42db6df4571c66cd0aabe8291a7a506512b12007d2", size = 168171, upload-time = "2026-09-18T13:22:55.152Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/de/748bd7609c71cae5d737f0ba9192f19329f70180ecda8fff3cac02c5abe3/psycopg-3.3.6-py3-none-any.whl", hash = "sha256:a1db9f7148b06a28606767efaca51fa6f9398c5c0a3810519be69d7000bdb631", size = 215490, upload-time = "2026-09-18T13:15:29.374Z" }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, +] +pool = [ + { name = "psycopg-pool" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.3.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/92/00350a66de0af05e41d01aa3134e3970045e816afed3f99d58ec1abe15b2/psycopg_binary-3.3.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7beb3e41c9a1e509f3ed85263386588cbe3e975aa67be21f79f44fd35ffaeefc", size = 4728682, upload-time = "2026-09-18T13:15:36.605Z" }, + { url = "https://files.pythonhosted.org/packages/91/fc/afa9c7fd316a469af7ede6ebb020eac482f5d827fae57d5310c9bc0c41ae/psycopg_binary-3.3.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:aa73160077345ec21b3f51e8e24b3de2e99586217e497629326eb9b2ea88c52e", size = 4777111, upload-time = "2026-09-18T13:15:46.566Z" }, + { url = "https://files.pythonhosted.org/packages/f2/44/7c1e015f1bc56b36ff1369f09e852b2d83ccefd5a669a42633a916cdedc4/psycopg_binary-3.3.6-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f87dbdc42e78ee0f7ea180c03f8c78e80a949e373066629bd90fefff10552dff", size = 5592755, upload-time = "2026-09-18T13:15:52.886Z" }, + { url = "https://files.pythonhosted.org/packages/3b/ae/314a251ca918cdac380bce1b87839ade9355382ea749e6ef3ba75ba0c09f/psycopg_binary-3.3.6-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a9348c5b43a3bb5ef8c2e89d5237c9c87eeafb01d338c84a7aebbc5cd0313299", size = 5267949, upload-time = "2026-09-18T13:16:00.53Z" }, + { url = "https://files.pythonhosted.org/packages/b6/9f/3bb0cfe9bb0f31ca57cf486ddc8c9ac51251aed8181bf88ff870b2623105/psycopg_binary-3.3.6-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a52991594ac4db888c7d39bccef331797e30cb31a95cae02cf2607f83a42dc2", size = 6862403, upload-time = "2026-09-18T13:16:10.385Z" }, + { url = "https://files.pythonhosted.org/packages/c4/d6/7032c10309c3155e9b24300fdcc9a1afa539cfd20ce52fdef74a46f10161/psycopg_binary-3.3.6-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5ea8beeb5541780b4b50b462eeacbc4f594ce3b911dc20c81c75f267876f71d2", size = 5107271, upload-time = "2026-09-18T13:16:16.843Z" }, + { url = "https://files.pythonhosted.org/packages/61/cc/79add2cf92684cf1a81da134b32caa662c25c72d0cc905d181ef4455f834/psycopg_binary-3.3.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:198a48e68cc99ccac03ba95ac857e73aa66f3bf6be77019fafb0832a05f7ad03", size = 4627984, upload-time = "2026-09-18T13:16:23.889Z" }, + { url = "https://files.pythonhosted.org/packages/c9/48/6dfb14f9350c14af6a2edb3c31262051b8cd94e2186e4b831e46dbbe8cd9/psycopg_binary-3.3.6-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:fa34eb47969297471db7b7f193622c7e3ee839ec05abd05f1fe104d5b1b1dcf4", size = 4322054, upload-time = "2026-09-18T13:16:29.33Z" }, + { url = "https://files.pythonhosted.org/packages/29/35/2982338716a91cbb4dfc866be015be4457ee8106a445aabf3d1fb6a270e0/psycopg_binary-3.3.6-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b979a42815410432420275412633960807178b1ce26591a16ce06e78a5bd4bb2", size = 4048471, upload-time = "2026-09-18T13:16:34.119Z" }, + { url = "https://files.pythonhosted.org/packages/24/e1/171b1db1542c5f76a678b7ee0a7800bebc9735a0a03417c76cf948bfd63c/psycopg_binary-3.3.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:889e42acec10450185e0cdfb396f375e2c1a8d7737c114830a7fde4654f59e30", size = 4353931, upload-time = "2026-09-18T13:16:38.692Z" }, + { url = "https://files.pythonhosted.org/packages/08/89/4424e62a944eef40bd9326ada4ae23802b28eab6502af91e84ef7bba74fb/psycopg_binary-3.3.6-cp310-cp310-win_amd64.whl", hash = "sha256:cbd5f73073ed19c378d4c35499db1e3e703a5b1a324e521204065967bfaa7a18", size = 3674362, upload-time = "2026-09-18T13:16:44.454Z" }, + { url = "https://files.pythonhosted.org/packages/70/86/b71166048974d49c6d136b2ed1c0e5bec0b974d8c4de5cbce7e86a9e412a/psycopg_binary-3.3.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:be4f9b3c9338ac5dd217c5847e21521b396c8117f78dc420d495a5c49bbef874", size = 4728002, upload-time = "2026-09-18T13:16:53.393Z" }, + { url = "https://files.pythonhosted.org/packages/12/1d/1e06c0de7ed5aed898acb87544eac6ef0bc7d752a67ec6e5d6b835e9b40c/psycopg_binary-3.3.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f0535693ce476a722b718b002d5d2c27d47e71ca945276ac194409c98e74c492", size = 4775403, upload-time = "2026-09-18T13:16:58.939Z" }, + { url = "https://files.pythonhosted.org/packages/84/02/2ffcbc43f8e4bbc38e5286a22013bcac01898d13cd38325f60dd5428a8af/psycopg_binary-3.3.6-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3c9e663b2e800e3218994cf948c11bcc2844e6491b34aa80d089baf6531827bf", size = 5594145, upload-time = "2026-09-18T13:17:08.515Z" }, + { url = "https://files.pythonhosted.org/packages/e1/25/031dae2c7d2e7e77dcf5b1962c1e0684fa548d7af0ff6707b6b5e6054ca7/psycopg_binary-3.3.6-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a2e44a342d2aee40508e28a563d8961c39d9bbd8cae36d8578f0a3c6658aab0f", size = 5267985, upload-time = "2026-09-18T13:17:16.24Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e5/94c89ada3c003a4d858178f3bba49a35e0297ef2aad659b80eb5e380e690/psycopg_binary-3.3.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f598f19fa9a91540b5cee17932ffd227b7b53a481605bcc4573c0eafa647300", size = 6860986, upload-time = "2026-09-18T13:17:23.348Z" }, + { url = "https://files.pythonhosted.org/packages/9d/a0/81bf499d095adee8413bd19822a6872fbfa21663ec78014a68d83a8db83c/psycopg_binary-3.3.6-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6ff05561e4a067d35507dc5c90f1deb2ec1c9703ac5cccc1bc26e08a197f9c5a", size = 5106702, upload-time = "2026-09-18T13:17:28.847Z" }, + { url = "https://files.pythonhosted.org/packages/00/75/99d56da64c27bd985fd82c6ecbf7976b724ac638fdd1654ef995323a1a26/psycopg_binary-3.3.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:566dd827f17728efdf7d88a5b066f815170f6fdad13967ae952842d90e6aaa9f", size = 4627090, upload-time = "2026-09-18T13:17:36.668Z" }, + { url = "https://files.pythonhosted.org/packages/3e/0c/0222171d11233332c6a24b1cef1578215f0ffddf3642eb8dd8c4448ad69f/psycopg_binary-3.3.6-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9b2f11794e017ce340934e35de46181c46ef71ec75ea3d85dd75cd836761c01e", size = 4320353, upload-time = "2026-09-18T13:17:42.526Z" }, + { url = "https://files.pythonhosted.org/packages/62/6f/e1cc2a28dd1228c67c969ba6fd37cd8726b312e2ff51380f847ddb38ccde/psycopg_binary-3.3.6-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:910ace140e3e7b7596898d083f37a8fe90c5c40684252ad4e682364b2cd3deba", size = 4047253, upload-time = "2026-09-18T13:17:47.068Z" }, + { url = "https://files.pythonhosted.org/packages/d8/fd/38b64790ce7a515b1dbd2bab3d119637a858aeb22c380cf4859bc4ce0e42/psycopg_binary-3.3.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37e517c146b185f9c0c6e8d0a0ebbdeeeb67896af28466e032bc810d0c7dc7a7", size = 4353208, upload-time = "2026-09-18T13:17:52.41Z" }, + { url = "https://files.pythonhosted.org/packages/f7/dc/45386530ceb2a8c789a226de9b9b34eca8fccf1feba2e4ef68a6aca50c56/psycopg_binary-3.3.6-cp311-cp311-win_amd64.whl", hash = "sha256:c7f92daa0d2a1c76f07264abddf8cbabd30152a2f09c3270e50f0c7efdf5dcac", size = 3675638, upload-time = "2026-09-18T13:17:58.112Z" }, + { url = "https://files.pythonhosted.org/packages/e6/01/2cdd1824e58b4467ee0b9498664cd28c42d8794db6b1e35b6bcb834f0044/psycopg_binary-3.3.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3f84dab25e0385692ee13274c68678377e0b1a70ab9d14e56264cbf61f60c62d", size = 4707086, upload-time = "2026-09-18T13:18:05.138Z" }, + { url = "https://files.pythonhosted.org/packages/f6/76/de9948ac06895261c84d5b9fbe283d8f3c5bc9f070691b8d9eaa1b51e322/psycopg_binary-3.3.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:612382ac3ed13651c7fa44b5fee9fbf7baaa2ddbc6f500391672682c5f1df9e0", size = 4769607, upload-time = "2026-09-18T13:18:12.83Z" }, + { url = "https://files.pythonhosted.org/packages/76/a9/72436c9915ee4905964689e7f0e182ce7767cc0a0390b3ce703be8177625/psycopg_binary-3.3.6-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:366db6e97e66b37211475f20c4c1324a2dc0dd825e46d4e87f9d599304d276f9", size = 5554134, upload-time = "2026-09-18T13:18:21.175Z" }, + { url = "https://files.pythonhosted.org/packages/0a/42/948bb3d2617795093512613fd96ba380e922992c7908fbc073858147d196/psycopg_binary-3.3.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1679a1cb93fbe5a6d1fd58d82cbddcc6fcb8c61446ba7cae6eb2a7b19bc585de", size = 5235723, upload-time = "2026-09-18T13:18:27.071Z" }, + { url = "https://files.pythonhosted.org/packages/99/47/93e823ff1b0088400703410939c9bda3e63ed9c850b3ee088e8769f4c10b/psycopg_binary-3.3.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:37d40450659401600e6d043ff586c89a71a69f33cbb8bcdba6cdb2569beecdbe", size = 6833587, upload-time = "2026-09-18T13:18:33.794Z" }, + { url = "https://files.pythonhosted.org/packages/5e/2d/ecc69c847795aa704041a9f5667a6b0938a088cf1853636d762a6938e493/psycopg_binary-3.3.6-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a5165300324efd5a772c48a88ab3a928513ab3979fca76553e62ee815f7b2b9c", size = 5070013, upload-time = "2026-09-18T13:18:39.628Z" }, + { url = "https://files.pythonhosted.org/packages/92/36/6126f0dac21713dcae91404f2a76da18598a6252339a8c669c46370d43b2/psycopg_binary-3.3.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d636338c8f21b0df2f84657b00bc34f9313f826ef93f1155bc743607e4a0c5eb", size = 4597367, upload-time = "2026-09-18T13:18:45.023Z" }, + { url = "https://files.pythonhosted.org/packages/4d/29/7ecfc04243b46c89ffd49924e9c5634ea904ef96c7d0f37e4073623584c1/psycopg_binary-3.3.6-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a4ee3bdd5468a725f2a4d9aab8a74b6d0279f768c8b5d3aeb102c5307ff3d59c", size = 4275419, upload-time = "2026-09-18T13:18:49.299Z" }, + { url = "https://files.pythonhosted.org/packages/6e/90/2f46d2e0de79706ac170df0a3637fe63c4498fc04f131f6049520b78b806/psycopg_binary-3.3.6-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:289aadd6a00e151203c081f708348ec89f1e483c9b510ef4ac3981f847f01f79", size = 4007358, upload-time = "2026-09-18T13:18:53.944Z" }, + { url = "https://files.pythonhosted.org/packages/03/48/6744e91291b751a8cf12d63d719977974bb94c84ceba913e7ddb2e478e51/psycopg_binary-3.3.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f21d057f3e5f5491067e5b292498073b73847d48799b099803fef100775fcc52", size = 4320156, upload-time = "2026-09-18T13:18:59.258Z" }, + { url = "https://files.pythonhosted.org/packages/1a/9b/94ff7fce53a64d5b286e2ec454e0a025cf3d6e6b4a9189bef16aa5de98b2/psycopg_binary-3.3.6-cp312-cp312-win_amd64.whl", hash = "sha256:e23a66a763fbe83fcc210bc77c27e5a5ea380ebf091c06f34d8561b695e5a40f", size = 3658864, upload-time = "2026-09-18T13:19:06.503Z" }, + { url = "https://files.pythonhosted.org/packages/b4/c3/c072584b69ad44a747b448cfc9766fecb8aae56e372a017e2ef668790057/psycopg_binary-3.3.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ad8f35e67cc16d1fad1fa8c88972dc9b3a3141ea67897399904edab96a301b6", size = 4712284, upload-time = "2026-09-18T13:19:13.451Z" }, + { url = "https://files.pythonhosted.org/packages/0a/b9/4283b785339e8e2318d03048994b093d650ea6289fabaa806b765dc0d449/psycopg_binary-3.3.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:373704aea331d3f3e3402c125a1543f5875e2986ebb54f97d1647942161f803f", size = 4772031, upload-time = "2026-09-18T13:19:18.524Z" }, + { url = "https://files.pythonhosted.org/packages/6f/72/7a1321d359246769fff1affffbd0132785a28f7f63c18524c15a502398f4/psycopg_binary-3.3.6-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b82491019b884d62318b5f30706c3d7e6d4e5a6cb7eabcb3edc0c1b0fdaceae9", size = 5556392, upload-time = "2026-09-18T13:19:24.418Z" }, + { url = "https://files.pythonhosted.org/packages/de/b0/c6f8a0585a5dacbea74e130bcfc66629390e8f5bbc79d2a8e806e8952150/psycopg_binary-3.3.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cec5ea900390897d0b46130f60bc2883bf19c314f9044235217c8be88b0ef269", size = 5237855, upload-time = "2026-09-18T13:19:31.257Z" }, + { url = "https://files.pythonhosted.org/packages/e2/fc/c3a7a8bbef7e945ec584ac61d460a612363ea398511cd0e220242b1d69f1/psycopg_binary-3.3.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98c02090d88f2ebc0ec1e8da538f77d225ce0fffecf372aa39262e62a1b054ef", size = 6833856, upload-time = "2026-09-18T13:19:43.622Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f2/8e80b921db728ebb68fc105bd7c4277f908210ad755bd6481d5ea7add740/psycopg_binary-3.3.6-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ee2c4728c691245e24501fcd7a97b5b381236b9985bc445bba88cdce7d1b5784", size = 5070730, upload-time = "2026-09-18T13:19:49.968Z" }, + { url = "https://files.pythonhosted.org/packages/54/6a/5b313e0c5348244f0e973aff3258bf86766656256d5ece8d541a53e35b4a/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f19cc87343eaa55255e76b31259a570072ac95d6ae82c92dd34b97691f5e49dc", size = 4598089, upload-time = "2026-09-18T13:19:56.426Z" }, + { url = "https://files.pythonhosted.org/packages/32/e9/db7f76ec24bf6699e92bf604e5c4bae10664a681a8999ef42aa0faf0f2c6/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fdccb3a0e184b03e9baa673b15a809cf36c339c85dbda0ebc25a698846dfbee8", size = 4278481, upload-time = "2026-09-18T13:20:04.681Z" }, + { url = "https://files.pythonhosted.org/packages/61/83/72c67013656f4d6b547caabffb193e91d57e63f90eefdcc6d045c400e97d/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:9892188bb15e5803beb51afe8a25add6b56be391a53058e8bca03b74e1e6bf22", size = 4009229, upload-time = "2026-09-18T13:20:11.905Z" }, + { url = "https://files.pythonhosted.org/packages/82/35/5e4500df2c999eb0faed8b184e6958b834172128274f06167a5deef4c19c/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3af90f92769d8cc10f94515ee7a0aef36ea85ca733a0ce22858f6e0953f41138", size = 4321467, upload-time = "2026-09-18T13:20:17.949Z" }, + { url = "https://files.pythonhosted.org/packages/55/7f/e350e1cf498ba2565c3f87b12f429d2012eb86b76c2b3845a19ee5fbb4d6/psycopg_binary-3.3.6-cp313-cp313-win_amd64.whl", hash = "sha256:0ebfad5d131de9f892ae9e70cc7616207768b6714b66a52d4612b8ceaf78b372", size = 3658179, upload-time = "2026-09-18T13:20:22.691Z" }, + { url = "https://files.pythonhosted.org/packages/6d/b9/60711317c284a442511644ea7185b56ebe627606d6741e732cd16108c47b/psycopg_binary-3.3.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:b3f75dee0f9afafabe4edc52c4842f1e1878ed2069bd05b22d6fe961e97e4dba", size = 4720512, upload-time = "2026-09-18T13:20:29.278Z" }, + { url = "https://files.pythonhosted.org/packages/63/da/28befc84454cbc6374550de7746f591f8fe1b6165c1fce249652cc8291c4/psycopg_binary-3.3.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5927b7ba63153cd8e9862987290a2b783a5c590daf2a4ef981700cc3569166d4", size = 4782318, upload-time = "2026-09-18T13:20:35.401Z" }, + { url = "https://files.pythonhosted.org/packages/a4/8a/0d21c2c833cdc0d4244c77e858e0ed37fa2abec2623be4fd686f617109ce/psycopg_binary-3.3.6-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:0bf08b749cc144f33b44a91b78e3f71c60eb07963746a0df5a100b36ce3d7475", size = 5567460, upload-time = "2026-09-18T13:20:41.902Z" }, + { url = "https://files.pythonhosted.org/packages/49/6d/7692d0d4e656b6cc9868d8acc2e3b42f17a0db4a625400a6d093cb0533a1/psycopg_binary-3.3.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:31cd942c23f613276b81a6e6598cefa12960058b0f46e1e874b540c793f6aca5", size = 5246902, upload-time = "2026-09-18T13:20:47.661Z" }, + { url = "https://files.pythonhosted.org/packages/d4/c1/b8a1f18fb1b7558a17f57f7cb3fc8bc93189feea2958925950b3acb15743/psycopg_binary-3.3.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4690cf67738f0e0e49a32aeec99bf0e4595cc2b4f1af984a4345394b1dcff91a", size = 6847192, upload-time = "2026-09-18T13:20:56.874Z" }, + { url = "https://files.pythonhosted.org/packages/a5/76/404f33519167c65cca88ec4998776f1dbebccc301ee977f0e62c47fb0826/psycopg_binary-3.3.6-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ad1c785e784cfd87e8436c6b7702f2d321fc39601bbaf29bc63a41a867091638", size = 5079573, upload-time = "2026-09-18T13:21:04.155Z" }, + { url = "https://files.pythonhosted.org/packages/f0/d9/79e8fbc8f37262a415f3550f0bcc5f98037442bf3d12ef6cbae2056655ae/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:79a2a1c3449f6c3409427078ed1cec10de79f3023cb5f2504f0597d350ad46c7", size = 4613633, upload-time = "2026-09-18T13:21:10.664Z" }, + { url = "https://files.pythonhosted.org/packages/d4/47/96225db74be7d2ce04b3a58678b53cda610225055edf5faa775c9f501d8b/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:86147cb5d140341c3363fb5bacce31f8d5543902a46699d3c536b101bbceaf9e", size = 4293375, upload-time = "2026-09-18T13:21:16.027Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d2/18e9c779a5efd565250329adaf529ecc2b8b2ed5be5cb0f6ccee208cbfd9/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:7308c93cf0b19bbaf8e6ff0a6ad50d3c442385739245fe15a8d593bf841734a6", size = 4019883, upload-time = "2026-09-18T13:21:21.587Z" }, + { url = "https://files.pythonhosted.org/packages/ef/28/0cc654afc6c2cda982767f5679d3646b30b1ec86545bdaa9402202d6776c/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:05a83ac9fd52b9bca7cb5ab04b3691163170bd16f53defa27216ea3aa07ee781", size = 4332607, upload-time = "2026-09-18T13:21:27.63Z" }, + { url = "https://files.pythonhosted.org/packages/f1/3e/0a753a74fbd7aef120f286c016e09d3cc3f1daf7688f4a145d27281260b2/psycopg_binary-3.3.6-cp314-cp314-win_amd64.whl", hash = "sha256:1fbd30e537dab22cafdf080608f10148fe2a5f3a61294ddb5113caac8a623840", size = 3755671, upload-time = "2026-09-18T13:21:33.855Z" }, + { url = "https://files.pythonhosted.org/packages/0e/b1/a372b9c02aea50148e71c9853e19efca8fa5ae2010a8e27243b9b8f790c0/psycopg_binary-3.3.6-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:bf8c8481d026b85dd70c5fa7dde85b2333aed0b32a2602bcd38a900cbd78a49c", size = 4719571, upload-time = "2026-09-18T13:21:41.437Z" }, + { url = "https://files.pythonhosted.org/packages/65/7c/811e3828c6b82e2f10c6c9cdd963cfc66f3e024026e5a69ac18530bad984/psycopg_binary-3.3.6-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:b599defe9190b17e9907c8b4d114c181e702c87efcd1b8a0ad40971cdcc4634a", size = 4781230, upload-time = "2026-09-18T13:21:49.516Z" }, + { url = "https://files.pythonhosted.org/packages/3e/15/9a784eed813ea9e97c294af3ead63d02b7b203502c66380336c50065e441/psycopg_binary-3.3.6-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b8ece331509f7a975b90501f41e83ad905e4141753fedf3f2711b2bc70a8efbc", size = 5566111, upload-time = "2026-09-18T13:21:58.089Z" }, + { url = "https://files.pythonhosted.org/packages/68/16/47194e002007c27337b11e49bf459c4b19727463f9aff2e1a90917bcc806/psycopg_binary-3.3.6-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c61617eaae0112ca154da87ffb99b73af2c74067acac28dfb9a4455b019dff2e", size = 5249963, upload-time = "2026-09-18T13:22:06.695Z" }, + { url = "https://files.pythonhosted.org/packages/53/84/5dcf9f310b11f0675cd860c6b2c70f58ce61798a3ee3f6f962b53fa358ca/psycopg_binary-3.3.6-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c6d19cb4999d03231e8730a5f66c8f5068bc3b532677eb39dab0f600bff3e312", size = 6847925, upload-time = "2026-09-18T13:22:13.088Z" }, + { url = "https://files.pythonhosted.org/packages/f3/06/1957a06dc22963c418c27b284929579de84f29c37ad1abe6dc6ee9e8cf25/psycopg_binary-3.3.6-cp315-cp315-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8cbb54454dbf1bbf2ff08dd7693e8d94ac94b1a20f70f4b3b813d52ecb5cbc1", size = 5087720, upload-time = "2026-09-18T13:22:17.959Z" }, + { url = "https://files.pythonhosted.org/packages/21/43/ac07d042bae99b57bf123bb473632f29af544008094da0ffd285ab8011e2/psycopg_binary-3.3.6-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:dc75da5a20951049f7b773145f998f69d181adad9c58a0ff36e0cf1d73c10e10", size = 4613412, upload-time = "2026-09-18T13:22:26.719Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b1/019156fbeafcefb4cccc9d109de4699493bceb8313c7545c8349e089dfbc/psycopg_binary-3.3.6-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:955e3dd94da361e052d2e49acf591017158dc8f8ed2c8a42c2e3943403c39dc2", size = 4292618, upload-time = "2026-09-18T13:22:33.042Z" }, + { url = "https://files.pythonhosted.org/packages/5d/0f/62113dc6b1df65983a1f2fc816c04b1edfa22f2ae9d4abee74ed267f4a96/psycopg_binary-3.3.6-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:c7753871eb57e6a5f4646f6168590c6653073dea5e9e720b201c8875332df4c8", size = 4027121, upload-time = "2026-09-18T13:22:38.334Z" }, + { url = "https://files.pythonhosted.org/packages/5d/d5/cf0cbd1ea5a7d8167fe2c6953efde19101f7b193bd61a23e6d622ad6854c/psycopg_binary-3.3.6-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:303732e798fe6729f8e12021b9c96107df8e95ecec4dd487c67b98ec2a59435e", size = 4336388, upload-time = "2026-09-18T13:22:45.576Z" }, + { url = "https://files.pythonhosted.org/packages/98/33/e2a5b36edf8aa422f6fa4b894756eb33dc93b36df5f65121280bb8b929c4/psycopg_binary-3.3.6-cp315-cp315-win_amd64.whl", hash = "sha256:2f122603f36050937982abf9668d8bc4769a79f7c93a65013b1c49f1cab7b56b", size = 3756154, upload-time = "2026-09-18T13:22:51.283Z" }, +] + +[[package]] +name = "psycopg-pool" +version = "3.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/5e/c0664b968b102ff68b811d999c728546c48d5c1eec03e3bbaf88c0cb4472/psycopg_pool-3.3.3.tar.gz", hash = "sha256:df87b5d9d0ad7db37f6cdad4fa8ce113d250f5997f6db38e9a99192fb67f9e1d", size = 32006, upload-time = "2026-09-22T15:53:24.947Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/b4/452c6607a0f479465cd8a9b0d9956919fcb150050c1f83f9f11e6b8ee8dc/psycopg_pool-3.3.3-py3-none-any.whl", hash = "sha256:9b9cd6a4fcec47a410f7e82d408540e7f77b478509e91b44c1a5457a13e5ff37", size = 40304, upload-time = "2026-09-22T15:53:23.712Z" }, ] [[package]] @@ -2334,7 +2382,7 @@ name = "python-dateutil" version = "2.9.0.post0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "six" }, + { name = "six", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } wheels = [ @@ -2468,18 +2516,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fc/3c/480c61cc8d5a3e76bb44e86231f408c93643e5498beadbbeb381ab55d02b/qdrant_client-1.19.0-py3-none-any.whl", hash = "sha256:13602a2b3478a95ecdf42f97b93d7f703b63a3361cd912a04495a33a5ac14121", size = 396157, upload-time = "2026-08-04T14:32:55.734Z" }, ] -[[package]] -name = "redis" -version = "8.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "async-timeout", marker = "python_full_version < '3.11.3'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a8/99/604f0b666d4c616d891cf77ebb9db6bb21601344c051aebf1b72b9ff915f/redis-8.1.0.tar.gz", hash = "sha256:6e1a19beef9225c83efd689c7e6b7da2d5215b1f42cd13b7fc3714d0a09c7b25", size = 5254356, upload-time = "2026-07-30T08:51:00.269Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/66/9d/c5731f6e3608663d4d3656fd8d3aecee8b509c3082818f5a13eae925baea/redis-8.1.0-py3-none-any.whl", hash = "sha256:a4fe1aac3d3b3cc791d4b3d5931c5a956045dc951ee74d1c913ee3ac4d2ee9fb", size = 560618, upload-time = "2026-07-30T08:50:58.497Z" }, -] - [[package]] name = "regex" version = "2026.7.19" @@ -2686,10 +2722,10 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform == 'darwin'", ] dependencies = [ - { name = "joblib" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" } }, - { name = "threadpoolctl" }, + { name = "joblib", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "threadpoolctl", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/98/c2/a7855e41c9d285dfe86dc50b250978105dce513d6e459ea66a6aeb0e1e0c/scikit_learn-1.7.2.tar.gz", hash = "sha256:20e9e49ecd130598f1ca38a1d85090e1a600147b9c02fa6f15d69cb53d968fda", size = 7193136, upload-time = "2025-09-09T08:21:29.075Z" } wheels = [ @@ -2740,13 +2776,13 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform == 'darwin'", ] dependencies = [ - { name = "joblib" }, - { name = "narwhals" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "joblib", marker = "python_full_version >= '3.11'" }, + { name = "narwhals", marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "scipy", version = "1.18.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "threadpoolctl" }, + { name = "threadpoolctl", marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" } wheels = [ @@ -2791,7 +2827,7 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform == 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" } wheels = [ @@ -2851,7 +2887,7 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform == 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -2930,7 +2966,7 @@ resolution-markers = [ "python_full_version == '3.12.*' and sys_platform == 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7e/74/66de6258867beb2ef08f35f9f2ac017a52cacd5081714d239ff1a442d458/scipy-1.18.1.tar.gz", hash = "sha256:52c4b7422442aba924d03ad4019852b08a92e64ea187b933135687bfe2747307", size = 30781235, upload-time = "2026-08-21T23:28:50.599Z" } wheels = [ @@ -3123,6 +3159,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, ] +[[package]] +name = "testcontainers" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docker" }, + { name = "python-dotenv" }, + { name = "typing-extensions" }, + { name = "urllib3" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4b/13/2cc466bddf26d0085f30a2b2bd56b7f8708b54a54db833eec97c5c69129b/testcontainers-4.15.0.tar.gz", hash = "sha256:085cde086337632e19002719460b7b80bbab2bdd51bb3ea04f77d0de96504706", size = 95340, upload-time = "2026-07-24T23:08:01.731Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/7e/424aac8b355597835deb333e757a0e94b5ccf38ad00f07fe6ed1f4e17c88/testcontainers-4.15.0-py3-none-any.whl", hash = "sha256:8796c14e76604031ad39cf0ed3b8e9806283a1fbf5270965c2b1c594caa31b74", size = 160771, upload-time = "2026-07-24T23:08:00.13Z" }, +] + [[package]] name = "threadpoolctl" version = "3.6.0" @@ -3296,14 +3348,14 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform == 'darwin'", ] dependencies = [ - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "setuptools" }, - { name = "sympy" }, - { name = "typing-extensions" }, + { name = "filelock", marker = "sys_platform == 'darwin'" }, + { name = "fsspec", marker = "sys_platform == 'darwin'" }, + { name = "jinja2", marker = "sys_platform == 'darwin'" }, + { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform == 'darwin'" }, + { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and sys_platform == 'darwin'" }, + { name = "setuptools", marker = "sys_platform == 'darwin'" }, + { name = "sympy", marker = "sys_platform == 'darwin'" }, + { name = "typing-extensions", marker = "sys_platform == 'darwin'" }, ] wheels = [ { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.13.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:94f0de129916f77b8dc2c7a8eff644cfeddfe59e39c9f55e9f6e17543410281d", upload-time = "2026-07-08T12:26:07Z" }, @@ -3326,14 +3378,14 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "setuptools" }, - { name = "sympy" }, - { name = "typing-extensions" }, + { name = "filelock", marker = "sys_platform != 'darwin'" }, + { name = "fsspec", marker = "sys_platform != 'darwin'" }, + { name = "jinja2", marker = "sys_platform != 'darwin'" }, + { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform != 'darwin'" }, + { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and sys_platform != 'darwin'" }, + { name = "setuptools", marker = "sys_platform != 'darwin'" }, + { name = "sympy", marker = "sys_platform != 'darwin'" }, + { name = "typing-extensions", marker = "sys_platform != 'darwin'" }, ] wheels = [ { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.13.0%2Bcpu-cp310-cp310-linux_s390x.whl", hash = "sha256:0555fde6108ca90247ae33d4e1237cbae475c86a223bb2f0f91d9addf1f611bd", upload-time = "2026-07-08T19:27:11Z" }, @@ -3443,6 +3495,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, ] +[[package]] +name = "tzdata" +version = "2026.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/68/f1b440335057bfce71b6e50a9d09445aa2ecbd08359a337976627b8409e7/tzdata-2026.5.tar.gz", hash = "sha256:8cc73c0a0bfca7dbfa59235d60b2eff82231dee33f53d206db1acd9173cfc0a7", size = 200404, upload-time = "2026-10-03T09:23:14.143Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/21/1e5995a1c920cce14e4bffae20c665ec10e7ed03ab25e006cd741092b718/tzdata-2026.5-py2.py3-none-any.whl", hash = "sha256:b683bd1b6659ddcd810ff02ad09ba821d4bf1065072805063eb35c49617905ac", size = 347996, upload-time = "2026-10-03T09:23:12.535Z" }, +] + [[package]] name = "urllib3" version = "2.7.0" From c2c25f2402400269336f266b15e3f1f90c2f331e Mon Sep 17 00:00:00 2001 From: Naveen Date: Mon, 26 Oct 2026 13:18:48 +0530 Subject: [PATCH 05/53] feat(db): add Postgres schema, migration runner and schema docs 0001_initial_schema.sql creates users, catalog_entries, documents, workflows, workflow_artifacts, pages, chunks, audit_logs, search_settings, email_otps and webhook_records, plus the document_current_workflow and catalog views. pipeline/migrate.py applies versioned SQL files under an advisory lock. docs/POSTGRES_SCHEMA.md describes the schema; docs/SQLITE_SCHEMA.md is kept as the legacy reference. --- docs/POSTGRES_SCHEMA.md | 553 +++++++++++++++++++++++++++++ docs/SQLITE_SCHEMA.md | 374 +++++++++++++++++++ migrations/0001_initial_schema.sql | 292 +++++++++++++++ pipeline/migrate.py | 76 ++++ 4 files changed, 1295 insertions(+) create mode 100644 docs/POSTGRES_SCHEMA.md create mode 100644 docs/SQLITE_SCHEMA.md create mode 100644 migrations/0001_initial_schema.sql create mode 100644 pipeline/migrate.py diff --git a/docs/POSTGRES_SCHEMA.md b/docs/POSTGRES_SCHEMA.md new file mode 100644 index 0000000..d76307a --- /dev/null +++ b/docs/POSTGRES_SCHEMA.md @@ -0,0 +1,553 @@ +# Postgres Schema — knowledge-provider + +Postgres schema behind `pipeline/db.py`. A document is content and metadata. A workflow is one run of processing against a document. Source file and vector index state are stored as JSONB on the document. Translations are stored as JSONB on each page. + +The schema is created by `migrations/0001_initial_schema.sql` and applied by the api and worker at startup (`pipeline/migrate.py`). `pipeline/db.py` is the only code that reads or writes it. The previous SQLite store is described in `docs/SQLITE_SCHEMA.md` (legacy) and is moved over once by `scripts/migrate_sqlite_to_postgres.py`. + +## Conventions + +- Every table has a primary key `id`, a UUID v4 (`DEFAULT gen_random_uuid()`, built in from Postgres 13). +- A foreign key is named `_id` (for example `document_id`). A role prefix is added when the role matters (`uploaded_by_user_id`, `started_by_user_id`). +- Text identifiers other systems rely on (`catalog_entries.code`, `temporal_workflow_id`) are ordinary columns, not keys. +- `created_at` and `updated_at` are the last two columns of a table. `updated_at` is maintained by one shared trigger. Append-only or short-lived tables skip `updated_at`, as noted per table. +- Timestamps are `TIMESTAMPTZ` (UTC), booleans are `BOOLEAN`, JSON is `JSONB`, lists of strings are `TEXT[]`. +- JSONB columns have a `CHECK` that the value is an object (or array). Value rules inside them live in Pydantic models, not in the database. +- Fixed value sets are enums. Open sets (`artifact_type`, `action_type`) stay `TEXT`. +- Counts have `CHECK >= 0`. Slugs and identifiers have pattern `CHECK`s. + +### Shared trigger + +```sql +CREATE FUNCTION set_updated_at() RETURNS trigger AS $$ +BEGIN + NEW.updated_at := now(); + RETURN NEW; +END $$ LANGUAGE plpgsql; +-- one per table that has updated_at: +-- CREATE TRIGGER trg__updated_at BEFORE UPDATE ON
+-- FOR EACH ROW EXECUTE FUNCTION set_updated_at(); +``` + +### Enums + +| Enum | Values | +|---|---| +| `workflow_stage` | `registered`, `ocr_processing`, `ocr_review`, `translation_processing`, `translation_review`, `chunking`, `chunk_review`, `ready_for_ingestion`, `ingesting`, `publishing_to_network`, `approval_for_prod`, `ingesting_prod`, `completed`, `failed` | +| `workflow_type_enum` | `pipeline`, `ocr_only`, `ocr_retry`, `translation_retry`, `chunking_retry`, `reingest`, `promote_prod` | +| `workflow_status_enum` | `running`, `waiting_review`, `completed`, `failed` | +| `audit_entity_type` | `document`, `page`, `chunk`, `setting`, `catalog` | + +Adding a value later needs `ALTER TYPE ... ADD VALUE`. + +## Tables + +| # | Table | Purpose | +|---|---|---| +| 1 | `users` | People who upload, review and act, mirrored from Keycloak | +| 2 | `catalog_entries` | Generic named entries that documents belong to. Type `scheme` today, `advisory` and others later | +| 3 | `documents` | One row per document. Content, metadata, source file, index state | +| 4 | `workflows` | One row per processing run of a document. Stage, status, config | +| 5 | `workflow_artifacts` | Files each workflow produces, stored in MinIO | +| 6 | `pages` | OCR layer and translations of each page | +| 7 | `chunks` | Chunked text and review state | +| 8 | `audit_logs` | Append-only action history | +| 9 | `search_settings` | Single-row search configuration | +| 10 | `email_otps` | Email OTP login codes | +| 11 | `webhook_records` | Records pushed to `POST /webhook` | +| — | `schema_migrations` | Applied migration files (created by the runner, not by a migration) | +| — | `catalog` (view) | Computed catalog of all entries. `/catalog/v1/*` serves the `scheme` type | +| — | `document_current_workflow` (view) | Latest workflow of each document | + +Relationships: + +``` +catalog_entries 1 ── * documents 1 ── * workflows 1 ── * workflow_artifacts + │ + ├── * pages + └── * chunks +``` + +Dependency order for creation: `users`, `catalog_entries`, `documents`, `workflows`, `workflow_artifacts`, `pages`, `chunks`, `audit_logs`, `search_settings`, `email_otps`, `webhook_records`, views. + +--- + +## 1. `users` + +Uploader, reviewer and actor identity, looked up once from the Keycloak token. + +Unique: `keycloak_user_id`; unique index on `lower(email)` where `email IS NOT NULL`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Internal user ID. | +| keycloak_user_id | TEXT | NOT NULL UNIQUE | Keycloak `sub` claim. `local-admin` when auth is disabled (not a UUID, so it cannot be the PK). | +| username | TEXT | | Current username. | +| email | TEXT | | Current email. | +| roles | TEXT[] | NOT NULL DEFAULT '{}' | Current roles. | +| first_seen_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | First login or upload. | +| last_seen_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Most recent activity. | + +Notes +- Upserted on each authenticated upload or login. A `local-admin` row is seeded for `AUTH_DISABLED=true`. +- Point-in-time identity is kept in `audit_logs.actor`. + +--- + +## 2. `catalog_entries` + +Generic named entries that documents belong to. One table for every entry type: `scheme` today, `advisory` and others later. A new type adds rows, not tables. + +Unique: `(type, code)` and `(id, type)` (the second is the target of the composite FK from `documents`). + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Entry ID. | +| type | TEXT | NOT NULL, CHECK `^[a-z0-9]+([_-][a-z0-9]+)*$` | Entry type (`scheme`, later `advisory`, and so on). Open set, so not an enum. | +| code | TEXT | NOT NULL, CHECK `^[a-z0-9]+(-[a-z0-9]+)*$` | Short code (`pm-kmy`) that catalog consumers see. Unique within a type. | +| name | TEXT | NOT NULL | Display name, shared by all documents of the entry. | +| aliases | TEXT[] | NOT NULL DEFAULT '{}', CHECK no empty string | Ordered alternate names used to match queries. The first one is the prompt hint. | +| metadata | JSONB | NOT NULL DEFAULT '{"source": "pipeline"}', CHECK object | `source` and any type-specific fields. See keys below. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Created. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). | + +### `metadata` keys + +| Key | Type | Default | Purpose | +|---|---|---|---| +| source | string | `pipeline` | `bootstrap` (seeded from code), `pipeline` (built from documents) or `import`. | + +Type-specific fields are added as keys, with no migration. Pydantic validates the keys per type. + +Notes +- Computed catalog values (`status`, `instances`, `workflow_ids`, `chunk_count`, `network_visible`, `promoted_at`) come from the `catalog` view, not stored here. +- Case-insensitive alias uniqueness is enforced in the app (`_dedupe_aliases`). Add a GIN index on `aliases` only if "find entry by alias" is needed in SQL. +- No join table: an entry has many documents and a document has one entry, which `documents.catalog_entry_id` expresses. +- Plain documents (kind `document`) have no entry. + +--- + +## 3. `documents` + +One row per document. 13 columns. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Document ID. | +| instance | TEXT | NOT NULL | Tenant or state code. | +| source | JSONB | NOT NULL, CHECK `jsonb_typeof(source) = 'object'`, CHECK `source->>'fingerprint' ~ '^[0-9a-f]{32}$'` | The uploaded file. See shape below. | +| document_kind | TEXT | NOT NULL DEFAULT 'document', CHECK `^[a-z0-9]+([_-][a-z0-9]+)*$` | Knowledge kind (`document`, `scheme`, `advisory`, `video`, or custom). | +| catalog_entry_id | UUID | | Catalog entry (for example a scheme) the document belongs to. Nullable. Composite FK, see below. | +| valid_from | DATE | | Start of the period chunks are searchable. | +| valid_to | DATE | CHECK `valid_from <= valid_to` | End of the period. NULL means no end. | +| is_disabled | BOOLEAN | NOT NULL DEFAULT FALSE | Soft delete. Restore flips it back. | +| index_status | JSONB | NOT NULL DEFAULT '{}', CHECK `jsonb_typeof(index_status) = 'object'` | Vector index state per index. Keys `dev` and `prod`. | +| metadata | JSONB | NOT NULL, DEFAULT see below, CHECK object, CHECK `(metadata->>'chunk_count')::int >= 0` | Display and flag fields. See keys below. | +| uploaded_by_user_id | UUID | FK → `users(id)` | Who uploaded. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Row created. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). | + +Composite FK `(catalog_entry_id, document_kind)` → `catalog_entries(id, type)`, `ON UPDATE CASCADE ON DELETE RESTRICT`. It guarantees a document's kind equals its entry's type. When `catalog_entry_id` is NULL the FK is not checked, so plain documents need no entry. + +Suggested indexes: +- `(instance)` +- `(created_at DESC)` +- `(catalog_entry_id) WHERE catalog_entry_id IS NOT NULL` +- `(instance, (source->>'fingerprint'))` for duplicate checks and identifier lookups +- `(((metadata->>'is_demo')::boolean))` for the demo filter on every list + +### `source` + +Facts about the uploaded file, not about processing. + +```json +{ + "fingerprint": "", + "original_filename": "pmkisan.pdf", + "storage_uri": "s3://...", + "source_type": "document", + "canonical_input_type": "pdf", + "page_count": null +} +``` + +| Key | Type | Notes | +|---|---|---| +| fingerprint | string | MD5, 32 lowercase hex. Enforced by CHECK. Also the key the document has inside the vector index. | +| original_filename | string | Name as uploaded. | +| storage_uri | string | Location of the uploaded file. | +| source_type | string | `spreadsheet`, `document` or `unknown`. | +| canonical_input_type | string | `spreadsheet` or `pdf`. Normalized pipeline path. | +| page_count | integer or null | NULL until OCR runs. | + +### `index_status` + +One entry per vector index. DEV and PROD are separate Qdrant collections. + +```json +{ + "dev": {"status": "indexed", "chunk_count_indexed": 42, + "last_indexed_at": "2026-10-19T06:20:00Z", "last_verified_at": "2026-10-19T06:25:00Z", + "schema_version": "passage-v1", "details": {}}, + "prod": {"...": "..."} +} +``` + +- A missing key means the document was never ingested to that index. +- `status` is `unknown`, `indexed` or `missing`. +- The document's key inside the vector index is `source.fingerprint`. +- The key is `prod` for the PROD collection names (`PROD_VECTOR_DB_COLLECTION_NAME`, `PROD_SCHEME_VECTOR_DB_COLLECTION_NAME`) and `dev` for every other collection. The collection name is kept inside the entry as `index_name`, next to `doc_id`, the id the index uses for the document. +- Update one index atomically: `SET index_status = index_status || jsonb_build_object('dev', $1::jsonb)`. +- A third index is a new key, with no migration. + +### `metadata` + +| Key | Type | Default | Purpose | +|---|---|---|---| +| display_name | string | absent | Human-friendly title. | +| catalog_visible | boolean | true | Counts toward the catalog. | +| network_visible | boolean | true | May be published to the Beckn network. | +| is_demo | boolean | false | Demo content, hidden by default. | +| chunk_count | integer ≥ 0 | 0 | Cached number of chunks. Updated in the same transaction as chunk writes. | +| reindex_required | boolean | false | Approved content changed and the vector index is stale. | +| reindex_reason | string | absent | Why a reindex is needed. | +| chunking | object `{run_number, provider, model, config, created_at}` | absent | Settings of the run that produced the current chunks. | + +Column default: `'{"catalog_visible": true, "network_visible": true, "is_demo": false, "chunk_count": 0, "reindex_required": false}'`. + +### Notes + +- A document's stage is the `current_stage` of its latest workflow. List and filter queries join the `document_current_workflow` view. +- SQL does not check types inside the JSONB columns. The insert path writes every default `metadata` key, and Pydantic validates `source`, `index_status` and `metadata`. +- `metadata.chunk_count` is rewritten in every chunk transaction and `metadata.is_demo` is filtered on every list. If performance suffers, move these two back to columns first. +- Each document keeps its own `source`. Two uploads of the same content from different paths keep their own `storage_uri`. There is no unique key on `(instance, fingerprint)`; the upload path reuses a document by its Temporal workflow id instead. Add `UNIQUE (instance, (source->>'fingerprint'))` only if one document per file per tenant must be enforced. +- Identifier lookup (`find_document_by_doc_identifier`) matches `documents.id`, `workflows.temporal_workflow_id`, `source->>'fingerprint'` and `source->>'original_filename'`. The legacy `md5(identifier)` check is `md5(source->>'fingerprint') = $1` in SQL. +- No `deleted_at`: soft delete is `is_disabled`, hard delete is `purge_document`. + +--- + +## 4. `workflows` + +One row per processing run of a document: the main pipeline run, retries, reingest and PROD promotion. Holds all processing state. + +Indexes: `(document_id, started_at DESC)`, `(status, started_at DESC)`, `(temporal_workflow_id)`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Workflow ID. | +| document_id | UUID | NOT NULL, FK → `documents(id)` ON DELETE CASCADE | Document the workflow runs against. | +| workflow_type | workflow_type_enum | NOT NULL | Kind of run. | +| temporal_workflow_id | TEXT | NOT NULL | Temporal workflow ID. `doc-` for the main run, suffixed for retries. Not unique (a restart reuses the ID). Used by the API, UI and Temporal. | +| status | workflow_status_enum | NOT NULL DEFAULT 'running' | `running`, `waiting_review`, `completed` or `failed`. | +| current_stage | workflow_stage | NOT NULL DEFAULT 'registered' | Stage the workflow is in now. This is the document's stage when the workflow is the latest. | +| stage_timestamps | JSONB | NOT NULL DEFAULT '{}', CHECK object with only keys `ocr`, `translation`, `chunks` | When each stage first completed in this run, ISO-8601 UTC. Set-once per key. | +| stop_after_ocr | BOOLEAN | NOT NULL DEFAULT FALSE | Halt after OCR (OCR-only batch runs). | +| error_message | TEXT | | Failure reason when status is `failed`. | +| config | JSONB | NOT NULL DEFAULT '{}', CHECK object | Settings the run used, plus `chunking_progress` while chunking. | +| started_by_user_id | UUID | FK → `users(id)` ON DELETE SET NULL | Who started it. For `promote_prod`, the person who requested promotion. NULL for system-started runs. | +| started_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | When the workflow started. Plays the role of `created_at`. | +| completed_at | TIMESTAMPTZ | | When it completed or failed. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). Progress updates touch the row often. | + +13 columns. + +Notes +- `stage_timestamps` is set-once: `stage_timestamps = jsonb_build_object('ocr', $1) || stage_timestamps` (right side wins, so existing keys are kept). +- Progress updates are atomic: `UPDATE ... SET config = config || jsonb_build_object('chunking_progress', ...)`. +- A PROD request inserts a `promote_prod` workflow in `waiting_review`, with `started_by_user_id` set to the requester and `started_at` to the request time. Approval moves it to `running`; when approval starts a `PromoteToProdWorkflow`, that run takes over the waiting row instead of adding a second one. +- The current workflow of a document is the latest by `started_at`. A retry started while an older run is still `running` takes over the stage display, and so does a retry on a `completed` document: its stage goes back to the retry's stage until the retry finishes. +- No CHECK ties `completed_at` to status: `reconcile_materialized_state` sets `waiting_review` without clearing `completed_at`. +- `cancelled` is not a status because nothing writes it. +- Do not copy the uploader snapshot into `config`; `started_by_user_id` and `audit_logs` cover it. + +### View: `document_current_workflow` + +Latest workflow per document. Stores nothing. + +```sql +SELECT DISTINCT ON (document_id) * +FROM workflows +ORDER BY document_id, started_at DESC; +``` + +--- + +## 5. `workflow_artifacts` + +Ledger of every file a workflow produces or stores. Files live in MinIO; this table records where and what. Append-only. + +Indexes: `(workflow_id, created_at DESC)`, `(workflow_id, artifact_type, created_at DESC)`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Artifact ID, used in `/documents/{id}/artifacts/{artifact_id}`. | +| workflow_id | UUID | NOT NULL, FK → `workflows(id)` ON DELETE CASCADE | Workflow that produced it. | +| artifact_type | TEXT | NOT NULL, CHECK not blank | Kind of output (`original_upload`, `normalized_pdf`, `normalized_spreadsheet`, `ocr_pages_json`, `translation_pages_json`, `chunk_json_export`, `vector_payload_export`, network publish exchange). Open set, so not an enum. | +| stage | workflow_stage | | Stage that created it. | +| storage_uri | TEXT | NOT NULL | MinIO location of the blob. | +| mime_type | TEXT | | MIME type. | +| filename | TEXT | | File name of the blob. | +| size_bytes | BIGINT | CHECK ≥ 0 | Blob size. | +| metadata | JSONB | NOT NULL DEFAULT '{}', CHECK object | Chunking config for chunk exports, request and response for network publishes, source path for uploads. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | When the artifact was recorded. | + +10 columns. + +Notes +- Every artifact belongs to a workflow. The upload path creates the first `pipeline` workflow before recording `original_upload`. +- The document is reached through `workflows.document_id`. Per-document listing: `SELECT a.* FROM workflow_artifacts a JOIN workflows w ON w.id = a.workflow_id WHERE w.document_id = $1 ORDER BY a.created_at DESC`. +- The latest original or normalized artifact uses the same join with `artifact_type = 'original_upload'` (or `normalized_%`). +- Deleting a workflow deletes its artifact rows. Blobs in MinIO are not removed by the database; cleanup stays in `purge_document`. +- No `updated_at`: rows are append-only. +- Chunking run history lives in `chunk_json_export` metadata. + +--- + +## 6. `pages` + +OCR layer and translations of a page. + +Unique: `(document_id, page_number)`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Page ID. | +| document_id | UUID | NOT NULL, FK → `documents(id)` ON DELETE CASCADE | Owning document. | +| page_number | INTEGER | NOT NULL, CHECK ≥ 1 | 1-based page position. | +| original_markdown | TEXT | | Raw OCR output, kept unchanged. | +| edited_markdown | TEXT | | Reviewer-corrected OCR text. Used instead of the original when present. | +| translations | JSONB | NOT NULL DEFAULT '{}', CHECK `jsonb_typeof(translations) = 'object'` | Translations of this page, keyed by target language. See shape below. | +| metadata | JSONB | NOT NULL DEFAULT '{"is_reviewed": false}', CHECK object | OCR review and language fields. See keys below. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | When the page row was first created. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). | + +### `metadata` keys + +| Key | Type | Default | Purpose | +|---|---|---|---| +| is_reviewed | boolean | false | OCR review done. | +| reviewer_notes | string | absent | Reviewer comments on the OCR text. | +| detected_language | string | absent | Language code detected. Non-`en` means the page needs translation. | + +SQL does not check these types; Pydantic does. The review stage gate counts reviewed pages with `(metadata->>'is_reviewed')::boolean`, and the translation step finds pages to translate with `metadata->>'detected_language'`. Add expression indexes on both if those queries are slow: `((metadata->>'is_reviewed')::boolean)` and `((metadata->>'detected_language'))`. + +### `translations` + +```json +{ + "gu": { + "translated_markdown": "...", + "edited_translation": null, + "is_reviewed": false, + "reviewer_notes": null, + "provider": "azure", + "model": "gpt-5.4-mini", + "translated_at": "2026-10-19T06:14:00Z" + } +} +``` + +- Key is the target language. Several languages per page are possible. +- Pages that need no translation (English) keep `'{}'`. An entry exists only for pages actually translated. +- `edited_translation` takes precedence over `translated_markdown` when present. +- Writes target one path, for example `SET translations = jsonb_set(translations, '{gu,edited_translation}', to_jsonb($1::text))`. +- Re-running OCR on a page resets its `translations` to `'{}'`, since the page text changed. +- Counting translated pages is `WHERE translations <> '{}'`. Finding unreviewed translations needs `jsonb_each` over the small per-page value. + +Notes +- `save_pages` replaces the whole row, so it must pass `translations` through unchanged unless the translation activity is writing it. Otherwise an OCR save would wipe translations. +- `save_pages` is `INSERT ... ON CONFLICT (document_id, page_number) DO UPDATE SET `. It is called during OCR segments, after OCR, after translation and at persist time, and each call replaces the whole row, so the caller passes every field. +- `save_pages` also passes `metadata` through whole. Update single keys with `metadata = metadata || jsonb_build_object('is_reviewed', true)` instead of rewriting the object. +- Page edits are audited in `audit_logs` (`page_edit`, `page_reset`), so no edit-history columns. +- Pages belong to a document, not to the source file: edits, review flags and OCR retries are per document. + +--- + +## 7. `chunks` + +Unique: `(document_id, chunk_number)`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Chunk ID. | +| document_id | UUID | NOT NULL, FK → `documents(id)` ON DELETE CASCADE | Owning document. | +| chunk_number | INTEGER | NOT NULL, CHECK ≥ 1 | Order within the document. | +| original_text | TEXT | NOT NULL | Chunk text as produced by the chunker. | +| edited_text | TEXT | | Reviewer-edited text. Indexed when present. | +| token_count | INTEGER | NOT NULL DEFAULT 0, CHECK ≥ 0 | Length in tokens. | +| page_start | INTEGER | NOT NULL DEFAULT 1, CHECK ≥ 1 | First page covered. Kept for display and provenance links. | +| page_end | INTEGER | NOT NULL DEFAULT 1, CHECK ≥ `page_start` | Last page covered. | +| source_spans | JSONB | NOT NULL DEFAULT '[]', CHECK array | List of `{page_number, start_char, end_char}` pointing back into the page text. | +| section_title | TEXT | | Heading of the section. | +| content_type | TEXT | | Kind of content (`heading`, `body`, ...). Only those two seen in the chunkers, so left as `TEXT`. | +| is_reference | BOOLEAN | NOT NULL DEFAULT FALSE | Reference material. Search can exclude it. | +| is_reviewed | BOOLEAN | NOT NULL DEFAULT FALSE | Reviewed by a human. | +| is_excluded | BOOLEAN | NOT NULL DEFAULT FALSE | Excluded from indexing. | +| reviewer_notes | TEXT | | Reviewer comments. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Created. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). | + +Notes +- Chunking provider, model, config and run number are stored once per document in `documents.metadata.chunking`, not per chunk. +- The distinct page numbers a chunk draws from are derived from `source_spans`. +- Re-chunk runs in one transaction: delete the document's chunks, insert the new ones, update `metadata.chunking` (with `run_number` incremented) and `metadata.chunk_count`. Chunk UUIDs change on every re-chunk; nothing references `chunks.id` (audit uses chunk number). +- `search_chunks` does a substring match on `lower(coalesce(edited_text, original_text))`, served by a trigram GIN index on that expression (the migration creates the `pg_trgm` extension). + +> **Why `pages` and `chunks` stay separate tables:** a page is a physical unit from OCR and a chunk is a semantic unit from the chunker, so their boundaries do not align. One chunk can span several pages and one page can hold several chunks, which makes the relationship many-to-many. Storing page text inside each chunk would copy a page into every chunk it overlaps, so any page edit, OCR retry or translation change would have to be rewritten in all of those chunks. + + +--- + +## 8. `audit_logs` + +Append-only history of actions. Must outlive documents, so there is no cascade delete. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Audit entry ID. | +| document_id | UUID | FK → `documents(id)` ON DELETE SET NULL | Document the action applied to. NULL for system-level entries and after a purge. | +| workflow_id | TEXT | | Temporal workflow ID snapshot. Survives a purge. | +| instance | TEXT | | Tenant snapshot, so tenant scoping works after the document is gone. | +| action_type | TEXT | NOT NULL, CHECK not blank | What happened (`page_edit`, `purge_document`, ...). 22 values and growing, so not an enum. | +| entity_type | audit_entity_type | | `document`, `page`, `chunk`, `setting` or `catalog`. | +| entity_number | INTEGER | | Page number or chunk number the action applied to. | +| field_name | TEXT | | Field that changed. | +| old_value | TEXT | | Value before the change. | +| new_value | TEXT | | Value after the change. | +| metadata | JSONB | NOT NULL DEFAULT '{}', CHECK object | Extra context. | +| actor_user_id | UUID | FK → `users(id)` ON DELETE SET NULL | Who acted. NULL for system actions. | +| actor | JSONB | NOT NULL DEFAULT '{}', CHECK object | Identity at the time: `{username, email, roles[]}`. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | When it happened. | + +Table CHECK: `entity_type IN ('setting','catalog') OR workflow_id IS NOT NULL`. + +Indexes: `(document_id, created_at DESC)`, `(created_at DESC)`, `(action_type, created_at DESC)`, `(created_at DESC) WHERE entity_type = 'setting'`. + +Notes +- The FK to `documents` is `SET NULL`, not cascade. A cascade would erase the audit trail on purge, and a strict FK would reject setting and catalog entries that have no document. +- Setting and catalog entries have NULL `document_id` and `workflow_id`, with `entity_type` set to `setting` or `catalog`. +- `instance` is a snapshot, so a purged document's history stays visible only to its own tenant. +- `entity_number` holds a page or chunk number, not an ID (chunk UUIDs would dangle after a re-chunk). +- `get_settings_audit_logs` is `WHERE entity_type = 'setting'`. +- `log_audit` fills `instance`, resolves the actor to a `users` row and builds the `actor` JSON. +- No `updated_at`: entries are immutable. +- The table only grows. Monthly partitioning is possible later without changing this design. + +--- + +## 9. `search_settings` + +Single row of search configuration. + +Unique: `singleton`. + +| Column | Type | Constraints / Default | Keys inside | Purpose | +|---|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | | Row ID. | +| singleton | BOOLEAN | NOT NULL DEFAULT TRUE UNIQUE, CHECK `singleton` | | Guarantees only one row. | +| retrieval | JSONB | NOT NULL DEFAULT '{}', CHECK object | `searchMethod`, `alpha`, `rankingMethod`, `hybridRrfK` | How semantic and keyword matching are combined and ranked. | +| candidates | JSONB | NOT NULL DEFAULT '{}', CHECK object | `indexName`, `efSearch`, `candidateCap`, `candidateMultiplier` | Which vector index is searched and how many candidates are pulled. | +| results | JSONB | NOT NULL DEFAULT '{}', CHECK object | `limit`, `maxChunksPerDoc`, `excludeReference`, `showHighlights`, `rerankMode` | How the final result list is shaped. | +| query | JSONB | NOT NULL DEFAULT '{}', CHECK object | `useE5Prefix`, `queryExpansionProfile` | How the query is prepared before searching. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | | Created. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | | Last change (trigger). | + +15 settings in four groups (4 + 4 + 5 + 2). All are global, with no per-tenant scope. + +Notes +- The database enforces only "is a JSON object". Value rules live in Pydantic: `SearchSettings` has four nested sub-models (`alpha` 0 to 1, integers ≥ 1, `searchMethod` in `TENSOR|LEXICAL|HYBRID`, `rankingMethod` in `rrf|normalize_linear`, unknown keys rejected). +- The API stays flat. `db.py` keeps one dict mapping each flat key to its group; reads flatten the columns and writes route each key to its column. +- Defaults live only in the Pydantic models. Reads merge stored JSON over the defaults. The default for `alpha` is 0.6. +- Partial update is an atomic merge per group: `UPDATE search_settings SET retrieval = retrieval || $1::jsonb`. +- Audit: one `audit_logs` row per changed key, `entity_type = 'setting'`, `field_name` such as `retrieval.alpha`. +- `rerankMode` has at least `none` and `bm25lite`; the full set was not seen, so no enum. +- Adding a setting to an existing group needs no migration; a new group needs a column. + +--- + +## 10. `email_otps` + +Server-side email-OTP login codes. Short-lived. + +Unique: `email`; index on `(expires_at)`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Row ID. | +| email | TEXT | NOT NULL UNIQUE, CHECK `email = lower(email)` | Address the code was requested for. One live code per email. | +| code_hash | TEXT | NOT NULL | Hash of the one-time code. The plain code is never stored. Random for unknown accounts. | +| expires_at | TIMESTAMPTZ | NOT NULL | When the code stops being valid. | +| attempts | INTEGER | NOT NULL DEFAULT 0, CHECK ≥ 0 | Failed verifications. Reset to 0 on resend. | +| last_sent_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | When the code was last sent. Used for the resend cooldown. | + +Notes +- No `created_at` or `updated_at`: rows are overwritten on each send. +- `email` is deliberately not an FK to `users`. A row is written for every request, even for unknown addresses (anti-enumeration, `email_otp.py:276-310`). +- Periodic cleanup (`DELETE FROM email_otps WHERE expires_at < now() - interval '1 day'`) is not scheduled yet. The endpoint is unauthenticated, so unverified rows stay until it is. +- `verify_otp` increments attempts atomically: `UPDATE ... SET attempts = attempts + 1 WHERE email = $1 RETURNING attempts`, or `SELECT ... FOR UPDATE`. +- Code constants (length 6, TTL 300 s, resend cooldown 30 s, max 5 attempts) stay in code. +- Optional: make the table `UNLOGGED`. Losing it in a crash only forces users to request a new code. + +--- + +## View: `catalog` + +Computed from `catalog_entries`, `documents`, `workflows` and `document_current_workflow`. Stores nothing. The `/catalog/v1/*` snapshot API reads it with `type = 'scheme'`, and its fields are named `scheme_code`, `scheme_name` and so on, so consumers see no change. + +| Column | Type | How it is computed | +|---|---|---| +| entry_id | UUID | From `catalog_entries`. | +| type | TEXT | From `catalog_entries`. | +| code | TEXT | From `catalog_entries`. | +| name | TEXT | From `catalog_entries`. | +| aliases | TEXT[] | From `catalog_entries`. | +| source | TEXT | `catalog_entries.metadata->>'source'`. | +| status | TEXT | `pending_reindex` if a live document needs reindex. Else `live` if any live document exists. Else `pending_prod` if a document awaits PROD approval. Else `live` for a bootstrap entry with no documents. Else `disabled`. | +| instances | TEXT[] | Distinct `instance` of the entry's documents, or `{default}` if none. | +| workflow_ids | TEXT[] | `temporal_workflow_id` of each entry document's main `pipeline` workflow. | +| chunk_count | INTEGER | Sum of `metadata->>'chunk_count'` over the entry's documents. | +| network_visible | BOOLEAN | True if any of its documents has `metadata->>'network_visible'` true. For an entry with no documents, true if bootstrap. | +| promoted_at | TIMESTAMPTZ | Latest `index_status->'prod'->>'last_indexed_at'` of the live documents. | + +"Live document" means the latest workflow's `current_stage = 'completed'`, `is_disabled` false, and `metadata->>'catalog_visible'` true. The entry's `type` already ties it to the document's kind through the composite FK. + +--- + +## 11. `webhook_records` + +Records pushed to `POST /webhook` (`x-api-key` = `WEBHOOK_API_KEY`). Carried over from the previous store unchanged apart from the types. + +Unique: `(source, external_id)`; index on `(source, status)`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Record ID. | +| source | TEXT | NOT NULL DEFAULT 'webhook' | Sender. | +| external_id | TEXT | NOT NULL | `contentID`, else `id`, else `slug`, else a hash of the payload. | +| title | TEXT | | From the payload. | +| slug | TEXT | | From the payload. | +| status | TEXT | | From the payload. | +| is_published | BOOLEAN | | From `isPublished`. | +| category | TEXT | | From `domainName`. | +| payload | JSONB | NOT NULL, CHECK object | The full record. | +| payload_hash | TEXT | | SHA-256 of the canonical payload; an upsert with the same hash writes nothing. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | The payload's `createdAt` when it has one. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | The payload's `updatedAt` when it has one. | + +The payload's own timestamps are stored as given, so these two columns are not maintained by the shared trigger. + +--- + +## `schema_migrations` + +Created by the runner, not by a migration file: `version` (the `NNNN` prefix, primary key), `name` (the file name) and `applied_at`. The runner takes a session advisory lock, so concurrent api and worker starts apply each file once, and runs each file in its own transaction. + +--- + +## Implementation notes + +- **Identifiers.** `db.py` functions take a `workflow_id` argument that may be the document UUID, a Temporal workflow id, the fingerprint, the filename, an index `doc_id` or the legacy `md5(fingerprint)` doc id (`find_document_by_doc_identifier`). Documents, jobs and artifacts come back as flat dicts; `id` is the UUID and `workflow_id` is the Temporal workflow id of the document's main run (the earliest `pipeline` or `ocr_only` workflow). The API rewrites `/documents/{id}/...` to that workflow id before routing, because handlers signal and query Temporal with it. +- **Jobs.** The API's "jobs" and "runs" are `workflows` rows; job ids and artifact ids are the UUIDs of `workflows` and `workflow_artifacts`. `latest_job_id`, `original_artifact_id` and `normalized_artifact_id` are derived by join, not stored. +- **Catalog version.** `/catalog/v1/version` counts `audit_logs` rows with `entity_type = 'catalog'` and `action_type = 'catalog_version_bump'`, so it only grows. `updated_at` is the newest such row. +- **Document stage.** The stage a document shows is `document_current_workflow.current_stage`; `update_document_stage` writes to that workflow. +- **Page translations.** `save_pages` keeps stored translations unless the page carries `translations` (replaces them) or flat `translated_*` fields (merged into `translation_target_language`). A page whose OCR text changed loses its translations. +- **Users.** Rows are upserted from the auth dependency on write requests (at most once per five minutes per user) and from uploads, workflow starts and audit entries. `local-admin` is seeded by the migration. diff --git a/docs/SQLITE_SCHEMA.md b/docs/SQLITE_SCHEMA.md new file mode 100644 index 0000000..c3fa539 --- /dev/null +++ b/docs/SQLITE_SCHEMA.md @@ -0,0 +1,374 @@ +# SQLite Schema — knowledge-provider (legacy) + +> Legacy. The store is now Postgres, described in `docs/POSTGRES_SCHEMA.md`. This file documents the shape of the old SQLite file that `scripts/migrate_sqlite_to_postgres.py` reads. + +## Overview + +- One SQLite file is the canonical store for metadata, review state and editable content. MinIO holds blobs, Qdrant holds search projections, Temporal only orchestrates. +- 12 tables. `documents` is the root. `pages`, `chunks`, `document_jobs`, `document_artifacts`, `document_index_status` and `audit_logs` hang off it by `workflow_id`. +- Connection settings: WAL journal mode, `foreign_keys=ON`, `IMMEDIATE` transactions, 30s lock timeout. Writes are serialized by a process-local lock. +- Schema is created on startup. Changes are applied as idempotent `ALTER TABLE ... ADD COLUMN` calls, so the tables below show the final shape with migration-added columns merged in. +- Timestamps are ISO-8601 strings in TEXT. Booleans are INTEGER 0/1. Columns ending in `_json` hold JSON-encoded TEXT. +- `workflow_id` is the key used by the API, UI and Temporal. + +## Drawbacks + +### Integrity +- **No foreign keys.** Tables are linked by convention on `workflow_id`. Nothing stops orphan rows, and purging a document means deleting from each table by hand. +- **Pointer columns are unenforced.** `latest_job_id`, `original_artifact_id` and `normalized_artifact_id` on `documents` point into tables that point back, with no constraint. +- **No constraints on values.** Stages, statuses, job types, routing and flags are free TEXT or INTEGER. `created_at` and `updated_at` are nullable. Nothing checks that `valid_from <= valid_to` or that counts are non-negative. +- **Audit rows use sentinel strings** (`__system__`, `__settings__`, `__catalog__`) in key columns for non-document events, which a real foreign key could not hold. +- **Settings are untyped.** Every value is TEXT and cast on read. Nothing stops an out-of-range `alpha` or an invalid search method. Defaults disagree across three places (`alpha` is 0.7 in the model, 0.6 in the seed and reset). + +### Data modeling +- **Same fact stored in several places.** + - Uploader username, email and roles are copied onto each document and again into the audit log. + - Scheme name, aliases and routing are copied onto every document of a scheme. + - The scheme-to-document relationship is stored twice: a JSON list on the catalog entry and `scheme_code` on the document. + - The file's MD5 is stored as `document_id`, `canonical_document_id` and `source_file_fingerprint`; `filename` equals `source_filename`. +- **`documents` is too wide.** About 45 columns mix file facts, processing state, uploader identity, PROD request, validity and scheme data. +- **JSON in TEXT.** Lists and objects (`scheme_aliases_json`, `workflow_ids_json`, `source_page_numbers_json`, `config_json`, `details_json`) cannot be validated or queried. +- **Run metadata repeated on every chunk.** Chunking provider, model, config, run ID and version are identical for all chunks of a run. +- **Translation columns sit on `pages`.** Eight mostly-NULL columns for pages that need no translation, and only one target language per page. +- **Roles are comma-separated strings**, with no array semantics. +- **Dead or constant columns.** `marqo_doc_id`, `temporal_run_id`, `content_hash`, `collection_name` and `tool_routing` carry no used data. The manifest table has no writer in the code. +- **Hardcoded language.** The manifest table has a Gujarati title column and the search settings default to a Gujarati query profile. +- **`entity_id` in the audit log is not an ID.** It holds a page or chunk number. +- **Catalog table is a cache** that is fully recomputed from documents and can drift. + +## Tables + +| # | Table | Purpose | +|---|-------|---------| +| 1 | `documents` | One row per ingested document. Root table, holds stage and metadata. | +| 2 | `pages` | OCR and translated page content plus review state. | +| 3 | `chunks` | Chunked text plus review state. | +| 4 | `audit_logs` | Per-document change and action history. | +| 5 | `settings` | Key/value app config, including search defaults. | +| 6 | `document_jobs` | Temporal job runs per document. | +| 7 | `document_artifacts` | Pointers to blobs stored in MinIO. | +| 8 | `document_index_status` | Vector index state per document and index. | +| 9 | `document_manifest_entries` | Imported manifest (CSV) metadata. | +| 10 | `scheme_catalog_meta` | Single-row catalog version counter. | +| 11 | `scheme_catalog_entries` | Scheme master catalog (served by `/catalog/v1/*`). | +| 12 | `email_otps` | Email OTP login codes. | + +## 1. documents + +Primary key: `workflow_id` + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| workflow_id | TEXT | PRIMARY KEY | Temporal workflow ID. Unique key for the document and the join key used by every other table. | +| document_id | TEXT | NOT NULL | Document identifier carried into chunk payloads and index records. | +| canonical_document_id | TEXT | | Stable ID shared across re-uploads or re-ingestions of the same document. | +| filename | TEXT | NOT NULL | Name of the file as stored by the pipeline. | +| display_name | TEXT | | Human-friendly title shown in the UI. | +| source_filename | TEXT | | Original filename as uploaded by the user. | +| source_manifest_name | TEXT | | Filename of the matching entry in `document_manifest_entries`, used to link imported metadata. | +| source_file_fingerprint | TEXT | | Hash of the source file, used to detect duplicate uploads. | +| filepath | TEXT | NOT NULL | Local path of the uploaded file. | +| stage | TEXT | NOT NULL DEFAULT 'registered' | Current pipeline stage (registered, ocr_processing, ocr_review, translation_processing, translation_review, chunking, chunk_review, ready_for_ingestion, ingesting, approval_for_prod, ingesting_prod, completed, failed). | +| page_count | INTEGER | DEFAULT 0 | Number of pages extracted by OCR. | +| chunk_count | INTEGER | DEFAULT 0 | Number of chunks produced. | +| error_message | TEXT | | Failure reason when the stage is `failed`. | +| is_demo | INTEGER | DEFAULT 0 | Marks the document as demo content so it can be filtered out of summaries and index lists. | +| is_disabled | INTEGER | DEFAULT 0 | Soft-disable flag. A disabled document is hidden from lists and search. | +| created_at | TEXT | | When the row was registered. | +| updated_at | TEXT | | Last time the row changed. | +| ocr_completed_at | TEXT | | When OCR finished. | +| translation_completed_at | TEXT | | When translation finished. | +| chunks_completed_at | TEXT | | When chunking finished. | +| ingested_at | TEXT | | When the document was ingested into the vector index. | +| source_type | TEXT | | Input family derived from the file extension: `spreadsheet` (csv/xlsx), `document` (pdf, image, office) or `unknown`. | +| canonical_input_type | TEXT | | Form the input is normalized to: `spreadsheet` or `pdf`. Drives which pipeline path runs. | +| stop_after_ocr | INTEGER | DEFAULT 0 | When 1, the workflow halts after OCR (OCR-only batch runs). | +| reindex_required | INTEGER | DEFAULT 0 | Flags that approved content changed and the vector index is stale. | +| reindex_reason | TEXT | | Why a reindex is needed. | +| original_artifact_id | INTEGER | | Points to the `document_artifacts` row for the original upload. | +| normalized_artifact_id | INTEGER | | Points to the `document_artifacts` row for the normalized PDF or spreadsheet. | +| latest_job_id | INTEGER | | Points to the most recent `document_jobs` row. | +| instance | TEXT | | Tenant or state code (`bv` is platform-wide, others are state codes such as `mh`). Used for scoping and list filters. NULL or empty is backfilled with `DEFAULT_INSTANCE`. | +| uploaded_by_user_id | TEXT | | Keycloak user ID of the uploader. | +| uploaded_by_username | TEXT | | Username of the uploader. | +| uploaded_by_email | TEXT | | Email of the uploader. | +| uploaded_by_roles | TEXT | | Comma-separated roles of the uploader at upload time. | +| prod_ready_requested_at | TEXT | | When someone requested promotion to PROD. | +| prod_ready_requested_by_user_id | TEXT | | User ID of the PROD promotion requester. | +| prod_ready_requested_by_username | TEXT | | Username of the PROD promotion requester. | +| valid_from | TEXT | | Start of the period in which the document's chunks are searchable. Stamped with the upload day, editable by the approver. NULL means always current. | +| valid_to | TEXT | | End of the validity period. NULL means no end. | +| document_kind | TEXT | DEFAULT 'document' | Knowledge kind slug: `document`, `scheme`, `advisory`, `video` or a custom slug. Decides which Beckn catalog is announced. | +| scheme_code | TEXT | | Scheme identifier (for example `pmkisan`). Used when `document_kind` is `scheme`. Links to `scheme_catalog_entries`. | +| scheme_name | TEXT | | Human-readable scheme name. | +| scheme_aliases_json | TEXT | | JSON list of alternate names for the scheme, used for matching queries. | +| tool_routing | TEXT | | Which AI tool path serves the scheme: `qdrant`, `legacy` or `both`. | +| catalog_visible | INTEGER | DEFAULT 1 | Whether the document appears in the pull-based catalog API. | +| network_visible | INTEGER | DEFAULT 1 | Whether the document may be published to the Beckn network (Discovery Service). | + +Indexes: + +| Index | Columns | Purpose | +|-------|---------|---------| +| idx_documents_stage | stage | Filter and count documents by stage. | +| idx_documents_instance | instance | Tenant-scoped lists. | +| idx_documents_created | created_at DESC | Newest-first listing. | +| idx_documents_scheme_code | scheme_code | Look up documents by scheme. | + +## 2. pages + +Primary key: `id`. Unique: `(workflow_id, page_number)`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| id | INTEGER | PRIMARY KEY AUTOINCREMENT | Row ID. | +| workflow_id | TEXT | NOT NULL | Owning document (`documents.workflow_id`). | +| page_number | INTEGER | NOT NULL | 1-based page position in the document. | +| original_markdown | TEXT | | Raw OCR output for the page, kept unchanged. | +| edited_markdown | TEXT | | Reviewer-corrected OCR text. Takes precedence over the original when present. | +| is_reviewed | INTEGER | DEFAULT 0 | Whether the OCR text for this page has been reviewed. | +| reviewer_notes | TEXT | | Reviewer comments on the OCR text. | +| detected_language | TEXT | | Language detected for the page (via the `lang-detect` service). | +| translated_markdown | TEXT | | Machine translation of the page. | +| edited_translation | TEXT | | Reviewer-corrected translation. | +| translation_reviewed | INTEGER | DEFAULT 0 | Whether the translation has been reviewed. | +| translation_notes | TEXT | | Reviewer comments on the translation. | +| translation_provider | TEXT | | Provider that produced the translation. | +| translation_model | TEXT | | Model used for the translation. | +| translation_target_language | TEXT | | Language the page was translated into. | +| translated_at | TEXT | | When the translation was produced. | + +Indexes: `idx_pages_workflow (workflow_id)` — fetch all pages of a document. + +## 3. chunks + +Primary key: `id`. Unique: `(workflow_id, chunk_number)`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| id | INTEGER | PRIMARY KEY AUTOINCREMENT | Row ID. | +| workflow_id | TEXT | NOT NULL | Owning document (`documents.workflow_id`). | +| chunk_number | INTEGER | NOT NULL | Order of the chunk within the document. | +| original_text | TEXT | | Chunk text as produced by the chunker. | +| edited_text | TEXT | | Reviewer-edited chunk text. This is what gets indexed when present. | +| token_count | INTEGER | DEFAULT 0 | Token length of the chunk. | +| page_start | INTEGER | DEFAULT 1 | First page the chunk covers. | +| page_end | INTEGER | DEFAULT 1 | Last page the chunk covers. | +| source_page_numbers_json | TEXT | | JSON list of every page the chunk draws from. | +| source_spans_json | TEXT | | JSON list of text spans in the source pages, used to trace a chunk back to its provenance. | +| section_title | TEXT | | Heading of the section the chunk belongs to. | +| content_type | TEXT | | Kind of content in the chunk, for example `heading` or `body`. | +| is_reference | INTEGER | DEFAULT 0 | Marks reference material. Search can exclude these (`search_exclude_reference`). | +| chunking_provider | TEXT | | Chunking provider that produced the chunk. | +| chunking_model | TEXT | | Model used for chunking, if any. | +| chunking_config_json | TEXT | | JSON snapshot of the chunking settings used. | +| chunking_run_id | TEXT | | ID of the chunking run, to group chunks created together. | +| chunk_version | INTEGER | DEFAULT 1 | Version counter for the chunk. | +| is_reviewed | INTEGER | DEFAULT 0 | Whether the chunk has been reviewed. | +| is_excluded | INTEGER | DEFAULT 0 | Reviewer excluded the chunk. Excluded chunks are not indexed. | +| reviewer_notes | TEXT | | Reviewer comments on the chunk. | + +Indexes: `idx_chunks_workflow (workflow_id)` — fetch all chunks of a document. + +## 4. audit_logs + +Primary key: `id`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| id | INTEGER | PRIMARY KEY AUTOINCREMENT | Row ID. | +| workflow_id | TEXT | NOT NULL | Document the action applied to. | +| document_id | TEXT | NOT NULL | Document identifier at the time of the action. | +| action_type | TEXT | NOT NULL | What happened, for example `promote_to_prod`. | +| entity_type | TEXT | | Kind of record changed, for example `document`, page or chunk. | +| entity_id | INTEGER | | ID or number of the record changed. | +| field_name | TEXT | | Field that was changed. | +| old_value | TEXT | | Value before the change. | +| new_value | TEXT | | Value after the change. | +| metadata | TEXT | | Extra context, stored as JSON text. | +| timestamp | TEXT | NOT NULL | When the action happened. | +| actor_user_id | TEXT | | Keycloak user ID of whoever acted. | +| actor_username | TEXT | | Username of the actor. | +| actor_email | TEXT | | Email of the actor. | +| actor_roles | TEXT | | Roles of the actor at the time. | + +Indexes: + +| Index | Columns | Purpose | +|-------|---------|---------| +| idx_audit_workflow | workflow_id | Per-document audit trail. | +| idx_audit_timestamp | timestamp DESC | Newest-first global audit log. | + +## 5. settings + +Primary key: `key`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| key | TEXT | PRIMARY KEY | Setting name. | +| value | TEXT | NOT NULL | Setting value, always stored as text. | +| description | TEXT | | What the setting does. | +| updated_at | TEXT | NOT NULL | When the setting last changed. | + +Default rows (`INSERT OR IGNORE`): + +| key | default value | Purpose | +|-----|---------------|---------| +| search_method | HYBRID | Search mode: TENSOR, LEXICAL or HYBRID. | +| search_limit | 12 | Number of search results to return. | +| search_alpha | 0.6 | Hybrid weighting: 0 is lexical only, 1 is semantic only. | +| search_ranking_method | rrf | Hybrid ranking: `rrf` or `normalize_linear`. | +| search_show_highlights | true | Show highlighted matches in results. | +| search_ef_search | 256 | HNSW search accuracy parameter. | +| search_index_name | documents-index | Default vector DB collection name. | +| search_candidate_cap | 120 | Cap on the candidate retrieval pool. | +| search_candidate_multiplier | 10 | Candidate pool multiplier before the final cut. | +| search_max_chunks_per_doc | 2 | Result diversity cap per document. | +| search_use_e5_prefix | true | Prefix queries with `query:` for e5 embedding models. | +| search_exclude_reference | true | Exclude reference chunks when the index supports it. | +| search_query_expansion_profile | gu-v1 | Query expansion profile. | +| search_rerank_mode | none | Post-search reranking mode. | +| search_hybrid_rrfk | 60 | RRF tuning parameter for hybrid search. | + +## 6. document_jobs + +Primary key: `id`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| id | INTEGER | PRIMARY KEY AUTOINCREMENT | Job ID. Referenced by `documents.latest_job_id` and `document_artifacts.job_id`. | +| workflow_id | TEXT | NOT NULL | Document the job ran for. | +| job_type | TEXT | NOT NULL | Kind of run: `ocr_only`, `ocr_retry`, `translation_retry`, `chunking_retry`, `reingest`, `promote_prod`, plus the main pipeline run. | +| temporal_workflow_id | TEXT | | Temporal workflow ID, to find the run in the Temporal UI. | +| temporal_run_id | TEXT | | Temporal run ID. | +| status | TEXT | NOT NULL DEFAULT 'running' | Job state. | +| current_stage | TEXT | | Stage the job is in right now. | +| started_at | TEXT | NOT NULL | When the job started. | +| completed_at | TEXT | | When the job ended. | +| error_message | TEXT | | Failure reason. | +| config_json | TEXT | | JSON snapshot of the settings the job ran with. | + +Indexes: `idx_document_jobs_workflow (workflow_id, started_at DESC)` — latest jobs for a document. + +## 7. document_artifacts + +Primary key: `id`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| id | INTEGER | PRIMARY KEY AUTOINCREMENT | Artifact ID. Referenced by `documents.original_artifact_id` and `normalized_artifact_id`. | +| workflow_id | TEXT | NOT NULL | Owning document. | +| job_id | INTEGER | | Job that produced the artifact (`document_jobs.id`). | +| artifact_type | TEXT | NOT NULL | Kind of output, for example `original_upload`, `normalized_pdf`, `normalized_spreadsheet`, `ocr_pages_json`, `translation_pages_json`, `chunk_json_export`, `vector_payload_export`. | +| stage | TEXT | | Pipeline stage that created it. | +| storage_uri | TEXT | NOT NULL | MinIO location of the blob. | +| mime_type | TEXT | | MIME type of the blob. | +| filename | TEXT | | File name of the blob. | +| size_bytes | INTEGER | | Blob size in bytes. | +| metadata_json | TEXT | | Extra details as JSON, for example the source path or input type. | +| created_at | TEXT | NOT NULL | When the artifact was recorded. | + +Indexes: `idx_document_artifacts_workflow (workflow_id, created_at DESC)` — artifacts of a document, newest first. + +## 8. document_index_status + +Primary key: `(workflow_id, index_name)`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| workflow_id | TEXT | NOT NULL | Document that was indexed. | +| index_name | TEXT | NOT NULL | Vector index or collection (for example DEV or PROD). One row per document and index. | +| doc_id | TEXT | | Document ID as stored in the vector index. Replaces legacy `marqo_doc_id`. | +| chunk_count_indexed | INTEGER | DEFAULT 0 | Number of chunks currently in the index. | +| last_indexed_at | TEXT | | When the document was last written to the index. | +| last_verified_at | TEXT | | When the index state was last checked against the vector store. | +| schema_version | TEXT | | Index payload schema version, for spotting stale payloads. | +| status | TEXT | NOT NULL DEFAULT 'unknown' | Index state, for example `indexed` or `unknown`. | +| details_json | TEXT | | Extra details as JSON. | + +Older DBs may still carry `marqo_doc_id` until a cleanup script is run. Startup backfills `doc_id` from it when present. + +## 9. document_manifest_entries + +Primary key: `id`. Unique: `manifest_filename`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| id | INTEGER | PRIMARY KEY AUTOINCREMENT | Row ID. | +| manifest_filename | TEXT | NOT NULL UNIQUE | Filename the entry describes. Matches `documents.source_manifest_name`. | +| title_en | TEXT | | English title. | +| title_gu | TEXT | | Gujarati title. | +| doc_language | TEXT | | Language of the document. | +| category_tags | TEXT | | Category tags for the document. | +| description | TEXT | | Short description. | +| quality_score | TEXT | | Quality rating from the manifest. | +| priority_rank | TEXT | | Ingestion priority from the manifest. | +| ingestion_status | TEXT | | Ingestion status recorded in the manifest. | +| feedback | TEXT | | Reviewer feedback from the manifest. | +| source_csv_path | TEXT | | CSV file the row was imported from. | +| imported_at | TEXT | NOT NULL | When the row was imported. | + +Indexes: `idx_manifest_entries_filename (manifest_filename)` — look up by filename. + +## 10. scheme_catalog_meta + +Single-row table, enforced by `CHECK (id = 1)`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| id | INTEGER | PRIMARY KEY, CHECK (id = 1) | Fixed to 1 so only one row can exist. | +| version | INTEGER | NOT NULL DEFAULT 0 | Catalog version counter. Bumped on catalog changes so pull clients can detect updates. | +| updated_at | TEXT | | When the version last changed. | +| notes | TEXT | | Reason for the last bump. | + +## 11. scheme_catalog_entries + +Primary key: `scheme_code`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| scheme_code | TEXT | PRIMARY KEY | Scheme identifier (for example `pmkisan`). | +| scheme_name | TEXT | | Human-readable scheme name. | +| scheme_aliases_json | TEXT | | JSON list of alternate names for matching queries. | +| instances_json | TEXT | | JSON list of instances (tenants) that have this scheme. | +| workflow_ids_json | TEXT | | JSON list of `documents.workflow_id` values that back the scheme. | +| collection_name | TEXT | | Vector collection holding the scheme's chunks. | +| chunk_count | INTEGER | DEFAULT 0 | Number of chunks across the scheme's documents. | +| status | TEXT | NOT NULL DEFAULT 'disabled' | Catalog state: `live`, `pending_prod`, `pending_reindex` or `disabled`. Only live schemes are served as vector schemes. | +| network_visible | INTEGER | DEFAULT 0 | Whether the scheme is exposed on the network. | +| promoted_at | TEXT | | When the scheme was promoted to PROD. | +| content_hash | TEXT | | Hash of code, name, aliases, visibility and status, used to detect changes. | +| source | TEXT | | Where the entry came from: `pipeline` (built from scheme documents) or `import` (bootstrapped). | +| updated_at | TEXT | | When the entry last changed. | + +Indexes: `idx_scheme_catalog_status (status)` — filter entries by status. + +## 12. email_otps + +Primary key: `email`. One live code per email, overwritten on resend. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| email | TEXT | PRIMARY KEY | Email the code was sent to. | +| code_hash | TEXT | NOT NULL | Hash of the one-time code. The plain code is never stored. | +| expires_at | TEXT | NOT NULL | Expiry time of the code. | +| attempts | INTEGER | NOT NULL DEFAULT 0 | Failed verification attempts. Reset to 0 on resend. | +| created_at | TEXT | NOT NULL | When the code was first issued. | +| last_sent_at | TEXT | NOT NULL | When the code was last sent. Used to throttle resends. | + +## Relationships (by convention, not enforced) + +| From | Column | To | +|------|--------|----| +| pages | workflow_id | documents.workflow_id | +| chunks | workflow_id | documents.workflow_id | +| audit_logs | workflow_id | documents.workflow_id | +| document_jobs | workflow_id | documents.workflow_id | +| document_artifacts | workflow_id | documents.workflow_id | +| document_artifacts | job_id | document_jobs.id | +| document_index_status | workflow_id | documents.workflow_id | +| documents | original_artifact_id, normalized_artifact_id | document_artifacts.id | +| documents | latest_job_id | document_jobs.id | +| documents | source_manifest_name | document_manifest_entries.manifest_filename | +| documents | scheme_code | scheme_catalog_entries.scheme_code | +| scheme_catalog_entries | workflow_ids_json (JSON list) | documents.workflow_id | diff --git a/migrations/0001_initial_schema.sql b/migrations/0001_initial_schema.sql new file mode 100644 index 0000000..3fc75d0 --- /dev/null +++ b/migrations/0001_initial_schema.sql @@ -0,0 +1,292 @@ +CREATE EXTENSION IF NOT EXISTS pg_trgm; + +CREATE FUNCTION set_updated_at() RETURNS trigger AS $$ +BEGIN + NEW.updated_at := now(); + RETURN NEW; +END $$ LANGUAGE plpgsql; + +CREATE TYPE workflow_stage AS ENUM ( + 'registered', 'ocr_processing', 'ocr_review', 'translation_processing', + 'translation_review', 'chunking', 'chunk_review', 'ready_for_ingestion', + 'ingesting', 'publishing_to_network', 'approval_for_prod', 'ingesting_prod', + 'completed', 'failed' +); +CREATE TYPE workflow_type_enum AS ENUM ( + 'pipeline', 'ocr_only', 'ocr_retry', 'translation_retry', 'chunking_retry', + 'reingest', 'promote_prod' +); +CREATE TYPE workflow_status_enum AS ENUM ('running', 'waiting_review', 'completed', 'failed'); +CREATE TYPE audit_entity_type AS ENUM ('document', 'page', 'chunk', 'setting', 'catalog'); + +-- 1. users +CREATE TABLE users ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + keycloak_user_id TEXT NOT NULL UNIQUE, + username TEXT, + email TEXT, + roles TEXT[] NOT NULL DEFAULT '{}', + first_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_seen_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE UNIQUE INDEX users_email_lower_key ON users (lower(email)) WHERE email IS NOT NULL; + +INSERT INTO users (keycloak_user_id, username, roles) +VALUES ('local-admin', 'local-admin', '{superadmin}'); + +-- 2. catalog_entries +CREATE TABLE catalog_entries ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + type TEXT NOT NULL CHECK (type ~ '^[a-z0-9]+([_-][a-z0-9]+)*$'), + code TEXT NOT NULL CHECK (code ~ '^[a-z0-9]+(-[a-z0-9]+)*$'), + name TEXT NOT NULL, + aliases TEXT[] NOT NULL DEFAULT '{}' CHECK (array_position(aliases, '') IS NULL), + metadata JSONB NOT NULL DEFAULT '{"source": "pipeline"}' CHECK (jsonb_typeof(metadata) = 'object'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (type, code), + UNIQUE (id, type) +); +CREATE TRIGGER trg_catalog_entries_updated_at BEFORE UPDATE ON catalog_entries + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); + +-- 3. documents +CREATE TABLE documents ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + instance TEXT NOT NULL, + source JSONB NOT NULL + CHECK (jsonb_typeof(source) = 'object') + CHECK (source->>'fingerprint' ~ '^[0-9a-f]{32}$'), + document_kind TEXT NOT NULL DEFAULT 'document' CHECK (document_kind ~ '^[a-z0-9]+([_-][a-z0-9]+)*$'), + catalog_entry_id UUID, + valid_from DATE, + valid_to DATE CHECK (valid_from <= valid_to), + is_disabled BOOLEAN NOT NULL DEFAULT FALSE, + index_status JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(index_status) = 'object'), + metadata JSONB NOT NULL + DEFAULT '{"catalog_visible": true, "network_visible": true, "is_demo": false, "chunk_count": 0, "reindex_required": false}' + CHECK (jsonb_typeof(metadata) = 'object') + CHECK ((metadata->>'chunk_count')::int >= 0), + uploaded_by_user_id UUID REFERENCES users(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + FOREIGN KEY (catalog_entry_id, document_kind) REFERENCES catalog_entries (id, type) + ON UPDATE CASCADE ON DELETE RESTRICT +); +CREATE INDEX documents_instance_idx ON documents (instance); +CREATE INDEX documents_created_at_idx ON documents (created_at DESC); +CREATE INDEX documents_catalog_entry_idx ON documents (catalog_entry_id) WHERE catalog_entry_id IS NOT NULL; +CREATE INDEX documents_fingerprint_idx ON documents (instance, (source->>'fingerprint')); +CREATE INDEX documents_is_demo_idx ON documents (((metadata->>'is_demo')::boolean)); +CREATE TRIGGER trg_documents_updated_at BEFORE UPDATE ON documents + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); + +-- 4. workflows +CREATE TABLE workflows ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + workflow_type workflow_type_enum NOT NULL, + temporal_workflow_id TEXT NOT NULL, + status workflow_status_enum NOT NULL DEFAULT 'running', + current_stage workflow_stage NOT NULL DEFAULT 'registered', + stage_timestamps JSONB NOT NULL DEFAULT '{}' + CHECK (jsonb_typeof(stage_timestamps) = 'object' + AND (stage_timestamps - ARRAY['ocr', 'translation', 'chunks']) = '{}'::jsonb), + stop_after_ocr BOOLEAN NOT NULL DEFAULT FALSE, + error_message TEXT, + config JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(config) = 'object'), + started_by_user_id UUID REFERENCES users(id) ON DELETE SET NULL, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + completed_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX workflows_document_started_idx ON workflows (document_id, started_at DESC); +CREATE INDEX workflows_status_started_idx ON workflows (status, started_at DESC); +CREATE INDEX workflows_temporal_id_idx ON workflows (temporal_workflow_id); +CREATE TRIGGER trg_workflows_updated_at BEFORE UPDATE ON workflows + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); + +CREATE VIEW document_current_workflow AS +SELECT DISTINCT ON (document_id) * +FROM workflows +ORDER BY document_id, started_at DESC, id DESC; + +-- 5. workflow_artifacts +CREATE TABLE workflow_artifacts ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + workflow_id UUID NOT NULL REFERENCES workflows(id) ON DELETE CASCADE, + artifact_type TEXT NOT NULL CHECK (btrim(artifact_type) <> ''), + stage workflow_stage, + storage_uri TEXT NOT NULL, + mime_type TEXT, + filename TEXT, + size_bytes BIGINT CHECK (size_bytes >= 0), + metadata JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(metadata) = 'object'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX workflow_artifacts_workflow_idx ON workflow_artifacts (workflow_id, created_at DESC); +CREATE INDEX workflow_artifacts_type_idx ON workflow_artifacts (workflow_id, artifact_type, created_at DESC); + +-- 6. pages +CREATE TABLE pages ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + page_number INTEGER NOT NULL CHECK (page_number >= 1), + original_markdown TEXT, + edited_markdown TEXT, + translations JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(translations) = 'object'), + metadata JSONB NOT NULL DEFAULT '{"is_reviewed": false}' CHECK (jsonb_typeof(metadata) = 'object'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (document_id, page_number) +); +CREATE TRIGGER trg_pages_updated_at BEFORE UPDATE ON pages + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); + +-- 7. chunks +CREATE TABLE chunks ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + chunk_number INTEGER NOT NULL CHECK (chunk_number >= 1), + original_text TEXT NOT NULL, + edited_text TEXT, + token_count INTEGER NOT NULL DEFAULT 0 CHECK (token_count >= 0), + page_start INTEGER NOT NULL DEFAULT 1 CHECK (page_start >= 1), + page_end INTEGER NOT NULL DEFAULT 1, + source_spans JSONB NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(source_spans) = 'array'), + section_title TEXT, + content_type TEXT, + is_reference BOOLEAN NOT NULL DEFAULT FALSE, + is_reviewed BOOLEAN NOT NULL DEFAULT FALSE, + is_excluded BOOLEAN NOT NULL DEFAULT FALSE, + reviewer_notes TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (document_id, chunk_number), + CHECK (page_end >= page_start) +); +CREATE INDEX chunks_text_trgm_idx ON chunks + USING gin (lower(coalesce(edited_text, original_text)) gin_trgm_ops); +CREATE TRIGGER trg_chunks_updated_at BEFORE UPDATE ON chunks + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); + +-- 8. audit_logs +CREATE TABLE audit_logs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + document_id UUID REFERENCES documents(id) ON DELETE SET NULL, + workflow_id TEXT, + instance TEXT, + action_type TEXT NOT NULL CHECK (btrim(action_type) <> ''), + entity_type audit_entity_type, + entity_number INTEGER, + field_name TEXT, + old_value TEXT, + new_value TEXT, + metadata JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(metadata) = 'object'), + actor_user_id UUID REFERENCES users(id) ON DELETE SET NULL, + actor JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(actor) = 'object'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK (entity_type IN ('setting', 'catalog') OR workflow_id IS NOT NULL) +); +CREATE INDEX audit_logs_document_idx ON audit_logs (document_id, created_at DESC); +CREATE INDEX audit_logs_created_idx ON audit_logs (created_at DESC); +CREATE INDEX audit_logs_action_idx ON audit_logs (action_type, created_at DESC); +CREATE INDEX audit_logs_settings_idx ON audit_logs (created_at DESC) WHERE entity_type = 'setting'; + +-- 9. search_settings +CREATE TABLE search_settings ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + singleton BOOLEAN NOT NULL DEFAULT TRUE UNIQUE CHECK (singleton), + retrieval JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(retrieval) = 'object'), + candidates JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(candidates) = 'object'), + results JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(results) = 'object'), + query JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(query) = 'object'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE TRIGGER trg_search_settings_updated_at BEFORE UPDATE ON search_settings + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); +INSERT INTO search_settings DEFAULT VALUES; + +-- 10. email_otps +CREATE TABLE email_otps ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + email TEXT NOT NULL UNIQUE CHECK (email = lower(email)), + code_hash TEXT NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0), + last_sent_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX email_otps_expires_idx ON email_otps (expires_at); + +-- webhook_records +CREATE TABLE webhook_records ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + source TEXT NOT NULL DEFAULT 'webhook', + external_id TEXT NOT NULL, + title TEXT, + slug TEXT, + status TEXT, + is_published BOOLEAN, + category TEXT, + payload JSONB NOT NULL CHECK (jsonb_typeof(payload) = 'object'), + payload_hash TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (source, external_id) +); +CREATE INDEX webhook_records_source_status_idx ON webhook_records (source, status); + +-- catalog view +CREATE VIEW catalog AS +SELECT + e.id AS entry_id, + e.type, + e.code, + e.name, + e.aliases, + e.metadata->>'source' AS source, + CASE + WHEN COALESCE(a.has_pending_reindex, FALSE) THEN 'pending_reindex' + WHEN COALESCE(a.live_count, 0) > 0 THEN 'live' + WHEN COALESCE(a.pending_prod_count, 0) > 0 THEN 'pending_prod' + WHEN COALESCE(a.doc_count, 0) = 0 AND e.metadata->>'source' = 'bootstrap' THEN 'live' + ELSE 'disabled' + END AS status, + CASE WHEN COALESCE(cardinality(a.instances), 0) = 0 THEN ARRAY['default']::TEXT[] ELSE a.instances END AS instances, + COALESCE(a.workflow_ids, '{}'::TEXT[]) AS workflow_ids, + COALESCE(a.chunk_count, 0)::INTEGER AS chunk_count, + CASE + WHEN COALESCE(a.doc_count, 0) = 0 THEN COALESCE(e.metadata->>'source' = 'bootstrap', FALSE) + ELSE COALESCE(a.network_visible, FALSE) + END AS network_visible, + a.promoted_at +FROM catalog_entries e +LEFT JOIN LATERAL ( + SELECT + COUNT(*) AS doc_count, + COUNT(*) FILTER (WHERE x.is_live) AS live_count, + BOOL_OR(x.is_live AND COALESCE((x.metadata->>'reindex_required')::boolean, FALSE)) AS has_pending_reindex, + COUNT(*) FILTER (WHERE x.current_stage = 'approval_for_prod' AND NOT x.is_disabled) AS pending_prod_count, + ARRAY(SELECT DISTINCT d2.instance FROM documents d2 + WHERE d2.catalog_entry_id = e.id ORDER BY d2.instance) AS instances, + ARRAY(SELECT w.temporal_workflow_id FROM documents d3 + JOIN LATERAL ( + SELECT w2.temporal_workflow_id FROM workflows w2 + WHERE w2.document_id = d3.id AND w2.workflow_type = 'pipeline' + ORDER BY w2.started_at ASC, w2.id ASC LIMIT 1 + ) w ON TRUE + WHERE d3.catalog_entry_id = e.id + ORDER BY d3.created_at, d3.id) AS workflow_ids, + COALESCE(SUM((x.metadata->>'chunk_count')::int), 0) AS chunk_count, + BOOL_OR(COALESCE((x.metadata->>'network_visible')::boolean, TRUE)) AS network_visible, + MAX((x.index_status->'prod'->>'last_indexed_at')::timestamptz) FILTER (WHERE x.is_live) AS promoted_at + FROM ( + SELECT d.*, cw.current_stage, + (cw.current_stage = 'completed' AND NOT d.is_disabled + AND COALESCE((d.metadata->>'catalog_visible')::boolean, TRUE)) AS is_live + FROM documents d + LEFT JOIN document_current_workflow cw ON cw.document_id = d.id + WHERE d.catalog_entry_id = e.id + ) x +) a ON TRUE; diff --git a/pipeline/migrate.py b/pipeline/migrate.py new file mode 100644 index 0000000..fe1a870 --- /dev/null +++ b/pipeline/migrate.py @@ -0,0 +1,76 @@ +"""Versioned SQL migration runner. + +Applies ``migrations/NNNN_.sql`` files in order, once each, recording +them in ``schema_migrations``. A session-level advisory lock serialises +concurrent api/worker startups so only one process applies a given file. +""" + +import logging +import re +from pathlib import Path + +import psycopg + +logger = logging.getLogger(__name__) + +MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations" +_MIGRATION_FILE_RE = re.compile(r"^(\d{4})_[a-z0-9_]+\.sql$") +_ADVISORY_LOCK_KEY = 7_204_911_338_105_221 + + +def list_migrations(directory: Path = MIGRATIONS_DIR) -> list[tuple[str, Path]]: + found = [] + for path in sorted(directory.glob("*.sql")): + match = _MIGRATION_FILE_RE.match(path.name) + if not match: + raise ValueError(f"Migration file name must be NNNN_name.sql: {path.name}") + found.append((match.group(1), path)) + return found + + +def apply_migrations(conninfo: str, directory: Path = MIGRATIONS_DIR) -> list[str]: + """Apply every pending migration. Returns the versions applied by this call.""" + applied: list[str] = [] + with psycopg.connect(conninfo, autocommit=True) as conn: + conn.execute("SELECT pg_advisory_lock(%s)", (_ADVISORY_LOCK_KEY,)) + try: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")} + for version, path in list_migrations(directory): + if version in done: + continue + with conn.transaction(): + conn.execute(path.read_text(encoding="utf-8")) + conn.execute( + "INSERT INTO schema_migrations (version, name) VALUES (%s, %s)", + (version, path.name), + ) + logger.info("applied migration %s", path.name) + applied.append(version) + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (_ADVISORY_LOCK_KEY,)) + return applied + + +def current_version(conninfo: str) -> str | None: + with psycopg.connect(conninfo, autocommit=True) as conn: + exists = conn.execute("SELECT to_regclass('schema_migrations')").fetchone()[0] + if not exists: + return None + row = conn.execute("SELECT max(version) FROM schema_migrations").fetchone() + return row[0] + + +def is_current(conninfo: str, directory: Path = MIGRATIONS_DIR) -> bool: + migrations = list_migrations(directory) + if not migrations: + return True + return current_version(conninfo) == migrations[-1][0] From 10011df3c263713e3415abc0e206aa70b08bb110 Mon Sep 17 00:00:00 2001 From: Naveen Date: Mon, 26 Oct 2026 13:18:48 +0530 Subject: [PATCH 06/53] feat(db): replace SQLite store with Postgres Rewrite pipeline/db.py on a psycopg connection pool, splitting documents from workflows. Functions keep their names and flat dict shapes and accept the document UUID or any legacy identifier (doc-, temporal workflow id, fingerprint, filename). Artifact and job ids are now UUIDs. Scheme catalog is served from catalog_entries and the catalog view. Authenticated callers are upserted into users. Tests run against a real Postgres (TEST_DATABASE_URL or testcontainers). --- pipeline/activities.py | 16 +- pipeline/api.py | 172 +- pipeline/auth/deps.py | 36 +- pipeline/db.py | 4302 +++++++++++++++-------------- pipeline/models.py | 136 +- pipeline/scheme_catalog.py | 258 +- pipeline/worker.py | 4 +- pipeline/workflows.py | 2 +- tests/conftest.py | 63 +- tests/test_activities.py | 2 +- tests/test_api.py | 142 +- tests/test_db.py | 680 ++++- tests/test_document_repository.py | 2 +- tests/test_provenance.py | 2 +- tests/test_scheme_catalog.py | 24 +- tests/test_tenancy.py | 14 +- tests/test_webhook.py | 6 +- ui/src/views/DocumentOpsView.jsx | 8 +- 18 files changed, 3445 insertions(+), 2424 deletions(-) diff --git a/pipeline/activities.py b/pipeline/activities.py index b2ec3e4..421013f 100644 --- a/pipeline/activities.py +++ b/pipeline/activities.py @@ -911,7 +911,7 @@ async def run_ocr(filepath: str) -> list[dict]: @activity.defn async def run_ocr_and_store(workflow_id: str, filepath: str) -> dict: - """Run OCR and persist pages to SQLite to avoid large Temporal payloads.""" + """Run OCR and persist pages to Postgres to avoid large Temporal payloads.""" from . import db local_path, cleanup_local = _resolve_local_path(filepath) @@ -1093,7 +1093,7 @@ async def create_chunks_from_db( chunk_overlap: int = 128, min_tokens: int = 100, ) -> dict: - """Create chunks from persisted pages and persist chunks in SQLite.""" + """Create chunks from persisted pages and persist chunks in Postgres.""" from . import db pages = db.get_pages(workflow_id) @@ -1191,7 +1191,7 @@ async def _persist_chunking_progress(event: dict) -> None: "stats": result.stats, }, ) - # Reconcile the document row immediately after chunk persistence so SQLite + # Reconcile the document row immediately after chunk persistence so Postgres # remains truthful even if the workflow fails before its final state update. db.reconcile_materialized_state(workflow_id) await _persist_chunking_progress( @@ -1270,7 +1270,7 @@ async def promote_document_to_prod_qdrant( filename: str, batch_size: int = 10, ) -> dict: - """Re-upsert document chunks into the PROD Qdrant collection (from SQLite).""" + """Re-upsert document chunks into the PROD Qdrant collection (from Postgres).""" from . import db from .vector_store.qdrant_store import QdrantVectorStore, get_qdrant_client @@ -1391,7 +1391,7 @@ async def ingest_document_from_db( index_name: str = "documents-index", batch_size: int = 10, ) -> dict: - """Prepare and ingest chunks directly from SQLite by workflow_id.""" + """Prepare and ingest chunks directly from Postgres by workflow_id.""" from . import db chunks = db.get_chunks(workflow_id, include_excluded=True) @@ -1541,7 +1541,7 @@ async def update_document_state( chunk_count: int = 0, error_message: str = None, ) -> dict: - """Update document state in SQLite.""" + """Update document state in Postgres.""" from . import db activity.logger.info(f"Updating state for {workflow_id}: stage={stage}") @@ -1578,7 +1578,7 @@ async def update_document_state( @activity.defn async def persist_document_content(workflow_id: str, pages: list[dict], chunks: list[dict]) -> dict: - """Persist pages and chunks to SQLite.""" + """Persist pages and chunks to Postgres.""" from . import db activity.logger.info(f"Persisting content for {workflow_id}: {len(pages)} pages, {len(chunks)} chunks") @@ -1611,7 +1611,7 @@ async def detect_and_translate_pages_from_db( target_language: str = "en", source_language: str = None, ) -> dict: - """Detect and translate pages loaded from SQLite; persist updated pages back to SQLite.""" + """Detect and translate pages loaded from Postgres; persist updated pages back to Postgres.""" from . import db pages = db.get_pages(workflow_id) diff --git a/pipeline/api.py b/pipeline/api.py index 1d86056..b8f0bfd 100644 --- a/pipeline/api.py +++ b/pipeline/api.py @@ -154,8 +154,7 @@ async def lifespan(app: FastAPI): auth_cfg.keycloak_jwks_url, ) - # Initialize SQLite database - print("Initializing SQLite database...") + print("Initializing database...") db.init_db() # Temporal @@ -250,6 +249,41 @@ async def lifespan(app: FastAPI): # own much tighter budget than the general default. RATE_LIMIT_OTP = os.environ.get("RATE_LIMIT_OTP", "10/minute") +_DOCUMENT_PATH_RE = re.compile(r"^/documents/([^/]+)(/.*)?$") +_static_document_segments: Optional[set[str]] = None + + +def _static_document_path_segments() -> set[str]: + """Literal second segments of ``/documents/...`` routes (``upload``, ``batch``...).""" + global _static_document_segments + if _static_document_segments is None: + segments = set() + for route in app.routes: + match = _DOCUMENT_PATH_RE.match(getattr(route, "path", "")) + if match and "{" not in match.group(1): + segments.add(match.group(1)) + _static_document_segments = segments + return _static_document_segments + + +@app.middleware("http") +async def canonicalize_document_identifier(request: Request, call_next): + """Let ``/documents/{id}/...`` take the document UUID or any legacy identifier. + + Handlers address a document by its Temporal workflow id (they signal and + query Temporal with it), so the path is rewritten to that id before routing. + An identifier that matches no document is left alone and 404s in the handler. + """ + match = _DOCUMENT_PATH_RE.match(request.scope["path"]) + if match and match.group(1) not in _static_document_path_segments(): + document = await asyncio.to_thread(db.find_document_by_doc_identifier, match.group(1)) + if document and document["workflow_id"] and document["workflow_id"] != match.group(1): + new_path = f"/documents/{document['workflow_id']}{match.group(2) or ''}" + request.scope["path"] = new_path + request.scope["raw_path"] = quote(new_path, safe="/:").encode("ascii") + return await call_next(request) + + limiter = Limiter(key_func=get_remote_address) app.state.limiter = limiter app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) @@ -334,7 +368,7 @@ def _instance_scope_for_user(user: AuthUser) -> Optional[list[str]]: def _document_cohorts_payload(summary: dict, by_instance: Optional[list[dict]] = None) -> dict: - """Shape SQLite aggregate counts for DocumentCohortsResponse (all ints).""" + """Shape Postgres aggregate counts for DocumentCohortsResponse (all ints).""" def _i(key: str, default: int = 0) -> int: val = summary.get(key, default) return int(val or 0) @@ -419,6 +453,7 @@ def _actor_from_user(user) -> dict: def _document_summary_from_row(doc: dict, current_job: Optional[dict] = None) -> DocumentSummary: return DocumentSummary( + id=doc.get("id"), document_id=doc["document_id"], canonical_document_id=doc.get("canonical_document_id"), workflow_id=doc["workflow_id"], @@ -957,8 +992,8 @@ async def start_document_workflow( workflow_id = get_workflow_id(str(filepath)) document_id = canonical_document_id - # Reuse only when SQLite still tracks this workflow. - # If SQLite was purged, avoid returning stale Temporal state and create a fresh run ID. + # Reuse only when Postgres still tracks this workflow. + # If Postgres was purged, avoid returning stale Temporal state and create a fresh run ID. existing_doc = db.get_document(workflow_id) if existing_doc: # Same fingerprint/path must not leak or restart another tenant's doc. @@ -1007,13 +1042,13 @@ async def start_document_workflow( task_queue=TASK_QUEUE, ) - # Save to SQLite for visibility during processing + # Save to Postgres for visibility during processing actor = _actor_from_user(user) # The period this document is searchable in starts here, at the one moment # that knows the upload day. A reviewer confirms or moves it later, on the # form that sets the document type. validity = document_validity.default_period() - db.upsert_document( + document_uuid = db.upsert_document( workflow_id=workflow_id, document_id=document_id, canonical_document_id=canonical_document_id, @@ -1030,9 +1065,10 @@ async def start_document_workflow( uploaded_by_roles=actor.get("roles_csv") or None, valid_from=validity.start_date, valid_to=validity.end_date, + with_workflow=False, ) - job_id = db.create_document_job( - workflow_id=workflow_id, + db.create_document_job( + workflow_id=document_uuid, job_type="ocr_only" if stop_after_ocr else "pipeline", temporal_workflow_id=workflow_id, status="running", @@ -1046,8 +1082,8 @@ async def start_document_workflow( "stop_after_ocr": stop_after_ocr, "uploaded_by": actor, }, + actor=actor, ) - db.update_document_fields(workflow_id, latest_job_id=job_id) _log_audit( workflow_id=workflow_id, action_type="document_upload", @@ -1135,8 +1171,8 @@ async def upload_and_process( document_id = file_hash canonical_document_id = file_hash - # Reuse only when SQLite still tracks this workflow. - # If SQLite was purged, avoid returning stale Temporal state and create a fresh run ID. + # Reuse only when Postgres still tracks this workflow. + # If Postgres was purged, avoid returning stale Temporal state and create a fresh run ID. existing_doc = db.get_document(workflow_id) if existing_doc: existing_doc = assert_document_instance_access(user, existing_doc) @@ -1184,11 +1220,11 @@ async def upload_and_process( task_queue=TASK_QUEUE, ) - # Save to SQLite for visibility during processing + # Save to Postgres for visibility during processing actor = _actor_from_user(user) # See the manifest upload path above: the period starts at the upload. validity = document_validity.default_period() - db.upsert_document( + document_uuid = db.upsert_document( workflow_id=workflow_id, document_id=document_id, canonical_document_id=canonical_document_id, @@ -1205,9 +1241,10 @@ async def upload_and_process( uploaded_by_roles=actor.get("roles_csv") or None, valid_from=validity.start_date, valid_to=validity.end_date, + with_workflow=False, ) job_id = db.create_document_job( - workflow_id=workflow_id, + workflow_id=document_uuid, job_type="ocr_only" if stop_after_ocr else "pipeline", temporal_workflow_id=workflow_id, status="running", @@ -1221,8 +1258,9 @@ async def upload_and_process( "stop_after_ocr": stop_after_ocr, "uploaded_by": actor, }, + actor=actor, ) - original_artifact_id = db.add_document_artifact( + db.add_document_artifact( workflow_id=workflow_id, job_id=job_id, artifact_type="original_upload", @@ -1251,8 +1289,6 @@ async def upload_and_process( canonical_input_type = "spreadsheet" if suffix in {".csv", ".xlsx"} else "pdf" db.update_document_fields( workflow_id, - latest_job_id=job_id, - original_artifact_id=original_artifact_id, source_type=source_type, canonical_input_type=canonical_input_type, stop_after_ocr=1 if stop_after_ocr else 0, @@ -1344,7 +1380,7 @@ async def list_documents( """ List all document workflows. - Uses SQLite for fast listing (no Temporal queries for performance). + Uses Postgres for fast listing (no Temporal queries for performance). Demo documents are excluded by default - use X-Include-Demo: true header to show them. Disabled (soft-deleted) documents are excluded by default - use X-Include-Disabled: true to show them. Results are limited to instances the caller can access. @@ -1357,7 +1393,7 @@ async def list_documents( include_demo = x_include_demo and x_include_demo.lower() == "true" include_disabled = x_include_disabled and x_include_disabled.lower() == "true" - # Use SQLite only for fast listing - no Temporal queries + # Use Postgres only for fast listing - no Temporal queries docs = db.list_documents( stage=stage_filter, limit=limit, @@ -1376,7 +1412,7 @@ async def get_documents_summary( x_include_demo: Optional[str] = Header(None, alias="X-Include-Demo"), x_include_disabled: Optional[str] = Header(None, alias="X-Include-Disabled") ): - """Return aggregate SQLite counts for dashboard totals and migration planning.""" + """Return aggregate Postgres counts for dashboard totals and migration planning.""" include_demo = x_include_demo and x_include_demo.lower() == "true" include_disabled = x_include_disabled and x_include_disabled.lower() == "true" instance_scope = _instance_scope_for_user(user) @@ -1476,7 +1512,7 @@ async def list_runs( @app.get("/runs/{job_id}") -async def get_run(job_id: int, user: RequireSearch): +async def get_run(job_id: str, user: RequireSearch): """Get a specific document job/run (404 if missing or outside tenant scope).""" run = db.get_document_job(job_id) if not run: @@ -1498,6 +1534,7 @@ def _build_document_detail(doc: dict) -> DocumentDetail: workflow_id = doc["workflow_id"] current_job = db.get_latest_document_job(workflow_id) return DocumentDetail( + id=doc.get("id"), document_id=doc["document_id"], canonical_document_id=doc.get("canonical_document_id"), workflow_id=workflow_id, @@ -1760,7 +1797,7 @@ async def get_workflow_error_details(workflow_id: str, user: RequireSearch): This endpoint queries Temporal directly to get the most detailed error information available, which may be more complete than what's - stored in SQLite. + stored in Postgres. """ import traceback @@ -1838,7 +1875,7 @@ async def get_workflow_error_details(workflow_id: str, user: RequireSearch): @app.get("/documents/{workflow_id}/runtime") async def get_document_runtime(workflow_id: str, user: RequireSearch): - """Return live runtime status by combining SQLite state and Temporal workflow state.""" + """Return live runtime status by combining Postgres state and Temporal workflow state.""" doc = _require_document_for_user(workflow_id, user) return await _get_runtime_payload(workflow_id, doc=doc) @@ -1850,7 +1887,7 @@ async def list_document_artifacts(workflow_id: str, user: RequireSearch): @app.get("/documents/{workflow_id}/artifacts/{artifact_id}") -async def get_document_artifact(workflow_id: str, user: RequireSearch, artifact_id: int): +async def get_document_artifact(workflow_id: str, user: RequireSearch, artifact_id: str): _require_document_for_user(workflow_id, user) artifact = db.get_document_artifact(workflow_id, artifact_id) if not artifact: @@ -1859,7 +1896,7 @@ async def get_document_artifact(workflow_id: str, user: RequireSearch, artifact_ @app.get("/documents/{workflow_id}/artifacts/{artifact_id}/content") -async def get_document_artifact_content(workflow_id: str, user: RequireSearch, artifact_id: int): +async def get_document_artifact_content(workflow_id: str, user: RequireSearch, artifact_id: str): _require_document_for_user(workflow_id, user) artifact = db.get_document_artifact(workflow_id, artifact_id) if not artifact: @@ -1929,7 +1966,7 @@ async def get_document_graph(workflow_id: str, user: RequireSearch): def _delete_artifacts_from_minio(workflow_id: str) -> dict: """Remove every MinIO object backing a document's artifacts. - Artifacts are the only part of a document that lives outside SQLite and the + Artifacts are the only part of a document that lives outside Postgres and the vector store — the original upload, the normalized PDF, and the OCR / translation / chunk JSON snapshots. Nothing else reads them once the document is gone, so a hard delete has to sweep them explicitly. @@ -1977,15 +2014,15 @@ async def disable_document( - Remove all chunks from the DEV vector index - Delete PROD Qdrant points too, if this document was ever promoted (schemes-index for scheme documents, documents-index otherwise) - - For scheme documents, also rebuild the legacy SQLite scheme catalog + - For scheme documents, also rebuild the scheme catalog - Stop the workflow if it is still running - Soft delete (default) then marks the document disabled in SQLite and keeps + Soft delete (default) then marks the document disabled in Postgres and keeps every row and artifact, so ``POST /documents/{id}/restore`` can bring it back. Use X-Include-Disabled: true in list_documents to see it. ``purge=true`` additionally deletes the MinIO artifacts (original upload, - normalized PDF, OCR/translation/chunk JSON) and every SQLite row (pages, + normalized PDF, OCR/translation/chunk JSON) and every Postgres row (pages, chunks, artifacts, jobs, index status, the document itself). This is irreversible — restore has nothing left to restore. Audit rows are kept unless ``keep_audit=false``, since they record the deletion itself. @@ -1993,7 +2030,7 @@ async def disable_document( Args: workflow_id: The document workflow ID remove_from_search: If True (default), removes chunks from the index - purge: If True, hard delete — also drops MinIO artifacts and SQLite rows + purge: If True, hard delete — also drops MinIO artifacts and Postgres rows keep_audit: If False (and purge=True), also deletes the audit trail Requires ``admin`` (super admin — any document), or ``delete_own`` @@ -2046,7 +2083,7 @@ async def disable_document( except Exception: pass # Workflow already completed/cancelled - # Mark as disabled in SQLite (a purge deletes the row outright below, but + # Mark as disabled in Postgres (a purge deletes the row outright below, but # this keeps the document hidden if a later step fails partway through) db.set_document_disabled(workflow_id, True) @@ -2073,7 +2110,7 @@ async def disable_document( else: result["prod_qdrant_deleted"] = _delete_document_points_from_prod(doc) - # Hard delete: drop the artifacts and the SQLite rows themselves. Ordered + # Hard delete: drop the artifacts and the Postgres rows themselves. Ordered # storage-then-database so a failure mid-way leaves rows pointing at # already-deleted objects (visible, fixable) rather than orphaned objects # no row references any more (invisible, unreclaimable). @@ -2188,7 +2225,6 @@ async def patch_scheme_metadata( async def catalog_version(request: Request, user: CurrentUser): """Cheap poll for cache warmers: { version, updated_at }.""" _require_catalog_read(request, user) - scheme_catalog.ensure_catalog_schema() scheme_catalog.bootstrap_catalog_if_empty() meta = db.get_catalog_meta() return { @@ -2274,7 +2310,6 @@ async def catalog_rebuild(request: Request, user: RequireAdmin): # Admin JWT always OK; service key also accepted for automation if not _service_key_ok(request) and not user.has_permission(Permission.ADMIN): raise HTTPException(403, "Rebuild requires admin") - scheme_catalog.ensure_catalog_schema() # Ensure bootstrap seeds exist before rebuild merge scheme_catalog.bootstrap_catalog_if_empty() version = scheme_catalog.rebuild_catalog_from_documents(reason="admin_rebuild") @@ -2306,7 +2341,7 @@ async def restore_document(workflow_id: str, user: RequireAdmin): """ Restore a soft-deleted (disabled) document. - Note: This only restores the document in SQLite. Chunks that were removed + Note: This only restores the document in Postgres. Chunks that were removed from the vector index will NOT be automatically re-indexed. To re-index, you would need to re-run the ingestion process. """ @@ -2336,12 +2371,12 @@ async def reingest_document( Use this to re-ingest documents that completed but weren't properly indexed (e.g., due to index schema changes). This starts a lightweight - workflow that uses chunks already stored in SQLite. + workflow that uses chunks already stored in Postgres. - The document must have chunks stored in SQLite (typically from a + The document must have chunks stored in Postgres (typically from a completed or previously ingested document). """ - # Get document from SQLite + # Get document from Postgres doc = _require_document_for_user(workflow_id, user) # Reingest skips the entire review pipeline (including ready_for_ingestion, @@ -2362,7 +2397,7 @@ async def reingest_document( "review stage entirely and would otherwise ingest with no scheme metadata.", ) - # Get chunks from SQLite + # Get chunks from Postgres chunks = db.get_chunks(workflow_id, include_excluded=False) if not chunks: raise HTTPException(400, "No chunks found for document. The document may need to be reprocessed from scratch.") @@ -2381,7 +2416,7 @@ async def reingest_document( args=[ document_id, filename, - workflow_id, # original workflow_id for SQLite updates + workflow_id, # original workflow_id for Postgres updates page_count, len(chunks), index_name @@ -2396,6 +2431,7 @@ async def reingest_document( status="running", current_stage="ingesting", config={"index_name": index_name, "chunk_count": len(chunks)}, + actor=_actor_from_user(user), ) # Log audit @@ -2443,15 +2479,16 @@ async def retry_ocr(workflow_id: str, user: RequirePipeline): id=temporal_workflow_id, task_queue=TASK_QUEUE, ) - job_id = db.create_document_job( + db.create_document_job( workflow_id=workflow_id, job_type="ocr_retry", temporal_workflow_id=temporal_workflow_id, status="running", current_stage="ocr_processing", config={"source": "api_retry_ocr"}, + actor=_actor_from_user(user), ) - db.update_document_fields(workflow_id, latest_job_id=job_id, error_message=None) + db.update_document_fields(workflow_id, error_message=None) _log_audit( workflow_id=workflow_id, action_type="retry_ocr", @@ -2475,15 +2512,16 @@ async def retry_translation(workflow_id: str, user: RequirePipeline): id=temporal_workflow_id, task_queue=TASK_QUEUE, ) - job_id = db.create_document_job( + db.create_document_job( workflow_id=workflow_id, job_type="translation_retry", temporal_workflow_id=temporal_workflow_id, status="running", current_stage="translation_processing", config={"source": "api_retry_translation"}, + actor=_actor_from_user(user), ) - db.update_document_fields(workflow_id, latest_job_id=job_id, error_message=None) + db.update_document_fields(workflow_id, error_message=None) _log_audit( workflow_id=workflow_id, action_type="retry_translation", @@ -2521,7 +2559,7 @@ async def retry_chunking( id=temporal_workflow_id, task_queue=TASK_QUEUE, ) - job_id = db.create_document_job( + db.create_document_job( workflow_id=workflow_id, job_type="chunking_retry", temporal_workflow_id=temporal_workflow_id, @@ -2533,8 +2571,9 @@ async def retry_chunking( "chunk_overlap": chunk_overlap, "min_tokens": min_tokens, }, + actor=_actor_from_user(user), ) - db.update_document_fields(workflow_id, latest_job_id=job_id, error_message=None) + db.update_document_fields(workflow_id, error_message=None) _log_audit( workflow_id=workflow_id, action_type="retry_chunking", @@ -2677,7 +2716,7 @@ async def _reconcile_single_document(doc: dict) -> dict: @app.post("/documents/{workflow_id}/reconcile") async def reconcile_single_document(workflow_id: str, user: RequirePipeline): - """Reconcile SQLite stage with Temporal state for one document.""" + """Reconcile Postgres stage with Temporal state for one document.""" doc = _require_document_for_user(workflow_id, user) return await _reconcile_single_document(doc) @@ -2715,7 +2754,7 @@ async def _validate_approval_stage(workflow_id: str, expected_stage: str): except HTTPException: raise except Exception: - # Try SQLite fallback to check if workflow exists but is completed/failed + # Try Postgres fallback to check if workflow exists but is completed/failed doc = db.get_document(workflow_id) if doc: current = doc.get("stage") @@ -3055,6 +3094,7 @@ async def approve_prod( temporal_workflow_id=promote_workflow_id, status="running", current_stage="ingesting_prod", + actor=_actor_from_user(user), ) db.update_document_stage( workflow_id=workflow_id, @@ -3118,7 +3158,7 @@ def _log_audit( metadata: Additional context as dict user: Optional AuthUser whose email/roles are stored on the audit row """ - # Get document_id from SQLite + # Get document_id from Postgres doc = db.get_document(workflow_id) document_id = doc["document_id"] if doc else workflow_id @@ -3230,7 +3270,7 @@ async def get_document_audit_log( @app.get("/documents/{workflow_id}/pages") async def list_pages(workflow_id: str, user: RequireSearch): - """Get all pages for a document. SQLite-first for speed.""" + """Get all pages for a document. Postgres-first for speed.""" # Enforce tenant scope up front (404 hides other tenants' documents). _require_document_for_user(workflow_id, user) return db.get_pages(workflow_id) @@ -3238,7 +3278,7 @@ async def list_pages(workflow_id: str, user: RequireSearch): @app.get("/documents/{workflow_id}/pages/{page_num}") async def get_page(workflow_id: str, user: RequireSearch, page_num: int = PathParam(..., ge=1, le=10000, description="Page number (1-indexed)")): - """Get a specific page. SQLite-first for speed.""" + """Get a specific page. Postgres-first for speed.""" _require_document_for_user(workflow_id, user) page = db.get_page(workflow_id, page_num) if page: @@ -3435,14 +3475,14 @@ async def search_chunks_across_documents( @app.get("/documents/{workflow_id}/chunks") async def list_chunks(workflow_id: str, user: RequireSearch, include_excluded: bool = False): - """Get all chunks for a document. SQLite-first for speed.""" + """Get all chunks for a document. Postgres-first for speed.""" _require_document_for_user(workflow_id, user) return db.get_chunks(workflow_id, include_excluded=include_excluded) @app.get("/documents/{workflow_id}/chunks/{chunk_num}") async def get_chunk(workflow_id: str, user: RequireSearch, chunk_num: int = PathParam(..., ge=1, le=10000, description="Chunk number (1-indexed)")): - """Get a specific chunk. SQLite-first for speed.""" + """Get a specific chunk. Postgres-first for speed.""" _require_document_for_user(workflow_id, user) chunk = db.get_chunk(workflow_id, chunk_num) if chunk: @@ -3661,9 +3701,9 @@ def _inline_content_disposition(filename: str) -> str: async def get_document_pdf(workflow_id: str, user: RequireSearch): """ Get the original PDF file for a document. - Returns the PDF as a streaming response. SQLite-first for speed. + Returns the PDF as a streaming response. Postgres-first for speed. """ - # SQLite-first - instant lookup, tenant-scoped (404 hides other tenants). + # Postgres-first - instant lookup, tenant-scoped (404 hides other tenants). doc = _require_document_for_user(workflow_id, user) filepath = doc.get("filepath", "") @@ -3719,7 +3759,7 @@ def file_iterator(): async def resolve_provenance_chunk( request: Request, user: RequireSearch, - doc_id: Optional[str] = Query(None, description="workflow slug, SQLite document_id, or legacy vector-index doc_id"), + doc_id: Optional[str] = Query(None, description="workflow slug, Postgres document_id, or legacy vector-index doc_id"), chunk_num: Optional[int] = Query(None, alias="chunk_num"), ): """ @@ -3786,14 +3826,14 @@ async def get_document_qdrant_status( ) normalized_hit.setdefault("chunk_number", chunk_num) hits.append(normalized_hit) - sqlite_chunks = db.get_chunks(workflow_id, include_excluded=True) + db_chunks = db.get_chunks(workflow_id, include_excluded=True) status = { "workflow_id": workflow_id, "index_name": resolved_index, "backend": backend, "index_doc_id": index_doc_id, "doc_id": index_doc_id, - "sqlite_chunk_count": len([c for c in sqlite_chunks if not c.get("is_excluded")]), + "sqlite_chunk_count": len([c for c in db_chunks if not c.get("is_excluded")]), "indexed_chunk_count": len(hits), "status": "indexed" if hits else "missing", "hits": hits, @@ -3846,7 +3886,7 @@ async def get_vector_indexes_summary( x_include_demo: Optional[str] = Header(None, alias="X-Include-Demo"), x_include_disabled: Optional[str] = Header(None, alias="X-Include-Disabled"), ): - """Summarize index coverage from SQLite-backed index status plus live vector-store stats.""" + """Summarize index coverage from Postgres-backed index status plus live vector-store stats.""" from .vector_store import get_vector_store include_demo = x_include_demo and x_include_demo.lower() == "true" @@ -4016,7 +4056,8 @@ async def health(): """Health check.""" return { "status": "ok", - "temporal_connected": temporal_client is not None + "temporal_connected": temporal_client is not None, + "database": await asyncio.to_thread(db.check_health), } @@ -4158,10 +4199,10 @@ async def get_ingest_info(user: RequireAdmin): @app.post("/documents/reconcile") async def reconcile_document_states(user: RequirePipeline): """ - Reconcile SQLite document states with Temporal workflow states. + Reconcile Postgres document states with Temporal workflow states. This endpoint checks all documents in processing/review stages and updates - SQLite if the Temporal workflow has terminated or failed. This fixes + Postgres if the Temporal workflow has terminated or failed. This fixes inconsistencies caused by external workflow termination or worker crashes. Returns a summary of documents checked and updated. @@ -4258,7 +4299,10 @@ async def update_search_settings_endpoint(settings: SearchSettingsUpdate, user: if not updates: raise HTTPException(400, "No settings provided to update") - return db.update_search_settings(updates) + try: + return db.update_search_settings(updates) + except ValueError as exc: + raise HTTPException(400, str(exc)) from exc @app.get("/settings/search/audit", response_model=SettingsAuditResponse) diff --git a/pipeline/auth/deps.py b/pipeline/auth/deps.py index c09dae9..4b192f3 100644 --- a/pipeline/auth/deps.py +++ b/pipeline/auth/deps.py @@ -5,10 +5,13 @@ import asyncio import base64 import json +import logging +import time from typing import Annotated, Any, Callable from fastapi import Depends, HTTPException, Request +from .. import db from .config import load_auth_config from .jwt import claims_to_user, decode_and_validate_token from .models import AuthUser, local_bypass_user @@ -72,6 +75,25 @@ def _bypass_user_with_optional_jwt(token: str | None) -> AuthUser: ) +_USER_SYNC_INTERVAL_SECONDS = 300 +_user_synced_at: dict[str, float] = {} +_WRITE_METHODS = {"POST", "PUT", "PATCH", "DELETE"} + + +def _sync_user_row(user: AuthUser) -> None: + """Mirror the caller into ``users`` (at most once per interval per user).""" + now = time.monotonic() + last = _user_synced_at.get(user.user_id) + if last is not None and now - last < _USER_SYNC_INTERVAL_SECONDS: + return + try: + db.upsert_user(user.user_id, user.username, user.email, user.roles) + except Exception: + logging.warning("users upsert failed for %s", user.user_id, exc_info=True) + return + _user_synced_at[user.user_id] = now + + async def get_current_user(request: Request) -> AuthUser: """Resolve the caller. When AUTH_DISABLED=true, returns a local bypass user. @@ -83,12 +105,16 @@ async def get_current_user(request: Request) -> AuthUser: token = _bearer_token(request, required=False) if config.disabled: - return _bypass_user_with_optional_jwt(token) + user = _bypass_user_with_optional_jwt(token) + else: + if not token: + raise HTTPException(401, "Missing Bearer token") + # JWKS fetch/refresh is sync and can block; keep the event loop free. + user = await asyncio.to_thread(decode_and_validate_token, token, config) - if not token: - raise HTTPException(401, "Missing Bearer token") - # JWKS fetch/refresh is sync and can block; keep the event loop free. - return await asyncio.to_thread(decode_and_validate_token, token, config) + if request.method in _WRITE_METHODS and user.user_id: + await asyncio.to_thread(_sync_user_row, user) + return user def require_permission(permission: Permission | str) -> Callable[..., AuthUser]: diff --git a/pipeline/db.py b/pipeline/db.py index 22fd9bc..0799b44 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -1,414 +1,483 @@ """ -SQLite state persistence for document visibility. +Postgres persistence for documents, workflows, pages, chunks, audit and settings. -This provides a fallback when Temporal workflow queries fail during -long-running activities, ensuring the dashboard always shows document status. +Schema: ``migrations/`` (see docs/POSTGRES_SCHEMA.md). One function per query. + +A document is content and metadata; a workflow is one processing run of a +document. Callers still address a document by ``workflow_id`` - the Temporal +workflow id of its main run (``doc-``) - and every function here also +accepts the document UUID, the fingerprint or the filename (see +``find_document_by_doc_identifier``). Documents, jobs and artifacts come back +as flat dicts so call sites need not know how the tables are split; ``id`` +carries the UUID. """ import hashlib import json +import logging import os -import sqlite3 +import re +import threading import uuid from contextlib import contextmanager -from datetime import datetime -from pathlib import Path -from threading import Lock -from typing import Optional +from datetime import date, datetime, timezone +from typing import Any, Optional + +import psycopg +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb +from psycopg_pool import ConnectionPool + +from . import migrate +from .models import ( + CandidateSettings, + ChunkingRun, + DocumentMetadata, + DocumentSource, + IndexState, + PageMetadata, + PageTranslation, + QuerySettings, + ResultSettings, + RetrievalSettings, +) -# Database path - can be configured via environment -DB_PATH = os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db") +logger = logging.getLogger(__name__) -# Lock for thread-safe operations -_db_lock = Lock() +DEFAULT_DATABASE_URL = "postgresql://postgres:postgres@localhost:5432/knowledge_provider" +_pool: Optional[ConnectionPool] = None +_pool_url: Optional[str] = None +_pool_lock = threading.Lock() -def _add_column_if_missing(conn: sqlite3.Connection, table: str, column: str, definition: str) -> None: - """Best-effort SQLite migration helper.""" - try: - conn.execute(f"ALTER TABLE {table} ADD COLUMN {column} {definition}") - except sqlite3.OperationalError: - pass + +def database_url() -> str: + return os.environ.get("DATABASE_URL") or DEFAULT_DATABASE_URL -def _ensure_db_dir(): - """Ensure the database directory exists.""" - db_dir = Path(DB_PATH).parent - if not db_dir.exists(): - db_dir.mkdir(parents=True, exist_ok=True) +def get_pool() -> ConnectionPool: + global _pool, _pool_url + url = database_url() + if _pool is not None and _pool_url == url: + return _pool + with _pool_lock: + if _pool is not None and _pool_url != url: + _pool.close() + _pool = None + if _pool is None: + pool = ConnectionPool( + conninfo=url, + min_size=int(os.environ.get("DB_POOL_MIN_SIZE", "1")), + max_size=int(os.environ.get("DB_POOL_MAX_SIZE", "10")), + timeout=float(os.environ.get("DB_POOL_TIMEOUT_SECONDS", "30")), + kwargs={"row_factory": dict_row, "options": "-c timezone=UTC"}, + open=False, + ) + pool.open(wait=True) + _pool = pool + _pool_url = url + logger.info("db pool opened: %s", pool.get_stats()) + return _pool + + +def close_pool() -> None: + global _pool, _pool_url + with _pool_lock: + if _pool is not None: + _pool.close() + _pool = None + _pool_url = None @contextmanager def get_connection(): - """Get a database connection with proper cleanup and isolation.""" - _ensure_db_dir() - conn = sqlite3.connect( - DB_PATH, - check_same_thread=False, - isolation_level="IMMEDIATE", # Acquire lock immediately on write - timeout=30.0 # Wait up to 30s for locks - ) - conn.row_factory = sqlite3.Row - # Enable WAL mode for better concurrent access - conn.execute("PRAGMA journal_mode=WAL") - # Ensure foreign keys are enforced - conn.execute("PRAGMA foreign_keys=ON") - try: + """A pooled connection; the transaction commits on exit and rolls back on error.""" + with get_pool().connection() as conn: yield conn - finally: - conn.close() -def init_db(): - """Initialize the database schema.""" - with _db_lock: +def init_db() -> None: + """Apply pending migrations and open the connection pool.""" + applied = migrate.apply_migrations(database_url()) + current = migrate.current_version(database_url()) + logger.info("db migrations current=%s applied_now=%s", current, applied) + get_pool() + + +def check_health() -> dict: + """Reachability and migration state, for the /health endpoint.""" + try: with get_connection() as conn: - conn.execute(""" - CREATE TABLE IF NOT EXISTS documents ( - workflow_id TEXT PRIMARY KEY, - document_id TEXT NOT NULL, - canonical_document_id TEXT, - filename TEXT NOT NULL, - display_name TEXT, - source_filename TEXT, - source_manifest_name TEXT, - source_file_fingerprint TEXT, - filepath TEXT NOT NULL, - stage TEXT NOT NULL DEFAULT 'registered', - page_count INTEGER DEFAULT 0, - chunk_count INTEGER DEFAULT 0, - error_message TEXT, - is_demo INTEGER DEFAULT 0, - created_at TEXT, - updated_at TEXT, - ocr_completed_at TEXT, - translation_completed_at TEXT, - chunks_completed_at TEXT, - ingested_at TEXT, - source_type TEXT, - canonical_input_type TEXT, - stop_after_ocr INTEGER DEFAULT 0, - reindex_required INTEGER DEFAULT 0, - reindex_reason TEXT, - original_artifact_id INTEGER, - normalized_artifact_id INTEGER, - latest_job_id INTEGER - ) - """) - _add_column_if_missing(conn, "documents", "is_demo", "INTEGER DEFAULT 0") - _add_column_if_missing(conn, "documents", "is_disabled", "INTEGER DEFAULT 0") - _add_column_if_missing(conn, "documents", "display_name", "TEXT") - _add_column_if_missing(conn, "documents", "canonical_document_id", "TEXT") - _add_column_if_missing(conn, "documents", "source_filename", "TEXT") - _add_column_if_missing(conn, "documents", "source_manifest_name", "TEXT") - _add_column_if_missing(conn, "documents", "source_file_fingerprint", "TEXT") - _add_column_if_missing(conn, "documents", "translation_completed_at", "TEXT") - _add_column_if_missing(conn, "documents", "source_type", "TEXT") - _add_column_if_missing(conn, "documents", "canonical_input_type", "TEXT") - _add_column_if_missing(conn, "documents", "stop_after_ocr", "INTEGER DEFAULT 0") - _add_column_if_missing(conn, "documents", "reindex_required", "INTEGER DEFAULT 0") - _add_column_if_missing(conn, "documents", "reindex_reason", "TEXT") - _add_column_if_missing(conn, "documents", "original_artifact_id", "INTEGER") - _add_column_if_missing(conn, "documents", "normalized_artifact_id", "INTEGER") - _add_column_if_missing(conn, "documents", "latest_job_id", "INTEGER") - _add_column_if_missing(conn, "documents", "instance", "TEXT") - _add_column_if_missing(conn, "documents", "uploaded_by_user_id", "TEXT") - _add_column_if_missing(conn, "documents", "uploaded_by_username", "TEXT") - _add_column_if_missing(conn, "documents", "uploaded_by_email", "TEXT") - _add_column_if_missing(conn, "documents", "uploaded_by_roles", "TEXT") - _add_column_if_missing(conn, "documents", "prod_ready_requested_at", "TEXT") - _add_column_if_missing(conn, "documents", "prod_ready_requested_by_user_id", "TEXT") - _add_column_if_missing(conn, "documents", "prod_ready_requested_by_username", "TEXT") - # Period this document's chunks are searchable in, stamped on - # upload and editable by the approver. NULL on documents uploaded - # before validity existed - search reads that as always current - # (see pipeline/document_validity.period_from_row). - _add_column_if_missing(conn, "documents", "valid_from", "TEXT") - _add_column_if_missing(conn, "documents", "valid_to", "TEXT") - # Stamp NULL/empty rows with the configured default so list filters - # (which coalesce to DEFAULT_INSTANCE) match migrated data. - default_instance = ( - (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" - ) - conn.execute( - """ - UPDATE documents - SET instance = ? - WHERE instance IS NULL OR trim(instance) = '' - """, - (default_instance,), - ) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_documents_stage - ON documents(stage) - """) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_documents_instance - ON documents(instance) - """) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_documents_created - ON documents(created_at DESC) - """) - # Audit logs table - conn.execute(""" - CREATE TABLE IF NOT EXISTS audit_logs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - workflow_id TEXT NOT NULL, - document_id TEXT NOT NULL, - action_type TEXT NOT NULL, - entity_type TEXT, - entity_id INTEGER, - field_name TEXT, - old_value TEXT, - new_value TEXT, - metadata TEXT, - timestamp TEXT NOT NULL, - actor_user_id TEXT, - actor_username TEXT, - actor_email TEXT, - actor_roles TEXT - ) - """) - _add_column_if_missing(conn, "audit_logs", "actor_user_id", "TEXT") - _add_column_if_missing(conn, "audit_logs", "actor_username", "TEXT") - _add_column_if_missing(conn, "audit_logs", "actor_email", "TEXT") - _add_column_if_missing(conn, "audit_logs", "actor_roles", "TEXT") - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_audit_workflow - ON audit_logs(workflow_id) - """) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_audit_timestamp - ON audit_logs(timestamp DESC) - """) - # Pages table for persistence after workflow completion - conn.execute(""" - CREATE TABLE IF NOT EXISTS pages ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - workflow_id TEXT NOT NULL, - page_number INTEGER NOT NULL, - original_markdown TEXT, - edited_markdown TEXT, - is_reviewed INTEGER DEFAULT 0, - reviewer_notes TEXT, - detected_language TEXT, - translated_markdown TEXT, - edited_translation TEXT, - translation_reviewed INTEGER DEFAULT 0, - translation_notes TEXT, - translation_provider TEXT, - translation_model TEXT, - translation_target_language TEXT, - translated_at TEXT, - UNIQUE(workflow_id, page_number) - ) - """) - _add_column_if_missing(conn, "pages", "translation_provider", "TEXT") - _add_column_if_missing(conn, "pages", "translation_model", "TEXT") - _add_column_if_missing(conn, "pages", "translation_target_language", "TEXT") - _add_column_if_missing(conn, "pages", "translated_at", "TEXT") - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_pages_workflow - ON pages(workflow_id) - """) - # Chunks table for persistence after workflow completion - conn.execute(""" - CREATE TABLE IF NOT EXISTS chunks ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - workflow_id TEXT NOT NULL, - chunk_number INTEGER NOT NULL, - original_text TEXT, - edited_text TEXT, - token_count INTEGER DEFAULT 0, - page_start INTEGER DEFAULT 1, - page_end INTEGER DEFAULT 1, - source_page_numbers_json TEXT, - source_spans_json TEXT, - section_title TEXT, - content_type TEXT, - is_reference INTEGER DEFAULT 0, - chunking_provider TEXT, - chunking_model TEXT, - chunking_config_json TEXT, - chunking_run_id TEXT, - chunk_version INTEGER DEFAULT 1, - is_reviewed INTEGER DEFAULT 0, - is_excluded INTEGER DEFAULT 0, - reviewer_notes TEXT, - UNIQUE(workflow_id, chunk_number) - ) - """) - _add_column_if_missing(conn, "chunks", "source_page_numbers_json", "TEXT") - _add_column_if_missing(conn, "chunks", "source_spans_json", "TEXT") - _add_column_if_missing(conn, "chunks", "section_title", "TEXT") - _add_column_if_missing(conn, "chunks", "content_type", "TEXT") - _add_column_if_missing(conn, "chunks", "is_reference", "INTEGER DEFAULT 0") - _add_column_if_missing(conn, "chunks", "chunking_provider", "TEXT") - _add_column_if_missing(conn, "chunks", "chunking_model", "TEXT") - _add_column_if_missing(conn, "chunks", "chunking_config_json", "TEXT") - _add_column_if_missing(conn, "chunks", "chunking_run_id", "TEXT") - _add_column_if_missing(conn, "chunks", "chunk_version", "INTEGER DEFAULT 1") - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_chunks_workflow - ON chunks(workflow_id) - """) - # Settings table for application configuration - conn.execute(""" - CREATE TABLE IF NOT EXISTS settings ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL, - description TEXT, - updated_at TEXT NOT NULL - ) - """) - conn.execute(""" - CREATE TABLE IF NOT EXISTS document_jobs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - workflow_id TEXT NOT NULL, - job_type TEXT NOT NULL, - temporal_workflow_id TEXT, - temporal_run_id TEXT, - status TEXT NOT NULL DEFAULT 'running', - current_stage TEXT, - started_at TEXT NOT NULL, - completed_at TEXT, - error_message TEXT, - config_json TEXT - ) - """) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_document_jobs_workflow - ON document_jobs(workflow_id, started_at DESC) - """) - conn.execute(""" - CREATE TABLE IF NOT EXISTS document_artifacts ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - workflow_id TEXT NOT NULL, - job_id INTEGER, - artifact_type TEXT NOT NULL, - stage TEXT, - storage_uri TEXT NOT NULL, - mime_type TEXT, - filename TEXT, - size_bytes INTEGER, - metadata_json TEXT, - created_at TEXT NOT NULL - ) - """) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_document_artifacts_workflow - ON document_artifacts(workflow_id, created_at DESC) - """) - conn.execute(""" - CREATE TABLE IF NOT EXISTS document_index_status ( - workflow_id TEXT NOT NULL, - index_name TEXT NOT NULL, - doc_id TEXT, - chunk_count_indexed INTEGER DEFAULT 0, - last_indexed_at TEXT, - last_verified_at TEXT, - schema_version TEXT, - status TEXT NOT NULL DEFAULT 'unknown', - details_json TEXT, - PRIMARY KEY (workflow_id, index_name) - ) - """) - # doc_id supersedes the legacy marqo_doc_id column (generic - # naming, no DB name baked into the schema) on databases that - # predate this change. marqo_doc_id is left in place, unwritten, - # until scripts/drop_legacy_marqo_columns.py is run — so this - # backfill is best-effort: a fresh database, or one that's already - # had the column dropped, simply has nothing to migrate. - _add_column_if_missing(conn, "document_index_status", "doc_id", "TEXT") - try: - conn.execute(""" - UPDATE document_index_status - SET doc_id = marqo_doc_id - WHERE doc_id IS NULL AND marqo_doc_id IS NOT NULL - """) - except sqlite3.OperationalError: - pass - conn.execute(""" - CREATE TABLE IF NOT EXISTS document_manifest_entries ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - manifest_filename TEXT NOT NULL UNIQUE, - title_en TEXT, - title_gu TEXT, - doc_language TEXT, - category_tags TEXT, - description TEXT, - quality_score TEXT, - priority_rank TEXT, - ingestion_status TEXT, - feedback TEXT, - source_csv_path TEXT, - imported_at TEXT NOT NULL - ) - """) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_manifest_entries_filename - ON document_manifest_entries(manifest_filename) - """) - # Insert default search settings if not exists - default_settings = [ - ("search_method", "HYBRID", "Search method: TENSOR, LEXICAL, or HYBRID"), - ("search_limit", "12", "Number of search results to return"), - ("search_alpha", "0.6", "Hybrid search alpha: 0=lexical, 1=semantic"), - ("search_ranking_method", "rrf", "Hybrid ranking: rrf or normalize_linear"), - ("search_show_highlights", "true", "Show highlighted matches in results"), - ("search_ef_search", "256", "HNSW search accuracy parameter"), - ("search_index_name", "documents-index", "Default vector DB collection name"), - ("search_candidate_cap", "120", "Candidate retrieval pool cap"), - ("search_candidate_multiplier", "10", "Candidate pool multiplier before final cut"), - ("search_max_chunks_per_doc", "2", "Final result diversity cap per document"), - ("search_use_e5_prefix", "true", "Prefix search queries with e5 query:"), - ("search_exclude_reference", "true", "Exclude reference chunks when index supports it"), - ("search_query_expansion_profile", "gu-v1", "Query expansion profile"), - ("search_rerank_mode", "none", "Post-search reranking mode"), - ("search_hybrid_rrfk", "60", "RRF tuning parameter for hybrid search"), - ] - for key, value, description in default_settings: - conn.execute(""" - INSERT OR IGNORE INTO settings (key, value, description, updated_at) - VALUES (?, ?, ?, ?) - """, (key, value, description, datetime.utcnow().isoformat())) - # Scheme metadata + master catalog tables (AI tool sync) - init_scheme_catalog_schema(conn=conn) - # Email-OTP login codes (one live code per email; overwritten on resend). - conn.execute(""" - CREATE TABLE IF NOT EXISTS email_otps ( - email TEXT PRIMARY KEY, - code_hash TEXT NOT NULL, - expires_at TEXT NOT NULL, - attempts INTEGER NOT NULL DEFAULT 0, - created_at TEXT NOT NULL, - last_sent_at TEXT NOT NULL - ) - """) - # Generic inbound webhook records (full body kept as JSON text in payload). - conn.execute(""" - CREATE TABLE IF NOT EXISTS webhook_records ( - id TEXT PRIMARY KEY, -- uuid4 - source TEXT NOT NULL DEFAULT 'webhook', - external_id TEXT NOT NULL, - title TEXT, - slug TEXT, - status TEXT, - is_published INTEGER, - category TEXT, - payload TEXT NOT NULL, - payload_hash TEXT, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - UNIQUE (source, external_id) - ) - """) + conn.execute("SELECT 1") + return {"reachable": True, "migrations_current": migrate.is_current(database_url())} + except Exception as exc: + return {"reachable": False, "migrations_current": False, "error": str(exc)} + + +# ============================================================================= +# Helpers +# ============================================================================= + +_UUID_RE = re.compile( + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" +) +_FINGERPRINT_RE = re.compile(r"^[0-9a-f]{32}$") + +_REVIEW_STAGES = { + "ocr_review", + "translation_review", + "chunk_review", + "ready_for_ingestion", + "approval_for_prod", +} + +PROD_INDEX_KEY = "prod" +DEV_INDEX_KEY = "dev" + + +def _default_instance() -> str: + return (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + + +def _normalize_instances(instances: list[str]) -> list[str]: + default = _default_instance() + return sorted({(i or "").strip().lower() or default for i in instances}) + + +def _iso_now() -> str: + return datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + + +def _is_uuid(value: Any) -> bool: + return isinstance(value, str) and bool(_UUID_RE.match(value)) + + +def _clean(value: Any) -> Any: + if isinstance(value, uuid.UUID): + return str(value) + if isinstance(value, datetime): + if value.tzinfo is not None: + value = value.astimezone(timezone.utc).replace(tzinfo=None) + return value.isoformat() + if isinstance(value, date): + return value.isoformat() + return value + + +def _clean_row(row: Optional[dict]) -> Optional[dict]: + if row is None: + return None + return {k: _clean(v) for k, v in row.items()} + + +def _csv_roles(roles: Any) -> list[str]: + if roles is None: + return [] + if isinstance(roles, (list, tuple, set)): + return [str(r).strip() for r in roles if str(r).strip()] + return [part.strip() for part in str(roles).split(",") if part.strip()] + + +def _fingerprint_for( + source_file_fingerprint: Optional[str], + document_id: Optional[str], + workflow_id: Optional[str], +) -> str: + for candidate in (source_file_fingerprint, document_id): + if candidate and _FINGERPRINT_RE.match(candidate): + return candidate + return hashlib.md5((document_id or workflow_id or "").encode()).hexdigest() + + +def _prod_collection_names() -> set[str]: + names = { + os.environ.get("PROD_VECTOR_DB_COLLECTION_NAME"), + os.environ.get("PROD_SCHEME_VECTOR_DB_COLLECTION_NAME"), + } + return {n.strip() for n in names if n and n.strip()} + + +def _index_key_for(index_name: str) -> str: + return PROD_INDEX_KEY if index_name in _prod_collection_names() else DEV_INDEX_KEY + + +# ============================================================================= +# Identifier resolution +# ============================================================================= + +def _resolve_document_uuid(conn, identifier: Optional[str]) -> Optional[str]: + """Document UUID for a UUID, Temporal workflow id, fingerprint or filename.""" + if not identifier: + return None + identifier = str(identifier) + + if _is_uuid(identifier): + row = conn.execute( + "SELECT id FROM documents WHERE id = %s::uuid", (identifier,) + ).fetchone() + if row: + return str(row["id"]) + + row = conn.execute( + """ + SELECT document_id FROM workflows + WHERE temporal_workflow_id = %s + ORDER BY started_at DESC LIMIT 1 + """, + (identifier,), + ).fetchone() + if row: + return str(row["document_id"]) + + row = conn.execute( + """ + SELECT id FROM documents + WHERE source->>'fingerprint' = %s OR source->>'original_filename' = %s + ORDER BY created_at DESC LIMIT 1 + """, + (identifier, identifier), + ).fetchone() + if row: + return str(row["id"]) + + row = conn.execute( + """ + SELECT id FROM documents d + WHERE EXISTS ( + SELECT 1 FROM jsonb_each(d.index_status) e WHERE e.value->>'doc_id' = %s + ) OR md5(d.source->>'fingerprint') = %s + ORDER BY created_at DESC LIMIT 1 + """, + (identifier, identifier), + ).fetchone() + return str(row["id"]) if row else None + + +def _require_document_uuid(conn, identifier: str) -> str: + doc_uuid = _resolve_document_uuid(conn, identifier) + if doc_uuid is None: + raise LookupError(f"Document not found: {identifier}") + return doc_uuid + + +# ============================================================================= +# Document row <-> flat dict +# ============================================================================= + +_DOC_SELECT = """ + SELECT + d.id, d.instance, d.source, d.document_kind, d.catalog_entry_id, + d.valid_from, d.valid_to, d.is_disabled, d.index_status, d.metadata, + d.created_at, d.updated_at, + root.temporal_workflow_id AS root_workflow_id, + root.stop_after_ocr AS root_stop_after_ocr, + cw.id AS current_workflow_pk, + cw.current_stage AS current_stage, + cw.error_message AS current_error_message, + ce.code AS entry_code, ce.name AS entry_name, ce.aliases AS entry_aliases, + u.keycloak_user_id AS up_user_id, u.username AS up_username, + u.email AS up_email, u.roles AS up_roles, + ts.ocr_completed_at, ts.translation_completed_at, ts.chunks_completed_at, + ts.ingested_at, + req.started_at AS prod_ready_requested_at, + ru.keycloak_user_id AS prod_ready_requested_by_user_id, + ru.username AS prod_ready_requested_by_username, + oa.id AS original_artifact_id, + na.id AS normalized_artifact_id + FROM documents d + LEFT JOIN document_current_workflow cw ON cw.document_id = d.id + LEFT JOIN LATERAL ( + SELECT w.temporal_workflow_id, w.stop_after_ocr FROM workflows w + WHERE w.document_id = d.id + ORDER BY (w.workflow_type IN ('pipeline', 'ocr_only')) DESC, w.started_at ASC, w.id ASC + LIMIT 1 + ) root ON TRUE + LEFT JOIN catalog_entries ce ON ce.id = d.catalog_entry_id + LEFT JOIN users u ON u.id = d.uploaded_by_user_id + LEFT JOIN LATERAL ( + SELECT min(w.stage_timestamps->>'ocr') AS ocr_completed_at, + min(w.stage_timestamps->>'translation') AS translation_completed_at, + min(w.stage_timestamps->>'chunks') AS chunks_completed_at, + max(COALESCE(w.completed_at, w.updated_at)) + FILTER (WHERE w.current_stage = 'completed') AS ingested_at + FROM workflows w WHERE w.document_id = d.id + ) ts ON TRUE + LEFT JOIN LATERAL ( + SELECT w.started_at, w.started_by_user_id FROM workflows w + WHERE w.document_id = d.id AND w.workflow_type = 'promote_prod' + AND w.status = 'waiting_review' + ORDER BY w.started_at DESC LIMIT 1 + ) req ON TRUE + LEFT JOIN users ru ON ru.id = req.started_by_user_id + LEFT JOIN LATERAL ( + SELECT a.id FROM workflow_artifacts a JOIN workflows w ON w.id = a.workflow_id + WHERE w.document_id = d.id AND a.artifact_type = 'original_upload' + ORDER BY a.created_at DESC, a.id DESC LIMIT 1 + ) oa ON TRUE + LEFT JOIN LATERAL ( + SELECT a.id FROM workflow_artifacts a JOIN workflows w ON w.id = a.workflow_id + WHERE w.document_id = d.id + AND a.artifact_type IN ('normalized_pdf', 'normalized_spreadsheet') + ORDER BY a.created_at DESC, a.id DESC LIMIT 1 + ) na ON TRUE +""" + + +def _document_from_row(row: Optional[dict]) -> Optional[dict]: + if row is None: + return None + source = row["source"] or {} + meta = row["metadata"] or {} + fingerprint = source.get("fingerprint") + roles = row["up_roles"] or [] + entry_aliases = row["entry_aliases"] + doc = { + "id": row["id"], + "workflow_id": row["root_workflow_id"], + "document_id": fingerprint, + "canonical_document_id": fingerprint, + "filename": source.get("original_filename") or "", + "display_name": meta.get("display_name"), + "source_filename": source.get("original_filename"), + "source_manifest_name": None, + "source_file_fingerprint": fingerprint, + "filepath": source.get("storage_uri") or "", + "instance": row["instance"], + "stage": row["current_stage"] or "registered", + "page_count": source.get("page_count") or 0, + "chunk_count": meta.get("chunk_count") or 0, + "error_message": row["current_error_message"], + "is_demo": bool(meta.get("is_demo")), + "is_disabled": bool(row["is_disabled"]), + "source_type": source.get("source_type"), + "canonical_input_type": source.get("canonical_input_type"), + "stop_after_ocr": bool(row["root_stop_after_ocr"]), + "reindex_required": bool(meta.get("reindex_required")), + "reindex_reason": meta.get("reindex_reason"), + "document_kind": row["document_kind"], + "scheme_code": row["entry_code"], + "scheme_name": row["entry_name"], + "scheme_aliases_json": json.dumps(list(entry_aliases)) if entry_aliases is not None else None, + "tool_routing": "qdrant" if row["entry_code"] else None, + "catalog_visible": bool(meta.get("catalog_visible", True)), + "network_visible": bool(meta.get("network_visible", True)), + "valid_from": row["valid_from"], + "valid_to": row["valid_to"], + "uploaded_by_user_id": row["up_user_id"], + "uploaded_by_username": row["up_username"], + "uploaded_by_email": row["up_email"], + "uploaded_by_roles": ",".join(roles) if roles else None, + "prod_ready_requested_at": row["prod_ready_requested_at"], + "prod_ready_requested_by_user_id": row["prod_ready_requested_by_user_id"], + "prod_ready_requested_by_username": row["prod_ready_requested_by_username"], + "ocr_completed_at": row["ocr_completed_at"], + "translation_completed_at": row["translation_completed_at"], + "chunks_completed_at": row["chunks_completed_at"], + "ingested_at": row["ingested_at"], + "original_artifact_id": row["original_artifact_id"], + "normalized_artifact_id": row["normalized_artifact_id"], + "latest_job_id": row["current_workflow_pk"], + "catalog_entry_id": row["catalog_entry_id"], + "index_status": row["index_status"] or {}, + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } + return {k: _clean(v) for k, v in doc.items()} + + +def _fetch_document(conn, doc_uuid: Optional[str]) -> Optional[dict]: + if doc_uuid is None: + return None + row = conn.execute(_DOC_SELECT + " WHERE d.id = %s::uuid", (doc_uuid,)).fetchone() + return _document_from_row(row) + + +def _document_filters( + include_demo: bool, + include_disabled: bool, + instances: Optional[list[str]], + alias: str = "d", +) -> tuple[list[str], list]: + clauses: list[str] = [] + params: list = [] + if not include_demo: + clauses.append(f"NOT COALESCE(({alias}.metadata->>'is_demo')::boolean, FALSE)") + if not include_disabled: + clauses.append(f"NOT {alias}.is_disabled") + if instances is not None: + clauses.append(f"lower({alias}.instance) = ANY(%s)") + params.append(_normalize_instances(instances)) + return clauses, params + + +# ============================================================================= +# Users +# ============================================================================= + +def upsert_user( + keycloak_user_id: str, + username: Optional[str] = None, + email: Optional[str] = None, + roles: Any = None, + conn=None, +) -> Optional[str]: + """Create or refresh the ``users`` row for a Keycloak subject; returns its UUID.""" + keycloak_user_id = (keycloak_user_id or "").strip() + if not keycloak_user_id: + return None + if conn is None: + with get_connection() as own: + return upsert_user(keycloak_user_id, username, email, roles, conn=own) + + role_list = _csv_roles(roles) + email = (email or "").strip() or None + username = (username or "").strip() or None + for candidate_email in ([email, None] if email else [None]): + try: + with conn.transaction(): + row = conn.execute( + """ + INSERT INTO users (keycloak_user_id, username, email, roles) + VALUES (%s, %s, %s, %s) + ON CONFLICT (keycloak_user_id) DO UPDATE SET + username = COALESCE(EXCLUDED.username, users.username), + email = COALESCE(EXCLUDED.email, users.email), + roles = CASE WHEN cardinality(EXCLUDED.roles) > 0 + THEN EXCLUDED.roles ELSE users.roles END, + last_seen_at = now() + RETURNING id + """, + (keycloak_user_id, username, candidate_email, role_list), + ).fetchone() + return str(row["id"]) + except psycopg.errors.UniqueViolation: + continue + return None + + +def get_user_by_keycloak_id(keycloak_user_id: str) -> Optional[dict]: + with get_connection() as conn: + return _clean_row( conn.execute( - "CREATE INDEX IF NOT EXISTS idx_webhook_records_source_status " - "ON webhook_records (source, status)" - ) - conn.commit() + "SELECT * FROM users WHERE keycloak_user_id = %s", (keycloak_user_id,) + ).fetchone() + ) + + +# ============================================================================= +# Webhook records +# ============================================================================= + +def _parse_timestamp(value: Any) -> Optional[datetime]: + if not value: + return None + try: + parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed def upsert_webhook_records(source: str, records: list) -> int: @@ -417,13 +486,13 @@ def upsert_webhook_records(source: str, records: list) -> int: external_id comes from contentID, else id, else slug, else a hash of the payload. Unchanged payloads (same payload_hash) are not rewritten but still counted. """ - now = datetime.utcnow().isoformat() + now = datetime.now(timezone.utc) rows = [] for rec in records: if not isinstance(rec, dict): raise ValueError("Each record must be a JSON object") - payload = json.dumps(rec, sort_keys=True, ensure_ascii=False, default=str) - payload_hash = hashlib.sha256(payload.encode("utf-8")).hexdigest() + payload_text = json.dumps(rec, sort_keys=True, ensure_ascii=False, default=str) + payload_hash = hashlib.sha256(payload_text.encode("utf-8")).hexdigest() ext = rec.get("contentID") if ext in (None, ""): ext = rec.get("id") @@ -433,296 +502,296 @@ def upsert_webhook_records(source: str, records: list) -> int: ext = payload_hash pub = rec.get("isPublished") rows.append(( - str(uuid.uuid4()), source, str(ext), rec.get("title"), rec.get("slug"), rec.get("status"), - int(pub) if isinstance(pub, (bool, int)) else None, - rec.get("domainName"), payload, payload_hash, - rec.get("createdAt") or now, rec.get("updatedAt") or now, + source, str(ext), rec.get("title"), rec.get("slug"), rec.get("status"), + bool(pub) if isinstance(pub, (bool, int)) else None, + rec.get("domainName"), Jsonb(json.loads(payload_text)), payload_hash, + _parse_timestamp(rec.get("createdAt")) or now, + _parse_timestamp(rec.get("updatedAt")) or now, )) - with _db_lock: - with get_connection() as conn: - for r in rows: - conn.execute( - """ - INSERT INTO webhook_records - (id, source, external_id, title, slug, status, is_published, category, - payload, payload_hash, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(source, external_id) DO UPDATE SET - title = excluded.title, - slug = excluded.slug, - status = excluded.status, - is_published = excluded.is_published, - category = excluded.category, - payload = excluded.payload, - payload_hash = excluded.payload_hash, - created_at = excluded.created_at, - updated_at = excluded.updated_at - WHERE webhook_records.payload_hash IS NOT excluded.payload_hash - """, - r, - ) - conn.commit() + with get_connection() as conn: + for r in rows: + conn.execute( + """ + INSERT INTO webhook_records + (source, external_id, title, slug, status, is_published, category, + payload, payload_hash, created_at, updated_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (source, external_id) DO UPDATE SET + title = EXCLUDED.title, + slug = EXCLUDED.slug, + status = EXCLUDED.status, + is_published = EXCLUDED.is_published, + category = EXCLUDED.category, + payload = EXCLUDED.payload, + payload_hash = EXCLUDED.payload_hash, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at + WHERE webhook_records.payload_hash IS DISTINCT FROM EXCLUDED.payload_hash + """, + r, + ) return len(rows) +# ============================================================================= +# Email OTPs +# ============================================================================= + def store_email_otp( *, email: str, code_hash: str, expires_at: str, created_at: str, last_sent_at: str ) -> None: """Upsert the one live OTP for an email, resetting attempts on resend.""" - with _db_lock: - with get_connection() as conn: - conn.execute( - """ - INSERT INTO email_otps (email, code_hash, expires_at, attempts, created_at, last_sent_at) - VALUES (?, ?, ?, 0, ?, ?) - ON CONFLICT(email) DO UPDATE SET - code_hash = excluded.code_hash, - expires_at = excluded.expires_at, - attempts = 0, - created_at = excluded.created_at, - last_sent_at = excluded.last_sent_at - """, - (email, code_hash, expires_at, created_at, last_sent_at), - ) - conn.commit() + with get_connection() as conn: + conn.execute( + """ + INSERT INTO email_otps (email, code_hash, expires_at, attempts, last_sent_at) + VALUES (%s, %s, %s, 0, %s) + ON CONFLICT (email) DO UPDATE SET + code_hash = EXCLUDED.code_hash, + expires_at = EXCLUDED.expires_at, + attempts = 0, + last_sent_at = EXCLUDED.last_sent_at + """, + (email.strip().lower(), code_hash, expires_at, last_sent_at), + ) -def get_email_otp(email: str) -> Optional[sqlite3.Row]: +def get_email_otp(email: str) -> Optional[dict]: + """The live OTP row; timestamps are timezone-aware ISO strings (UTC).""" with get_connection() as conn: - return conn.execute( - "SELECT * FROM email_otps WHERE email = ?", (email,) + row = conn.execute( + "SELECT * FROM email_otps WHERE email = %s", (email.strip().lower(),) ).fetchone() + if row is None: + return None + row = _clean_row(row) | { + "expires_at": row["expires_at"].astimezone(timezone.utc).isoformat(), + "last_sent_at": row["last_sent_at"].astimezone(timezone.utc).isoformat(), + } + return row -def increment_email_otp_attempts(email: str) -> None: - with _db_lock: - with get_connection() as conn: - conn.execute( - "UPDATE email_otps SET attempts = attempts + 1 WHERE email = ?", (email,) - ) - conn.commit() +def increment_email_otp_attempts(email: str) -> Optional[int]: + """Atomically count one failed attempt; returns the new total.""" + with get_connection() as conn: + row = conn.execute( + "UPDATE email_otps SET attempts = attempts + 1 WHERE email = %s RETURNING attempts", + (email.strip().lower(),), + ).fetchone() + return int(row["attempts"]) if row else None def delete_email_otp(email: str) -> None: - with _db_lock: - with get_connection() as conn: - conn.execute("DELETE FROM email_otps WHERE email = ?", (email,)) - conn.commit() - - -def init_scheme_catalog_schema(conn: Optional[sqlite3.Connection] = None) -> None: - """Idempotent: scheme document columns + scheme_catalog_* tables.""" - - def _apply(c: sqlite3.Connection) -> None: - for column, definition in ( - ("document_kind", "TEXT DEFAULT 'document'"), - ("scheme_code", "TEXT"), - ("scheme_name", "TEXT"), - ("scheme_aliases_json", "TEXT"), - ("tool_routing", "TEXT"), - ("catalog_visible", "INTEGER DEFAULT 1"), - ("network_visible", "INTEGER DEFAULT 1"), - ): - _add_column_if_missing(c, "documents", column, definition) - - c.execute(""" - CREATE TABLE IF NOT EXISTS scheme_catalog_meta ( - id INTEGER PRIMARY KEY CHECK (id = 1), - version INTEGER NOT NULL DEFAULT 0, - updated_at TEXT, - notes TEXT - ) - """) - c.execute(""" - INSERT OR IGNORE INTO scheme_catalog_meta (id, version, updated_at, notes) - VALUES (1, 0, ?, ?) - """, (datetime.utcnow().isoformat(), "init")) - - c.execute(""" - CREATE TABLE IF NOT EXISTS scheme_catalog_entries ( - scheme_code TEXT PRIMARY KEY, - scheme_name TEXT, - scheme_aliases_json TEXT, - instances_json TEXT, - workflow_ids_json TEXT, - collection_name TEXT, - chunk_count INTEGER DEFAULT 0, - status TEXT NOT NULL DEFAULT 'disabled', - network_visible INTEGER DEFAULT 0, - promoted_at TEXT, - content_hash TEXT, - source TEXT, - updated_at TEXT - ) - """) - c.execute(""" - CREATE INDEX IF NOT EXISTS idx_scheme_catalog_status - ON scheme_catalog_entries(status) - """) - c.execute(""" - CREATE INDEX IF NOT EXISTS idx_documents_scheme_code - ON documents(scheme_code) - """) - - if conn is not None: - _apply(conn) - return - - with _db_lock: - with get_connection() as c: - _apply(c) - c.commit() + with get_connection() as conn: + conn.execute("DELETE FROM email_otps WHERE email = %s", (email.strip().lower(),)) + + +# ============================================================================= +# Catalog +# ============================================================================= + +CATALOG_VERSION_ACTION = "catalog_version_bump" def get_catalog_meta() -> dict: - init_scheme_catalog_schema() + """Catalog version: the number of version bumps recorded in ``audit_logs``.""" with get_connection() as conn: - row = conn.execute("SELECT * FROM scheme_catalog_meta WHERE id = 1").fetchone() - if not row: - return {"id": 1, "version": 0, "updated_at": None, "notes": None} - return dict(row) + row = conn.execute( + """ + SELECT count(*) AS version, max(created_at) AS updated_at, + (array_agg(metadata->>'reason' ORDER BY created_at DESC))[1] AS notes + FROM audit_logs + WHERE entity_type = 'catalog' AND action_type = %s + """, + (CATALOG_VERSION_ACTION,), + ).fetchone() + return { + "id": 1, + "version": int(row["version"] or 0), + "updated_at": _clean(row["updated_at"]), + "notes": row["notes"], + } def bump_catalog_version(reason: str = "") -> int: """Monotonic catalog version bump. Returns new version.""" - now = datetime.utcnow().isoformat() - init_scheme_catalog_schema() - with _db_lock: - with get_connection() as conn: - conn.execute( - """ - UPDATE scheme_catalog_meta - SET version = version + 1, updated_at = ?, notes = ? - WHERE id = 1 - """, - (now, (reason or "")[:500]), - ) - conn.commit() - row = conn.execute( - "SELECT version FROM scheme_catalog_meta WHERE id = 1" - ).fetchone() - return int(row["version"]) if row else 0 + with get_connection() as conn: + conn.execute( + """ + INSERT INTO audit_logs (action_type, entity_type, metadata) + VALUES (%s, 'catalog', %s) + """, + (CATALOG_VERSION_ACTION, Jsonb({"reason": (reason or "")[:500]})), + ) + row = conn.execute( + "SELECT count(*) AS version FROM audit_logs " + "WHERE entity_type = 'catalog' AND action_type = %s", + (CATALOG_VERSION_ACTION,), + ).fetchone() + return int(row["version"]) def upsert_catalog_entry( *, - scheme_code: str, - scheme_name: str, - scheme_aliases: list, - instances: list, - workflow_ids: list, - collection_name: str, - chunk_count: int, - status: str, - network_visible: int, - promoted_at: Optional[str], - content_hash: str, - source: str, - updated_at: str, -) -> None: - init_scheme_catalog_schema() - with _db_lock: - with get_connection() as conn: - conn.execute( - """ - INSERT INTO scheme_catalog_entries ( - scheme_code, scheme_name, scheme_aliases_json, instances_json, - workflow_ids_json, collection_name, chunk_count, status, - network_visible, promoted_at, content_hash, source, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(scheme_code) DO UPDATE SET - scheme_name = excluded.scheme_name, - scheme_aliases_json = excluded.scheme_aliases_json, - instances_json = excluded.instances_json, - workflow_ids_json = excluded.workflow_ids_json, - collection_name = excluded.collection_name, - chunk_count = excluded.chunk_count, - status = excluded.status, - network_visible = excluded.network_visible, - promoted_at = excluded.promoted_at, - content_hash = excluded.content_hash, - source = excluded.source, - updated_at = excluded.updated_at - """, - ( - scheme_code, - scheme_name, - json.dumps(scheme_aliases or []), - json.dumps(instances or []), - json.dumps(workflow_ids or []), - collection_name, - int(chunk_count or 0), - status, - int(network_visible or 0), - promoted_at, - content_hash, - source, - updated_at, - ), + type: str, + code: str, + name: Optional[str] = None, + aliases: Optional[list[str]] = None, + source: Optional[str] = None, + conn=None, +) -> dict: + """Insert or update one ``catalog_entries`` row. ``None`` fields keep stored values.""" + if conn is None: + with get_connection() as own: + return upsert_catalog_entry( + type=type, code=code, name=name, aliases=aliases, source=source, conn=own ) - conn.commit() + alias_list = [a for a in (aliases or []) if a] if aliases is not None else None + row = conn.execute( + """ + INSERT INTO catalog_entries (type, code, name, aliases, metadata) + VALUES (%s, %s, COALESCE(%s, %s), COALESCE(%s, '{}'::text[]), %s) + ON CONFLICT (type, code) DO UPDATE SET + name = COALESCE(%s, catalog_entries.name), + aliases = COALESCE(%s, catalog_entries.aliases), + metadata = CASE WHEN %s::text IS NULL THEN catalog_entries.metadata + ELSE catalog_entries.metadata || jsonb_build_object('source', %s::text) END + RETURNING id, type, code, name, aliases, metadata + """, + ( + type, code, name, code, alias_list, Jsonb({"source": source or "pipeline"}), + name, alias_list, source, source, + ), + ).fetchone() + return _clean_row(row) + + +def list_catalog_entry_codes(type: str) -> set[str]: + with get_connection() as conn: + rows = conn.execute("SELECT code FROM catalog_entries WHERE type = %s", (type,)).fetchall() + return {r["code"] for r in rows} -def list_catalog_entries(status: Optional[str] = None) -> list[dict]: - init_scheme_catalog_schema() +def _catalog_row(row: dict) -> dict: + cleaned = _clean_row(row) + cleaned["aliases"] = list(row["aliases"] or []) + cleaned["instances"] = list(row["instances"] or []) + cleaned["workflow_ids"] = list(row["workflow_ids"] or []) + return cleaned + + +def list_catalog_entries(status: Optional[str] = None, type: str = "scheme") -> list[dict]: + """Rows of the ``catalog`` view for one entry type.""" with get_connection() as conn: if status: rows = conn.execute( - "SELECT * FROM scheme_catalog_entries WHERE status = ? ORDER BY scheme_code", - (status,), + "SELECT * FROM catalog WHERE type = %s AND status = %s ORDER BY code", + (type, status), ).fetchall() else: rows = conn.execute( - "SELECT * FROM scheme_catalog_entries ORDER BY scheme_code" + "SELECT * FROM catalog WHERE type = %s ORDER BY code", (type,) ).fetchall() - return [dict(r) for r in rows] + return [_catalog_row(r) for r in rows] -def get_catalog_entry(scheme_code: str) -> Optional[dict]: - init_scheme_catalog_schema() +def get_catalog_entry(code: str, type: str = "scheme") -> Optional[dict]: with get_connection() as conn: row = conn.execute( - "SELECT * FROM scheme_catalog_entries WHERE scheme_code = ?", - (scheme_code,), + "SELECT * FROM catalog WHERE type = %s AND code = %s", (type, code) ).fetchone() - return dict(row) if row else None + return _catalog_row(row) if row else None -def list_scheme_documents_for_catalog() -> list[dict]: - """All documents that look scheme-related (for rebuild).""" - init_scheme_catalog_schema() +def list_used_scheme_codes(exclude_workflow_id: Optional[str] = None) -> set[str]: + """Every scheme code already taken - the collision domain for auto-generated codes. + + The entry the excluded document alone holds is not counted, so re-deriving a + document's own code does not collide with itself. + """ with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, exclude_workflow_id) if exclude_workflow_id else None rows = conn.execute( """ - SELECT * FROM documents - WHERE document_kind = 'scheme' - OR (scheme_code IS NOT NULL AND trim(scheme_code) != '') - """ + SELECT e.code FROM catalog_entries e + WHERE e.type = 'scheme' + AND NOT ( + %s::uuid IS NOT NULL + AND e.metadata->>'source' <> 'bootstrap' + AND EXISTS (SELECT 1 FROM documents d + WHERE d.catalog_entry_id = e.id AND d.id = %s::uuid) + AND NOT EXISTS (SELECT 1 FROM documents d + WHERE d.catalog_entry_id = e.id AND d.id <> %s::uuid) + ) + """, + (doc_uuid, doc_uuid, doc_uuid), ).fetchall() - return [dict(r) for r in rows] + return {r["code"] for r in rows} -def list_used_scheme_codes(exclude_workflow_id: Optional[str] = None) -> set[str]: - """Every scheme_code currently in use, across both the live documents table - and the scheme_catalog_entries registry (PROD-published codes may outlive - the document row that first defined them) — the full collision domain for - auto-generating a new code from a title.""" - init_scheme_catalog_schema() +def list_scheme_catalog_entries_for_workflow(workflow_id: str) -> list[dict]: + """Catalog rows whose documents still include this workflow.""" with get_connection() as conn: - doc_rows = conn.execute( - "SELECT scheme_code, workflow_id FROM documents " - "WHERE scheme_code IS NOT NULL AND trim(scheme_code) != ''" + rows = conn.execute( + "SELECT code, workflow_ids FROM catalog WHERE type = 'scheme' AND %s = ANY(workflow_ids)", + (workflow_id,), ).fetchall() - entry_rows = conn.execute("SELECT scheme_code FROM scheme_catalog_entries").fetchall() - codes = { - row["scheme_code"].strip().lower() - for row in doc_rows - if row["workflow_id"] != exclude_workflow_id - } - codes.update(row["scheme_code"].strip().lower() for row in entry_rows) - return codes + return [{"scheme_code": r["code"], "workflow_ids": list(r["workflow_ids"])} for r in rows] + + +def _link_catalog_entry( + conn, + doc_uuid: str, + updates: dict, +) -> dict[str, Any]: + """Resolve document_kind / scheme_* updates into documents column changes. + + Returns the extra ``documents`` assignments (``document_kind``, + ``catalog_entry_id``) so the kind and its entry change in one statement, + which the composite foreign key requires. + """ + current = conn.execute( + """ + SELECT d.document_kind, d.catalog_entry_id, ce.type AS entry_type, + ce.code AS entry_code, ce.name AS entry_name, ce.aliases AS entry_aliases + FROM documents d LEFT JOIN catalog_entries ce ON ce.id = d.catalog_entry_id + WHERE d.id = %s::uuid + """, + (doc_uuid,), + ).fetchone() + kind = updates.get("document_kind") or current["document_kind"] + code = updates["scheme_code"] if "scheme_code" in updates else current["entry_code"] + name = updates["scheme_name"] if "scheme_name" in updates else None + aliases = None + if "scheme_aliases_json" in updates: + raw = updates["scheme_aliases_json"] + aliases = json.loads(raw) if isinstance(raw, str) else list(raw or []) + + changes: dict[str, Any] = {} + if "document_kind" in updates: + changes["document_kind"] = kind + + current_entry_id = str(current["catalog_entry_id"]) if current["catalog_entry_id"] else None + new_entry_id = None + if kind != "document" and code: + moved = current["entry_type"] != kind or current["entry_code"] != code + if moved: + if name is None: + name = current["entry_name"] + if aliases is None and current["entry_aliases"] is not None: + aliases = list(current["entry_aliases"]) + new_entry_id = upsert_catalog_entry( + type=kind, code=code, name=name, aliases=aliases, source="pipeline", conn=conn + )["id"] + if new_entry_id != current_entry_id: + changes["catalog_entry_id"] = new_entry_id + return changes +# ============================================================================= +# Documents +# ============================================================================= + def upsert_document( workflow_id: str, document_id: str, @@ -744,9 +813,9 @@ def upsert_document( source_type: Optional[str] = None, canonical_input_type: Optional[str] = None, stop_after_ocr: bool = False, - original_artifact_id: Optional[int] = None, - normalized_artifact_id: Optional[int] = None, - latest_job_id: Optional[int] = None, + original_artifact_id: Optional[str] = None, + normalized_artifact_id: Optional[str] = None, + latest_job_id: Optional[str] = None, instance: Optional[str] = None, uploaded_by_user_id: Optional[str] = None, uploaded_by_username: Optional[str] = None, @@ -754,8 +823,9 @@ def upsert_document( uploaded_by_roles: Optional[str] = None, valid_from: Optional[str] = None, valid_to: Optional[str] = None, -): - """Insert or update a document record. + with_workflow: bool = True, +) -> str: + """Insert or update a document record; returns the document UUID. ``instance`` is applied on INSERT only. Updates leave the existing tenant stamp alone so a re-upload / restart cannot silently reassign tenants. @@ -766,94 +836,100 @@ def upsert_document( restart that re-registers the same workflow. What the period should default to is the caller's call (see ``pipeline/document_validity.py``); omitting both stores NULL, which search reads as "no stated period". - """ - now = datetime.utcnow().isoformat() - instance_value = (instance or os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" - with _db_lock: - with get_connection() as conn: - # Check if exists - row = conn.execute( - "SELECT created_at FROM documents WHERE workflow_id = ?", - (workflow_id,) + A new document also gets its first workflow row (``temporal_workflow_id`` = + ``workflow_id``) unless ``with_workflow`` is False, for callers that create + it themselves with ``create_document_job`` (addressing the document by the + returned UUID, since no workflow id resolves to it yet). + """ + instance_value = (instance or _default_instance()).strip().lower() or _default_instance() + fingerprint = _fingerprint_for( + source_file_fingerprint or canonical_document_id, document_id, workflow_id + ) + source_patch = { + "fingerprint": fingerprint, + "original_filename": filename, + "storage_uri": filepath, + "source_type": source_type, + "canonical_input_type": canonical_input_type, + "page_count": page_count, + } + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) if workflow_id else None + if doc_uuid: + patch = {k: v for k, v in source_patch.items() if v is not None} + DocumentSource.model_validate({**_load(conn, doc_uuid, "source"), **patch}) + metadata_patch: dict[str, Any] = {"chunk_count": chunk_count} + if display_name: + metadata_patch["display_name"] = display_name + uploader_uuid = ( + upsert_user(uploaded_by_user_id, uploaded_by_username, uploaded_by_email, + uploaded_by_roles, conn=conn) + if uploaded_by_user_id else None + ) + conn.execute( + """ + UPDATE documents SET + source = source || %s::jsonb, + metadata = metadata || %s::jsonb, + uploaded_by_user_id = COALESCE(uploaded_by_user_id, %s::uuid) + WHERE id = %s::uuid + """, + (Jsonb(patch), Jsonb(metadata_patch), uploader_uuid, doc_uuid), + ) + cw = conn.execute( + "SELECT id FROM document_current_workflow WHERE document_id = %s::uuid", (doc_uuid,) ).fetchone() + if cw: + conn.execute( + """ + UPDATE workflows SET current_stage = %s::workflow_stage, error_message = %s, + stop_after_ocr = %s + WHERE id = %s::uuid + """, + (stage, error_message, bool(stop_after_ocr), str(cw["id"])), + ) + return doc_uuid + + source = DocumentSource.model_validate( + {k: v for k, v in source_patch.items() if v is not None} + ).model_dump(exclude_none=True) + metadata = DocumentMetadata( + display_name=display_name, chunk_count=chunk_count + ).model_dump(exclude_none=True) + user_uuid = ( + upsert_user(uploaded_by_user_id, uploaded_by_username, uploaded_by_email, + uploaded_by_roles, conn=conn) + if uploaded_by_user_id else None + ) + row = conn.execute( + """ + INSERT INTO documents (instance, source, metadata, valid_from, valid_to, uploaded_by_user_id) + VALUES (%s, %s, %s, %s::date, %s::date, %s::uuid) + RETURNING id + """, + (instance_value, Jsonb(source), Jsonb(metadata), valid_from, valid_to, user_uuid), + ).fetchone() + if with_workflow: + conn.execute( + """ + INSERT INTO workflows (document_id, workflow_type, temporal_workflow_id, + current_stage, error_message, stop_after_ocr, + started_by_user_id) + VALUES (%s::uuid, %s::workflow_type_enum, %s, %s::workflow_stage, %s, %s, %s::uuid) + """, + ( + str(row["id"]), + "ocr_only" if stop_after_ocr else "pipeline", + workflow_id, stage, error_message, bool(stop_after_ocr), user_uuid, + ), + ) + return str(row["id"]) - if row: - # Update existing — do not overwrite instance (tenant ownership). - conn.execute(""" - UPDATE documents SET - document_id = ?, - canonical_document_id = COALESCE(?, canonical_document_id), - filename = ?, - display_name = COALESCE(?, display_name), - source_filename = COALESCE(?, source_filename), - source_manifest_name = COALESCE(?, source_manifest_name), - source_file_fingerprint = COALESCE(?, source_file_fingerprint), - filepath = ?, - stage = ?, - page_count = ?, - chunk_count = ?, - error_message = ?, - updated_at = ?, - ocr_completed_at = COALESCE(?, ocr_completed_at), - translation_completed_at = COALESCE(?, translation_completed_at), - chunks_completed_at = COALESCE(?, chunks_completed_at), - ingested_at = COALESCE(?, ingested_at), - source_type = COALESCE(?, source_type), - canonical_input_type = COALESCE(?, canonical_input_type), - stop_after_ocr = COALESCE(?, stop_after_ocr), - original_artifact_id = COALESCE(?, original_artifact_id), - normalized_artifact_id = COALESCE(?, normalized_artifact_id), - latest_job_id = COALESCE(?, latest_job_id), - uploaded_by_user_id = COALESCE(uploaded_by_user_id, ?), - uploaded_by_username = COALESCE(uploaded_by_username, ?), - uploaded_by_email = COALESCE(uploaded_by_email, ?), - uploaded_by_roles = COALESCE(uploaded_by_roles, ?) - WHERE workflow_id = ? - """, ( - document_id, canonical_document_id, filename, display_name, - source_filename, source_manifest_name, source_file_fingerprint, - filepath, stage, - page_count, chunk_count, error_message, now, - ocr_completed_at, translation_completed_at, chunks_completed_at, ingested_at, - source_type, canonical_input_type, 1 if stop_after_ocr else 0, - original_artifact_id, normalized_artifact_id, latest_job_id, - uploaded_by_user_id, uploaded_by_username, uploaded_by_email, uploaded_by_roles, - workflow_id - )) - else: - # Insert new - conn.execute(""" - INSERT INTO documents ( - workflow_id, document_id, filename, filepath, - canonical_document_id, display_name, source_filename, source_manifest_name, - source_file_fingerprint, - stage, page_count, chunk_count, error_message, - created_at, updated_at, - ocr_completed_at, translation_completed_at, chunks_completed_at, ingested_at, - source_type, canonical_input_type, stop_after_ocr, - reindex_required, reindex_reason, - original_artifact_id, normalized_artifact_id, latest_job_id, - instance, - uploaded_by_user_id, uploaded_by_username, uploaded_by_email, uploaded_by_roles, - valid_from, valid_to - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, ( - workflow_id, document_id, filename, filepath, - canonical_document_id, display_name, source_filename, source_manifest_name, - source_file_fingerprint, - stage, page_count, chunk_count, error_message, - now, now, - ocr_completed_at, translation_completed_at, chunks_completed_at, ingested_at, - source_type, canonical_input_type, 1 if stop_after_ocr else 0, - 0, None, - original_artifact_id, normalized_artifact_id, latest_job_id, - instance_value, - uploaded_by_user_id, uploaded_by_username, uploaded_by_email, uploaded_by_roles, - valid_from, valid_to, - )) - - conn.commit() + +def _load(conn, doc_uuid: str, column: str) -> dict: + row = conn.execute(f"SELECT {column} FROM documents WHERE id = %s::uuid", (doc_uuid,)).fetchone() + return dict(row[column]) if row else {} def update_document_stage( @@ -863,64 +939,61 @@ def update_document_stage( chunk_count: Optional[int] = None, error_message: Optional[str] = None ): - """Update just the stage and counts for a document.""" - now = datetime.utcnow().isoformat() - - with _db_lock: - with get_connection() as conn: - # Get previous state for audit logging - row = conn.execute( - "SELECT stage, document_id FROM documents WHERE workflow_id = ?", - (workflow_id,) - ).fetchone() - old_stage = row["stage"] if row else None - document_id = row["document_id"] if row else workflow_id - - # Build dynamic update - updates = ["stage = ?", "updated_at = ?"] - values = [stage, now] - - if page_count is not None: - updates.append("page_count = ?") - values.append(page_count) - - if chunk_count is not None: - updates.append("chunk_count = ?") - values.append(chunk_count) - - if error_message is not None: - updates.append("error_message = ?") - values.append(error_message) - - # Set timestamp based on stage - if stage == "ocr_review": - updates.append("ocr_completed_at = ?") - values.append(now) - elif stage == "translation_review": - updates.append("translation_completed_at = ?") - values.append(now) - elif stage == "chunk_review": - updates.append("chunks_completed_at = ?") - values.append(now) - elif stage == "completed": - updates.append("ingested_at = ?") - values.append(now) - - values.append(workflow_id) - - # Note: updates list contains only hardcoded field names, not user input - # Values are properly parameterized via ? - conn.execute( - f"UPDATE documents SET {', '.join(updates)} WHERE workflow_id = ?", - values + """Update the stage (of the document's current workflow) and counts.""" + old_stage = None + audit_workflow_id = workflow_id + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return + cw = conn.execute( + "SELECT id, current_stage FROM document_current_workflow WHERE document_id = %s::uuid", + (doc_uuid,), + ).fetchone() + if cw is None: + return + old_stage = cw["current_stage"] + + assignments = ["current_stage = %s::workflow_stage"] + params: list = [stage] + if error_message is not None: + assignments.append("error_message = %s") + params.append(error_message) + stamp_key = { + "ocr_review": "ocr", + "translation_review": "translation", + "chunk_review": "chunks", + }.get(stage) + if stamp_key: + assignments.append( + "stage_timestamps = jsonb_build_object(%s::text, %s::text) || stage_timestamps" ) - conn.commit() + params.extend([stamp_key, _iso_now()]) + if stage == "completed": + assignments.append("completed_at = now()") + params.append(str(cw["id"])) + conn.execute( + f"UPDATE workflows SET {', '.join(assignments)} WHERE id = %s::uuid", params + ) + + doc_assignments = ["updated_at = now()"] + doc_params: list = [] + if page_count is not None: + doc_assignments.append("source = source || jsonb_build_object('page_count', %s::int)") + doc_params.append(page_count) + if chunk_count is not None: + doc_assignments.append("metadata = metadata || jsonb_build_object('chunk_count', %s::int)") + doc_params.append(chunk_count) + doc_params.append(doc_uuid) + conn.execute( + f"UPDATE documents SET {', '.join(doc_assignments)} WHERE id = %s::uuid", doc_params + ) + audit_workflow_id = _root_workflow_id(conn, doc_uuid) or workflow_id - # Log stage transition if stage changed (outside lock to avoid deadlock) if old_stage and old_stage != stage: log_audit( - workflow_id=workflow_id, - document_id=document_id, + workflow_id=audit_workflow_id, + document_id=audit_workflow_id, action_type="stage_change", field_name="stage", old_value=old_stage, @@ -929,6 +1002,18 @@ def update_document_stage( ) +def _root_workflow_id(conn, doc_uuid: str) -> Optional[str]: + row = conn.execute( + """ + SELECT temporal_workflow_id FROM workflows WHERE document_id = %s::uuid + ORDER BY (workflow_type IN ('pipeline', 'ocr_only')) DESC, started_at ASC, id ASC + LIMIT 1 + """, + (doc_uuid,), + ).fetchone() + return row["temporal_workflow_id"] if row else None + + def reconcile_materialized_state(workflow_id: str) -> Optional[dict]: """ Reconcile a document row against already-persisted page/chunk content. @@ -939,23 +1024,19 @@ def reconcile_materialized_state(workflow_id: str) -> Optional[dict]: doc = get_document(workflow_id) if not doc: return None + workflow_id = doc["workflow_id"] with get_connection() as conn: page_stats = conn.execute( """ SELECT COUNT(*) AS page_rows, COALESCE(MAX(page_number), 0) AS max_page_number - FROM pages - WHERE workflow_id = ? + FROM pages WHERE document_id = %s::uuid """, - (workflow_id,), + (doc["id"],), ).fetchone() chunk_stats = conn.execute( - """ - SELECT COUNT(*) AS chunk_rows - FROM chunks - WHERE workflow_id = ? - """, - (workflow_id,), + "SELECT COUNT(*) AS chunk_rows FROM chunks WHERE document_id = %s::uuid", + (doc["id"],), ).fetchone() page_count = int((page_stats["page_rows"] if page_stats else 0) or 0) @@ -966,7 +1047,7 @@ def reconcile_materialized_state(workflow_id: str) -> Optional[dict]: current_stage = doc.get("stage") latest_job = get_latest_document_job(workflow_id) updated = False - now = datetime.utcnow().isoformat() + now = _iso_now() # Highest-confidence reconciliation first: if chunks exist, the document has # already reached chunk review materiality even if the workflow stalled. @@ -1049,14 +1130,9 @@ def reconcile_materialized_state(workflow_id: str) -> Optional[dict]: def get_document(workflow_id: str) -> Optional[dict]: - """Get a single document by workflow ID.""" + """Get a single document by workflow id, UUID or any other identifier.""" with get_connection() as conn: - row = conn.execute( - "SELECT * FROM documents WHERE workflow_id = ?", - (workflow_id,) - ).fetchone() - - return dict(row) if row else None + return _fetch_document(conn, _resolve_document_uuid(conn, workflow_id)) def list_documents( @@ -1076,43 +1152,40 @@ def list_documents( include_demo: If False (default), excludes demo documents from results include_disabled: If False (default), excludes soft-deleted documents from results instances: If provided, only return docs whose instance is in this list - (NULL/empty instance treated as DEFAULT_INSTANCE / 'default') """ - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + if instances is not None and not _normalize_instances(instances): + return [] + clauses, params = _document_filters(include_demo, include_disabled, instances) + if stage: + clauses.append("cw.current_stage = %s::workflow_stage") + params.append(stage) + where = f"WHERE {' AND '.join(clauses)}" if clauses else "" with get_connection() as conn: - # Note: filters are hardcoded SQL fragments based on boolean flags, not user input - demo_filter = "" if include_demo else "AND (is_demo = 0 OR is_demo IS NULL)" - disabled_filter = "" if include_disabled else "AND (is_disabled = 0 OR is_disabled IS NULL)" - instance_filter = "" - params: list = [] - - if instances is not None: - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) - if not normalized: - return [] - placeholders = ",".join("?" for _ in normalized) - instance_filter = ( - f"AND lower(COALESCE(NULLIF(trim(instance), ''), ?)) IN ({placeholders})" - ) - params.append(default_instance) - params.extend(normalized) - - if stage: - rows = conn.execute(f""" - SELECT * FROM documents - WHERE stage = ? {demo_filter} {disabled_filter} {instance_filter} - ORDER BY created_at DESC - LIMIT ? OFFSET ? - """, (stage, *params, limit, offset)).fetchall() - else: - rows = conn.execute(f""" - SELECT * FROM documents - WHERE 1=1 {demo_filter} {disabled_filter} {instance_filter} - ORDER BY created_at DESC - LIMIT ? OFFSET ? - """, (*params, limit, offset)).fetchall() - - return [dict(row) for row in rows] + rows = conn.execute( + f"{_DOC_SELECT} {where} ORDER BY d.created_at DESC, d.id LIMIT %s OFFSET %s", + (*params, limit, offset), + ).fetchall() + return [_document_from_row(r) for r in rows] + + +_REVIEW_QUEUE_STAGES = ("ocr_review", "translation_review", "chunk_review", "approval_for_prod") + +_SUMMARY_KEYS = ( + "total_documents", + "authoritative_documents", + "legacy_documents", + "completed_documents", + "review_queue", + "ocr_review_documents", + "translation_review_documents", + "chunk_review_documents", + "translation_processing_documents", + "chunking_documents", + "ready_for_ingestion_documents", + "failed_documents", + "needs_reindex", + "running_jobs", +) def get_document_summary_counts( @@ -1121,90 +1194,37 @@ def get_document_summary_counts( instances: Optional[list[str]] = None, ) -> dict: """Return aggregate document counts for dashboard and migration planning.""" - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + if instances is not None and not _normalize_instances(instances): + return {key: 0 for key in _SUMMARY_KEYS} + clauses, params = _document_filters(include_demo, include_disabled, instances) + where = f"WHERE {' AND '.join(clauses)}" if clauses else "" with get_connection() as conn: - demo_filter = "" if include_demo else "AND (is_demo = 0 OR is_demo IS NULL)" - disabled_filter = "" if include_disabled else "AND (is_disabled = 0 OR is_disabled IS NULL)" - job_demo_filter = "" if include_demo else "AND (d.is_demo = 0 OR d.is_demo IS NULL)" - job_disabled_filter = "" if include_disabled else "AND (d.is_disabled = 0 OR d.is_disabled IS NULL)" - instance_filter = "" - job_instance_filter = "" - params: list = [] - job_params: list = [] - - if instances is not None: - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) - if not normalized: - return { - "total_documents": 0, - "authoritative_documents": 0, - "legacy_documents": 0, - "completed_documents": 0, - "review_queue": 0, - "ocr_review_documents": 0, - "translation_review_documents": 0, - "chunk_review_documents": 0, - "translation_processing_documents": 0, - "chunking_documents": 0, - "ready_for_ingestion_documents": 0, - "failed_documents": 0, - "needs_reindex": 0, - "running_jobs": 0, - } - placeholders = ",".join("?" for _ in normalized) - instance_filter = ( - f"AND lower(COALESCE(NULLIF(trim(instance), ''), ?)) IN ({placeholders})" - ) - job_instance_filter = ( - f"AND lower(COALESCE(NULLIF(trim(d.instance), ''), ?)) IN ({placeholders})" - ) - params = [default_instance, *normalized] - job_params = [default_instance, *normalized] - - row = conn.execute(f""" + row = conn.execute( + f""" SELECT COUNT(*) AS total_documents, - -- SQLite SUM() returns NULL when no rows match; COALESCE keeps response ints valid. - COALESCE(SUM(CASE WHEN source_manifest_name IS NOT NULL THEN 1 ELSE 0 END), 0) AS authoritative_documents, - COALESCE(SUM(CASE WHEN source_manifest_name IS NULL THEN 1 ELSE 0 END), 0) AS legacy_documents, - COALESCE(SUM(CASE WHEN stage = 'completed' THEN 1 ELSE 0 END), 0) AS completed_documents, - COALESCE(SUM(CASE WHEN stage IN ('ocr_review', 'translation_review', 'chunk_review', 'approval_for_prod') THEN 1 ELSE 0 END), 0) AS review_queue, - COALESCE(SUM(CASE WHEN stage = 'ocr_review' THEN 1 ELSE 0 END), 0) AS ocr_review_documents, - COALESCE(SUM(CASE WHEN stage = 'translation_review' THEN 1 ELSE 0 END), 0) AS translation_review_documents, - COALESCE(SUM(CASE WHEN stage = 'chunk_review' THEN 1 ELSE 0 END), 0) AS chunk_review_documents, - COALESCE(SUM(CASE WHEN stage = 'translation_processing' THEN 1 ELSE 0 END), 0) AS translation_processing_documents, - COALESCE(SUM(CASE WHEN stage = 'chunking' THEN 1 ELSE 0 END), 0) AS chunking_documents, - COALESCE(SUM(CASE WHEN stage = 'ready_for_ingestion' THEN 1 ELSE 0 END), 0) AS ready_for_ingestion_documents, - COALESCE(SUM(CASE WHEN stage = 'failed' THEN 1 ELSE 0 END), 0) AS failed_documents, - COALESCE(SUM(CASE WHEN reindex_required = 1 THEN 1 ELSE 0 END), 0) AS needs_reindex, - ( - SELECT COUNT(*) - FROM document_jobs j - JOIN documents d ON d.workflow_id = j.workflow_id - WHERE j.status = 'running' {job_demo_filter} {job_disabled_filter} {job_instance_filter} - ) AS running_jobs - FROM documents - WHERE 1=1 {demo_filter} {disabled_filter} {instance_filter} - """, (*job_params, *params)).fetchone() - result = dict(row) if row else {} - # Defensive: every count field must be an int for DocumentCohortsResponse. - int_keys = ( - "total_documents", - "authoritative_documents", - "legacy_documents", - "completed_documents", - "review_queue", - "ocr_review_documents", - "translation_review_documents", - "chunk_review_documents", - "translation_processing_documents", - "chunking_documents", - "ready_for_ingestion_documents", - "failed_documents", - "needs_reindex", - "running_jobs", - ) - return {key: int(result.get(key) or 0) for key in int_keys} + 0 AS authoritative_documents, + COUNT(*) AS legacy_documents, + COUNT(*) FILTER (WHERE cw.current_stage = 'completed') AS completed_documents, + COUNT(*) FILTER (WHERE cw.current_stage IN + ('ocr_review', 'translation_review', 'chunk_review', 'approval_for_prod')) AS review_queue, + COUNT(*) FILTER (WHERE cw.current_stage = 'ocr_review') AS ocr_review_documents, + COUNT(*) FILTER (WHERE cw.current_stage = 'translation_review') AS translation_review_documents, + COUNT(*) FILTER (WHERE cw.current_stage = 'chunk_review') AS chunk_review_documents, + COUNT(*) FILTER (WHERE cw.current_stage = 'translation_processing') AS translation_processing_documents, + COUNT(*) FILTER (WHERE cw.current_stage = 'chunking') AS chunking_documents, + COUNT(*) FILTER (WHERE cw.current_stage = 'ready_for_ingestion') AS ready_for_ingestion_documents, + COUNT(*) FILTER (WHERE cw.current_stage = 'failed') AS failed_documents, + COUNT(*) FILTER (WHERE COALESCE((d.metadata->>'reindex_required')::boolean, FALSE)) AS needs_reindex, + COALESCE(SUM((SELECT COUNT(*) FROM workflows j + WHERE j.document_id = d.id AND j.status = 'running')), 0) AS running_jobs + FROM documents d + LEFT JOIN document_current_workflow cw ON cw.document_id = d.id + {where} + """, + params, + ).fetchone() + return {key: int(row[key] or 0) for key in _SUMMARY_KEYS} def get_document_counts_by_instance( @@ -1213,57 +1233,50 @@ def get_document_counts_by_instance( instances: Optional[list[str]] = None, ) -> list[dict]: """Return document counts grouped by instance (state), for dashboard breakdowns.""" - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + if instances is not None and not _normalize_instances(instances): + return [] + clauses, params = _document_filters(include_demo, include_disabled, instances) + where = f"WHERE {' AND '.join(clauses)}" if clauses else "" with get_connection() as conn: - demo_filter = "" if include_demo else "AND (is_demo = 0 OR is_demo IS NULL)" - disabled_filter = "" if include_disabled else "AND (is_disabled = 0 OR is_disabled IS NULL)" - instance_filter = "" - params: list = [default_instance] - - if instances is not None: - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) - if not normalized: - return [] - placeholders = ",".join("?" for _ in normalized) - instance_filter = ( - f"AND lower(COALESCE(NULLIF(trim(instance), ''), ?)) IN ({placeholders})" - ) - params.append(default_instance) - params.extend(normalized) - - rows = conn.execute(f""" + rows = conn.execute( + f""" SELECT - lower(COALESCE(NULLIF(trim(instance), ''), ?)) AS instance, + lower(d.instance) AS instance, COUNT(*) AS count, - COALESCE(SUM(CASE WHEN stage = 'completed' THEN 1 ELSE 0 END), 0) AS success, - COALESCE(SUM(CASE WHEN stage = 'failed' THEN 1 ELSE 0 END), 0) AS failed, - COALESCE(SUM(CASE WHEN stage IN ('ocr_review', 'translation_review', 'chunk_review', 'approval_for_prod') THEN 1 ELSE 0 END), 0) AS dev_approval - FROM documents - WHERE 1=1 {demo_filter} {disabled_filter} {instance_filter} - GROUP BY instance + COUNT(*) FILTER (WHERE cw.current_stage = 'completed') AS success, + COUNT(*) FILTER (WHERE cw.current_stage = 'failed') AS failed, + COUNT(*) FILTER (WHERE cw.current_stage IN + ('ocr_review', 'translation_review', 'chunk_review', 'approval_for_prod')) AS dev_approval + FROM documents d + LEFT JOIN document_current_workflow cw ON cw.document_id = d.id + {where} + GROUP BY lower(d.instance) ORDER BY count DESC, instance ASC - """, params).fetchall() - return [ - { - "instance": row["instance"], - "count": int(row["count"] or 0), - "success": int(row["success"] or 0), - "failed": int(row["failed"] or 0), - "dev_approval": int(row["dev_approval"] or 0), - } - for row in rows - ] + """, + params, + ).fetchall() + return [ + { + "instance": row["instance"], + "count": int(row["count"] or 0), + "success": int(row["success"] or 0), + "failed": int(row["failed"] or 0), + "dev_approval": int(row["dev_approval"] or 0), + } + for row in rows + ] def set_document_demo(workflow_id: str, is_demo: bool = True): """Mark a document as demo (filtered from UI by default).""" - with _db_lock: - with get_connection() as conn: + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid: conn.execute( - "UPDATE documents SET is_demo = ? WHERE workflow_id = ?", - (1 if is_demo else 0, workflow_id) + "UPDATE documents SET metadata = metadata || jsonb_build_object('is_demo', %s::boolean) " + "WHERE id = %s::uuid", + (bool(is_demo), doc_uuid), ) - conn.commit() def set_document_disabled(workflow_id: str, is_disabled: bool = True): @@ -1272,98 +1285,67 @@ def set_document_disabled(workflow_id: str, is_disabled: bool = True): Unlike hard delete, this preserves the document record for audit purposes. Use X-Include-Disabled: true header in API to see disabled documents. """ - with _db_lock: - with get_connection() as conn: + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid: conn.execute( - "UPDATE documents SET is_disabled = ?, updated_at = ? WHERE workflow_id = ?", - (1 if is_disabled else 0, datetime.utcnow().isoformat(), workflow_id) + "UPDATE documents SET is_disabled = %s WHERE id = %s::uuid", + (bool(is_disabled), doc_uuid), ) - conn.commit() def delete_document(workflow_id: str): - """Delete a document record.""" - with _db_lock: - with get_connection() as conn: - conn.execute( - "DELETE FROM documents WHERE workflow_id = ?", - (workflow_id,) - ) - conn.commit() - - -# Every table holding per-document rows, child-first so a partial failure never -# strands parent rows. `chunk_tags` is legacy — created by an older tagging -# build and absent from fresh databases — so it is probed rather than assumed. -_PURGE_TABLES = ( - "chunk_tags", - "chunks", - "pages", - "document_artifacts", - "document_index_status", - "document_jobs", - "documents", -) + """Delete a document record and everything that cascades from it.""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid: + conn.execute("DELETE FROM documents WHERE id = %s::uuid", (doc_uuid,)) def purge_document(workflow_id: str, keep_audit: bool = True) -> dict: - """Hard delete every SQLite row belonging to a document. + """Hard delete every row belonging to a document. - Unlike :func:`set_document_disabled` this is irreversible — there is + Unlike :func:`set_document_disabled` this is irreversible - there is nothing left for ``POST /documents/{id}/restore`` to bring back. Audit rows are kept by default: they are the record that the deletion - happened, not part of the document itself. + happened, not part of the document itself. Their ``document_id`` becomes + NULL; the ``workflow_id`` and ``instance`` snapshots stay. Returns a per-table count of deleted rows. """ deleted: dict[str, int] = {} - with _db_lock: - with get_connection() as conn: - existing = { - row[0] - for row in conn.execute( - "SELECT name FROM sqlite_master WHERE type = 'table'" - ) - } - for table in _PURGE_TABLES: - if table not in existing: - continue - columns = {r[1] for r in conn.execute(f"PRAGMA table_info([{table}])")} - if "workflow_id" not in columns: - continue - cursor = conn.execute( - f"DELETE FROM [{table}] WHERE workflow_id = ?", (workflow_id,) - ) - if cursor.rowcount > 0: - deleted[table] = cursor.rowcount - - if not keep_audit and "audit_logs" in existing: - cursor = conn.execute( - "DELETE FROM audit_logs WHERE workflow_id = ?", (workflow_id,) - ) - if cursor.rowcount > 0: - deleted["audit_logs"] = cursor.rowcount - - conn.commit() - return deleted - - -def list_scheme_catalog_entries_for_workflow(workflow_id: str) -> list[dict]: - """Scheme catalog rows that still reference this workflow.""" with get_connection() as conn: - rows = conn.execute( - "SELECT scheme_code, workflow_ids_json FROM scheme_catalog_entries" - ).fetchall() - matches = [] - for row in rows: - try: - ids = json.loads(row["workflow_ids_json"] or "[]") - except (ValueError, TypeError): - ids = [] - if workflow_id in ids: - matches.append({"scheme_code": row["scheme_code"], "workflow_ids": ids}) - return matches + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return deleted + root = _root_workflow_id(conn, doc_uuid) or workflow_id + + def _count(sql: str) -> int: + return int(conn.execute(sql, (doc_uuid,)).fetchone()["n"]) + + counts = { + "chunks": _count("SELECT count(*) AS n FROM chunks WHERE document_id = %s::uuid"), + "pages": _count("SELECT count(*) AS n FROM pages WHERE document_id = %s::uuid"), + "document_artifacts": _count( + "SELECT count(*) AS n FROM workflow_artifacts a JOIN workflows w ON w.id = a.workflow_id " + "WHERE w.document_id = %s::uuid" + ), + "document_jobs": _count("SELECT count(*) AS n FROM workflows WHERE document_id = %s::uuid"), + } + if not keep_audit: + row = conn.execute( + "DELETE FROM audit_logs WHERE document_id = %s::uuid OR workflow_id = %s", + (doc_uuid, root), + ) + if row.rowcount > 0: + deleted["audit_logs"] = row.rowcount + conn.execute("DELETE FROM documents WHERE id = %s::uuid", (doc_uuid,)) + for table, count in counts.items(): + if count > 0: + deleted[table] = count + deleted["documents"] = 1 + return deleted def get_document_count(stage: Optional[str] = None) -> int: @@ -1371,121 +1353,242 @@ def get_document_count(stage: Optional[str] = None) -> int: with get_connection() as conn: if stage: row = conn.execute( - "SELECT COUNT(*) as cnt FROM documents WHERE stage = ?", - (stage,) + """ + SELECT COUNT(*) AS cnt FROM documents d + JOIN document_current_workflow cw ON cw.document_id = d.id + WHERE cw.current_stage = %s::workflow_stage + """, + (stage,), ).fetchone() else: - row = conn.execute("SELECT COUNT(*) as cnt FROM documents").fetchone() - - return row["cnt"] if row else 0 - - -def upsert_manifest_entry( - manifest_filename: str, - title_en: Optional[str] = None, - title_gu: Optional[str] = None, - doc_language: Optional[str] = None, - category_tags: Optional[str] = None, - description: Optional[str] = None, - quality_score: Optional[str] = None, - priority_rank: Optional[str] = None, - ingestion_status: Optional[str] = None, - feedback: Optional[str] = None, - source_csv_path: Optional[str] = None, -) -> dict: - imported_at = datetime.utcnow().isoformat() - with _db_lock: - with get_connection() as conn: - conn.execute(""" - INSERT INTO document_manifest_entries ( - manifest_filename, title_en, title_gu, doc_language, - category_tags, description, quality_score, priority_rank, - ingestion_status, feedback, source_csv_path, imported_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(manifest_filename) DO UPDATE SET - title_en = COALESCE(excluded.title_en, document_manifest_entries.title_en), - title_gu = COALESCE(excluded.title_gu, document_manifest_entries.title_gu), - doc_language = COALESCE(excluded.doc_language, document_manifest_entries.doc_language), - category_tags = COALESCE(excluded.category_tags, document_manifest_entries.category_tags), - description = COALESCE(excluded.description, document_manifest_entries.description), - quality_score = COALESCE(excluded.quality_score, document_manifest_entries.quality_score), - priority_rank = COALESCE(excluded.priority_rank, document_manifest_entries.priority_rank), - ingestion_status = COALESCE(excluded.ingestion_status, document_manifest_entries.ingestion_status), - feedback = COALESCE(excluded.feedback, document_manifest_entries.feedback), - source_csv_path = COALESCE(excluded.source_csv_path, document_manifest_entries.source_csv_path), - imported_at = excluded.imported_at - """, ( - manifest_filename, title_en, title_gu, doc_language, - category_tags, description, quality_score, priority_rank, - ingestion_status, feedback, source_csv_path, imported_at - )) - conn.commit() - return get_manifest_entry(manifest_filename) or {} - - -def get_manifest_entry(manifest_filename: str) -> Optional[dict]: + row = conn.execute("SELECT COUNT(*) AS cnt FROM documents").fetchone() + return int(row["cnt"]) if row else 0 + + +def update_document_fields(workflow_id: str, **updates: object) -> Optional[dict]: + """Update selected document fields and return the document. + + Legacy flat names are mapped onto the tables: ``stage``/``error_message`` act + on the current workflow, ``scheme_*`` on the catalog entry, counts and + flags on the JSONB columns. Names derived from other rows (artifact and job + pointers, completion timestamps) are accepted and ignored. + """ with get_connection() as conn: - row = conn.execute( - "SELECT * FROM document_manifest_entries WHERE manifest_filename = ?", - (manifest_filename,), - ).fetchone() - return dict(row) if row else None + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + if not updates: + return _fetch_document(conn, doc_uuid) + + doc_sets: list[str] = [] + doc_params: list = [] + source_patch: dict[str, Any] = {} + meta_patch: dict[str, Any] = {} + meta_remove: list[str] = [] + wf_sets: list[str] = [] + wf_params: list = [] + + for key, value in updates.items(): + if key == "stage": + wf_sets.append("current_stage = %s::workflow_stage") + wf_params.append(value) + elif key == "error_message": + wf_sets.append("error_message = %s") + wf_params.append(value) + elif key == "page_count": + source_patch["page_count"] = value + elif key == "chunk_count": + meta_patch["chunk_count"] = value + elif key == "source_type": + source_patch["source_type"] = value + elif key == "canonical_input_type": + source_patch["canonical_input_type"] = value + elif key == "filepath": + source_patch["storage_uri"] = value + elif key in ("filename", "source_filename"): + if key == "filename" or "filename" not in updates: + source_patch["original_filename"] = value + elif key in ("document_id", "canonical_document_id", "source_file_fingerprint"): + if isinstance(value, str) and _FINGERPRINT_RE.match(value): + source_patch["fingerprint"] = value + elif key == "display_name": + if value: + meta_patch["display_name"] = value + else: + meta_remove.append("display_name") + elif key in ("reindex_required", "catalog_visible", "network_visible"): + meta_patch[key] = bool(value) + elif key == "reindex_reason": + if value: + meta_patch["reindex_reason"] = value + else: + meta_remove.append("reindex_reason") + elif key == "stop_after_ocr": + conn.execute( + "UPDATE workflows SET stop_after_ocr = %s WHERE id = (" + "SELECT id FROM workflows WHERE document_id = %s::uuid " + "ORDER BY (workflow_type IN ('pipeline', 'ocr_only')) DESC, started_at ASC, id ASC LIMIT 1)", + (bool(value), doc_uuid), + ) + elif key in ("valid_from", "valid_to"): + doc_sets.append(f"{key} = %s::date") + doc_params.append(value) + elif key == "uploaded_by_user_id" and value: + doc_sets.append("uploaded_by_user_id = COALESCE(uploaded_by_user_id, %s::uuid)") + doc_params.append( + upsert_user( + str(value), + updates.get("uploaded_by_username"), + updates.get("uploaded_by_email"), + updates.get("uploaded_by_roles"), + conn=conn, + ) + ) + + if any(k in updates for k in ("document_kind", "scheme_code", "scheme_name", "scheme_aliases_json")): + for column, new_value in _link_catalog_entry(conn, doc_uuid, updates).items(): + doc_sets.append(f"{column} = %s" + ("::uuid" if column == "catalog_entry_id" else "")) + doc_params.append(new_value) + + if source_patch: + doc_sets.append("source = source || %s::jsonb") + doc_params.append(Jsonb({k: v for k, v in source_patch.items() if v is not None})) + if meta_patch or meta_remove: + expr = "metadata" + for removed in meta_remove: + expr = f"({expr} - '{removed}')" + if meta_patch: + expr = f"{expr} || %s::jsonb" + doc_params.append(Jsonb(meta_patch)) + doc_sets.append(f"metadata = {expr}") + + if wf_sets: + conn.execute( + f"UPDATE workflows SET {', '.join(wf_sets)} WHERE id = (" + "SELECT id FROM document_current_workflow WHERE document_id = %s::uuid)", + (*wf_params, doc_uuid), + ) + if doc_sets: + conn.execute( + f"UPDATE documents SET {', '.join(doc_sets)} WHERE id = %s::uuid", + (*doc_params, doc_uuid), + ) + elif wf_sets: + conn.execute("UPDATE documents SET updated_at = now() WHERE id = %s::uuid", (doc_uuid,)) + return _fetch_document(conn, doc_uuid) + + +def mark_document_reindex_required( + workflow_id: str, + required: bool = True, + reason: Optional[str] = None, +) -> Optional[dict]: + return update_document_fields( + workflow_id, + reindex_required=bool(required), + reindex_reason=reason if required else None, + ) -def list_manifest_entries(limit: int = 1000, offset: int = 0) -> list[dict]: +def set_document_validity( + workflow_id: str, + valid_from: str, + valid_to: str, +) -> Optional[dict]: + """Record the period this document's chunks are searchable in. Both ends + are `YYYY-MM-DD`; validation belongs to the caller + (`pipeline/document_validity.parse_period`), not to this SQL layer.""" + return update_document_fields(workflow_id, valid_from=valid_from, valid_to=valid_to) + + +def mark_prod_ready_requested( + workflow_id: str, + user_id: Optional[str] = None, + username: Optional[str] = None, +) -> Optional[dict]: + """Flag that a state_admin has asked a super_admin to review this document + for prod promotion. Purely informational - does not fire the approve_prod + signal itself; the super_admin still makes that call separately. + + Recorded as a ``promote_prod`` workflow in ``waiting_review``.""" with get_connection() as conn: - rows = conn.execute(""" - SELECT * FROM document_manifest_entries - ORDER BY manifest_filename - LIMIT ? OFFSET ? - """, (limit, offset)).fetchall() - return [dict(row) for row in rows] + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + root = _root_workflow_id(conn, doc_uuid) or workflow_id + user_uuid = upsert_user(user_id, username, None, None, conn=conn) if user_id else None + conn.execute( + """ + UPDATE workflows SET status = 'running' + WHERE document_id = %s::uuid AND workflow_type = 'promote_prod' AND status = 'waiting_review' + """, + (doc_uuid,), + ) + conn.execute( + """ + INSERT INTO workflows (document_id, workflow_type, temporal_workflow_id, status, + current_stage, started_by_user_id) + VALUES (%s::uuid, 'promote_prod', %s, 'waiting_review', 'approval_for_prod', %s::uuid) + """, + (doc_uuid, f"{root}-prod-request", user_uuid), + ) + return _fetch_document(conn, doc_uuid) -def update_document_fields(workflow_id: str, **updates: object) -> Optional[dict]: - """Update selected document fields and return the document.""" - if not updates: - return get_document(workflow_id) +def clear_prod_ready_requested(workflow_id: str) -> Optional[dict]: + """Reset the request flag once prod approval actually happens (or the + document leaves the approval_for_prod gate for any other reason).""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + conn.execute( + """ + UPDATE workflows SET status = 'running' + WHERE document_id = %s::uuid AND workflow_type = 'promote_prod' AND status = 'waiting_review' + """, + (doc_uuid,), + ) + return _fetch_document(conn, doc_uuid) - allowed_fields = { - "stage", "page_count", "chunk_count", "error_message", "updated_at", - "ocr_completed_at", "translation_completed_at", "chunks_completed_at", "ingested_at", - "source_type", "canonical_input_type", "original_artifact_id", "normalized_artifact_id", - "latest_job_id", "filepath", "filename", "display_name", "document_id", - "canonical_document_id", "source_filename", "source_manifest_name", - "source_file_fingerprint", "stop_after_ocr", "reindex_required", "reindex_reason", - "uploaded_by_user_id", "uploaded_by_username", "uploaded_by_email", "uploaded_by_roles", - # Scheme / master catalog fields - "document_kind", "scheme_code", "scheme_name", "scheme_aliases_json", - "tool_routing", "catalog_visible", "network_visible", - "prod_ready_requested_at", "prod_ready_requested_by_user_id", "prod_ready_requested_by_username", - # Document validity period (searchable-from / searchable-to) - "valid_from", "valid_to", - } - set_clauses = [] - values: list[object] = [] - for key, value in updates.items(): - if key not in allowed_fields: - continue - set_clauses.append(f"{key} = ?") - values.append(value) - if not set_clauses: - return get_document(workflow_id) +# ============================================================================= +# Workflows (exposed as "jobs") +# ============================================================================= - if "updated_at" not in updates: - set_clauses.append("updated_at = ?") - values.append(datetime.utcnow().isoformat()) +_JOB_SELECT = """ + SELECT w.id, w.document_id, w.workflow_type, w.temporal_workflow_id, w.status, + w.current_stage, w.config, w.error_message, w.started_at, w.completed_at, + root.temporal_workflow_id AS doc_workflow_id + FROM workflows w + LEFT JOIN LATERAL ( + SELECT r.temporal_workflow_id FROM workflows r + WHERE r.document_id = w.document_id + ORDER BY (r.workflow_type IN ('pipeline', 'ocr_only')) DESC, r.started_at ASC, r.id ASC + LIMIT 1 + ) root ON TRUE +""" - values.append(workflow_id) - with _db_lock: - with get_connection() as conn: - conn.execute( - f"UPDATE documents SET {', '.join(set_clauses)} WHERE workflow_id = ?", - values, - ) - conn.commit() - return get_document(workflow_id) + +def _job_from_row(row: Optional[dict]) -> Optional[dict]: + if row is None: + return None + config = row["config"] or {} + job = { + "id": row["id"], + "workflow_id": row["doc_workflow_id"], + "job_type": row["workflow_type"], + "temporal_workflow_id": row["temporal_workflow_id"], + "temporal_run_id": config.get("temporal_run_id"), + "status": row["status"], + "current_stage": row["current_stage"], + "started_at": row["started_at"], + "completed_at": row["completed_at"], + "error_message": row["error_message"], + "config": config, + "config_json": json.dumps(config) if config else None, + } + return {k: _clean(v) for k, v in job.items()} def create_document_job( @@ -1496,32 +1599,65 @@ def create_document_job( status: str = "running", current_stage: Optional[str] = None, config: Optional[dict] = None, -) -> int: - now = datetime.utcnow().isoformat() - config_json = json.dumps(config) if config else None - with _db_lock: - with get_connection() as conn: - cursor = conn.execute(""" - INSERT INTO document_jobs ( - workflow_id, job_type, temporal_workflow_id, temporal_run_id, - status, current_stage, started_at, config_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) - """, ( - workflow_id, job_type, temporal_workflow_id, temporal_run_id, - status, current_stage, now, config_json - )) - job_id = cursor.lastrowid - conn.execute( - "UPDATE documents SET latest_job_id = ?, updated_at = ? WHERE workflow_id = ?", - (job_id, now, workflow_id), + actor: Optional[dict] = None, +) -> str: + """Record a workflow run for a document; returns the workflow UUID. + + A ``promote_prod`` run adopts the document's pending request row (see + ``mark_prod_ready_requested``) instead of adding a second one. + """ + cfg = dict(config or {}) + if temporal_run_id: + cfg["temporal_run_id"] = temporal_run_id + with get_connection() as conn: + doc_uuid = _require_document_uuid(conn, workflow_id) + root = _root_workflow_id(conn, doc_uuid) or workflow_id + user_uuid = None + if actor and actor.get("user_id"): + user_uuid = upsert_user( + actor["user_id"], actor.get("username"), actor.get("email"), actor.get("roles"), + conn=conn, ) - conn.commit() - return job_id + if job_type == "promote_prod": + adopted = conn.execute( + """ + UPDATE workflows SET temporal_workflow_id = %s, status = %s::workflow_status_enum, + current_stage = COALESCE(%s::workflow_stage, current_stage), + config = config || %s::jsonb + WHERE id = ( + SELECT id FROM workflows + WHERE document_id = %s::uuid AND workflow_type = 'promote_prod' + AND status = 'waiting_review' + ORDER BY started_at DESC LIMIT 1) + RETURNING id + """, + (temporal_workflow_id or root, status, current_stage, Jsonb(cfg), doc_uuid), + ).fetchone() + if adopted: + conn.execute("UPDATE documents SET updated_at = now() WHERE id = %s::uuid", (doc_uuid,)) + return str(adopted["id"]) + row = conn.execute( + """ + INSERT INTO workflows (document_id, workflow_type, temporal_workflow_id, status, + current_stage, stop_after_ocr, config, started_by_user_id) + VALUES (%s::uuid, %s::workflow_type_enum, %s, %s::workflow_status_enum, + COALESCE(%s, 'registered')::workflow_stage, %s, %s, %s::uuid) + RETURNING id + """, + ( + doc_uuid, job_type, temporal_workflow_id or root, status, current_stage, + bool(cfg.get("stop_after_ocr")), Jsonb(cfg), user_uuid, + ), + ).fetchone() + conn.execute("UPDATE documents SET updated_at = now() WHERE id = %s::uuid", (doc_uuid,)) + return str(row["id"]) -def update_document_job(job_id: int, **updates: object) -> Optional[dict]: +def update_document_job(job_id: str, **updates: object) -> Optional[dict]: if not updates: return get_document_job(job_id) + if not _is_uuid(str(job_id)): + return None allowed_fields = { "status", "current_stage", "completed_at", "error_message", "temporal_run_id", "config_json" } @@ -1530,52 +1666,62 @@ def update_document_job(job_id: int, **updates: object) -> Optional[dict]: for key, value in updates.items(): if key not in allowed_fields: continue - if key == "config_json" and isinstance(value, dict): - value = json.dumps(value) - set_clauses.append(f"{key} = ?") + if key == "status": + set_clauses.append("status = %s::workflow_status_enum") + elif key == "current_stage": + set_clauses.append("current_stage = %s::workflow_stage") + elif key == "completed_at": + set_clauses.append("completed_at = %s::timestamptz") + elif key == "temporal_run_id": + set_clauses.append("config = config || jsonb_build_object('temporal_run_id', %s::text)") + elif key == "config_json": + if isinstance(value, str): + value = json.loads(value) + set_clauses.append("config = config || %s::jsonb") + value = Jsonb(value or {}) + else: + set_clauses.append(f"{key} = %s") values.append(value) if not set_clauses: return get_document_job(job_id) - values.append(job_id) - with _db_lock: - with get_connection() as conn: - conn.execute( - f"UPDATE document_jobs SET {', '.join(set_clauses)} WHERE id = ?", - values, - ) - conn.commit() + values.append(str(job_id)) + with get_connection() as conn: + conn.execute(f"UPDATE workflows SET {', '.join(set_clauses)} WHERE id = %s::uuid", values) return get_document_job(job_id) -def get_document_job(job_id: int) -> Optional[dict]: +def get_document_job(job_id: str) -> Optional[dict]: + if not _is_uuid(str(job_id)): + return None with get_connection() as conn: - row = conn.execute( - "SELECT * FROM document_jobs WHERE id = ?", - (job_id,), - ).fetchone() - return dict(row) if row else None + return _job_from_row( + conn.execute(_JOB_SELECT + " WHERE w.id = %s::uuid", (str(job_id),)).fetchone() + ) def get_latest_document_job(workflow_id: str) -> Optional[dict]: with get_connection() as conn: - row = conn.execute(""" - SELECT * FROM document_jobs - WHERE workflow_id = ? - ORDER BY started_at DESC, id DESC - LIMIT 1 - """, (workflow_id,)).fetchone() - return dict(row) if row else None + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + return _job_from_row( + conn.execute( + _JOB_SELECT + " WHERE w.document_id = %s::uuid ORDER BY w.started_at DESC, w.id DESC LIMIT 1", + (doc_uuid,), + ).fetchone() + ) def list_document_jobs(workflow_id: str, limit: int = 50) -> list[dict]: with get_connection() as conn: - rows = conn.execute(""" - SELECT * FROM document_jobs - WHERE workflow_id = ? - ORDER BY started_at DESC, id DESC - LIMIT ? - """, (workflow_id, limit)).fetchall() - return [dict(row) for row in rows] + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return [] + rows = conn.execute( + _JOB_SELECT + " WHERE w.document_id = %s::uuid ORDER BY w.started_at DESC, w.id DESC LIMIT %s", + (doc_uuid, limit), + ).fetchall() + return [_job_from_row(r) for r in rows] def list_operations_queue( @@ -1588,64 +1734,51 @@ def list_operations_queue( """List documents needing action, optionally scoped to tenant instances. ``instances``: - - ``None`` — unrestricted (super admin / bypass) - - ``[]`` — restricted user with no states → empty result - - non-empty — only those document.instance values + - ``None`` - unrestricted (super admin / bypass) + - ``[]`` - restricted user with no states -> empty result + - non-empty - only those document.instance values + """ + if instances is not None and not _normalize_instances(instances): + return [], 0 + clauses, params = _document_filters(include_demo, include_disabled, instances) + clauses.append( + "(cw.current_stage IN ('ocr_review', 'translation_review', 'chunk_review', " + "'ready_for_ingestion', 'approval_for_prod', 'failed') " + "OR COALESCE((d.metadata->>'reindex_required')::boolean, FALSE) " + "OR cw.status = 'running')" + ) + where = f"WHERE {' AND '.join(clauses)}" + from_sql = """ + FROM documents d + JOIN document_current_workflow cw ON cw.document_id = d.id """ - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" with get_connection() as conn: - demo_filter = "" if include_demo else "AND (d.is_demo = 0 OR d.is_demo IS NULL)" - disabled_filter = "" if include_disabled else "AND (d.is_disabled = 0 OR d.is_disabled IS NULL)" - instance_filter = "" - instance_params: list = [] - if instances is not None: - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) - if not normalized: - return [], 0 - placeholders = ",".join("?" for _ in normalized) - instance_filter = ( - f"AND lower(COALESCE(NULLIF(trim(d.instance), ''), ?)) IN ({placeholders})" - ) - instance_params = [default_instance, *normalized] - where_clause = f""" - ( - d.stage IN ('ocr_review', 'translation_review', 'chunk_review', 'ready_for_ingestion', 'approval_for_prod', 'failed') - OR d.reindex_required = 1 - OR (j.status = 'running') - ) {demo_filter} {disabled_filter} {instance_filter} - """ - total_row = conn.execute( - f""" - SELECT COUNT(*) AS count - FROM documents d - LEFT JOIN document_jobs j ON j.id = d.latest_job_id - WHERE {where_clause} - """, - instance_params, - ).fetchone() + total_row = conn.execute(f"SELECT COUNT(*) AS count {from_sql} {where}", params).fetchone() rows = conn.execute( f""" SELECT - d.workflow_id, - d.filename, - d.stage, + d.id AS id, + (SELECT r.temporal_workflow_id FROM workflows r WHERE r.document_id = d.id + ORDER BY (r.workflow_type IN ('pipeline', 'ocr_only')) DESC, r.started_at ASC, r.id ASC + LIMIT 1) AS workflow_id, + d.source->>'original_filename' AS filename, + cw.current_stage AS stage, d.instance, - d.error_message, - d.reindex_required, - d.reindex_reason, - j.id AS job_id, - j.job_type, - j.status AS job_status, - j.started_at - FROM documents d - LEFT JOIN document_jobs j ON j.id = d.latest_job_id - WHERE {where_clause} - ORDER BY COALESCE(j.started_at, d.updated_at, d.created_at) DESC - LIMIT ? OFFSET ? + cw.error_message, + COALESCE((d.metadata->>'reindex_required')::boolean, FALSE) AS reindex_required, + d.metadata->>'reindex_reason' AS reindex_reason, + cw.id AS job_id, + cw.workflow_type AS job_type, + cw.status AS job_status, + cw.started_at AS started_at + {from_sql} + {where} + ORDER BY cw.started_at DESC, d.id + LIMIT %s OFFSET %s """, - (*instance_params, limit, offset), + (*params, limit, offset), ).fetchall() - return [dict(row) for row in rows], (total_row["count"] if total_row else 0) + return [_clean_row(r) for r in rows], int(total_row["count"] if total_row else 0) def list_runs( @@ -1654,190 +1787,89 @@ def list_runs( status: Optional[str] = None, instances: Optional[list[str]] = None, ) -> list[dict]: - """List jobs with document title fields for the Runs UI. - - Joins ``documents`` so the UI can show filename / display_name instead of - falling back to "Untitled document". + """List workflows with document title fields for the Runs UI. Args: - instances: If provided, only return jobs whose document instance is in - this list (NULL/empty instance treated as DEFAULT_INSTANCE). - Empty list returns no rows. None = unrestricted (all tenants). + instances: If provided, only return runs whose document instance is in + this list. Empty list returns no rows. None = unrestricted. """ - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + where_parts: list[str] = [] + params: list = [] + if status: + where_parts.append("w.status = %s::workflow_status_enum") + params.append(status) + if instances is not None: + normalized = _normalize_instances(instances) + if not normalized: + return [] + where_parts.append("lower(d.instance) = ANY(%s)") + params.append(normalized) + where_sql = f"WHERE {' AND '.join(where_parts)}" if where_parts else "" with get_connection() as conn: - # Prefer document identity columns for list labels. - select_sql = """ - SELECT - j.*, - d.filename AS filename, - d.display_name AS display_name, - d.source_filename AS source_filename, - d.stage AS document_stage, - d.instance AS instance - FROM document_jobs j - LEFT JOIN documents d ON d.workflow_id = j.workflow_id - """ - where_parts: list[str] = [] - params: list = [] - - if status: - where_parts.append("j.status = ?") - params.append(status) - - if instances is not None: - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) - if not normalized: - return [] - placeholders = ",".join("?" for _ in normalized) - # Scope via owning document.instance (legacy NULL/empty → default). - where_parts.append( - f"lower(COALESCE(NULLIF(trim(d.instance), ''), ?)) IN ({placeholders})" - ) - params.append(default_instance) - params.extend(normalized) - - where_sql = f"WHERE {' AND '.join(where_parts)}" if where_parts else "" rows = conn.execute( - select_sql - + f""" + f""" + SELECT w.id, w.document_id, w.workflow_type, w.temporal_workflow_id, w.status, + w.current_stage, w.config, w.error_message, w.started_at, w.completed_at, + (SELECT r.temporal_workflow_id FROM workflows r WHERE r.document_id = w.document_id + ORDER BY (r.workflow_type IN ('pipeline', 'ocr_only')) DESC, r.started_at ASC, r.id ASC + LIMIT 1) AS doc_workflow_id, + d.source->>'original_filename' AS filename, + d.metadata->>'display_name' AS display_name, + d.source->>'original_filename' AS source_filename, + cw.current_stage AS document_stage, + d.instance AS instance + FROM workflows w + JOIN documents d ON d.id = w.document_id + LEFT JOIN document_current_workflow cw ON cw.document_id = d.id {where_sql} - ORDER BY j.started_at DESC, j.id DESC - LIMIT ? OFFSET ? + ORDER BY w.started_at DESC, w.id DESC + LIMIT %s OFFSET %s """, (*params, limit, offset), ).fetchall() - return [dict(row) for row in rows] - - -def list_index_summaries(include_demo: bool = False, include_disabled: bool = False) -> list[dict]: - with get_connection() as conn: - demo_filter = "" if include_demo else "AND (d.is_demo = 0 OR d.is_demo IS NULL)" - disabled_filter = "" if include_disabled else "AND (d.is_disabled = 0 OR d.is_disabled IS NULL)" - rows = conn.execute(f""" - SELECT - s.index_name, - COUNT(*) AS documents, - SUM(COALESCE(s.chunk_count_indexed, 0)) AS indexed_chunks, - SUM(CASE WHEN d.reindex_required = 1 THEN 1 ELSE 0 END) AS stale_documents, - MAX(s.last_indexed_at) AS last_indexed_at, - GROUP_CONCAT(DISTINCT s.status) AS statuses - FROM document_index_status s - JOIN documents d ON d.workflow_id = s.workflow_id - WHERE 1=1 {demo_filter} {disabled_filter} - GROUP BY s.index_name - ORDER BY s.index_name - """).fetchall() - result: list[dict] = [] - for row in rows: - item = dict(row) - item["statuses"] = [part for part in (item.get("statuses") or "").split(",") if part] - result.append(item) - return result - - -def mark_document_reindex_required( - workflow_id: str, - required: bool = True, - reason: Optional[str] = None, -) -> Optional[dict]: - with _db_lock: - with get_connection() as conn: - conn.execute( - """ - UPDATE documents - SET reindex_required = ?, reindex_reason = ?, updated_at = ? - WHERE workflow_id = ? - """, - ( - 1 if required else 0, - reason if required else None, - datetime.utcnow().isoformat(), - workflow_id, - ), - ) - conn.commit() - return get_document(workflow_id) + result = [] + for row in rows: + job = _job_from_row(row) + job["document_uuid"] = str(row["document_id"]) + for extra in ("filename", "display_name", "source_filename", "document_stage", "instance"): + job[extra] = row[extra] + result.append(job) + return result -def mark_prod_ready_requested( - workflow_id: str, - user_id: Optional[str] = None, - username: Optional[str] = None, -) -> Optional[dict]: - """Flag that a state_admin has asked a super_admin to review this document - for prod promotion. Purely informational — does not fire the approve_prod - signal itself; the super_admin still makes that call separately.""" - with _db_lock: - with get_connection() as conn: - conn.execute( - """ - UPDATE documents - SET prod_ready_requested_at = ?, - prod_ready_requested_by_user_id = ?, - prod_ready_requested_by_username = ?, - updated_at = ? - WHERE workflow_id = ? - """, - ( - datetime.utcnow().isoformat(), - user_id, - username, - datetime.utcnow().isoformat(), - workflow_id, - ), - ) - conn.commit() - return get_document(workflow_id) - +# ============================================================================= +# Artifacts +# ============================================================================= -def set_document_validity( - workflow_id: str, - valid_from: str, - valid_to: str, -) -> Optional[dict]: - """Record the period this document's chunks are searchable in. Both ends - are `YYYY-MM-DD`; validation belongs to the caller - (`pipeline/document_validity.parse_period`), not to this SQL layer.""" - with _db_lock: - with get_connection() as conn: - conn.execute( - """ - UPDATE documents - SET valid_from = ?, - valid_to = ?, - updated_at = ? - WHERE workflow_id = ? - """, - ( - valid_from, - valid_to, - datetime.utcnow().isoformat(), - workflow_id, - ), - ) - conn.commit() - return get_document(workflow_id) +def _artifact_from_row(row: Optional[dict]) -> Optional[dict]: + if row is None: + return None + metadata = row["metadata"] or {} + artifact = { + "id": row["id"], + "workflow_id": row["doc_workflow_id"], + "job_id": row["workflow_pk"], + "artifact_type": row["artifact_type"], + "stage": row["stage"], + "storage_uri": row["storage_uri"], + "mime_type": row["mime_type"], + "filename": row["filename"], + "size_bytes": row["size_bytes"], + "metadata_json": json.dumps(metadata) if metadata else None, + "created_at": row["created_at"], + } + return {k: _clean(v) for k, v in artifact.items()} -def clear_prod_ready_requested(workflow_id: str) -> Optional[dict]: - """Reset the request flag once prod approval actually happens (or the - document leaves the approval_for_prod gate for any other reason).""" - with _db_lock: - with get_connection() as conn: - conn.execute( - """ - UPDATE documents - SET prod_ready_requested_at = NULL, - prod_ready_requested_by_user_id = NULL, - prod_ready_requested_by_username = NULL, - updated_at = ? - WHERE workflow_id = ? - """, - (datetime.utcnow().isoformat(), workflow_id), - ) - conn.commit() - return get_document(workflow_id) +_ARTIFACT_SELECT = """ + SELECT a.id, a.workflow_id AS workflow_pk, a.artifact_type, a.stage, a.storage_uri, + a.mime_type, a.filename, a.size_bytes, a.metadata, a.created_at, + (SELECT r.temporal_workflow_id FROM workflows r WHERE r.document_id = w.document_id + ORDER BY (r.workflow_type IN ('pipeline', 'ocr_only')) DESC, r.started_at ASC, r.id ASC + LIMIT 1) AS doc_workflow_id + FROM workflow_artifacts a + JOIN workflows w ON w.id = a.workflow_id +""" def add_document_artifact( @@ -1845,51 +1877,97 @@ def add_document_artifact( artifact_type: str, storage_uri: str, stage: Optional[str] = None, - job_id: Optional[int] = None, + job_id: Optional[str] = None, mime_type: Optional[str] = None, filename: Optional[str] = None, size_bytes: Optional[int] = None, metadata: Optional[dict] = None, -) -> int: - created_at = datetime.utcnow().isoformat() - metadata_json = json.dumps(metadata) if metadata else None - with _db_lock: - with get_connection() as conn: - cursor = conn.execute(""" - INSERT INTO document_artifacts ( - workflow_id, job_id, artifact_type, stage, storage_uri, - mime_type, filename, size_bytes, metadata_json, created_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, ( - workflow_id, job_id, artifact_type, stage, storage_uri, - mime_type, filename, size_bytes, metadata_json, created_at - )) - artifact_id = cursor.lastrowid - if artifact_type == "original_upload": - conn.execute("UPDATE documents SET original_artifact_id = ? WHERE workflow_id = ?", (artifact_id, workflow_id)) - elif artifact_type in {"normalized_pdf", "normalized_spreadsheet"}: - conn.execute("UPDATE documents SET normalized_artifact_id = ? WHERE workflow_id = ?", (artifact_id, workflow_id)) - conn.commit() - return artifact_id +) -> str: + """Record a file produced by a workflow; returns the artifact UUID. + + Belongs to ``job_id`` when given, else to the document's current workflow. + """ + with get_connection() as conn: + doc_uuid = _require_document_uuid(conn, workflow_id) + wf_uuid = None + if job_id and _is_uuid(str(job_id)): + row = conn.execute( + "SELECT id FROM workflows WHERE id = %s::uuid AND document_id = %s::uuid", + (str(job_id), doc_uuid), + ).fetchone() + wf_uuid = str(row["id"]) if row else None + if wf_uuid is None: + row = conn.execute( + "SELECT id FROM document_current_workflow WHERE document_id = %s::uuid", (doc_uuid,) + ).fetchone() + if row is None: + row = conn.execute( + """ + INSERT INTO workflows (document_id, workflow_type, temporal_workflow_id) + VALUES (%s::uuid, 'pipeline', %s) RETURNING id + """, + (doc_uuid, workflow_id), + ).fetchone() + wf_uuid = str(row["id"]) + created = conn.execute( + """ + INSERT INTO workflow_artifacts (workflow_id, artifact_type, stage, storage_uri, + mime_type, filename, size_bytes, metadata) + VALUES (%s::uuid, %s, %s::workflow_stage, %s, %s, %s, %s, %s) + RETURNING id + """, + ( + wf_uuid, artifact_type, stage, storage_uri, mime_type, filename, size_bytes, + Jsonb(metadata or {}), + ), + ).fetchone() + return str(created["id"]) def list_document_artifacts(workflow_id: str) -> list[dict]: with get_connection() as conn: - rows = conn.execute(""" - SELECT * FROM document_artifacts - WHERE workflow_id = ? - ORDER BY created_at DESC, id DESC - """, (workflow_id,)).fetchall() - return [dict(row) for row in rows] + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return [] + rows = conn.execute( + _ARTIFACT_SELECT + " WHERE w.document_id = %s::uuid ORDER BY a.created_at DESC, a.id DESC", + (doc_uuid,), + ).fetchall() + return [_artifact_from_row(r) for r in rows] -def get_document_artifact(workflow_id: str, artifact_id: int) -> Optional[dict]: +def get_document_artifact(workflow_id: str, artifact_id: str) -> Optional[dict]: + if not _is_uuid(str(artifact_id)): + return None with get_connection() as conn: - row = conn.execute(""" - SELECT * FROM document_artifacts - WHERE workflow_id = ? AND id = ? - """, (workflow_id, artifact_id)).fetchone() - return dict(row) if row else None + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + return _artifact_from_row( + conn.execute( + _ARTIFACT_SELECT + " WHERE w.document_id = %s::uuid AND a.id = %s::uuid", + (doc_uuid, str(artifact_id)), + ).fetchone() + ) + + +# ============================================================================= +# Vector index state (documents.index_status) +# ============================================================================= + +def _index_entry_to_legacy(workflow_id: str, key: str, entry: dict) -> dict: + details = entry.get("details") or {} + return { + "workflow_id": workflow_id, + "index_name": entry.get("index_name") or key, + "doc_id": entry.get("doc_id"), + "chunk_count_indexed": entry.get("chunk_count_indexed") or 0, + "last_indexed_at": entry.get("last_indexed_at"), + "last_verified_at": entry.get("last_verified_at"), + "schema_version": entry.get("schema_version"), + "status": entry.get("status") or "unknown", + "details_json": json.dumps(details) if details else None, + } def upsert_document_index_status( @@ -1903,111 +1981,102 @@ def upsert_document_index_status( status: str = "unknown", details: Optional[dict] = None, ) -> dict: - details_json = json.dumps(details) if details else None - with _db_lock: - with get_connection() as conn: - conn.execute(""" - INSERT INTO document_index_status ( - workflow_id, index_name, doc_id, chunk_count_indexed, - last_indexed_at, last_verified_at, schema_version, status, details_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(workflow_id, index_name) DO UPDATE SET - doc_id = COALESCE(excluded.doc_id, document_index_status.doc_id), - chunk_count_indexed = COALESCE(excluded.chunk_count_indexed, document_index_status.chunk_count_indexed), - last_indexed_at = COALESCE(excluded.last_indexed_at, document_index_status.last_indexed_at), - last_verified_at = COALESCE(excluded.last_verified_at, document_index_status.last_verified_at), - schema_version = COALESCE(excluded.schema_version, document_index_status.schema_version), - status = excluded.status, - details_json = COALESCE(excluded.details_json, document_index_status.details_json) - """, ( - workflow_id, index_name, doc_id, chunk_count_indexed, - last_indexed_at, last_verified_at, schema_version, status, details_json - )) - conn.commit() - return get_document_index_status(workflow_id, index_name) or {} - + """Merge one index's state into ``documents.index_status``. -def get_document_index_status(workflow_id: str, index_name: str) -> Optional[dict]: + The key is ``prod`` for the PROD collection names and ``dev`` otherwise; + the collection name is kept inside the entry. ``None`` arguments leave the + stored value alone, except ``status`` which always replaces it. + """ + key = _index_key_for(index_name) with get_connection() as conn: - row = conn.execute(""" - SELECT * FROM document_index_status - WHERE workflow_id = ? AND index_name = ? - """, (workflow_id, index_name)).fetchone() - return dict(row) if row else None + doc_uuid = _require_document_uuid(conn, workflow_id) + row = conn.execute( + "SELECT index_status->%s::text AS entry FROM documents WHERE id = %s::uuid FOR UPDATE", + (key, doc_uuid), + ).fetchone() + existing = dict(row["entry"] or {}) + incoming = { + "index_name": index_name, + "doc_id": doc_id, + "chunk_count_indexed": chunk_count_indexed, + "last_indexed_at": last_indexed_at, + "last_verified_at": last_verified_at, + "schema_version": schema_version, + "details": details or None, + } + merged = {**existing, **{k: v for k, v in incoming.items() if v is not None}, "status": status} + validated = IndexState.model_validate(merged).model_dump() + conn.execute( + "UPDATE documents SET index_status = index_status || jsonb_build_object(%s::text, %s::jsonb) " + "WHERE id = %s::uuid", + (key, Jsonb(validated), doc_uuid), + ) + root = _root_workflow_id(conn, doc_uuid) or workflow_id + return _index_entry_to_legacy(root, key, validated) def list_document_index_status(workflow_id: str) -> list[dict]: - with get_connection() as conn: - rows = conn.execute(""" - SELECT * FROM document_index_status - WHERE workflow_id = ? - ORDER BY last_verified_at DESC, last_indexed_at DESC - """, (workflow_id,)).fetchall() - return [dict(row) for row in rows] - - -def find_document_by_doc_identifier(identifier: str) -> Optional[dict]: - """Resolve a vector-store/chat doc identifier to a SQLite document row.""" - if not identifier: - return None + doc = get_document(workflow_id) + if not doc: + return [] + entries = [ + _index_entry_to_legacy(doc["workflow_id"], key, entry) + for key, entry in doc["index_status"].items() + ] + entries.sort( + key=lambda e: (e["last_verified_at"] or "", e["last_indexed_at"] or ""), reverse=True + ) + return entries - with get_connection() as conn: - row = conn.execute( - "SELECT * FROM documents WHERE workflow_id = ?", - (identifier,), - ).fetchone() - if row: - return dict(row) - row = conn.execute( - "SELECT * FROM documents WHERE document_id = ?", - (identifier,), - ).fetchone() - if row: - return dict(row) +def get_document_index_status(workflow_id: str, index_name: str) -> Optional[dict]: + for entry in list_document_index_status(workflow_id): + if entry["index_name"] == index_name: + return entry + return None - row = conn.execute( - "SELECT * FROM documents WHERE filename = ?", - (identifier,), - ).fetchone() - if row: - return dict(row) - row = conn.execute( - """ - SELECT d.* - FROM document_index_status s - JOIN documents d ON d.workflow_id = s.workflow_id - WHERE s.doc_id = ? - LIMIT 1 +def list_index_summaries(include_demo: bool = False, include_disabled: bool = False) -> list[dict]: + clauses, params = _document_filters(include_demo, include_disabled, None) + where = f"AND {' AND '.join(clauses)}" if clauses else "" + with get_connection() as conn: + rows = conn.execute( + f""" + SELECT + COALESCE(e.value->>'index_name', e.key) AS index_name, + COUNT(*) AS documents, + SUM(COALESCE((e.value->>'chunk_count_indexed')::int, 0)) AS indexed_chunks, + SUM(CASE WHEN COALESCE((d.metadata->>'reindex_required')::boolean, FALSE) + THEN 1 ELSE 0 END) AS stale_documents, + MAX(e.value->>'last_indexed_at') AS last_indexed_at, + array_agg(DISTINCT COALESCE(e.value->>'status', 'unknown')) AS statuses + FROM documents d, jsonb_each(d.index_status) e + WHERE TRUE {where} + GROUP BY COALESCE(e.value->>'index_name', e.key) + ORDER BY index_name """, - (identifier,), - ).fetchone() - if row: - return dict(row) + params, + ).fetchall() + result = [] + for row in rows: + item = _clean_row(row) + item["indexed_chunks"] = int(item["indexed_chunks"] or 0) + item["stale_documents"] = int(item["stale_documents"] or 0) + item["statuses"] = sorted(s for s in row["statuses"] if s) + result.append(item) + return result - legacy_doc_id_hash = hashlib.md5(identifier.encode()).hexdigest() - if legacy_doc_id_hash != identifier: - row = conn.execute( - """ - SELECT d.* - FROM document_index_status s - JOIN documents d ON d.workflow_id = s.workflow_id - WHERE s.doc_id = ? - LIMIT 1 - """, - (legacy_doc_id_hash,), - ).fetchone() - if row: - return dict(row) - rows = conn.execute("SELECT * FROM documents").fetchall() - for candidate in rows: - document_id = candidate["document_id"] - if hashlib.md5(document_id.encode()).hexdigest() == identifier: - return dict(candidate) +def find_document_by_doc_identifier(identifier: str) -> Optional[dict]: + """Resolve a vector-store/chat doc identifier to a document. - return None + Matches the document UUID, a Temporal workflow id, the fingerprint, the + filename, an index ``doc_id``, or the legacy ``md5(fingerprint)`` doc id. + """ + if not identifier: + return None + with get_connection() as conn: + return _fetch_document(conn, _resolve_document_uuid(conn, identifier)) def resolve_chunk_provenance( @@ -2052,6 +2121,9 @@ def resolve_chunk_provenance( # Audit Log Functions # ============================================================================= +_AUDIT_ENTITY_TYPES = {"document", "page", "chunk", "setting", "catalog"} + + def log_audit( workflow_id: str, document_id: str, @@ -2066,50 +2138,68 @@ def log_audit( actor_username: Optional[str] = None, actor_email: Optional[str] = None, actor_roles: Optional[str] = None, -) -> int: +) -> str: """ Log an audit entry. Args: - workflow_id: The Temporal workflow ID - document_id: The document hash ID + workflow_id: The Temporal workflow ID (or document identifier). Ids that + start with ``__`` mark system-level entries and are not stored. + document_id: Unused; kept for call-site compatibility. The document is + resolved from ``workflow_id``. action_type: Type of action (stage_change, page_edit, chunk_edit, approval, reset) - entity_type: Type of entity (page, chunk, document) - entity_id: ID of the entity (page_number or chunk_number) + entity_type: Type of entity (document, page, chunk, setting, catalog) + entity_id: Page number or chunk number the action applied to field_name: Name of the field that changed old_value: Previous value (as string or JSON) new_value: New value (as string or JSON) - metadata: Additional context as dict (will be JSON serialized) + metadata: Additional context as dict actor_*: Identity of the human/system that performed the action Returns: - The ID of the created audit log entry + The UUID of the created audit log entry """ - now = datetime.utcnow().isoformat() - metadata_json = json.dumps(metadata) if metadata else None - - with _db_lock: - with get_connection() as conn: - cursor = conn.execute(""" - INSERT INTO audit_logs ( - workflow_id, document_id, action_type, - entity_type, entity_id, field_name, - old_value, new_value, metadata, timestamp, - actor_user_id, actor_username, actor_email, actor_roles - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, ( - workflow_id, document_id, action_type, - entity_type, entity_id, field_name, - old_value, new_value, metadata_json, now, - actor_user_id, actor_username, actor_email, actor_roles, - )) - conn.commit() - return cursor.lastrowid + entity = entity_type if entity_type in _AUDIT_ENTITY_TYPES else None + system_entry = (workflow_id or "").startswith("__") + roles = _csv_roles(actor_roles) + with get_connection() as conn: + doc_uuid = None if system_entry else _resolve_document_uuid(conn, workflow_id) + instance = None + stored_workflow_id = None if system_entry else workflow_id + if doc_uuid: + doc_row = conn.execute( + "SELECT instance FROM documents WHERE id = %s::uuid", (doc_uuid,) + ).fetchone() + instance = doc_row["instance"] + if _is_uuid(workflow_id): + stored_workflow_id = _root_workflow_id(conn, doc_uuid) or workflow_id + if stored_workflow_id is None and entity not in ("setting", "catalog"): + stored_workflow_id = workflow_id + actor_uuid = ( + upsert_user(actor_user_id, actor_username, actor_email, roles, conn=conn) + if actor_user_id else None + ) + row = conn.execute( + """ + INSERT INTO audit_logs (document_id, workflow_id, instance, action_type, entity_type, + entity_number, field_name, old_value, new_value, metadata, + actor_user_id, actor) + VALUES (%s::uuid, %s, %s, %s, %s::audit_entity_type, %s, %s, %s, %s, %s, %s::uuid, %s) + RETURNING id + """, + ( + doc_uuid, stored_workflow_id, instance, action_type, entity, entity_id, + field_name, old_value, new_value, Jsonb(metadata or {}), actor_uuid, + Jsonb({"username": actor_username or None, "email": actor_email or None, "roles": roles}) + if (actor_user_id or actor_username or actor_email or roles) else Jsonb({}), + ), + ).fetchone() + return str(row["id"]) # Keycloak attaches these to every account regardless of what the user can # actually do here. Rendering them made the actor column a wall of noise -# ("…manage-account,manage-account-links,offline_access,uma_authorization…") +# ("...manage-account,manage-account-links,offline_access,uma_authorization...") # that buried the one role a reviewer cares about. _NOISE_ROLES = { "manage-account", @@ -2168,6 +2258,67 @@ def _normalize_audit_row(row) -> dict: return entry +_AUDIT_SELECT = """ + SELECT a.id, a.workflow_id, a.action_type, a.entity_type, a.entity_number, a.field_name, + a.old_value, a.new_value, a.metadata, a.created_at, a.actor, a.instance AS audit_instance, + au.keycloak_user_id AS actor_keycloak_id, + d.source->>'fingerprint' AS doc_fingerprint, + d.source->>'original_filename' AS filename, + d.metadata->>'display_name' AS display_name, + d.instance AS doc_instance, + uu.username AS uploaded_by_username, uu.email AS uploaded_by_email + FROM audit_logs a + LEFT JOIN documents d ON d.id = a.document_id + LEFT JOIN users au ON au.id = a.actor_user_id + LEFT JOIN users uu ON uu.id = d.uploaded_by_user_id +""" + + +def _audit_row_to_legacy(row: dict) -> dict: + actor = row["actor"] or {} + metadata = row["metadata"] or {} + entity_type = row["entity_type"] + if entity_type == "setting": + workflow_id, document_id = "__system__", "__settings__" + elif entity_type == "catalog" and not row["workflow_id"]: + workflow_id, document_id = "__catalog__", "__catalog__" + else: + workflow_id = row["workflow_id"] + document_id = row["doc_fingerprint"] or workflow_id + roles = actor.get("roles") or [] + entry = { + "id": row["id"], + "workflow_id": workflow_id, + "document_id": document_id, + "action_type": row["action_type"], + "entity_type": entity_type, + "entity_id": row["entity_number"], + "field_name": row["field_name"], + "old_value": row["old_value"], + "new_value": row["new_value"], + "metadata": json.dumps(metadata) if metadata else None, + "timestamp": row["created_at"], + "actor_user_id": row["actor_keycloak_id"], + "actor_username": actor.get("username"), + "actor_email": actor.get("email"), + "actor_roles": ",".join(roles) if roles else None, + "filename": row["filename"], + "display_name": row["display_name"], + "instance": row["doc_instance"] or row["audit_instance"], + "uploaded_by_username": row["uploaded_by_username"], + "uploaded_by_email": row["uploaded_by_email"], + } + return _normalize_audit_row({k: _clean(v) for k, v in entry.items()}) + + +def _audit_scope(workflow_id: str, conn) -> tuple[str, list]: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid: + root = _root_workflow_id(conn, doc_uuid) or workflow_id + return "(a.document_id = %s::uuid OR a.workflow_id = %s)", [doc_uuid, root] + return "a.workflow_id = %s", [workflow_id] + + def get_audit_logs( workflow_id: str, action_type: Optional[str] = None, @@ -2178,7 +2329,7 @@ def get_audit_logs( Get audit logs for a document. Args: - workflow_id: The Temporal workflow ID + workflow_id: The Temporal workflow ID (or document UUID) action_type: Optional filter by action type limit: Maximum number of entries to return offset: Offset for pagination @@ -2187,39 +2338,40 @@ def get_audit_logs( List of audit log entries as dicts """ with get_connection() as conn: + where, params = _audit_scope(workflow_id, conn) if action_type: - rows = conn.execute(""" - SELECT * FROM audit_logs - WHERE workflow_id = ? AND action_type = ? - ORDER BY timestamp DESC - LIMIT ? OFFSET ? - """, (workflow_id, action_type, limit, offset)).fetchall() - else: - rows = conn.execute(""" - SELECT * FROM audit_logs - WHERE workflow_id = ? - ORDER BY timestamp DESC - LIMIT ? OFFSET ? - """, (workflow_id, limit, offset)).fetchall() - - return [_normalize_audit_row(row) for row in rows] + where += " AND a.action_type = %s" + params.append(action_type) + rows = conn.execute( + f"{_AUDIT_SELECT} WHERE {where} ORDER BY a.created_at DESC, a.id LIMIT %s OFFSET %s", + (*params, limit, offset), + ).fetchall() + return [_audit_row_to_legacy(r) for r in rows] def get_audit_log_count(workflow_id: str, action_type: Optional[str] = None) -> int: """Get count of audit logs for a document.""" with get_connection() as conn: + where, params = _audit_scope(workflow_id, conn) if action_type: - row = conn.execute( - "SELECT COUNT(*) as cnt FROM audit_logs WHERE workflow_id = ? AND action_type = ?", - (workflow_id, action_type) - ).fetchone() - else: - row = conn.execute( - "SELECT COUNT(*) as cnt FROM audit_logs WHERE workflow_id = ?", - (workflow_id,) - ).fetchone() - - return row["cnt"] if row else 0 + where += " AND a.action_type = %s" + params.append(action_type) + row = conn.execute(f"SELECT COUNT(*) AS cnt FROM audit_logs a WHERE {where}", params).fetchone() + return int(row["cnt"]) if row else 0 + + +def _global_audit_filters( + action_type: Optional[str], instances: Optional[list[str]] +) -> tuple[list[str], list]: + where_parts: list[str] = ["a.action_type <> %s"] + params: list = [CATALOG_VERSION_ACTION] + if action_type: + where_parts.append("a.action_type = %s") + params.append(action_type) + if instances is not None: + where_parts.append("lower(COALESCE(a.instance, d.instance, %s)) = ANY(%s)") + params.extend([_default_instance(), _normalize_instances(instances)]) + return where_parts, params def get_all_audit_logs( @@ -2240,38 +2392,16 @@ def get_all_audit_logs( Returns: List of audit log entries as dicts, with document filename included """ - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + if instances is not None and not _normalize_instances(instances): + return [] + where_parts, params = _global_audit_filters(action_type, instances) with get_connection() as conn: - where_parts: list[str] = [] - params: list = [] - if action_type: - where_parts.append("a.action_type = ?") - params.append(action_type) - if instances is not None: - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) - if not normalized: - return [] - placeholders = ",".join("?" for _ in normalized) - where_parts.append( - f"lower(COALESCE(NULLIF(trim(d.instance), ''), ?)) IN ({placeholders})" - ) - params.append(default_instance) - params.extend(normalized) - where_sql = f"WHERE {' AND '.join(where_parts)}" if where_parts else "" rows = conn.execute( - f""" - SELECT a.*, d.filename, d.display_name, d.instance, - d.uploaded_by_username, d.uploaded_by_email - FROM audit_logs a - LEFT JOIN documents d ON a.workflow_id = d.workflow_id - {where_sql} - ORDER BY a.timestamp DESC - LIMIT ? OFFSET ? - """, + f"{_AUDIT_SELECT} WHERE {' AND '.join(where_parts)} " + "ORDER BY a.created_at DESC, a.id LIMIT %s OFFSET %s", (*params, limit, offset), ).fetchall() - - return [_normalize_audit_row(row) for row in rows] + return [_audit_row_to_legacy(r) for r in rows] def get_all_audit_log_count( @@ -2279,85 +2409,153 @@ def get_all_audit_log_count( instances: Optional[list[str]] = None, ) -> int: """Get total count of all audit logs (optionally tenant-scoped).""" - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + if instances is not None and not _normalize_instances(instances): + return 0 + where_parts, params = _global_audit_filters(action_type, instances) with get_connection() as conn: - where_parts: list[str] = [] - params: list = [] - if action_type: - where_parts.append("a.action_type = ?") - params.append(action_type) - if instances is not None: - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) - if not normalized: - return 0 - placeholders = ",".join("?" for _ in normalized) - where_parts.append( - f"lower(COALESCE(NULLIF(trim(d.instance), ''), ?)) IN ({placeholders})" - ) - params.append(default_instance) - params.extend(normalized) - where_sql = f"WHERE {' AND '.join(where_parts)}" if where_parts else "" - # Join documents when instance-scoped so tenant filter applies. - if instances is not None: - row = conn.execute( - f""" - SELECT COUNT(*) as cnt - FROM audit_logs a - LEFT JOIN documents d ON a.workflow_id = d.workflow_id - {where_sql} - """, - params, - ).fetchone() - elif action_type: - row = conn.execute( - "SELECT COUNT(*) as cnt FROM audit_logs a WHERE a.action_type = ?", - (action_type,), - ).fetchone() - else: - row = conn.execute("SELECT COUNT(*) as cnt FROM audit_logs").fetchone() - - return row["cnt"] if row else 0 + row = conn.execute( + f"SELECT COUNT(*) AS cnt FROM audit_logs a LEFT JOIN documents d ON d.id = a.document_id " + f"WHERE {' AND '.join(where_parts)}", + params, + ).fetchone() + return int(row["cnt"]) if row else 0 # ============================================================================= -# Page Functions (for persistence after workflow completion) +# Page Functions # ============================================================================= +_PAGE_SELECT = """ + SELECT p.id, p.document_id, p.page_number, p.original_markdown, p.edited_markdown, + p.translations, p.metadata, p.created_at, p.updated_at + FROM pages p +""" + + +def _primary_translation(translations: dict) -> tuple[Optional[str], dict]: + if not translations: + return None, {} + lang = max( + translations, + key=lambda k: (translations[k].get("translated_at") or "", k == "en"), + ) + return lang, translations[lang] + + +def _page_from_row(row: Optional[dict], workflow_id: str) -> Optional[dict]: + if row is None: + return None + meta = row["metadata"] or {} + translations = row["translations"] or {} + lang, tr = _primary_translation(translations) + page = { + "id": row["id"], + "workflow_id": workflow_id, + "page_number": row["page_number"], + "original_markdown": row["original_markdown"], + "edited_markdown": row["edited_markdown"], + "is_reviewed": bool(meta.get("is_reviewed")), + "reviewer_notes": meta.get("reviewer_notes"), + "detected_language": meta.get("detected_language"), + "translated_markdown": tr.get("translated_markdown"), + "edited_translation": tr.get("edited_translation"), + "translation_reviewed": bool(tr.get("is_reviewed")), + "translation_notes": tr.get("reviewer_notes"), + "translation_provider": tr.get("provider"), + "translation_model": tr.get("model"), + "translation_target_language": lang, + "translated_at": tr.get("translated_at"), + "translations": translations, + "metadata": meta, + } + return {k: _clean(v) for k, v in page.items()} + + +_TRANSLATION_FLAT_KEYS = ( + "translated_markdown", "edited_translation", "translation_notes", + "translation_provider", "translation_model", "translated_at", +) + + +def _translation_from_flat(page: dict) -> Optional[tuple[str, dict]]: + if not any(page.get(k) not in (None, "") for k in _TRANSLATION_FLAT_KEYS) and not page.get( + "translation_reviewed" + ): + return None + lang = page.get("translation_target_language") or "en" + entry = PageTranslation( + translated_markdown=page.get("translated_markdown"), + edited_translation=page.get("edited_translation"), + is_reviewed=bool(page.get("translation_reviewed")), + reviewer_notes=page.get("translation_notes"), + provider=page.get("translation_provider"), + model=page.get("translation_model"), + translated_at=page.get("translated_at"), + ).model_dump() + return lang, entry + + def save_pages(workflow_id: str, pages: list[dict]): """ Save all pages for a document (bulk upsert). - Called when workflow completes to persist data. + + Each call replaces the whole row, so the caller passes every field. Page + ``translations`` are kept as stored unless the page carries ``translations`` + (replaces them) or flat ``translated_*`` fields (merged in under + ``translation_target_language``); a page whose OCR text changed loses its + translations because they no longer describe it. """ - with _db_lock: - with get_connection() as conn: - for page in pages: - conn.execute(""" - INSERT OR REPLACE INTO pages ( - workflow_id, page_number, original_markdown, edited_markdown, - is_reviewed, reviewer_notes, detected_language, - translated_markdown, edited_translation, - translation_reviewed, translation_notes, - translation_provider, translation_model, - translation_target_language, translated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, ( - workflow_id, - page.get("page_number"), - page.get("original_markdown"), - page.get("edited_markdown"), - 1 if page.get("is_reviewed") else 0, - page.get("reviewer_notes"), - page.get("detected_language"), - page.get("translated_markdown"), - page.get("edited_translation"), - 1 if page.get("translation_reviewed") else 0, - page.get("translation_notes"), - page.get("translation_provider"), - page.get("translation_model"), - page.get("translation_target_language"), - page.get("translated_at"), - )) - conn.commit() + with get_connection() as conn: + doc_uuid = _require_document_uuid(conn, workflow_id) + existing = { + r["page_number"]: r + for r in conn.execute( + "SELECT page_number, original_markdown, translations FROM pages " + "WHERE document_id = %s::uuid", + (doc_uuid,), + ).fetchall() + } + for page in pages: + number = page.get("page_number") + current = existing.get(number) + meta = dict(page.get("metadata") or {}) + for flat_key in ("is_reviewed", "reviewer_notes", "detected_language"): + if flat_key in page: + meta[flat_key] = page[flat_key] + meta["is_reviewed"] = bool(meta.get("is_reviewed")) + page_meta = PageMetadata.model_validate( + {k: v for k, v in meta.items() if v is not None} + ).model_dump(exclude_none=True) + + translations = dict(current["translations"]) if current else {} + if current and current["original_markdown"] != page.get("original_markdown"): + translations = {} + if isinstance(page.get("translations"), dict): + translations = { + lang: PageTranslation.model_validate(entry).model_dump() + for lang, entry in page["translations"].items() + } + else: + flat = _translation_from_flat(page) + if flat: + translations[flat[0]] = flat[1] + + conn.execute( + """ + INSERT INTO pages (document_id, page_number, original_markdown, edited_markdown, + translations, metadata) + VALUES (%s::uuid, %s, %s, %s, %s, %s) + ON CONFLICT (document_id, page_number) DO UPDATE SET + original_markdown = EXCLUDED.original_markdown, + edited_markdown = EXCLUDED.edited_markdown, + translations = EXCLUDED.translations, + metadata = EXCLUDED.metadata + """, + ( + doc_uuid, number, page.get("original_markdown"), page.get("edited_markdown"), + Jsonb(translations), Jsonb(page_meta), + ), + ) def persist_document_content(workflow_id: str, pages: list[dict], chunks: list[dict]): @@ -2367,71 +2565,64 @@ def persist_document_content(workflow_id: str, pages: list[dict], chunks: list[d def get_pages(workflow_id: str) -> list[dict]: - """Get all pages for a document from SQLite.""" + """Get all pages for a document.""" with get_connection() as conn: - rows = conn.execute(""" - SELECT * FROM pages - WHERE workflow_id = ? - ORDER BY page_number - """, (workflow_id,)).fetchall() - - pages = [] - for row in rows: - page = dict(row) - # Convert SQLite integers back to booleans - page["is_reviewed"] = bool(page.get("is_reviewed")) - page["translation_reviewed"] = bool(page.get("translation_reviewed")) - pages.append(page) - return pages + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return [] + root = _root_workflow_id(conn, doc_uuid) or workflow_id + rows = conn.execute( + _PAGE_SELECT + " WHERE p.document_id = %s::uuid ORDER BY p.page_number", (doc_uuid,) + ).fetchall() + return [_page_from_row(r, root) for r in rows] def count_translated_pages(workflow_id: str) -> int: """Count pages with translation text without loading full page bodies.""" with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return 0 row = conn.execute( """ - SELECT COUNT(*) AS n - FROM pages - WHERE workflow_id = ? - AND ( - (translated_markdown IS NOT NULL AND TRIM(translated_markdown) != '') - OR (edited_translation IS NOT NULL AND TRIM(edited_translation) != '') + SELECT COUNT(*) AS n FROM pages p + WHERE p.document_id = %s::uuid + AND EXISTS ( + SELECT 1 FROM jsonb_each(p.translations) t + WHERE nullif(btrim(t.value->>'translated_markdown'), '') IS NOT NULL + OR nullif(btrim(t.value->>'edited_translation'), '') IS NOT NULL ) """, - (workflow_id,), + (doc_uuid,), ).fetchone() - return int(row["n"] if row else 0) + return int(row["n"] if row else 0) def get_saved_page_numbers(workflow_id: str) -> list[int]: """Get saved page numbers for a document without loading full page content.""" with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return [] rows = conn.execute( - """ - SELECT page_number - FROM pages - WHERE workflow_id = ? - ORDER BY page_number - """, - (workflow_id,), + "SELECT page_number FROM pages WHERE document_id = %s::uuid ORDER BY page_number", + (doc_uuid,), ).fetchall() - return [int(row["page_number"]) for row in rows] + return [int(row["page_number"]) for row in rows] def get_page(workflow_id: str, page_num: int) -> Optional[dict]: - """Get a specific page from SQLite.""" + """Get a specific page.""" with get_connection() as conn: - row = conn.execute(""" - SELECT * FROM pages - WHERE workflow_id = ? AND page_number = ? - """, (workflow_id, page_num)).fetchone() - - if row: - page = dict(row) - page["is_reviewed"] = bool(page.get("is_reviewed")) - page["translation_reviewed"] = bool(page.get("translation_reviewed")) - return page - return None + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + root = _root_workflow_id(conn, doc_uuid) or workflow_id + row = conn.execute( + _PAGE_SELECT + " WHERE p.document_id = %s::uuid AND p.page_number = %s", + (doc_uuid, page_num), + ).fetchone() + return _page_from_row(row, root) def update_page( @@ -2444,152 +2635,210 @@ def update_page( translation_reviewed: Optional[bool] = None, translation_notes: Optional[str] = None, ) -> Optional[dict]: - """Update a page in SQLite. Returns updated page or None if not found.""" - with _db_lock: - with get_connection() as conn: - # Check if page exists - existing = conn.execute( - "SELECT id FROM pages WHERE workflow_id = ? AND page_number = ?", - (workflow_id, page_num) - ).fetchone() - - if not existing: - return None - - # Build dynamic update - updates = [] - values = [] - - if edited_markdown is not None: - updates.append("edited_markdown = ?") - values.append(edited_markdown) - - if is_reviewed is not None: - updates.append("is_reviewed = ?") - values.append(1 if is_reviewed else 0) - - if reviewer_notes is not None: - updates.append("reviewer_notes = ?") - values.append(reviewer_notes) - - if edited_translation is not None: - updates.append("edited_translation = ?") - values.append(edited_translation) - - if translation_reviewed is not None: - updates.append("translation_reviewed = ?") - values.append(1 if translation_reviewed else 0) - - if translation_notes is not None: - updates.append("translation_notes = ?") - values.append(translation_notes) + """Update a page. Returns updated page or None if not found.""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + existing = conn.execute( + "SELECT translations FROM pages WHERE document_id = %s::uuid AND page_number = %s FOR UPDATE", + (doc_uuid, page_num), + ).fetchone() + if not existing: + return None - if updates: - values.extend([workflow_id, page_num]) - # Note: updates list contains only hardcoded field names, not user input - conn.execute( - f"UPDATE pages SET {', '.join(updates)} WHERE workflow_id = ? AND page_number = ?", - values - ) - conn.commit() + assignments: list[str] = [] + params: list = [] + if edited_markdown is not None: + assignments.append("edited_markdown = %s") + params.append(edited_markdown) + + meta_patch: dict[str, Any] = {} + if is_reviewed is not None: + meta_patch["is_reviewed"] = bool(is_reviewed) + if reviewer_notes is not None: + meta_patch["reviewer_notes"] = reviewer_notes + if meta_patch: + assignments.append("metadata = metadata || %s::jsonb") + params.append(Jsonb(meta_patch)) + + tr_patch: dict[str, Any] = {} + if edited_translation is not None: + tr_patch["edited_translation"] = edited_translation + if translation_reviewed is not None: + tr_patch["is_reviewed"] = bool(translation_reviewed) + if translation_notes is not None: + tr_patch["reviewer_notes"] = translation_notes + if tr_patch: + lang, _ = _primary_translation(existing["translations"] or {}) + lang = lang or "en" + assignments.append( + "translations = jsonb_set(translations, ARRAY[%s::text], " + "COALESCE(translations->%s::text, '{}'::jsonb) || %s::jsonb, TRUE)" + ) + params.extend([lang, lang, Jsonb(tr_patch)]) + if assignments: + conn.execute( + f"UPDATE pages SET {', '.join(assignments)} " + "WHERE document_id = %s::uuid AND page_number = %s", + (*params, doc_uuid, page_num), + ) return get_page(workflow_id, page_num) def reset_page(workflow_id: str, page_num: int) -> Optional[dict]: - """Reset a page to original markdown in SQLite.""" - with _db_lock: - with get_connection() as conn: - conn.execute(""" + """Reset a page to its original OCR markdown.""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is not None: + conn.execute( + """ UPDATE pages SET edited_markdown = NULL, - is_reviewed = 0, - reviewer_notes = NULL - WHERE workflow_id = ? AND page_number = ? - """, (workflow_id, page_num)) - conn.commit() - + metadata = (metadata - 'reviewer_notes') || '{"is_reviewed": false}'::jsonb + WHERE document_id = %s::uuid AND page_number = %s + """, + (doc_uuid, page_num), + ) return get_page(workflow_id, page_num) # ============================================================================= -# Chunk Functions (for persistence after workflow completion) +# Chunk Functions # ============================================================================= +_CHUNK_SELECT = """ + SELECT c.id, c.document_id, c.chunk_number, c.original_text, c.edited_text, c.token_count, + c.page_start, c.page_end, c.source_spans, c.section_title, c.content_type, + c.is_reference, c.is_reviewed, c.is_excluded, c.reviewer_notes, + c.created_at, c.updated_at, d.metadata->'chunking' AS chunking + FROM chunks c + JOIN documents d ON d.id = c.document_id +""" + + +def _chunk_from_row(row: Optional[dict], workflow_id: str) -> Optional[dict]: + if row is None: + return None + spans = row["source_spans"] or [] + page_numbers = sorted({int(s["page_number"]) for s in spans if "page_number" in s}) or list( + range(int(row["page_start"]), int(row["page_end"]) + 1) + ) + run = row.get("chunking") or {} + chunk = { + "id": row["id"], + "workflow_id": workflow_id, + "chunk_number": row["chunk_number"], + "original_text": row["original_text"], + "edited_text": row["edited_text"], + "token_count": row["token_count"], + "page_start": row["page_start"], + "page_end": row["page_end"], + "source_page_numbers_json": json.dumps(page_numbers), + "source_spans_json": json.dumps(spans), + "section_title": row["section_title"], + "content_type": row["content_type"], + "is_reference": bool(row["is_reference"]), + "chunking_provider": run.get("provider"), + "chunking_model": run.get("model"), + "chunking_config_json": json.dumps(run["config"]) if run.get("config") else None, + "chunking_run_id": f"run-{run['run_number']}" if run.get("run_number") else None, + "chunk_version": run.get("run_number") or 1, + "is_reviewed": bool(row["is_reviewed"]), + "is_excluded": bool(row["is_excluded"]), + "reviewer_notes": row["reviewer_notes"], + } + return {k: _clean(v) for k, v in chunk.items()} + + +def _load_json_value(raw: Any, default: Any) -> Any: + if raw is None or raw == "": + return default + if isinstance(raw, str): + return json.loads(raw) + return raw + + def save_chunks(workflow_id: str, chunks: list[dict]): """ - Save all chunks for a document (bulk upsert). - Called when workflow completes to persist data. + Save all chunks for a document. + + Re-chunking runs in one transaction: the document's chunks are deleted, the + new ones inserted, and ``metadata.chunk_count`` and ``metadata.chunking`` + (with ``run_number`` incremented) updated together. """ - with _db_lock: - with get_connection() as conn: - existing_version = conn.execute( - "SELECT COALESCE(MAX(chunk_version), 0) AS max_version FROM chunks WHERE workflow_id = ?", - (workflow_id,), - ).fetchone() - next_version = int(existing_version["max_version"] or 0) + 1 - conn.execute("DELETE FROM chunks WHERE workflow_id = ?", (workflow_id,)) - for chunk in chunks: - chunk_version = int(chunk.get("chunk_version") or next_version) - conn.execute(""" - INSERT INTO chunks ( - workflow_id, chunk_number, original_text, edited_text, - token_count, page_start, page_end, - source_page_numbers_json, source_spans_json, - section_title, content_type, is_reference, - chunking_provider, chunking_model, chunking_config_json, - chunking_run_id, chunk_version, - is_reviewed, is_excluded, reviewer_notes - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, ( - workflow_id, + with get_connection() as conn: + doc_uuid = _require_document_uuid(conn, workflow_id) + previous = conn.execute( + "SELECT metadata->'chunking'->>'run_number' AS run_number FROM documents " + "WHERE id = %s::uuid FOR UPDATE", + (doc_uuid,), + ).fetchone() + run_number = int(previous["run_number"] or 0) + 1 + conn.execute("DELETE FROM chunks WHERE document_id = %s::uuid", (doc_uuid,)) + for chunk in chunks: + spans = _load_json_value( + chunk.get("source_spans") if chunk.get("source_spans") is not None + else chunk.get("source_spans_json"), + [], + ) + page_start = int(chunk.get("page_start") or 1) + page_end = max(int(chunk.get("page_end") or page_start), page_start) + conn.execute( + """ + INSERT INTO chunks (document_id, chunk_number, original_text, edited_text, + token_count, page_start, page_end, source_spans, + section_title, content_type, is_reference, is_reviewed, + is_excluded, reviewer_notes) + VALUES (%s::uuid, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + """, + ( + doc_uuid, chunk.get("chunk_number"), - chunk.get("original_text"), + chunk.get("original_text") or "", chunk.get("edited_text"), - chunk.get("token_count", 0), - chunk.get("page_start", 1), - chunk.get("page_end", 1), - chunk.get("source_page_numbers_json"), - chunk.get("source_spans_json"), + chunk.get("token_count") or 0, + page_start, + page_end, + Jsonb(spans), chunk.get("section_title"), chunk.get("content_type"), - 1 if chunk.get("is_reference") else 0, - chunk.get("chunking_provider"), - chunk.get("chunking_model"), - chunk.get("chunking_config_json"), - chunk.get("chunking_run_id"), - chunk_version, - 1 if chunk.get("is_reviewed") else 0, - 1 if chunk.get("is_excluded") else 0, - chunk.get("reviewer_notes") - )) - conn.commit() + bool(chunk.get("is_reference")), + bool(chunk.get("is_reviewed")), + bool(chunk.get("is_excluded")), + chunk.get("reviewer_notes"), + ), + ) + meta_patch: dict[str, Any] = {"chunk_count": len(chunks)} + if chunks: + first = chunks[0] + meta_patch["chunking"] = ChunkingRun( + run_number=run_number, + provider=first.get("chunking_provider"), + model=first.get("chunking_model"), + config=_load_json_value(first.get("chunking_config_json"), {}), + created_at=_iso_now(), + ).model_dump() + conn.execute( + "UPDATE documents SET metadata = metadata || %s::jsonb WHERE id = %s::uuid", + (Jsonb(meta_patch), doc_uuid), + ) def get_chunks(workflow_id: str, include_excluded: bool = False) -> list[dict]: - """Get all chunks for a document from SQLite.""" + """Get all chunks for a document.""" with get_connection() as conn: - if include_excluded: - rows = conn.execute(""" - SELECT * FROM chunks - WHERE workflow_id = ? - ORDER BY chunk_number - """, (workflow_id,)).fetchall() - else: - rows = conn.execute(""" - SELECT * FROM chunks - WHERE workflow_id = ? AND is_excluded = 0 - ORDER BY chunk_number - """, (workflow_id,)).fetchall() - - chunks = [] - for row in rows: - chunk = dict(row) - chunk["is_reviewed"] = bool(chunk.get("is_reviewed")) - chunk["is_excluded"] = bool(chunk.get("is_excluded")) - chunk["is_reference"] = bool(chunk.get("is_reference")) - chunks.append(chunk) - return chunks + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return [] + root = _root_workflow_id(conn, doc_uuid) or workflow_id + excluded = "" if include_excluded else "AND NOT c.is_excluded" + rows = conn.execute( + f"{_CHUNK_SELECT} WHERE c.document_id = %s::uuid {excluded} ORDER BY c.chunk_number", + (doc_uuid,), + ).fetchall() + return [_chunk_from_row(r, root) for r in rows] def search_chunks( @@ -2601,92 +2850,83 @@ def search_chunks( stage: Optional[str] = None, instances: Optional[list[str]] = None, ) -> tuple[list[dict], int]: - """Search chunks across documents (SQLite-first) for maintainer workflows. + """Search chunks across documents for maintainer workflows. instances: If provided, only return chunks whose owning document's instance is in this list (tenant scoping). ``None`` means no instance restriction. """ - where_clauses = ["1=1"] + where_clauses = ["TRUE"] params: list = [] if not include_excluded: - where_clauses.append("c.is_excluded = 0") + where_clauses.append("NOT c.is_excluded") if stage: - where_clauses.append("d.stage = ?") + where_clauses.append("cw.current_stage = %s::workflow_stage") params.append(stage) if instances is not None: - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) + normalized = _normalize_instances(instances) if not normalized: - # Restricted caller with no instances: match nothing. - where_clauses.append("1 = 0") + where_clauses.append("FALSE") else: - placeholders = ",".join("?" for _ in normalized) - where_clauses.append( - f"LOWER(COALESCE(d.instance, ?)) IN ({placeholders})" - ) - params.append(default_instance) - params.extend(normalized) + where_clauses.append("lower(d.instance) = ANY(%s)") + params.append(normalized) if query and query.strip(): - where_clauses.append("LOWER(COALESCE(c.edited_text, c.original_text, '')) LIKE ?") + where_clauses.append("lower(coalesce(c.edited_text, c.original_text)) LIKE %s") params.append(f"%{query.strip().lower()}%") - where_sql = " AND ".join(where_clauses) base_from_sql = f""" FROM chunks c - JOIN documents d ON d.workflow_id = c.workflow_id - WHERE {where_sql} + JOIN documents d ON d.id = c.document_id + LEFT JOIN document_current_workflow cw ON cw.document_id = d.id + WHERE {' AND '.join(where_clauses)} """ - with get_connection() as conn: - total_row = conn.execute( - f"SELECT COUNT(*) AS cnt {base_from_sql}", - params, - ).fetchone() + total_row = conn.execute(f"SELECT COUNT(*) AS cnt {base_from_sql}", params).fetchone() total = int(total_row["cnt"] if total_row and total_row["cnt"] is not None else 0) - rows = conn.execute( f""" SELECT - c.*, - d.document_id, - d.filename, - d.display_name, - d.stage, - d.updated_at AS document_updated_at + c.id, c.chunk_number, c.original_text, c.edited_text, c.token_count, + c.page_start, c.page_end, c.source_spans, c.section_title, c.content_type, + c.is_reference, c.is_reviewed, c.is_excluded, c.reviewer_notes, + d.metadata->'chunking' AS chunking, + d.source->>'fingerprint' AS document_id, + d.source->>'original_filename' AS filename, + d.metadata->>'display_name' AS display_name, + cw.current_stage AS stage, + d.updated_at AS document_updated_at, + (SELECT r.temporal_workflow_id FROM workflows r WHERE r.document_id = d.id + ORDER BY (r.workflow_type IN ('pipeline', 'ocr_only')) DESC, r.started_at ASC, r.id ASC + LIMIT 1) AS root_workflow_id {base_from_sql} - ORDER BY d.updated_at DESC, c.workflow_id ASC, c.chunk_number ASC - LIMIT ? OFFSET ? + ORDER BY d.updated_at DESC, d.id, c.chunk_number + LIMIT %s OFFSET %s """, [*params, int(limit), int(offset)], ).fetchall() chunks: list[dict] = [] for row in rows: - chunk = dict(row) - chunk["is_reviewed"] = bool(chunk.get("is_reviewed")) - chunk["is_excluded"] = bool(chunk.get("is_excluded")) - chunk["is_reference"] = bool(chunk.get("is_reference")) + row = dict(row) + chunk = _chunk_from_row(row, row["root_workflow_id"]) + for extra in ("document_id", "filename", "display_name", "stage", "document_updated_at"): + chunk[extra] = _clean(row[extra]) chunks.append(chunk) - return chunks, total def get_chunk(workflow_id: str, chunk_num: int) -> Optional[dict]: - """Get a specific chunk from SQLite.""" + """Get a specific chunk.""" with get_connection() as conn: - row = conn.execute(""" - SELECT * FROM chunks - WHERE workflow_id = ? AND chunk_number = ? - """, (workflow_id, chunk_num)).fetchone() - - if row: - chunk = dict(row) - chunk["is_reviewed"] = bool(chunk.get("is_reviewed")) - chunk["is_excluded"] = bool(chunk.get("is_excluded")) - chunk["is_reference"] = bool(chunk.get("is_reference")) - return chunk - return None + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + root = _root_workflow_id(conn, doc_uuid) or workflow_id + row = conn.execute( + _CHUNK_SELECT + " WHERE c.document_id = %s::uuid AND c.chunk_number = %s", + (doc_uuid, chunk_num), + ).fetchone() + return _chunk_from_row(row, root) def update_chunk( @@ -2697,64 +2937,53 @@ def update_chunk( is_excluded: Optional[bool] = None, reviewer_notes: Optional[str] = None ) -> Optional[dict]: - """Update a chunk in SQLite. Returns updated chunk or None if not found.""" - with _db_lock: - with get_connection() as conn: - # Check if chunk exists - existing = conn.execute( - "SELECT id FROM chunks WHERE workflow_id = ? AND chunk_number = ?", - (workflow_id, chunk_num) - ).fetchone() - - if not existing: - return None - - # Build dynamic update - updates = [] - values = [] - - if edited_text is not None: - updates.append("edited_text = ?") - values.append(edited_text) - - if is_reviewed is not None: - updates.append("is_reviewed = ?") - values.append(1 if is_reviewed else 0) - - if is_excluded is not None: - updates.append("is_excluded = ?") - values.append(1 if is_excluded else 0) - - if reviewer_notes is not None: - updates.append("reviewer_notes = ?") - values.append(reviewer_notes) - - if updates: - values.extend([workflow_id, chunk_num]) - # Note: updates list contains only hardcoded field names, not user input - conn.execute( - f"UPDATE chunks SET {', '.join(updates)} WHERE workflow_id = ? AND chunk_number = ?", - values - ) - conn.commit() - + """Update a chunk. Returns updated chunk or None if not found.""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + existing = conn.execute( + "SELECT id FROM chunks WHERE document_id = %s::uuid AND chunk_number = %s", + (doc_uuid, chunk_num), + ).fetchone() + if not existing: + return None + + updates = [] + values: list = [] + if edited_text is not None: + updates.append("edited_text = %s") + values.append(edited_text) + if is_reviewed is not None: + updates.append("is_reviewed = %s") + values.append(bool(is_reviewed)) + if is_excluded is not None: + updates.append("is_excluded = %s") + values.append(bool(is_excluded)) + if reviewer_notes is not None: + updates.append("reviewer_notes = %s") + values.append(reviewer_notes) + if updates: + conn.execute( + f"UPDATE chunks SET {', '.join(updates)} WHERE document_id = %s::uuid AND chunk_number = %s", + (*values, doc_uuid, chunk_num), + ) return get_chunk(workflow_id, chunk_num) def reset_chunk(workflow_id: str, chunk_num: int) -> Optional[dict]: - """Reset a chunk to original text in SQLite.""" - with _db_lock: - with get_connection() as conn: - conn.execute(""" - UPDATE chunks SET - edited_text = NULL, - is_reviewed = 0, - is_excluded = 0, - reviewer_notes = NULL - WHERE workflow_id = ? AND chunk_number = ? - """, (workflow_id, chunk_num)) - conn.commit() - + """Reset a chunk to its original text.""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is not None: + conn.execute( + """ + UPDATE chunks SET edited_text = NULL, is_reviewed = FALSE, is_excluded = FALSE, + reviewer_notes = NULL + WHERE document_id = %s::uuid AND chunk_number = %s + """, + (doc_uuid, chunk_num), + ) return get_chunk(workflow_id, chunk_num) @@ -2762,88 +2991,57 @@ def reset_chunk(workflow_id: str, chunk_num: int) -> Optional[dict]: # Settings Functions # ============================================================================= -def get_setting(key: str) -> Optional[dict]: - """Get a single setting by key.""" - with get_connection() as conn: - row = conn.execute( - "SELECT * FROM settings WHERE key = ?", - (key,) - ).fetchone() - return dict(row) if row else None - - -def get_all_settings() -> dict: - """Get all settings as a dict of key -> value.""" - with get_connection() as conn: - rows = conn.execute("SELECT * FROM settings").fetchall() - return {row["key"]: dict(row) for row in rows} +_SETTING_GROUPS: dict[str, tuple[str, type]] = { + "searchMethod": ("retrieval", RetrievalSettings), + "alpha": ("retrieval", RetrievalSettings), + "rankingMethod": ("retrieval", RetrievalSettings), + "hybridRrfK": ("retrieval", RetrievalSettings), + "indexName": ("candidates", CandidateSettings), + "efSearch": ("candidates", CandidateSettings), + "candidateCap": ("candidates", CandidateSettings), + "candidateMultiplier": ("candidates", CandidateSettings), + "limit": ("results", ResultSettings), + "maxChunksPerDoc": ("results", ResultSettings), + "excludeReference": ("results", ResultSettings), + "showHighlights": ("results", ResultSettings), + "rerankMode": ("results", ResultSettings), + "useE5Prefix": ("query", QuerySettings), + "queryExpansionProfile": ("query", QuerySettings), +} +_GROUP_MODELS: dict[str, type] = {group: model for group, model in _SETTING_GROUPS.values()} -def get_search_settings() -> dict: - """Get all search-related settings as a simple dict.""" - all_settings = get_all_settings() - return { - "searchMethod": all_settings.get("search_method", {}).get("value", "HYBRID"), - "limit": int(all_settings.get("search_limit", {}).get("value", "12")), - "alpha": float(all_settings.get("search_alpha", {}).get("value", "0.6")), - "rankingMethod": all_settings.get("search_ranking_method", {}).get("value", "rrf"), - "showHighlights": all_settings.get("search_show_highlights", {}).get("value", "true") == "true", - "efSearch": int(all_settings.get("search_ef_search", {}).get("value", "256")), - "indexName": all_settings.get("search_index_name", {}).get("value", "documents-index"), - "candidateCap": int(all_settings.get("search_candidate_cap", {}).get("value", "120")), - "candidateMultiplier": int(all_settings.get("search_candidate_multiplier", {}).get("value", "10")), - "maxChunksPerDoc": int(all_settings.get("search_max_chunks_per_doc", {}).get("value", "2")), - "useE5Prefix": all_settings.get("search_use_e5_prefix", {}).get("value", "true") == "true", - "excludeReference": all_settings.get("search_exclude_reference", {}).get("value", "true") == "true", - "queryExpansionProfile": all_settings.get("search_query_expansion_profile", {}).get("value", "gu-v1"), - "rerankMode": all_settings.get("search_rerank_mode", {}).get("value", "none"), - "hybridRrfK": int(all_settings.get("search_hybrid_rrfk", {}).get("value", "60")), - } +_SETTINGS_ROW_SQL = ( + "SELECT retrieval, candidates, results, query FROM search_settings WHERE singleton" +) -def update_setting(key: str, value: str, log_change: bool = True) -> dict: - """ - Update a setting value. +def _settings_row(conn, lock: bool = False) -> dict: + sql = _SETTINGS_ROW_SQL + (" FOR UPDATE" if lock else "") + row = conn.execute(sql).fetchone() + if row is None: + conn.execute("INSERT INTO search_settings DEFAULT VALUES ON CONFLICT (singleton) DO NOTHING") + row = conn.execute(sql).fetchone() + return row - Args: - key: Setting key - value: New value (as string) - log_change: Whether to log to audit (default True) - Returns: - Updated setting dict - """ - now = datetime.utcnow().isoformat() - old_value = None +def _flatten_settings(row: dict) -> dict: + flat: dict[str, Any] = {} + for group, model in _GROUP_MODELS.items(): + flat.update(model.model_validate(row[group] or {}).model_dump()) + return flat - with _db_lock: - with get_connection() as conn: - # Get old value for audit - row = conn.execute( - "SELECT value FROM settings WHERE key = ?", - (key,) - ).fetchone() - old_value = row["value"] if row else None - conn.execute(""" - UPDATE settings SET value = ?, updated_at = ? - WHERE key = ? - """, (value, now, key)) - conn.commit() +def get_search_settings() -> dict: + """Get all search settings as a flat dict (stored values merged over defaults).""" + with get_connection() as conn: + return _flatten_settings(_settings_row(conn)) - # Log the change to audit - if log_change and old_value != value: - log_audit( - workflow_id="__system__", - document_id="__settings__", - action_type="settings_change", - entity_type="setting", - field_name=key, - old_value=old_value, - new_value=value - ) - return get_setting(key) +def _setting_text(value: Any) -> str: + if isinstance(value, bool): + return "true" if value else "false" + return str(value) def update_search_settings(settings: dict) -> dict: @@ -2855,55 +3053,61 @@ def update_search_settings(settings: dict) -> dict: Returns: Updated search settings - """ - # Map UI keys to DB keys - key_map = { - "searchMethod": "search_method", - "limit": "search_limit", - "alpha": "search_alpha", - "rankingMethod": "search_ranking_method", - "showHighlights": "search_show_highlights", - "efSearch": "search_ef_search", - "indexName": "search_index_name", - "candidateCap": "search_candidate_cap", - "candidateMultiplier": "search_candidate_multiplier", - "maxChunksPerDoc": "search_max_chunks_per_doc", - "useE5Prefix": "search_use_e5_prefix", - "excludeReference": "search_exclude_reference", - "queryExpansionProfile": "search_query_expansion_profile", - "rerankMode": "search_rerank_mode", - "hybridRrfK": "search_hybrid_rrfk", - } - - for ui_key, db_key in key_map.items(): - if ui_key in settings: - value = settings[ui_key] - # Convert to string for storage - if isinstance(value, bool): - value = "true" if value else "false" - else: - value = str(value) - update_setting(db_key, value) - return get_search_settings() + Raises: + ValueError: a value fails the setting's validation rules. + """ + unknown = sorted(set(settings) - set(_SETTING_GROUPS)) + if unknown: + raise ValueError(f"Unknown search settings: {', '.join(unknown)}") + changes: list[tuple[str, str, str]] = [] + with get_connection() as conn: + row = _settings_row(conn, lock=True) + before = _flatten_settings(row) + for group, model in _GROUP_MODELS.items(): + patch = {k: v for k, v in settings.items() if _SETTING_GROUPS[k][0] == group} + if not patch: + continue + try: + validated = model.model_validate({**model.model_validate(row[group] or {}).model_dump(), **patch}) + except Exception as exc: + raise ValueError(str(exc)) from exc + stored = {**(row[group] or {}), **{k: getattr(validated, k) for k in patch}} + conn.execute( + f"UPDATE search_settings SET {group} = {group} || %s::jsonb WHERE singleton", + (Jsonb({k: getattr(validated, k) for k in patch}),), + ) + for key in patch: + if before[key] != stored[key]: + changes.append((f"{group}.{key}", _setting_text(before[key]), _setting_text(stored[key]))) + for field_name, old_value, new_value in changes: + conn.execute( + """ + INSERT INTO audit_logs (action_type, entity_type, field_name, old_value, new_value) + VALUES ('settings_change', 'setting', %s, %s, %s) + """, + (field_name, old_value, new_value), + ) + return _flatten_settings( + conn.execute(_SETTINGS_ROW_SQL).fetchone() + ) def get_settings_audit_logs(limit: int = 50, offset: int = 0) -> list[dict]: """Get audit logs for settings changes.""" with get_connection() as conn: - rows = conn.execute(""" - SELECT * FROM audit_logs - WHERE document_id = '__settings__' - ORDER BY timestamp DESC - LIMIT ? OFFSET ? - """, (limit, offset)).fetchall() - return [dict(row) for row in rows] + rows = conn.execute( + f"{_AUDIT_SELECT} WHERE a.entity_type = 'setting' " + "ORDER BY a.created_at DESC, a.id LIMIT %s OFFSET %s", + (limit, offset), + ).fetchall() + return [_audit_row_to_legacy(r) for r in rows] def get_settings_audit_count() -> int: """Get count of settings audit logs.""" with get_connection() as conn: row = conn.execute( - "SELECT COUNT(*) as cnt FROM audit_logs WHERE document_id = '__settings__'" + "SELECT COUNT(*) AS cnt FROM audit_logs WHERE entity_type = 'setting'" ).fetchone() - return row["cnt"] if row else 0 + return int(row["cnt"]) if row else 0 diff --git a/pipeline/models.py b/pipeline/models.py index 7f06afd..88e4e86 100644 --- a/pipeline/models.py +++ b/pipeline/models.py @@ -7,7 +7,7 @@ from enum import Enum from typing import Any, Optional -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, Field class DocumentStage(str, Enum): @@ -180,6 +180,7 @@ class ApprovalRequest(BaseModel): class DocumentSummary(BaseModel): + id: Optional[str] = None # documents.id (UUID); canonical identifier in API paths document_id: str canonical_document_id: Optional[str] = None workflow_id: str # The Temporal workflow ID (use this for API calls) @@ -242,9 +243,9 @@ class SchemeMetadataUpdate(BaseModel): class DocumentArtifact(BaseModel): - id: int + id: str workflow_id: str - job_id: Optional[int] = None + job_id: Optional[str] = None artifact_type: str stage: Optional[str] = None storage_uri: str @@ -256,7 +257,7 @@ class DocumentArtifact(BaseModel): class DocumentJob(BaseModel): - id: int + id: str workflow_id: str job_type: str temporal_workflow_id: Optional[str] = None @@ -293,9 +294,9 @@ class DocumentDetail(DocumentSummary): source_type: Optional[str] = None canonical_input_type: Optional[str] = None stop_after_ocr: bool = False - original_artifact_id: Optional[int] = None - normalized_artifact_id: Optional[int] = None - latest_job_id: Optional[int] = None + original_artifact_id: Optional[str] = None + normalized_artifact_id: Optional[str] = None + latest_job_id: Optional[str] = None current_job: Optional[dict[str, Any]] = None artifacts: list[DocumentArtifact] = [] index_status: list[DocumentIndexStatus] = [] @@ -336,7 +337,7 @@ class OperationQueueEntry(BaseModel): workflow_id: str filename: str stage: str - job_id: Optional[int] = None + job_id: Optional[str] = None job_type: Optional[str] = None job_status: Optional[str] = None started_at: Optional[str] = None @@ -380,7 +381,7 @@ class ReindexStateRequest(BaseModel): class AuditLogEntry(BaseModel): """A single audit log entry.""" - id: int + id: str workflow_id: str document_id: str action_type: str # stage_change, page_edit, chunk_edit, approval, page_reset, chunk_reset @@ -425,7 +426,7 @@ class SearchSettings(BaseModel): """Search configuration settings.""" searchMethod: str = "HYBRID" # TENSOR, LEXICAL, HYBRID limit: int = 12 - alpha: float = 0.7 # 0=lexical, 1=semantic + alpha: float = 0.6 # 0=lexical, 1=semantic rankingMethod: str = "rrf" # rrf, normalize_linear showHighlights: bool = True efSearch: int = 256 @@ -473,3 +474,118 @@ class SettingsAuditResponse(BaseModel): total: int limit: int offset: int + + +# ============================================================================= +# JSONB column models +# +# The database only checks that these columns hold a JSON object; the value +# rules live here. ``db.py`` validates every value through these models before +# it writes one. +# ============================================================================= + +class DocumentSource(BaseModel): + """``documents.source``: facts about the uploaded file.""" + model_config = ConfigDict(extra="forbid") + + fingerprint: str = Field(pattern=r"^[0-9a-f]{32}$") + original_filename: Optional[str] = None + storage_uri: Optional[str] = None + source_type: Optional[str] = None # spreadsheet | document | unknown + canonical_input_type: Optional[str] = None # spreadsheet | pdf + page_count: Optional[int] = Field(default=None, ge=0) + + +class IndexState(BaseModel): + """One entry of ``documents.index_status`` (keys ``dev`` / ``prod``).""" + model_config = ConfigDict(extra="forbid") + + status: str = "unknown" # unknown | indexed | missing + index_name: Optional[str] = None + doc_id: Optional[str] = None + chunk_count_indexed: int = Field(default=0, ge=0) + last_indexed_at: Optional[str] = None + last_verified_at: Optional[str] = None + schema_version: Optional[str] = None + details: dict[str, Any] = Field(default_factory=dict) + + +class ChunkingRun(BaseModel): + model_config = ConfigDict(extra="forbid") + + run_number: int = Field(default=1, ge=1) + provider: Optional[str] = None + model: Optional[str] = None + config: dict[str, Any] = Field(default_factory=dict) + created_at: Optional[str] = None + + +class DocumentMetadata(BaseModel): + """``documents.metadata``: display and flag fields.""" + model_config = ConfigDict(extra="forbid") + + display_name: Optional[str] = None + catalog_visible: bool = True + network_visible: bool = True + is_demo: bool = False + chunk_count: int = Field(default=0, ge=0) + reindex_required: bool = False + reindex_reason: Optional[str] = None + chunking: Optional[ChunkingRun] = None + + +class PageMetadata(BaseModel): + """``pages.metadata``: OCR review and language fields.""" + model_config = ConfigDict(extra="forbid") + + is_reviewed: bool = False + reviewer_notes: Optional[str] = None + detected_language: Optional[str] = None + + +class PageTranslation(BaseModel): + """One language entry of ``pages.translations``.""" + model_config = ConfigDict(extra="forbid") + + translated_markdown: Optional[str] = None + edited_translation: Optional[str] = None + is_reviewed: bool = False + reviewer_notes: Optional[str] = None + provider: Optional[str] = None + model: Optional[str] = None + translated_at: Optional[str] = None + + +class RetrievalSettings(BaseModel): + model_config = ConfigDict(extra="forbid") + + searchMethod: str = Field(default="HYBRID", pattern=r"^(TENSOR|LEXICAL|HYBRID)$") + alpha: float = Field(default=0.6, ge=0, le=1) + rankingMethod: str = Field(default="rrf", pattern=r"^(rrf|normalize_linear)$") + hybridRrfK: int = Field(default=60, ge=1) + + +class CandidateSettings(BaseModel): + model_config = ConfigDict(extra="forbid") + + indexName: str = "documents-index" + efSearch: int = Field(default=256, ge=1) + candidateCap: int = Field(default=120, ge=1) + candidateMultiplier: int = Field(default=10, ge=1) + + +class ResultSettings(BaseModel): + model_config = ConfigDict(extra="forbid") + + limit: int = Field(default=12, ge=1) + maxChunksPerDoc: int = Field(default=2, ge=1) + excludeReference: bool = True + showHighlights: bool = True + rerankMode: str = "none" + + +class QuerySettings(BaseModel): + model_config = ConfigDict(extra="forbid") + + useE5Prefix: bool = True + queryExpansionProfile: str = "gu-v1" diff --git a/pipeline/scheme_catalog.py b/pipeline/scheme_catalog.py index f99e6db..cdf2bb7 100644 --- a/pipeline/scheme_catalog.py +++ b/pipeline/scheme_catalog.py @@ -370,54 +370,26 @@ def resolve_scheme_aliases( return _dedupe_aliases(merged) -def _content_hash(code: str, name: str, aliases: list[str], network_visible: bool, status: str) -> str: - payload = json.dumps( - { - "code": code, - "name": name, - "aliases": sorted(aliases), - "network_visible": network_visible, - "status": status, - }, - sort_keys=True, +def _seed_bootstrap_entry(code: str, definition: dict[str, Any]) -> None: + db.upsert_catalog_entry( + type="scheme", + code=code, + name=definition["scheme_name"], + aliases=list(definition.get("scheme_aliases") or []), + source="bootstrap", ) - return hashlib.sha256(payload.encode()).hexdigest()[:16] - - -def ensure_catalog_schema() -> None: - """Create catalog tables and scheme metadata columns (idempotent).""" - db.init_scheme_catalog_schema() def bootstrap_catalog_if_empty(*, force: bool = False) -> dict[str, Any]: - """Seed bootstrap vector + legacy schemes when catalog is empty (or force).""" - ensure_catalog_schema() - meta = db.get_catalog_meta() - entries = db.list_catalog_entries() - if entries and not force: - return {"bootstrapped": False, "version": meta.get("version", 0), "entry_count": len(entries)} - - now = utc_now_iso() - collection = scheme_collection_name() - for code, definition in BOOTSTRAP_VECTOR_SCHEMES.items(): - aliases = list(definition.get("scheme_aliases") or []) - name = definition["scheme_name"] - status = "live" - db.upsert_catalog_entry( - scheme_code=code, - scheme_name=name, - scheme_aliases=aliases, - instances=["default"], - workflow_ids=[], - collection_name=collection, - chunk_count=0, - status=status, - network_visible=True, - promoted_at=None, - content_hash=_content_hash(code, name, aliases, True, status), - source="bootstrap", - updated_at=now, - ) + """Seed bootstrap vector schemes that have no catalog entry yet (all of them if force).""" + existing = db.list_catalog_entry_codes("scheme") + missing = {c: d for c, d in BOOTSTRAP_VECTOR_SCHEMES.items() if force or c not in existing} + if not missing: + meta = db.get_catalog_meta() + return {"bootstrapped": False, "version": meta.get("version", 0), "entry_count": len(existing)} + + for code, definition in missing.items(): + _seed_bootstrap_entry(code, definition) version = db.bump_catalog_version("bootstrap_vector_schemes") return { "bootstrapped": True, @@ -429,182 +401,16 @@ def bootstrap_catalog_if_empty(*, force: bool = False) -> dict[str, Any]: def rebuild_catalog_from_documents(*, reason: str = "rebuild") -> int: """ - Recompute scheme_catalog_entries from documents table, then bump version. + Re-seed bootstrap codes that have no entry, then bump the catalog version. - Pipeline-sourced live docs override bootstrap for the same scheme_code. - Bootstrap-only codes with no live docs remain live (migration phase A1). + Status, instances, chunk counts and the rest are computed by the ``catalog`` + view, so there is nothing to recompute here. Bootstrap-only codes with no + documents stay live. """ - ensure_catalog_schema() - now = utc_now_iso() - collection = scheme_collection_name() - - docs = db.list_scheme_documents_for_catalog() - by_code: dict[str, list[dict]] = {} - for doc in docs: - code = (doc.get("scheme_code") or "").strip().lower() - if not code: - continue - by_code.setdefault(code, []).append(doc) - - # Track which codes are fully owned by pipeline live docs - live_pipeline_codes: set[str] = set() - - for code, group in by_code.items(): - live_docs = [ - d - for d in group - if (d.get("stage") == "completed") - and not int(d.get("is_disabled") or 0) - and int(d.get("catalog_visible") if d.get("catalog_visible") is not None else 1) - and (d.get("document_kind") or "document") == "scheme" - ] - pending_reindex = any( - (d.get("stage") == "completed") - and not int(d.get("is_disabled") or 0) - and (d.get("reindex_required") or d.get("_pending_reindex")) - for d in group - ) - # Also treat reindex_reason containing scheme_code_rename as pending_reindex - for d in group: - reason_txt = (d.get("reindex_reason") or "") - if "scheme_code_rename" in reason_txt and d.get("stage") == "completed" and not int(d.get("is_disabled") or 0): - pending_reindex = True - - if pending_reindex and live_docs: - # If any live doc is waiting reindex after rename, publish as pending_reindex - # unless we only have non-rename live docs — keep simple: any reindex_required on scheme - if any(int(d.get("reindex_required") or 0) for d in live_docs): - status = "pending_reindex" - else: - status = "live" - elif live_docs: - status = "live" - live_pipeline_codes.add(code) - elif any(d.get("stage") == "approval_for_prod" and not int(d.get("is_disabled") or 0) for d in group): - status = "pending_prod" - else: - status = "disabled" - - # Prefer live docs for aggregation; else all group - use = live_docs or group - names = [d.get("scheme_name") for d in use if d.get("scheme_name")] - scheme_name = names[0] if names else code - aliases: list[str] = [] - seen_alias: set[str] = set() - for d in use: - for a in parse_aliases(d.get("scheme_aliases_json")): - key = a.lower() - if key not in seen_alias: - seen_alias.add(key) - aliases.append(a) - instances = sorted( - { - (d.get("instance") or "default").strip().lower() or "default" - for d in use - } - ) - workflow_ids = [d["workflow_id"] for d in use if d.get("workflow_id")] - chunk_count = sum(int(d.get("chunk_count") or 0) for d in use) - network_visible = any(int(d.get("network_visible") or 0) for d in use) # OR - promoted_times = [d.get("ingested_at") or d.get("updated_at") for d in live_docs if d.get("ingested_at") or d.get("updated_at")] - promoted_at = max(promoted_times) if promoted_times else None - source = "pipeline" if any((d.get("document_kind") or "") == "scheme" for d in group) else "import" - - # pending_reindex entries must not appear as live for consumers - if status == "pending_reindex": - # still store entry but status excludes from vector_schemes - pass - - db.upsert_catalog_entry( - scheme_code=code, - scheme_name=scheme_name, - scheme_aliases=aliases, - instances=instances, - workflow_ids=workflow_ids, - collection_name=collection, - chunk_count=chunk_count, - status=status if status != "disabled" or live_docs else "disabled", - network_visible=1 if network_visible else 0, - promoted_at=promoted_at, - content_hash=_content_hash(code, scheme_name, aliases, network_visible, status), - source=source, - updated_at=now, - ) - - # Preserve bootstrap live entries for codes with no document rows; - # disable orphan pipeline / import codes that have no docs left. - existing = {e["scheme_code"]: e for e in db.list_catalog_entries()} + existing = db.list_catalog_entry_codes("scheme") for code, definition in BOOTSTRAP_VECTOR_SCHEMES.items(): - if code in by_code: - continue - if code in existing and existing[code].get("source") == "pipeline": - # Was pipeline-owned but no longer eligible → mark disabled - if code not in live_pipeline_codes: - db.upsert_catalog_entry( - scheme_code=code, - scheme_name=existing[code].get("scheme_name") or definition["scheme_name"], - scheme_aliases=parse_aliases(existing[code].get("scheme_aliases_json")), - instances=parse_aliases(existing[code].get("instances_json")) or ["default"], - workflow_ids=parse_aliases(existing[code].get("workflow_ids_json")), - collection_name=collection, - chunk_count=int(existing[code].get("chunk_count") or 0), - status="disabled", - network_visible=int(existing[code].get("network_visible") or 0), - promoted_at=existing[code].get("promoted_at"), - content_hash=_content_hash( - code, - existing[code].get("scheme_name") or definition["scheme_name"], - parse_aliases(existing[code].get("scheme_aliases_json")), - bool(existing[code].get("network_visible")), - "disabled", - ), - source="pipeline", - updated_at=now, - ) - continue - # Keep or re-seed bootstrap - aliases = list(definition.get("scheme_aliases") or []) - name = definition["scheme_name"] - db.upsert_catalog_entry( - scheme_code=code, - scheme_name=name, - scheme_aliases=aliases, - instances=["default"], - workflow_ids=[], - collection_name=collection, - chunk_count=0, - status="live", - network_visible=1, - promoted_at=None, - content_hash=_content_hash(code, name, aliases, True, "live"), - source="bootstrap", - updated_at=now, - ) - - # Disable non-bootstrap entries that no longer appear in any document group - for code, entry in existing.items(): - if code in by_code or code in BOOTSTRAP_VECTOR_SCHEMES: - continue - if (entry.get("status") or "") == "disabled": - continue - name = entry.get("scheme_name") or code - aliases = parse_aliases(entry.get("scheme_aliases_json")) - db.upsert_catalog_entry( - scheme_code=code, - scheme_name=name, - scheme_aliases=aliases, - instances=parse_aliases(entry.get("instances_json")) or ["default"], - workflow_ids=[], - collection_name=collection, - chunk_count=int(entry.get("chunk_count") or 0), - status="disabled", - network_visible=int(entry.get("network_visible") or 0), - promoted_at=entry.get("promoted_at"), - content_hash=_content_hash(code, name, aliases, bool(entry.get("network_visible")), "disabled"), - source=entry.get("source") or "pipeline", - updated_at=now, - ) - + if code not in existing: + _seed_bootstrap_entry(code, definition) return db.bump_catalog_version(reason) @@ -667,7 +473,6 @@ def build_snapshot( include_pending: bool = False, network_visible: Optional[bool] = None, ) -> dict[str, Any]: - ensure_catalog_schema() meta = db.get_catalog_meta() version = int(meta.get("version") or 0) updated_at = meta.get("updated_at") or utc_now_iso() @@ -685,27 +490,25 @@ def build_snapshot( # Live snapshot for warmers: only status=live (pending excluded unless include_pending) if not include_pending and st != "live": continue - instances = parse_aliases(e.get("instances_json")) + instances = e["instances"] if instance: inst = instance.strip().lower() if inst not in {i.lower() for i in instances} and instances: - # bootstrap may have ["default"] only - if inst not in {i.lower() for i in instances}: - continue - nv = bool(int(e.get("network_visible") or 0)) + continue + nv = bool(e.get("network_visible")) if network_visible is not None and nv != network_visible: continue vector_schemes.append( { - "scheme_code": e["scheme_code"], - "scheme_name": e.get("scheme_name") or e["scheme_code"], - "scheme_aliases": parse_aliases(e.get("scheme_aliases_json")), + "scheme_code": e["code"], + "scheme_name": e.get("name") or e["code"], + "scheme_aliases": e["aliases"], "tool": "search_schemes", "tool_routing": "qdrant", "instances": instances or ["default"], "network_visible": nv, "chunk_count": int(e.get("chunk_count") or 0), - "workflow_ids": parse_aliases(e.get("workflow_ids_json")), + "workflow_ids": e["workflow_ids"], "promoted_at": e.get("promoted_at"), "source": e.get("source") or "bootstrap", "status": st, @@ -765,7 +568,6 @@ def apply_scheme_metadata( document has none - so a reviewer who only moves the end date does not have to restate the start. """ - ensure_catalog_schema() doc = db.get_document(workflow_id) if not doc: raise LookupError(f"Document not found: {workflow_id}") diff --git a/pipeline/worker.py b/pipeline/worker.py index 0d39634..697ebe2 100644 --- a/pipeline/worker.py +++ b/pipeline/worker.py @@ -59,8 +59,8 @@ async def main(): print("Error: TRANSLATION_VLLM_BASE_URL not set") return - # Initialize SQLite database - print("Initializing SQLite database...") + # Initialize Postgres database + print("Initializing database...") db.init_db() temporal_host = os.environ.get("TEMPORAL_HOST", "localhost:7233") diff --git a/pipeline/workflows.py b/pipeline/workflows.py index 4765534..d6ab900 100644 --- a/pipeline/workflows.py +++ b/pipeline/workflows.py @@ -96,7 +96,7 @@ async def _mirror_state( error_message: Optional[str] = None, ) -> dict: """ - Mirror workflow state into SQLite as a local activity. + Mirror workflow state into Postgres as a local activity. Local activities avoid starving tiny state updates behind long-running OCR, translation, or chunking activities on the shared task queue. diff --git a/tests/conftest.py b/tests/conftest.py index a0fa63b..49b2aa2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,8 @@ Pytest configuration and fixtures for the document ingestion pipeline. Provides fixtures for: -- SQLite database (in-memory for isolation) +- Postgres database (a throwaway container, or the server named by + TEST_DATABASE_URL; migrations applied once, tables truncated per test) - FastAPI TestClient - Mock Temporal client - Mock MinIO client @@ -15,31 +16,58 @@ import pytest # Set test environment before imports -os.environ["DOCUMENT_DB_PATH"] = ":memory:" os.environ["MINIO_ACCESS_KEY"] = "test-access-key" os.environ["MINIO_SECRET_KEY"] = "test-secret-key" os.environ["TRANSLATION_VLLM_BASE_URL"] = "http://localhost:8000/v1" os.environ["AUTH_DISABLED"] = "true" -@pytest.fixture(scope="function") -def temp_db_path(tmp_path): - """Create a temporary database path for each test.""" - db_path = tmp_path / "test.db" - os.environ["DOCUMENT_DB_PATH"] = str(db_path) - yield str(db_path) - # Cleanup happens automatically with tmp_path +_TRUNCATE_SQL = """ + TRUNCATE documents, users, catalog_entries, audit_logs, email_otps, + webhook_records, search_settings CASCADE +""" -@pytest.fixture(scope="function") -def db_connection(temp_db_path): - """Initialize database and provide connection for testing.""" +@pytest.fixture(scope="session") +def postgres_url(): + """URL of the Postgres the tests run against, with migrations applied. + + Every table is truncated before each test, so TEST_DATABASE_URL must name a + scratch database, never a real one. + """ + url = os.environ.get("TEST_DATABASE_URL") + container = None + if not url: + from testcontainers.postgres import PostgresContainer + + container = PostgresContainer("postgres:15-alpine", driver=None) + container.start() + url = container.get_connection_url() + os.environ["DATABASE_URL"] = url from pipeline import db - db.DB_PATH = temp_db_path + + db.close_pool() db.init_db() + yield url + db.close_pool() + if container is not None: + container.stop() + + +@pytest.fixture(scope="function") +def db_connection(postgres_url): + """The db module on an empty database (seed rows restored after each truncate).""" + from pipeline import db + + os.environ["DATABASE_URL"] = postgres_url + with db.get_connection() as conn: + conn.execute(_TRUNCATE_SQL) + conn.execute( + "INSERT INTO users (keycloak_user_id, username, roles) " + "VALUES ('local-admin', 'local-admin', '{superadmin}')" + ) + conn.execute("INSERT INTO search_settings DEFAULT VALUES") yield db - # Reset for next test - db.DB_PATH = os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db") @pytest.fixture @@ -78,7 +106,7 @@ def mock_minio_client(): @pytest.fixture -def test_client(mock_temporal_client, mock_minio_client): +def test_client(db_connection, mock_temporal_client, mock_minio_client): """Create FastAPI TestClient with mocked dependencies. Neutralizes the app's real lifespan (which otherwise connects to a live @@ -95,9 +123,6 @@ def test_client(mock_temporal_client, mock_minio_client): api.temporal_client = mock_temporal_client api.minio_client = mock_minio_client - # Initialize database - api.db.init_db() - @asynccontextmanager async def _noop_lifespan(_app): yield diff --git a/tests/test_activities.py b/tests/test_activities.py index 4f847f2..e17cd92 100644 --- a/tests/test_activities.py +++ b/tests/test_activities.py @@ -395,7 +395,7 @@ class TestUpdateDocumentState: @pytest.mark.unit def test_update_state(self, db_connection): - """Test updating document state in SQLite.""" + """Test updating document state in the database.""" from pipeline.activities import update_document_state workflow_id = "state-update-test" diff --git a/tests/test_api.py b/tests/test_api.py index 822ea84..3e319c0 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -10,12 +10,12 @@ - Error handling """ +import hashlib import os import pytest # Set test environment -os.environ["DOCUMENT_DB_PATH"] = ":memory:" os.environ["MINIO_ACCESS_KEY"] = "test-key" os.environ["MINIO_SECRET_KEY"] = "test-secret" os.environ["ALLOWED_FILE_PATHS"] = "/app/books,/tmp" @@ -101,7 +101,7 @@ def test_list_documents_with_stage_filter(self, test_client, db_connection): @pytest.mark.api @pytest.mark.unit def test_get_document_from_db(self, test_client, db_connection): - """Test getting document details from SQLite fallback.""" + """Test getting document details from the database.""" workflow_id = "get-test-001" db_connection.upsert_document( workflow_id=workflow_id, @@ -124,6 +124,144 @@ def test_get_document_not_found(self, test_client): assert response.status_code == 404 +class TestDocumentIdentifiers: + """`/documents/{id}` takes the UUID (canonical) or any legacy identifier.""" + + FINGERPRINT = hashlib.md5(b"identifiers").hexdigest() + WORKFLOW_ID = f"doc-{FINGERPRINT}" + + def _document(self, db_connection): + db_connection.upsert_document( + workflow_id=self.WORKFLOW_ID, + document_id=self.FINGERPRINT, + filename="identifiers.pdf", + filepath="minio://documents/identifiers.pdf", + stage="ocr_review", + ) + db_connection.save_pages( + self.WORKFLOW_ID, [{"page_number": 1, "original_markdown": "page one"}] + ) + return db_connection.get_document(self.WORKFLOW_ID) + + @pytest.mark.api + @pytest.mark.unit + def test_the_response_carries_the_uuid_and_the_temporal_id(self, test_client, db_connection): + doc = self._document(db_connection) + body = test_client.get(f"/documents/{doc['id']}").json() + assert body["id"] == doc["id"] + assert body["workflow_id"] == self.WORKFLOW_ID + assert body["document_id"] == self.FINGERPRINT + + @pytest.mark.api + @pytest.mark.unit + @pytest.mark.parametrize("form", ["uuid", "workflow", "fingerprint", "filename"]) + def test_every_identifier_form_reaches_the_same_document(self, test_client, db_connection, form): + doc = self._document(db_connection) + identifier = { + "uuid": doc["id"], + "workflow": self.WORKFLOW_ID, + "fingerprint": self.FINGERPRINT, + "filename": "identifiers.pdf", + }[form] + detail = test_client.get(f"/documents/{identifier}") + pages = test_client.get(f"/documents/{identifier}/pages") + assert detail.status_code == 200 and pages.status_code == 200 + assert detail.json()["id"] == doc["id"] + assert [p["page_number"] for p in pages.json()] == [1] + + @pytest.mark.api + @pytest.mark.unit + def test_the_uuid_is_rewritten_to_the_temporal_id_before_handlers_run( + self, test_client, db_connection, mock_temporal_client + ): + doc = self._document(db_connection) + db_connection.update_document_stage(doc["id"], "approval_for_prod") + response = test_client.post(f"/documents/{doc['id']}/approve-prod") + assert response.status_code == 200 + queried = [call.args[0] for call in mock_temporal_client.get_workflow_handle.call_args_list] + assert queried and set(queried) == {self.WORKFLOW_ID} + + @pytest.mark.api + @pytest.mark.unit + def test_static_document_routes_are_not_mistaken_for_identifiers(self, test_client, db_connection): + db_connection.upsert_document( + workflow_id="summary", document_id=hashlib.md5(b"summary").hexdigest(), + filename="summary", filepath="/tmp/s.pdf", + ) + response = test_client.get("/documents/summary") + assert response.status_code == 200 + assert "total_documents" in response.json() + + @pytest.mark.api + @pytest.mark.unit + def test_an_unknown_identifier_is_404(self, test_client, db_connection): + assert test_client.get("/documents/00000000-0000-0000-0000-000000000000").status_code == 404 + + +class TestArtifactAndJobIdentifiers: + """Artifact and job ids are UUIDs.""" + + def _document(self, db_connection): + db_connection.upsert_document( + workflow_id="wf-art-api", document_id=hashlib.md5(b"art").hexdigest(), + filename="a.pdf", filepath="minio://documents/a.pdf", stage="ocr_review", + ) + artifact_id = db_connection.add_document_artifact( + "wf-art-api", "original_upload", "minio://documents/a.pdf", stage="registered", + filename="a.pdf", + ) + return artifact_id + + @pytest.mark.api + @pytest.mark.unit + def test_artifacts_are_addressed_by_uuid(self, test_client, db_connection): + artifact_id = self._document(db_connection) + listed = test_client.get("/documents/wf-art-api/artifacts").json() + assert [a["id"] for a in listed] == [artifact_id] + assert test_client.get(f"/documents/wf-art-api/artifacts/{artifact_id}").json()["id"] == artifact_id + assert test_client.get("/documents/wf-art-api/artifacts/7").status_code == 404 + + @pytest.mark.api + @pytest.mark.unit + def test_document_detail_derives_artifact_and_job_pointers(self, test_client, db_connection): + artifact_id = self._document(db_connection) + body = test_client.get("/documents/wf-art-api").json() + job = db_connection.get_latest_document_job("wf-art-api") + assert body["original_artifact_id"] == artifact_id + assert body["normalized_artifact_id"] is None + assert body["latest_job_id"] == job["id"] + assert body["artifacts"][0]["job_id"] == job["id"] + + @pytest.mark.api + @pytest.mark.unit + def test_runs_are_addressed_by_uuid(self, test_client, db_connection): + self._document(db_connection) + job = db_connection.get_latest_document_job("wf-art-api") + assert test_client.get(f"/runs/{job['id']}").json()["id"] == job["id"] + assert test_client.get("/runs/3").status_code == 404 + assert [r["id"] for r in test_client.get("/runs").json()] == [job["id"]] + + +class TestHealthAndSettings: + @pytest.mark.api + @pytest.mark.unit + def test_health_reports_the_database(self, test_client): + body = test_client.get("/health").json() + assert body["database"] == {"reachable": True, "migrations_current": True} + + @pytest.mark.api + @pytest.mark.unit + def test_settings_default_alpha_and_validation(self, test_client): + assert test_client.get("/settings/search").json()["alpha"] == 0.6 + assert test_client.put("/settings/search", json={"alpha": 2}).status_code == 400 + updated = test_client.put("/settings/search", json={"limit": 7}).json() + assert updated["limit"] == 7 + reset = test_client.post("/settings/search/reset").json() + assert reset["limit"] == 12 and reset["alpha"] == 0.6 + audit = test_client.get("/settings/search/audit").json() + assert audit["total"] == 2 and audit["logs"][0]["document_id"] == "__settings__" + + class TestProdApproval: """Promoting to PROD takes no body. diff --git a/tests/test_db.py b/tests/test_db.py index b771611..87e00ea 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -1,5 +1,5 @@ """ -Unit tests for pipeline/db.py - SQLite database operations. +Unit tests for pipeline/db.py against a real Postgres. Tests cover: - Document CRUD operations @@ -7,12 +7,20 @@ - Chunk operations - Audit logging - Settings management +- Schema constraints, JSONB merges, workflows, the catalog view, identifier + resolution and the migration runner """ +import hashlib +import psycopg import pytest +def _md5(text): + return hashlib.md5(text.encode()).hexdigest() + + class TestDocumentOperations: """Tests for document CRUD operations.""" @@ -21,7 +29,7 @@ class TestDocumentOperations: def test_upsert_document_creates_new(self, db_connection): """Test creating a new document.""" workflow_id = "test-wf-001" - document_id = "test-doc-001" + document_id = _md5("test-doc-001") db_connection.upsert_document( workflow_id=workflow_id, @@ -586,3 +594,671 @@ def test_settings_audit_log(self, db_connection): db_connection.update_search_settings({"limit": 50}) logs = db_connection.get_settings_audit_logs() assert len(logs) >= 1 + + +def _new_doc(db, workflow_id, **kwargs): + kwargs.setdefault("document_id", _md5(workflow_id)) + kwargs.setdefault("filename", f"{workflow_id}.pdf") + kwargs.setdefault("filepath", f"minio://documents/{workflow_id}.pdf") + return db.upsert_document(workflow_id=workflow_id, **kwargs) + + +class TestSchemaConstraints: + """Rules the database itself enforces.""" + + @pytest.mark.db + @pytest.mark.unit + def test_document_kind_must_match_its_catalog_entry_type(self, db_connection): + with db_connection.get_connection() as conn: + entry = db_connection.upsert_catalog_entry(type="scheme", code="x-scheme", name="X", conn=conn) + with pytest.raises(psycopg.errors.ForeignKeyViolation): + conn.execute( + "INSERT INTO documents (instance, source, document_kind, catalog_entry_id) " + "VALUES ('default', %s, 'advisory', %s::uuid)", + (psycopg.types.json.Jsonb({"fingerprint": _md5("k")}), entry["id"]), + ) + + @pytest.mark.db + @pytest.mark.unit + def test_source_requires_a_hex_fingerprint(self, db_connection): + with db_connection.get_connection() as conn: + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute( + "INSERT INTO documents (instance, source) VALUES ('default', %s)", + (psycopg.types.json.Jsonb({"fingerprint": "not-hex"}),), + ) + + @pytest.mark.db + @pytest.mark.unit + def test_stage_timestamps_only_allow_known_keys(self, db_connection): + _new_doc(db_connection, "wf-ts") + with db_connection.get_connection() as conn: + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute( + "UPDATE workflows SET stage_timestamps = '{\"bogus\": \"x\"}'::jsonb " + "WHERE temporal_workflow_id = 'wf-ts'" + ) + + @pytest.mark.db + @pytest.mark.unit + def test_validity_period_must_not_end_before_it_starts(self, db_connection): + with pytest.raises(psycopg.errors.CheckViolation): + _new_doc(db_connection, "wf-bad-period", valid_from="2026-02-01", valid_to="2026-01-01") + + @pytest.mark.db + @pytest.mark.unit + def test_chunk_page_range_must_not_run_backwards(self, db_connection): + _new_doc(db_connection, "wf-range") + with pytest.raises(psycopg.errors.CheckViolation): + with db_connection.get_connection() as conn: + conn.execute( + "INSERT INTO chunks (document_id, chunk_number, original_text, page_start, page_end) " + "SELECT id, 1, 't', 3, 2 FROM documents" + ) + + @pytest.mark.db + @pytest.mark.unit + def test_audit_entry_needs_a_workflow_unless_system_level(self, db_connection): + with db_connection.get_connection() as conn: + conn.execute("INSERT INTO audit_logs (action_type, entity_type) VALUES ('x', 'setting')") + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute("INSERT INTO audit_logs (action_type, entity_type) VALUES ('x', 'page')") + + @pytest.mark.db + @pytest.mark.unit + def test_search_settings_is_a_single_row(self, db_connection): + with db_connection.get_connection() as conn: + with pytest.raises(psycopg.errors.UniqueViolation): + conn.execute("INSERT INTO search_settings DEFAULT VALUES") + + +class TestWorkflows: + @pytest.mark.db + @pytest.mark.unit + def test_upsert_creates_the_main_workflow(self, db_connection): + _new_doc(db_connection, "wf-main", stage="ocr_processing") + jobs = db_connection.list_document_jobs("wf-main") + assert [(j["job_type"], j["temporal_workflow_id"]) for j in jobs] == [("pipeline", "wf-main")] + + @pytest.mark.db + @pytest.mark.unit + def test_current_workflow_is_the_latest_by_start_time(self, db_connection): + _new_doc(db_connection, "wf-cur", stage="completed") + job_id = db_connection.create_document_job( + "wf-cur", "chunking_retry", temporal_workflow_id="wf-cur-retry-chunking-1", + current_stage="chunking", + ) + doc = db_connection.get_document("wf-cur") + assert doc["stage"] == "chunking" + assert doc["latest_job_id"] == job_id + assert doc["workflow_id"] == "wf-cur" + + @pytest.mark.db + @pytest.mark.unit + def test_stage_updates_follow_the_current_workflow(self, db_connection): + _new_doc(db_connection, "wf-follow", stage="completed") + db_connection.create_document_job("wf-follow", "ocr_retry", current_stage="ocr_processing") + db_connection.update_document_stage("wf-follow", "ocr_review") + by_type = {j["job_type"]: j["current_stage"] for j in db_connection.list_document_jobs("wf-follow")} + assert by_type == {"pipeline": "completed", "ocr_retry": "ocr_review"} + + @pytest.mark.db + @pytest.mark.unit + def test_stage_timestamps_are_set_once(self, db_connection): + _new_doc(db_connection, "wf-once") + db_connection.update_document_stage("wf-once", "ocr_review") + first = db_connection.get_document("wf-once")["ocr_completed_at"] + assert first + db_connection.update_document_stage("wf-once", "ocr_processing") + db_connection.update_document_stage("wf-once", "ocr_review") + assert db_connection.get_document("wf-once")["ocr_completed_at"] == first + + @pytest.mark.db + @pytest.mark.unit + def test_prod_request_is_a_waiting_promote_workflow_that_approval_adopts(self, db_connection): + _new_doc(db_connection, "wf-prod", stage="approval_for_prod") + doc = db_connection.mark_prod_ready_requested("wf-prod", user_id="kc-1", username="reviewer1") + assert doc["prod_ready_requested_by_username"] == "reviewer1" + latest = db_connection.get_latest_document_job("wf-prod") + assert (latest["job_type"], latest["status"]) == ("promote_prod", "waiting_review") + + db_connection.create_document_job( + "wf-prod", "promote_prod", temporal_workflow_id="wf-prod-promote-prod-1", + current_stage="ingesting_prod", + ) + jobs = [j for j in db_connection.list_document_jobs("wf-prod") if j["job_type"] == "promote_prod"] + assert len(jobs) == 1 + assert jobs[0]["temporal_workflow_id"] == "wf-prod-promote-prod-1" + assert db_connection.get_document("wf-prod")["prod_ready_requested_at"] is None + + @pytest.mark.db + @pytest.mark.unit + def test_job_config_merges(self, db_connection): + _new_doc(db_connection, "wf-cfg") + job = db_connection.get_latest_document_job("wf-cfg") + db_connection.update_document_job(job["id"], config_json={"a": 1}) + db_connection.update_document_job(job["id"], config_json={"chunking_progress": {"percent": 5}}) + config = db_connection.get_document_job(job["id"])["config"] + assert config == {"a": 1, "chunking_progress": {"percent": 5}} + + @pytest.mark.db + @pytest.mark.unit + def test_artifacts_belong_to_their_workflow_and_derive_pointers(self, db_connection): + _new_doc(db_connection, "wf-art") + job = db_connection.get_latest_document_job("wf-art") + art_id = db_connection.add_document_artifact( + "wf-art", "original_upload", "minio://documents/a.pdf", stage="registered", job_id=job["id"], + ) + norm_id = db_connection.add_document_artifact( + "wf-art", "normalized_pdf", "minio://documents/n.pdf", stage="ocr_processing", + ) + doc = db_connection.get_document("wf-art") + assert doc["original_artifact_id"] == art_id + assert doc["normalized_artifact_id"] == norm_id + artifact = db_connection.get_document_artifact(doc["id"], art_id) + assert artifact["job_id"] == job["id"] + assert artifact["workflow_id"] == "wf-art" + assert db_connection.get_document_artifact("wf-art", "123") is None + + +class TestPageTranslations: + @pytest.mark.db + @pytest.mark.unit + def test_an_ocr_save_keeps_existing_translations(self, db_connection): + _new_doc(db_connection, "wf-tr") + db_connection.save_pages("wf-tr", [{ + "page_number": 1, "original_markdown": "namaste", "detected_language": "hi", + "translated_markdown": "hello", "translation_target_language": "en", + "translation_provider": "p", "translation_model": "m", + }]) + db_connection.save_pages("wf-tr", [{"page_number": 1, "original_markdown": "namaste"}]) + page = db_connection.get_page("wf-tr", 1) + assert page["translated_markdown"] == "hello" + assert page["translations"]["en"]["provider"] == "p" + + @pytest.mark.db + @pytest.mark.unit + def test_changed_ocr_text_drops_stale_translations(self, db_connection): + _new_doc(db_connection, "wf-tr2") + db_connection.save_pages("wf-tr2", [{ + "page_number": 1, "original_markdown": "old", "translated_markdown": "t", + "translation_target_language": "en", + }]) + db_connection.save_pages("wf-tr2", [{"page_number": 1, "original_markdown": "new"}]) + assert db_connection.get_page("wf-tr2", 1)["translations"] == {} + + @pytest.mark.db + @pytest.mark.unit + def test_reading_and_saving_pages_back_is_lossless(self, db_connection): + _new_doc(db_connection, "wf-rt") + db_connection.save_pages("wf-rt", [{ + "page_number": 1, "original_markdown": "a", "detected_language": "gu", + "translated_markdown": "b", "translation_target_language": "en", "reviewer_notes": "n", + }]) + before = db_connection.get_pages("wf-rt") + db_connection.save_pages("wf-rt", before) + assert db_connection.get_pages("wf-rt") == before + + @pytest.mark.db + @pytest.mark.unit + def test_translation_edits_target_one_language_entry(self, db_connection): + _new_doc(db_connection, "wf-edit") + db_connection.save_pages("wf-edit", [{ + "page_number": 1, "original_markdown": "a", "translated_markdown": "b", + "translation_target_language": "en", + }]) + db_connection.update_page( + "wf-edit", 1, edited_translation="fixed", translation_reviewed=True, is_reviewed=True, + ) + page = db_connection.get_page("wf-edit", 1) + assert page["edited_translation"] == "fixed" + assert page["translated_markdown"] == "b" + assert page["translation_reviewed"] is True + assert page["is_reviewed"] is True + + @pytest.mark.db + @pytest.mark.unit + def test_page_metadata_merges_instead_of_replacing(self, db_connection): + _new_doc(db_connection, "wf-meta") + db_connection.save_pages("wf-meta", [ + {"page_number": 1, "original_markdown": "a", "detected_language": "hi"} + ]) + db_connection.update_page("wf-meta", 1, reviewer_notes="ok") + page = db_connection.get_page("wf-meta", 1) + assert page["detected_language"] == "hi" + assert page["reviewer_notes"] == "ok" + + @pytest.mark.db + @pytest.mark.unit + def test_count_translated_pages(self, db_connection): + _new_doc(db_connection, "wf-count") + db_connection.save_pages("wf-count", [ + {"page_number": 1, "original_markdown": "a", "translated_markdown": "x", + "translation_target_language": "en"}, + {"page_number": 2, "original_markdown": "b"}, + ]) + assert db_connection.count_translated_pages("wf-count") == 1 + + +class TestChunkRuns: + @pytest.mark.db + @pytest.mark.unit + def test_rechunking_replaces_chunks_and_bumps_the_run(self, db_connection): + _new_doc(db_connection, "wf-run") + first = [{"chunk_number": 1, "original_text": "a", "chunking_provider": "p1"}] + db_connection.save_chunks("wf-run", first) + old_id = db_connection.get_chunk("wf-run", 1)["id"] + db_connection.save_chunks("wf-run", [ + {"chunk_number": 1, "original_text": "b", "chunking_provider": "p2", + "chunking_config_json": '{"size": 450}'}, + {"chunk_number": 2, "original_text": "c", "chunking_provider": "p2"}, + ]) + doc = db_connection.get_document("wf-run") + chunk = db_connection.get_chunk("wf-run", 1) + assert doc["chunk_count"] == 2 + assert chunk["id"] != old_id + assert chunk["chunk_version"] == 2 + assert chunk["chunking_provider"] == "p2" + assert chunk["chunking_config_json"] == '{"size": 450}' + + @pytest.mark.db + @pytest.mark.unit + def test_a_failed_rechunk_leaves_the_previous_chunks(self, db_connection): + _new_doc(db_connection, "wf-atomic") + db_connection.save_chunks("wf-atomic", [{"chunk_number": 1, "original_text": "keep"}]) + with pytest.raises(psycopg.errors.UniqueViolation): + db_connection.save_chunks("wf-atomic", [ + {"chunk_number": 1, "original_text": "x"}, + {"chunk_number": 1, "original_text": "dup"}, + ]) + assert db_connection.get_chunk("wf-atomic", 1)["original_text"] == "keep" + assert db_connection.get_document("wf-atomic")["chunk_count"] == 1 + + @pytest.mark.db + @pytest.mark.unit + def test_source_spans_round_trip_and_derive_page_numbers(self, db_connection): + _new_doc(db_connection, "wf-spans") + db_connection.save_chunks("wf-spans", [{ + "chunk_number": 1, "original_text": "a", "page_start": 2, "page_end": 3, + "source_spans_json": '[{"page_number": 2, "start_char": 0, "end_char": 1}, ' + '{"page_number": 3, "start_char": 0, "end_char": 1}]', + }]) + chunk = db_connection.get_chunk("wf-spans", 1) + assert chunk["source_page_numbers_json"] == "[2, 3]" + + @pytest.mark.db + @pytest.mark.unit + def test_search_chunks_matches_edited_text_first(self, db_connection): + _new_doc(db_connection, "wf-search", instance="mh") + db_connection.save_chunks("wf-search", [ + {"chunk_number": 1, "original_text": "alpha beta", "edited_text": "gamma delta"}, + ]) + hits, total = db_connection.search_chunks(query="gamma") + assert total == 1 and hits[0]["workflow_id"] == "wf-search" + assert db_connection.search_chunks(query="alpha")[1] == 0 + assert db_connection.search_chunks(query="gamma", instances=["up"])[1] == 0 + + +class TestIndexStatus: + @pytest.mark.db + @pytest.mark.unit + def test_entries_merge_and_dev_prod_are_separate_keys(self, db_connection, monkeypatch): + monkeypatch.setenv("PROD_VECTOR_DB_COLLECTION_NAME", "prod-docs") + _new_doc(db_connection, "wf-idx") + db_connection.upsert_document_index_status( + "wf-idx", "documents-index", doc_id="abc", chunk_count_indexed=4, + last_indexed_at="2026-01-01T00:00:00", status="indexed", + ) + db_connection.upsert_document_index_status( + "wf-idx", "documents-index", last_verified_at="2026-01-02T00:00:00", status="indexed", + ) + db_connection.upsert_document_index_status("wf-idx", "prod-docs", status="missing") + doc = db_connection.get_document("wf-idx") + assert set(doc["index_status"]) == {"dev", "prod"} + dev = db_connection.get_document_index_status("wf-idx", "documents-index") + assert dev["chunk_count_indexed"] == 4 + assert dev["last_indexed_at"] == "2026-01-01T00:00:00" + assert dev["last_verified_at"] == "2026-01-02T00:00:00" + summaries = {s["index_name"]: s for s in db_connection.list_index_summaries()} + assert summaries["documents-index"]["indexed_chunks"] == 4 + + +class TestIdentifierResolution: + @pytest.mark.db + @pytest.mark.unit + def test_every_documented_form_resolves(self, db_connection): + fingerprint = _md5("content") + _new_doc(db_connection, "doc-" + fingerprint, document_id=fingerprint, filename="pm.pdf") + db_connection.create_document_job( + "doc-" + fingerprint, "ocr_retry", temporal_workflow_id=f"doc-{fingerprint}-retry-ocr-9", + ) + doc = db_connection.get_document("doc-" + fingerprint) + for identifier in ( + doc["id"], + "doc-" + fingerprint, + f"doc-{fingerprint}-retry-ocr-9", + fingerprint, + "pm.pdf", + _md5(fingerprint), + ): + found = db_connection.find_document_by_doc_identifier(identifier) + assert found and found["id"] == doc["id"], identifier + assert found["workflow_id"] == "doc-" + fingerprint + assert db_connection.find_document_by_doc_identifier("nope") is None + assert db_connection.get_document("00000000-0000-0000-0000-000000000000") is None + + +class TestAuditTrail: + @pytest.mark.db + @pytest.mark.unit + def test_audit_outlives_the_document_and_keeps_tenant_scope(self, db_connection): + _new_doc(db_connection, "wf-aud", instance="mh") + db_connection.log_audit( + workflow_id="wf-aud", document_id="x", action_type="purge_document", + entity_type="document", actor_user_id="kc-9", actor_username="boss", + actor_email="b@x.in", actor_roles="superadmin", + ) + db_connection.purge_document("wf-aud") + + logs = db_connection.get_audit_logs("wf-aud") + assert [entry["action_type"] for entry in logs] == ["purge_document"] + assert logs[0]["actor"] == "b@x.in (superadmin)" + assert logs[0]["actor_user_id"] == "kc-9" + assert db_connection.get_all_audit_log_count(instances=["mh"]) == 1 + assert db_connection.get_all_audit_log_count(instances=["up"]) == 0 + + @pytest.mark.db + @pytest.mark.unit + def test_system_entries_need_no_document(self, db_connection): + db_connection.log_audit( + workflow_id="__catalog__", document_id="__catalog__", action_type="catalog_rebuild", + entity_type="catalog", + ) + logs = db_connection.get_all_audit_logs(action_type="catalog_rebuild") + assert logs[0]["workflow_id"] == "__catalog__" + + @pytest.mark.db + @pytest.mark.unit + def test_settings_changes_are_logged_per_key_with_group_names(self, db_connection): + db_connection.update_search_settings({"alpha": 0.2, "limit": 3}) + fields = {entry["field_name"] for entry in db_connection.get_settings_audit_logs()} + assert fields == {"retrieval.alpha", "results.limit"} + assert db_connection.get_settings_audit_count() == 2 + db_connection.update_search_settings({"alpha": 0.2}) + assert db_connection.get_settings_audit_count() == 2 + + +class TestSearchSettings: + @pytest.mark.db + @pytest.mark.unit + def test_defaults_come_from_the_models(self, db_connection): + settings = db_connection.get_search_settings() + assert settings["alpha"] == 0.6 + assert len(settings) == 15 + + @pytest.mark.db + @pytest.mark.unit + def test_values_are_validated(self, db_connection): + with pytest.raises(ValueError): + db_connection.update_search_settings({"alpha": 3}) + with pytest.raises(ValueError): + db_connection.update_search_settings({"searchMethod": "FUZZY"}) + with pytest.raises(ValueError): + db_connection.update_search_settings({"nonsense": 1}) + + @pytest.mark.db + @pytest.mark.unit + def test_a_partial_update_keeps_the_rest_of_its_group(self, db_connection): + db_connection.update_search_settings({"alpha": 0.3}) + db_connection.update_search_settings({"hybridRrfK": 10}) + settings = db_connection.get_search_settings() + assert (settings["alpha"], settings["hybridRrfK"]) == (0.3, 10) + + +class TestCatalogView: + @staticmethod + def _scheme_doc(db, workflow_id, code, stage, **kwargs): + _new_doc(db, workflow_id, stage=stage, **kwargs) + db.update_document_fields( + workflow_id, document_kind="scheme", scheme_code=code, scheme_name=code.upper(), + ) + + @pytest.mark.db + @pytest.mark.unit + def test_status_rules(self, db_connection): + db = db_connection + self._scheme_doc(db, "wf-live", "live-one", "completed") + self._scheme_doc(db, "wf-stale", "stale-one", "completed") + db.mark_document_reindex_required("wf-stale", True, "edit") + self._scheme_doc(db, "wf-gate", "gate-one", "approval_for_prod") + self._scheme_doc(db, "wf-wip", "wip-one", "chunk_review") + self._scheme_doc(db, "wf-off", "off-one", "completed") + db.set_document_disabled("wf-off", True) + db.upsert_catalog_entry(type="scheme", code="boot-one", name="Boot", source="bootstrap") + + status = {e["code"]: e["status"] for e in db.list_catalog_entries()} + assert status == { + "live-one": "live", + "stale-one": "pending_reindex", + "gate-one": "pending_prod", + "wip-one": "disabled", + "off-one": "disabled", + "boot-one": "live", + } + + @pytest.mark.db + @pytest.mark.unit + def test_computed_columns(self, db_connection): + db = db_connection + self._scheme_doc(db, "wf-a", "multi", "completed", instance="mh") + self._scheme_doc(db, "wf-b", "multi", "completed", instance="up") + db.save_chunks("wf-a", [{"chunk_number": 1, "original_text": "x"}]) + db.save_chunks("wf-b", [{"chunk_number": 1, "original_text": "y"}, + {"chunk_number": 2, "original_text": "z"}]) + db.update_document_fields("wf-b", network_visible=False) + db.upsert_document_index_status("wf-a", "documents-index", last_indexed_at="2026-05-01T00:00:00") + + entry = db.get_catalog_entry("multi") + assert entry["instances"] == ["mh", "up"] + assert sorted(entry["workflow_ids"]) == ["wf-a", "wf-b"] + assert entry["chunk_count"] == 3 + assert entry["network_visible"] is True + assert entry["promoted_at"] is None + + @pytest.mark.db + @pytest.mark.unit + def test_entries_without_documents_default_to_the_default_instance(self, db_connection): + db_connection.upsert_catalog_entry(type="scheme", code="lonely", name="Lonely", source="import") + entry = db_connection.get_catalog_entry("lonely") + assert entry["instances"] == ["default"] + assert entry["status"] == "disabled" + assert entry["network_visible"] is False + + @pytest.mark.db + @pytest.mark.unit + def test_other_entry_types_stay_out_of_the_scheme_listing(self, db_connection): + _new_doc(db_connection, "wf-adv", stage="completed") + db_connection.update_document_fields( + "wf-adv", document_kind="advisory", scheme_code="kharif", scheme_name="Kharif", + ) + assert db_connection.list_catalog_entries() == [] + assert [e["code"] for e in db_connection.list_catalog_entries(type="advisory")] == ["kharif"] + + @pytest.mark.db + @pytest.mark.unit + def test_changing_the_kind_moves_or_unlinks_the_entry(self, db_connection): + _new_doc(db_connection, "wf-move") + db_connection.update_document_fields( + "wf-move", document_kind="scheme", scheme_code="movable", scheme_name="Movable", + ) + db_connection.update_document_fields("wf-move", document_kind="advisory") + doc = db_connection.get_document("wf-move") + assert (doc["document_kind"], doc["scheme_code"], doc["scheme_name"]) == ( + "advisory", "movable", "Movable", + ) + db_connection.update_document_fields("wf-move", document_kind="document") + doc = db_connection.get_document("wf-move") + assert doc["scheme_code"] is None and doc["catalog_entry_id"] is None + + @pytest.mark.db + @pytest.mark.unit + def test_version_counts_bumps(self, db_connection): + assert db_connection.get_catalog_meta()["version"] == 0 + assert db_connection.bump_catalog_version("a") == 1 + assert db_connection.bump_catalog_version("b") == 2 + meta = db_connection.get_catalog_meta() + assert (meta["version"], meta["notes"]) == (2, "b") + + +class TestUsersAndOtps: + @pytest.mark.db + @pytest.mark.unit + def test_upsert_user_refreshes_and_survives_an_email_clash(self, db_connection): + first = db_connection.upsert_user("kc-a", "ann", "ann@x.in", ["reviewer"]) + again = db_connection.upsert_user("kc-a", "ann2", None, ["admin"]) + assert first == again + user = db_connection.get_user_by_keycloak_id("kc-a") + assert (user["username"], user["email"], user["roles"]) == ("ann2", "ann@x.in", ["admin"]) + other = db_connection.upsert_user("kc-b", "bob", "ANN@x.in", []) + assert other and other != first + assert db_connection.get_user_by_keycloak_id("kc-b")["email"] is None + + @pytest.mark.db + @pytest.mark.unit + def test_uploader_is_a_user_row(self, db_connection): + _new_doc(db_connection, "wf-up", uploaded_by_user_id="kc-u", uploaded_by_username="uma", + uploaded_by_email="u@x.in", uploaded_by_roles="contributor,reviewer") + doc = db_connection.get_document("wf-up") + assert (doc["uploaded_by_user_id"], doc["uploaded_by_username"]) == ("kc-u", "uma") + assert doc["uploaded_by_roles"] == "contributor,reviewer" + + @pytest.mark.db + @pytest.mark.unit + def test_otp_attempts_increment_atomically(self, db_connection): + db_connection.store_email_otp( + email="A@X.in", code_hash="h", expires_at="2099-01-01T00:00:00+00:00", + created_at="x", last_sent_at="2026-01-01T00:00:00+00:00", + ) + assert db_connection.increment_email_otp_attempts("a@x.in") == 1 + assert db_connection.increment_email_otp_attempts("a@x.in") == 2 + row = db_connection.get_email_otp("a@x.in") + assert row["attempts"] == 2 + assert row["expires_at"].endswith("+00:00") + db_connection.store_email_otp( + email="a@x.in", code_hash="h2", expires_at="2099-01-01T00:00:00+00:00", + created_at="x", last_sent_at="2026-01-02T00:00:00+00:00", + ) + assert db_connection.get_email_otp("a@x.in")["attempts"] == 0 + db_connection.delete_email_otp("a@x.in") + assert db_connection.get_email_otp("a@x.in") is None + + +class TestOperationsViews: + @pytest.mark.db + @pytest.mark.unit + def test_summary_counts_and_queue_use_the_current_workflow(self, db_connection): + _new_doc(db_connection, "wf-q1", stage="ocr_review", instance="mh") + _new_doc(db_connection, "wf-q2", stage="completed", instance="up") + db_connection.create_document_job("wf-q2", "reingest", current_stage="ingesting") + counts = db_connection.get_document_summary_counts() + assert counts["total_documents"] == 2 + assert counts["ocr_review_documents"] == 1 + assert counts["completed_documents"] == 0 + assert counts["running_jobs"] == 3 + assert db_connection.get_document_summary_counts(instances=[])["total_documents"] == 0 + by_instance = {r["instance"]: r for r in db_connection.get_document_counts_by_instance()} + assert by_instance["mh"]["dev_approval"] == 1 + + rows, total = db_connection.list_operations_queue() + assert total == 2 + assert {r["workflow_id"]: r["job_type"] for r in rows} == { + "wf-q1": "pipeline", "wf-q2": "reingest", + } + assert db_connection.list_operations_queue(instances=["up"])[1] == 1 + + @pytest.mark.db + @pytest.mark.unit + def test_demo_and_disabled_documents_are_hidden_by_default(self, db_connection): + _new_doc(db_connection, "wf-demo") + _new_doc(db_connection, "wf-off") + db_connection.set_document_demo("wf-demo", True) + db_connection.set_document_disabled("wf-off", True) + assert db_connection.list_documents() == [] + assert len(db_connection.list_documents(include_demo=True, include_disabled=True)) == 2 + + @pytest.mark.db + @pytest.mark.unit + def test_reconcile_moves_a_stalled_document_into_review(self, db_connection): + _new_doc(db_connection, "wf-rec", stage="ocr_processing") + db_connection.save_pages("wf-rec", [{"page_number": 1, "original_markdown": "a"}]) + result = db_connection.reconcile_materialized_state("wf-rec") + assert result["updated"] is True and result["stage"] == "ocr_review" + assert db_connection.get_latest_document_job("wf-rec")["status"] == "waiting_review" + + @pytest.mark.db + @pytest.mark.unit + def test_webhook_upsert_skips_unchanged_payloads(self, db_connection): + db = db_connection + assert db.upsert_webhook_records("webhook", [{"contentID": "c1", "title": "T", "isPublished": True}]) == 1 + assert db.upsert_webhook_records("webhook", [{"contentID": "c1", "title": "T", "isPublished": True}]) == 1 + with db.get_connection() as conn: + rows = conn.execute("SELECT title, is_published FROM webhook_records").fetchall() + assert rows == [{"title": "T", "is_published": True}] + + +class TestMigrationRunner: + @pytest.mark.db + @pytest.mark.unit + def test_applied_migrations_are_recorded_and_not_reapplied(self, db_connection, postgres_url): + from pipeline import migrate + + assert migrate.apply_migrations(postgres_url) == [] + assert migrate.is_current(postgres_url) + assert migrate.current_version(postgres_url) == migrate.list_migrations()[-1][0] + + @pytest.mark.db + @pytest.mark.unit + def test_migration_files_are_numbered_and_unversioned_by_schema_name(self): + from pipeline import migrate + + names = [path.name for _, path in migrate.list_migrations()] + assert names[0] == "0001_initial_schema.sql" + + @pytest.mark.db + @pytest.mark.unit + def test_a_new_migration_runs_once(self, db_connection, postgres_url, tmp_path): + from pipeline import migrate + + (tmp_path / "0001_initial_schema.sql").write_text("SELECT 1;") + (tmp_path / "0002_probe.sql").write_text("CREATE TABLE mig_probe (n int);") + try: + assert migrate.apply_migrations(postgres_url, tmp_path) == ["0002"] + assert migrate.apply_migrations(postgres_url, tmp_path) == [] + finally: + with db_connection.get_connection() as conn: + conn.execute("DROP TABLE IF EXISTS mig_probe") + conn.execute("DELETE FROM schema_migrations WHERE version = '0002'") + + @pytest.mark.db + @pytest.mark.unit + def test_a_failing_migration_applies_nothing_of_itself(self, db_connection, postgres_url, tmp_path): + from pipeline import migrate + + (tmp_path / "0001_initial_schema.sql").write_text("SELECT 1;") + (tmp_path / "0002_bad.sql").write_text("CREATE TABLE mig_half (n int); SELECT 1/0;") + with pytest.raises(psycopg.errors.DivisionByZero): + migrate.apply_migrations(postgres_url, tmp_path) + with db_connection.get_connection() as conn: + assert conn.execute("SELECT to_regclass('mig_half') AS t").fetchone()["t"] is None + assert conn.execute( + "SELECT count(*) AS n FROM schema_migrations WHERE version = '0002'" + ).fetchone()["n"] == 0 + + @pytest.mark.db + @pytest.mark.unit + def test_migration_file_names_are_validated(self, tmp_path): + from pipeline import migrate + + (tmp_path / "init.sql").write_text("SELECT 1;") + with pytest.raises(ValueError): + migrate.list_migrations(tmp_path) diff --git a/tests/test_document_repository.py b/tests/test_document_repository.py index dbf6c4f..85d00d8 100644 --- a/tests/test_document_repository.py +++ b/tests/test_document_repository.py @@ -1,7 +1,7 @@ """Unit tests for DocumentRepository. Two styles on purpose: the `db_connection` fixture exercises the repository -against real SQLite (so a drifted db.py signature fails here), and the fake-db +against real the database (so a drifted db.py signature fails here), and the fake-db tests pin the None guards that are the repository's own behaviour. """ diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 898f51c..f7a1794 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -95,7 +95,7 @@ def test_find_document_by_legacy_doc_id_hash(db_connection): def test_provenance_chunk_endpoint(db_connection): db = db_connection workflow_id = "doc-provenance01" - document_id = "prov-doc-id-xyz" + document_id = hashlib.md5(b"prov-doc-id-xyz").hexdigest() db.upsert_document( workflow_id=workflow_id, document_id=document_id, diff --git a/tests/test_scheme_catalog.py b/tests/test_scheme_catalog.py index d4be742..a58810c 100644 --- a/tests/test_scheme_catalog.py +++ b/tests/test_scheme_catalog.py @@ -2,21 +2,14 @@ from __future__ import annotations -import json -import os - import pytest @pytest.fixture -def catalog_db(temp_db_path): - from pipeline import db, scheme_catalog +def catalog_db(db_connection): + from pipeline import scheme_catalog - db.DB_PATH = temp_db_path - db.init_db() - scheme_catalog.ensure_catalog_schema() - yield db, scheme_catalog - db.DB_PATH = os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db") + yield db_connection, scheme_catalog def test_bootstrap_seeds_13_vector_schemes(catalog_db): @@ -72,8 +65,7 @@ def test_scheme_metadata_and_promote_rebuild(catalog_db): assert entry is not None assert entry["status"] == "live" assert entry["source"] == "pipeline" - aliases = json.loads(entry["scheme_aliases_json"]) - assert "organic" in aliases or "PKVY" in aliases + assert "organic" in entry["aliases"] or "PKVY" in entry["aliases"] def test_code_rename_pending_reindex(catalog_db): @@ -237,19 +229,16 @@ def test_prepare_records_document_unchanged(): @pytest.fixture -def asgi_client(temp_db_path, mock_temporal_client, mock_minio_client): +def asgi_client(db_connection, mock_temporal_client, mock_minio_client): """HTTP client that does not run real Temporal/MinIO lifespan.""" from contextlib import asynccontextmanager from fastapi.testclient import TestClient - from pipeline import api, db, scheme_catalog + from pipeline import api - db.DB_PATH = temp_db_path - db.init_db() api.temporal_client = mock_temporal_client api.minio_client = mock_minio_client - scheme_catalog.ensure_catalog_schema() @asynccontextmanager async def _noop_lifespan(_app): @@ -262,7 +251,6 @@ async def _noop_lifespan(_app): yield client finally: api.app.router.lifespan_context = original - db.DB_PATH = os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db") def test_catalog_api_snapshot(asgi_client): diff --git a/tests/test_tenancy.py b/tests/test_tenancy.py index 60148c2..03780b5 100644 --- a/tests/test_tenancy.py +++ b/tests/test_tenancy.py @@ -212,7 +212,8 @@ def test_summary_counts_honor_instance_filter(db_connection): def test_list_runs_filters_by_instance(db_connection): """Runs must honor the same tenant scope as documents (role-based states).""" db = db_connection - db.upsert_document( + doc_a = db.upsert_document( + with_workflow=False, workflow_id="wf-run-a", document_id="d-run-a", filename="a.pdf", @@ -220,7 +221,8 @@ def test_list_runs_filters_by_instance(db_connection): stage="completed", instance="tenant-a", ) - db.upsert_document( + doc_b = db.upsert_document( + with_workflow=False, workflow_id="wf-run-b", document_id="d-run-b", filename="b.pdf", @@ -228,8 +230,8 @@ def test_list_runs_filters_by_instance(db_connection): stage="completed", instance="tenant-b", ) - job_a = db.create_document_job("wf-run-a", "ocr", status="completed") - job_b = db.create_document_job("wf-run-b", "ocr", status="running") + job_a = db.create_document_job(doc_a, "ocr_only", status="completed") + job_b = db.create_document_job(doc_b, "ocr_only", status="running") tenant_a_only = db.list_runs(instances=["tenant-a"]) assert {r["id"] for r in tenant_a_only} == {job_a} @@ -270,8 +272,8 @@ def test_state_user_cannot_see_bv_portal_documents(db_connection): stage="ocr_review", instance="mh", ) - db.create_document_job("wf-bv", "ocr", status="running") - db.create_document_job("wf-mh", "ocr", status="running") + db.create_document_job("wf-bv", "ocr_only", status="running") + db.create_document_job("wf-mh", "ocr_only", status="running") # MH contributor from Keycloak group path mh_user = claims_to_user( diff --git a/tests/test_webhook.py b/tests/test_webhook.py index d218885..47177b8 100644 --- a/tests/test_webhook.py +++ b/tests/test_webhook.py @@ -1,7 +1,6 @@ """Tests for POST /webhook (x-api-key protected).""" import os -import uuid from unittest.mock import patch import pytest @@ -30,6 +29,7 @@ def test_client(db_connection): """TestClient without lifespan (no Temporal/MinIO needed for /webhook).""" from fastapi.testclient import TestClient + from pipeline import api return TestClient(api.app) @@ -87,9 +87,9 @@ def test_valid_payload(self, test_client, db_connection, key_env): rows = _rows(db_connection) assert len(rows) == 1 assert rows[0]["external_id"] == "101" - assert uuid.UUID(rows[0]["id"]).version == 4 + assert rows[0]["id"].version == 4 assert rows[0]["category"] == "Agriculture" - assert '"popList"' in rows[0]["payload"] + assert "popList" in rows[0]["payload"] def test_repost_is_idempotent(self, test_client, db_connection, key_env): test_client.post("/webhook", json=SAMPLE, headers=self.H) diff --git a/ui/src/views/DocumentOpsView.jsx b/ui/src/views/DocumentOpsView.jsx index 892fabd..d5dec0f 100644 --- a/ui/src/views/DocumentOpsView.jsx +++ b/ui/src/views/DocumentOpsView.jsx @@ -349,7 +349,7 @@ const STAGE_POLL_INTERVAL_MS = 1500 const STAGE_POLL_MAX_ATTEMPTS = 40 // ~60s safety net so a stuck workflow doesn't wedge the button forever function stageWantsPages(stage, pageCount = 0) { - // Skip empty registered docs; otherwise pages are local SQLite (fast) and power OCR/translation. + // Skip empty registered docs; otherwise pages come from the database (fast) and power OCR/translation. if ((stage === 'registered' || stage === 'ocr_processing') && !pageCount) return false return true } @@ -544,7 +544,7 @@ export default function DocumentOpsView() { /** * Progressive load: * 1) document shell (unblocks UI immediately) - * 2) SQLite pages/chunks needed for the active review surface + * 2) Database pages/chunks needed for the active review surface * 3) heavy panels (Qdrant / Temporal / audit) only when tab or stage needs them */ const load = useCallback(async ({ soft = false, forcePanels = null } = {}) => { @@ -581,7 +581,7 @@ export default function DocumentOpsView() { attemptedPanelsRef.current.chunks = true localTasks.push(loadChunks(wid)) } - // Local SQLite panels in parallel; do not wait on remote index/Temporal for first paint. + // Database panels in parallel; do not wait on remote index/Temporal for first paint. await Promise.allSettled(localTasks) if (requestId !== requestIdRef.current) return @@ -1673,7 +1673,7 @@ export default function DocumentOpsView() {

Document State

- SQLite Stage + Database Stage
From 7b6eff23e557dc7536b582223db36189a2f9d886 Mon Sep 17 00:00:00 2001 From: Naveen Date: Mon, 26 Oct 2026 13:18:48 +0530 Subject: [PATCH 07/53] feat(scripts): add SQLite to Postgres migration and retire legacy scripts migrate_sqlite_to_postgres.py imports an existing documents.db (dry-run, row-count report, idempotent). Remove start_manifest_translation_batch.py and drop_legacy_marqo_columns.py, whose tables and columns no longer exist. Adapt the backup, failed-workflow, OCR-batch and validity e2e scripts to Postgres. --- scripts/backup_volumes.sh | 17 +- scripts/drop_legacy_marqo_columns.py | 89 -- scripts/list_failed_workflows.py | 52 +- scripts/migrate_sqlite_to_postgres.py | 848 ++++++++++++++++++++ scripts/start_manifest_translation_batch.py | 129 --- scripts/start_ocr_only_batch.py | 39 +- scripts/verify_document_validity_e2e.py | 21 +- tests/test_migrate_sqlite_to_postgres.py | 393 +++++++++ 8 files changed, 1297 insertions(+), 291 deletions(-) delete mode 100644 scripts/drop_legacy_marqo_columns.py create mode 100644 scripts/migrate_sqlite_to_postgres.py delete mode 100644 scripts/start_manifest_translation_batch.py create mode 100644 tests/test_migrate_sqlite_to_postgres.py diff --git a/scripts/backup_volumes.sh b/scripts/backup_volumes.sh index 4dc91c0..6a7b08e 100644 --- a/scripts/backup_volumes.sh +++ b/scripts/backup_volumes.sh @@ -4,7 +4,8 @@ set -euo pipefail # Backup all persistent data volumes used by the docs pipeline: # - Temporal Postgres database # - MinIO object storage -# - SQLite documents/chunks DB +# - Application Postgres volume (documents, workflows, pages, chunks, audit) +# (a logical pg_dump is also written when the postgres container is running) # # Usage: # ./scripts/backup_volumes.sh # writes to ./backups @@ -52,8 +53,18 @@ backup_volume "temporal-db" "temporal-db-data" # 2) MinIO object storage (all PDFs / objects) backup_volume "minio" "minio-data" -# 3) SQLite documents/chunks DB -backup_volume "sqlite" "sqlite-data" +# 3) Application Postgres volume +backup_volume "app-db" "app-db-data" + +# 4) Logical dump of the application database, consistent while the stack runs +PG_CONTAINER="$(docker ps --format '{{.Names}}' | grep 'postgres' | grep -v 'temporal\|keycloak' | head -n 1 || true)" +if [[ -n "$PG_CONTAINER" ]]; then + echo "Dumping application database from ${PG_CONTAINER}" + docker exec "$PG_CONTAINER" sh -c 'pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc' \ + > "${BACKUP_DIR}/app-db-${TIMESTAMP}.dump" +else + echo "WARNING: application postgres container not running; skipped pg_dump" >&2 +fi echo echo "Backup complete. Files in ${BACKUP_DIR}:" diff --git a/scripts/drop_legacy_marqo_columns.py b/scripts/drop_legacy_marqo_columns.py deleted file mode 100644 index f178e3f..0000000 --- a/scripts/drop_legacy_marqo_columns.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python3 -""" -Drop the legacy marqo_doc_id column from document_index_status. - -Context: the Marqo backend was removed and document_index_status.marqo_doc_id -was superseded by a generically-named doc_id column, backfilled once at -startup (see pipeline/db.py init_db()). marqo_doc_id has been left in -place, unwritten, so the schema stayed usable by the pre-removal code during -rollout. This script is the followup: run it manually, once you've confirmed -the app has been stable on the new code for a while — it's a one-way door -(SQLite's DROP COLUMN can't be undone short of restoring a backup). - -No other legacy Marqo-only tables/columns were found in this codebase as of -this writing (only this one column) — if a future migration needs the same -treatment, extend this script rather than writing a new one. - -Requires SQLite 3.35+ for ALTER TABLE ... DROP COLUMN (bundled with Python -3.9+ on most platforms; the script checks and aborts with a clear message on -older versions rather than attempting a table-rebuild workaround). - -Usage: - uv run python scripts/drop_legacy_marqo_columns.py [--yes] -""" -import argparse -import sqlite3 -import sys -from pathlib import Path - -project_root = Path(__file__).parent.parent -sys.path.insert(0, str(project_root)) - -from pipeline import db # noqa: E402 - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--yes", action="store_true", help="Skip the interactive confirmation prompt.") - args = parser.parse_args() - - if sqlite3.sqlite_version_info < (3, 35, 0): - print( - f"ERROR: SQLite {sqlite3.sqlite_version} does not support " - "ALTER TABLE ... DROP COLUMN (needs 3.35+). Aborting — this " - "script does not implement a table-rebuild fallback.", - file=sys.stderr, - ) - sys.exit(1) - - with db.get_connection() as conn: - columns = {row[1] for row in conn.execute("PRAGMA table_info(document_index_status)")} - if "marqo_doc_id" not in columns: - print("document_index_status.marqo_doc_id doesn't exist — nothing to drop.") - sys.exit(0) - - stale = conn.execute( - """ - SELECT COUNT(*) FROM document_index_status - WHERE doc_id IS NULL AND marqo_doc_id IS NOT NULL - """ - ).fetchone()[0] - if stale: - print( - f"ERROR: {stale} row(s) in document_index_status have " - "marqo_doc_id set but doc_id NULL — the backfill in " - "init_db() hasn't run against this database yet. Start the " - "app once (so init_db() runs) before dropping the column.", - file=sys.stderr, - ) - sys.exit(1) - - total = conn.execute("SELECT COUNT(*) FROM document_index_status").fetchone()[0] - print(f"document_index_status: {total} row(s), all backfilled to doc_id.") - - if not args.yes: - answer = input( - "Drop the legacy marqo_doc_id column now? This cannot be undone " - "short of restoring a backup. [y/N] " - ).strip().lower() - if answer != "y": - print("Aborted.") - sys.exit(0) - - conn.execute("ALTER TABLE document_index_status DROP COLUMN marqo_doc_id") - conn.commit() - print("Dropped document_index_status.marqo_doc_id.") - - -if __name__ == "__main__": - main() diff --git a/scripts/list_failed_workflows.py b/scripts/list_failed_workflows.py index 4978822..1a19271 100755 --- a/scripts/list_failed_workflows.py +++ b/scripts/list_failed_workflows.py @@ -3,7 +3,7 @@ Query failed workflows and write their IDs and error messages to a file. This script: -1. Queries SQLite database for all failed workflows +1. Queries the Postgres database (DATABASE_URL) for all failed workflows 2. Extracts workflow_id, filename, and error_message 3. Optionally queries Temporal for additional error details 4. Writes results to a CSV file @@ -12,7 +12,6 @@ import asyncio import csv import os -import sqlite3 import sys from pathlib import Path @@ -22,8 +21,7 @@ from temporalio.client import Client, WorkflowFailureError -# Database path - try environment variable first, then default -DB_PATH = os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db") +from pipeline import db # Configuration TEMPORAL_HOST = os.environ.get("TEMPORAL_HOST", "localhost:7233") @@ -94,38 +92,24 @@ async def get_temporal_error_details(workflow_id: str, client: Client) -> dict: } -def query_failed_workflows_from_db(db_path: str) -> list: - """Query SQLite database directly for failed workflows.""" - conn = sqlite3.connect(db_path) - conn.row_factory = sqlite3.Row - - try: - cursor = conn.execute(""" - SELECT workflow_id, document_id, filename, error_message, created_at - FROM documents - WHERE stage = 'failed' - ORDER BY created_at DESC - """) - rows = cursor.fetchall() - return [dict(row) for row in rows] - finally: - conn.close() +def query_failed_workflows_from_db() -> list: + """Query the database for failed workflows (including demo and disabled documents).""" + return db.list_documents( + stage="failed", limit=1_000_000, include_demo=True, include_disabled=True + ) async def main(): """Main function to query failed workflows and write to file.""" - print(f"Querying failed workflows from SQLite database at {DB_PATH}...") - - # Check if database exists - if not os.path.exists(DB_PATH): - print(f"Error: Database not found at {DB_PATH}") - print("Make sure you're running this script inside the Docker container or") - print("that the database path is correct.") + print(f"Querying failed workflows from the database at {db.database_url().rsplit('@', 1)[-1]}...") + + try: + failed_docs = query_failed_workflows_from_db() + except Exception as e: + print(f"Error: could not query the database: {e}") + print("Check DATABASE_URL, and run inside the Docker network if the database is not published.") sys.exit(1) - # Query SQLite for failed workflows - failed_docs = query_failed_workflows_from_db(DB_PATH) - print(f"Found {len(failed_docs)} failed workflows") # Prepare data for CSV @@ -139,7 +123,7 @@ async def main(): print("Connected to Temporal") except Exception as e: print(f"Warning: Could not connect to Temporal: {e}") - print("Will only use SQLite error messages") + print("Will only use database error messages") temporal_client = None # Process each failed workflow @@ -154,7 +138,7 @@ async def main(): "workflow_id": workflow_id, "document_id": document_id, "filename": filename, - "sqlite_error_message": error_message, + "db_error_message": error_message, "created_at": created_at, } @@ -180,7 +164,7 @@ async def main(): "document_id", "filename", "created_at", - "sqlite_error_message", + "db_error_message", "temporal_status", "temporal_error_message", "temporal_error_type", @@ -196,7 +180,7 @@ async def main(): print(f"✓ Successfully wrote {len(rows)} failed workflows to {OUTPUT_FILE}") print("\nSummary:") print(f" - Total failed workflows: {len(rows)}") - print(f" - With SQLite error messages: {sum(1 for r in rows if r.get('sqlite_error_message'))}") + print(f" - With database error messages: {sum(1 for r in rows if r.get('db_error_message'))}") if temporal_client: print(f" - With Temporal error messages: {sum(1 for r in rows if r.get('temporal_error_message'))}") else: diff --git a/scripts/migrate_sqlite_to_postgres.py b/scripts/migrate_sqlite_to_postgres.py new file mode 100644 index 0000000..dc583c0 --- /dev/null +++ b/scripts/migrate_sqlite_to_postgres.py @@ -0,0 +1,848 @@ +#!/usr/bin/env python3 +"""One-off migration of the legacy SQLite store into Postgres. + +Run in the maintenance window, after the Postgres migrations are applied and +with the api and worker stopped. The SQLite file is opened read-only and is +never modified, so rolling back is redeploying the previous images on it. + + python scripts/migrate_sqlite_to_postgres.py --dry-run + python scripts/migrate_sqlite_to_postgres.py + +Every migrated row gets a deterministic UUID derived from its legacy key, so a +second run finds the rows already there and changes nothing. A document is +migrated in one transaction. ``--dry-run`` runs everything and rolls it back. + +Mapping: one legacy ``documents`` row becomes one document plus its main +workflow (``temporal_workflow_id`` = the legacy ``workflow_id``); each +``document_jobs`` row becomes a workflow; ``scheme_catalog_*`` becomes +``catalog_entries``; ``document_manifest_entries`` is dropped. +""" + +import argparse +import json +import os +import re +import sqlite3 +import sys +import uuid +from collections import Counter +from datetime import date, datetime, timezone +from pathlib import Path +from typing import Any, Optional + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import psycopg # noqa: E402 +from psycopg.rows import dict_row # noqa: E402 +from psycopg.types.json import Jsonb # noqa: E402 + +from pipeline import db, migrate # noqa: E402 +from pipeline.models import ( # noqa: E402 + ChunkingRun, + DocumentMetadata, + DocumentSource, + IndexState, + PageMetadata, + PageTranslation, +) + +_NAMESPACE = uuid.UUID("3f2b6c0e-5d1a-4a53-9a77-0c6a5c1d7e10") +_FINGERPRINT_RE = re.compile(r"^[0-9a-f]{32}$") +_CODE_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") +_KIND_RE = re.compile(r"^[a-z0-9]+([_-][a-z0-9]+)*$") +_STAGES = { + "registered", "ocr_processing", "ocr_review", "translation_processing", + "translation_review", "chunking", "chunk_review", "ready_for_ingestion", + "ingesting", "publishing_to_network", "approval_for_prod", "ingesting_prod", + "completed", "failed", +} +_WORKFLOW_TYPES = { + "pipeline", "ocr_only", "ocr_retry", "translation_retry", "chunking_retry", + "reingest", "promote_prod", +} +_WORKFLOW_STATUSES = {"running", "waiting_review", "completed", "failed"} +_REVIEW_STAGES = { + "ocr_review", "translation_review", "chunk_review", "ready_for_ingestion", "approval_for_prod", +} +_ENTITY_TYPES = {"document", "page", "chunk", "setting", "catalog"} + +_LEGACY_SETTING_KEYS = { + "search_method": "searchMethod", + "search_limit": "limit", + "search_alpha": "alpha", + "search_ranking_method": "rankingMethod", + "search_show_highlights": "showHighlights", + "search_ef_search": "efSearch", + "search_index_name": "indexName", + "search_candidate_cap": "candidateCap", + "search_candidate_multiplier": "candidateMultiplier", + "search_max_chunks_per_doc": "maxChunksPerDoc", + "search_use_e5_prefix": "useE5Prefix", + "search_exclude_reference": "excludeReference", + "search_query_expansion_profile": "queryExpansionProfile", + "search_rerank_mode": "rerankMode", + "search_hybrid_rrfk": "hybridRrfK", +} +_BOOL_SETTINGS = {"showHighlights", "useE5Prefix", "excludeReference"} +_INT_SETTINGS = { + "limit", "efSearch", "candidateCap", "candidateMultiplier", "maxChunksPerDoc", "hybridRrfK", +} +_FLOAT_SETTINGS = {"alpha"} + + +class Report: + """Per-table counts: rows read from SQLite and what became of them.""" + + def __init__(self) -> None: + self.tables: dict[str, Counter] = {} + self.skip_reasons: list[dict[str, Any]] = [] + self.notes: list[str] = [] + + def count(self, table: str, outcome: str, n: int = 1) -> None: + self.tables.setdefault(table, Counter())[outcome] += n + + def skip(self, table: str, key: str, reason: str) -> None: + self.count(table, "skipped") + self.skip_reasons.append({"table": table, "key": key, "reason": reason}) + + def mismatches(self) -> list[str]: + problems = [] + for table, counts in sorted(self.tables.items()): + accounted = counts["inserted"] + counts["existing"] + counts["skipped"] + counts["dropped"] + if counts["read"] != accounted: + problems.append(f"{table}: read {counts['read']} but accounted for {accounted}") + return problems + + def as_dict(self) -> dict[str, Any]: + return { + "tables": {t: dict(c) for t, c in sorted(self.tables.items())}, + "skipped": self.skip_reasons, + "notes": self.notes, + "mismatches": self.mismatches(), + } + + def render(self) -> str: + lines = [f"{'table':<28}{'read':>8}{'inserted':>10}{'existing':>10}{'skipped':>9}{'dropped':>9}"] + for table, c in sorted(self.tables.items()): + lines.append( + f"{table:<28}{c['read']:>8}{c['inserted']:>10}{c['existing']:>10}" + f"{c['skipped']:>9}{c['dropped']:>9}" + ) + for item in self.skip_reasons: + lines.append(f" skipped {item['table']} {item['key']}: {item['reason']}") + lines.extend(f" note: {n}" for n in self.notes) + problems = self.mismatches() + lines.append("verification: OK" if not problems else "verification: FAILED") + lines.extend(f" {p}" for p in problems) + return "\n".join(lines) + + +def det_id(kind: str, key: str) -> str: + return str(uuid.uuid5(_NAMESPACE, f"{kind}:{key}")) + + +def parse_ts(value: Any) -> Optional[datetime]: + if not value: + return None + try: + parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def parse_date(value: Any) -> Optional[date]: + if not value: + return None + try: + return date.fromisoformat(str(value)[:10]) + except ValueError: + return None + + +def load_json(text: Any, default: Any) -> Any: + if text in (None, ""): + return default + try: + return json.loads(text) + except (TypeError, ValueError): + return default + + +def truthy(value: Any, default: bool = False) -> bool: + if value is None: + return default + return bool(int(value)) + + +def _default_instance() -> str: + return (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + + +def _workflow_status_for_stage(stage: str) -> str: + if stage == "completed": + return "completed" + if stage == "failed": + return "failed" + return "waiting_review" if stage in _REVIEW_STAGES else "running" + + +def _index_key(index_name: str) -> str: + prod_names = { + n.strip() + for n in ( + os.environ.get("PROD_VECTOR_DB_COLLECTION_NAME"), + os.environ.get("PROD_SCHEME_VECTOR_DB_COLLECTION_NAME"), + ) + if n and n.strip() + } + return "prod" if index_name in prod_names else "dev" + + +def _has_table(sq: sqlite3.Connection, name: str) -> bool: + return sq.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (name,) + ).fetchone() is not None + + +def _rows(sq: sqlite3.Connection, table: str, where: str = "", params: tuple = ()) -> list[sqlite3.Row]: + if not _has_table(sq, table): + return [] + return sq.execute(f"SELECT * FROM {table} {where}", params).fetchall() + + +def _get(row: sqlite3.Row, key: str, default: Any = None) -> Any: + return row[key] if key in row.keys() else default + + +# ----------------------------------------------------------------------------- +# Catalog +# ----------------------------------------------------------------------------- + +def migrate_catalog_registry(sq, conn, report: Report) -> None: + for row in _rows(sq, "scheme_catalog_entries"): + report.count("scheme_catalog_entries", "read") + code = (row["scheme_code"] or "").strip().lower() + if not _CODE_RE.match(code): + report.skip("scheme_catalog_entries", str(row["scheme_code"]), "code is not a valid slug") + continue + aliases = [a for a in load_json(_get(row, "scheme_aliases_json"), []) if isinstance(a, str) and a] + source = _get(row, "source") if _get(row, "source") in ("bootstrap", "pipeline", "import") else "pipeline" + inserted = conn.execute( + """ + INSERT INTO catalog_entries (id, type, code, name, aliases, metadata) + VALUES (%s::uuid, 'scheme', %s, %s, %s, %s) + ON CONFLICT (type, code) DO NOTHING RETURNING id + """, + (det_id("entry", f"scheme:{code}"), code, row["scheme_name"] or code, aliases, + Jsonb({"source": source})), + ).fetchone() + report.count("scheme_catalog_entries", "inserted" if inserted else "existing") + + +def migrate_catalog_version(sq, conn, report: Report) -> None: + rows = _rows(sq, "scheme_catalog_meta") + legacy_version = int(rows[0]["version"] or 0) if rows else 0 + if rows: + report.count("scheme_catalog_meta", "read") + report.count("scheme_catalog_meta", "inserted") + for n in range(1, legacy_version + 1): + conn.execute( + """ + INSERT INTO audit_logs (id, action_type, entity_type, metadata) + VALUES (%s::uuid, %s, 'catalog', %s) ON CONFLICT (id) DO NOTHING + """, + (det_id("catalog-bump", str(n)), db.CATALOG_VERSION_ACTION, + Jsonb({"reason": "sqlite_migration:carried_over"})), + ) + conn.execute( + """ + INSERT INTO audit_logs (id, action_type, entity_type, metadata) + VALUES (%s::uuid, %s, 'catalog', %s) ON CONFLICT (id) DO NOTHING + """, + (det_id("catalog-bump", "migration"), db.CATALOG_VERSION_ACTION, + Jsonb({"reason": "sqlite_migration"})), + ) + + +def _ensure_doc_catalog_entry(conn, kind: str, code: str, name: str, aliases: list[str]) -> str: + conn.execute( + """ + INSERT INTO catalog_entries (id, type, code, name, aliases, metadata) + VALUES (%s::uuid, %s, %s, %s, %s, %s) ON CONFLICT (type, code) DO NOTHING + """, + (det_id("entry", f"{kind}:{code}"), kind, code, name or code, aliases, + Jsonb({"source": "pipeline"})), + ) + return str( + conn.execute( + "SELECT id FROM catalog_entries WHERE type = %s AND code = %s", (kind, code) + ).fetchone()["id"] + ) + + +# ----------------------------------------------------------------------------- +# Documents +# ----------------------------------------------------------------------------- + +def _user_for(conn, user_id, username, email, roles) -> Optional[str]: + if not (user_id or "").strip(): + return None + return db.upsert_user(user_id, username, email, roles, conn=conn) + + +def _build_index_status(sq, workflow_id: str) -> dict[str, dict]: + entries: dict[str, dict] = {} + for row in _rows(sq, "document_index_status", "WHERE workflow_id = ?", (workflow_id,)): + key = _index_key(row["index_name"]) + details = load_json(row["details_json"], {}) + state = IndexState( + status=row["status"] if row["status"] in ("unknown", "indexed", "missing") else "unknown", + index_name=row["index_name"], + doc_id=_get(row, "doc_id") or _get(row, "marqo_doc_id"), + chunk_count_indexed=max(int(row["chunk_count_indexed"] or 0), 0), + last_indexed_at=row["last_indexed_at"], + last_verified_at=row["last_verified_at"], + schema_version=row["schema_version"], + details=details if isinstance(details, dict) else {}, + ).model_dump() + current = entries.get(key) + if current is None or (state["last_indexed_at"] or "") >= (current["last_indexed_at"] or ""): + entries[key] = state + return entries + + +def _build_workflows( + sq, doc: sqlite3.Row, started_by: Optional[str], report: Report +) -> tuple[list[dict], dict[Any, str]]: + workflow_id = doc["workflow_id"] + stage = doc["stage"] if doc["stage"] in _STAGES else "registered" + created_at = parse_ts(doc["created_at"]) or datetime.now(timezone.utc) + stamps = { + key: value + for key, value in ( + ("ocr", _get(doc, "ocr_completed_at")), + ("translation", _get(doc, "translation_completed_at")), + ("chunks", _get(doc, "chunks_completed_at")), + ) + if value + } + + workflows: list[dict] = [] + job_ids: dict[Any, str] = {} + has_root = False + for job in _rows(sq, "document_jobs", "WHERE workflow_id = ? ORDER BY started_at, id", (workflow_id,)): + job_type = job["job_type"] + if job_type not in _WORKFLOW_TYPES: + report.skip("document_jobs", f"{workflow_id}#{job['id']}", f"unknown job_type {job_type!r}") + continue + temporal_id = job["temporal_workflow_id"] or workflow_id + is_root = job_type in ("pipeline", "ocr_only") and temporal_id == workflow_id and not has_root + has_root = has_root or is_root + wf_id = det_id("workflow", f"{workflow_id}:job:{job['id']}") + job_ids[job["id"]] = wf_id + status = job["status"] if job["status"] in _WORKFLOW_STATUSES else "failed" + job_stage = job["current_stage"] if job["current_stage"] in _STAGES else stage + config = load_json(job["config_json"], {}) + config = config if isinstance(config, dict) else {} + if job["temporal_run_id"]: + config["temporal_run_id"] = job["temporal_run_id"] + workflows.append({ + "id": wf_id, "type": job_type, "temporal_id": temporal_id, "status": status, + "stage": job_stage, "stamps": stamps if is_root else {}, + "stop_after_ocr": truthy(_get(doc, "stop_after_ocr")) and is_root, + "error_message": job["error_message"], "config": config, + "started_by": started_by if is_root else None, + "started_at": parse_ts(job["started_at"]) or created_at, + "completed_at": parse_ts(job["completed_at"]), "job_id": job["id"], + "inserted_as": "document_jobs", + }) + + if not has_root: + workflows.append({ + "id": det_id("workflow", f"{workflow_id}:root"), + "type": "ocr_only" if truthy(_get(doc, "stop_after_ocr")) else "pipeline", + "temporal_id": workflow_id, "status": _workflow_status_for_stage(stage), "stage": stage, + "stamps": stamps, "stop_after_ocr": truthy(_get(doc, "stop_after_ocr")), + "error_message": doc["error_message"], "config": {}, "started_by": started_by, + "started_at": created_at, + "completed_at": parse_ts(_get(doc, "ingested_at")) if stage == "completed" else None, + "job_id": None, "inserted_as": None, + }) + + requested_at = parse_ts(_get(doc, "prod_ready_requested_at")) + if requested_at and stage not in ("completed", "ingesting_prod"): + workflows.append({ + "id": det_id("workflow", f"{workflow_id}:prod-request"), "type": "promote_prod", + "temporal_id": f"{workflow_id}-prod-request", "status": "waiting_review", + "stage": "approval_for_prod", "stamps": {}, "stop_after_ocr": False, + "error_message": None, "config": {}, "started_by": None, "started_at": requested_at, + "completed_at": None, "job_id": None, "inserted_as": None, + "requested_by": ( + _get(doc, "prod_ready_requested_by_user_id"), + _get(doc, "prod_ready_requested_by_username"), + ), + }) + + workflows.sort(key=lambda w: w["started_at"]) + latest = workflows[-1] + latest["stage"] = stage + latest["error_message"] = doc["error_message"] + return workflows, job_ids + + +def migrate_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workflow: dict[str, list]) -> bool: + workflow_id = doc["workflow_id"] + doc_uuid = det_id("document", workflow_id) + exists = conn.execute("SELECT 1 FROM documents WHERE id = %s::uuid", (doc_uuid,)).fetchone() + if exists: + report.count("documents", "existing") + for table, label in ( + ("document_jobs", "document_jobs"), ("document_artifacts", "document_artifacts"), + ("pages", "pages"), ("chunks", "chunks"), + ): + report.count(label, "existing", len(_rows(sq, table, "WHERE workflow_id = ?", (workflow_id,)))) + return False + + fingerprint = next( + ( + c for c in ( + _get(doc, "source_file_fingerprint"), _get(doc, "canonical_document_id"), doc["document_id"] + ) if c and _FINGERPRINT_RE.match(c) + ), + None, + ) + if fingerprint is None: + import hashlib + + fingerprint = hashlib.md5((doc["document_id"] or workflow_id).encode()).hexdigest() + report.notes.append(f"{workflow_id}: no hex fingerprint, derived md5 of document_id") + + page_count = max(int(doc["page_count"] or 0), 0) + chunk_rows = _rows(sq, "chunks", "WHERE workflow_id = ? ORDER BY chunk_number", (workflow_id,)) + kind = (_get(doc, "document_kind") or "document").strip().lower() + if not _KIND_RE.match(kind): + report.notes.append(f"{workflow_id}: invalid document_kind {kind!r}, stored as 'document'") + kind = "document" + + chunking = None + if chunk_rows: + first = chunk_rows[0] + chunking = ChunkingRun( + run_number=max(max(int(r["chunk_version"] or 1) for r in chunk_rows), 1), + provider=_get(first, "chunking_provider"), + model=_get(first, "chunking_model"), + config=load_json(_get(first, "chunking_config_json"), {}), + ) + metadata = DocumentMetadata( + display_name=_get(doc, "display_name") or None, + catalog_visible=truthy(_get(doc, "catalog_visible"), True), + network_visible=truthy(_get(doc, "network_visible"), True), + is_demo=truthy(_get(doc, "is_demo")), + chunk_count=len(chunk_rows) if chunk_rows else max(int(doc["chunk_count"] or 0), 0), + reindex_required=truthy(_get(doc, "reindex_required")), + reindex_reason=_get(doc, "reindex_reason") or None, + chunking=chunking, + ).model_dump(exclude_none=True) + source = DocumentSource( + fingerprint=fingerprint, + original_filename=_get(doc, "source_filename") or doc["filename"], + storage_uri=doc["filepath"], + source_type=_get(doc, "source_type"), + canonical_input_type=_get(doc, "canonical_input_type"), + page_count=page_count, + ).model_dump(exclude_none=True) + + valid_from, valid_to = parse_date(_get(doc, "valid_from")), parse_date(_get(doc, "valid_to")) + if valid_from and valid_to and valid_from > valid_to: + report.notes.append(f"{workflow_id}: valid_from after valid_to, valid_to cleared") + valid_to = None + + uploader = _user_for( + conn, _get(doc, "uploaded_by_user_id"), _get(doc, "uploaded_by_username"), + _get(doc, "uploaded_by_email"), _get(doc, "uploaded_by_roles"), + ) + + entry_id = None + code = (_get(doc, "scheme_code") or "").strip().lower() + if code and kind != "document": + if _CODE_RE.match(code): + aliases = [a for a in load_json(_get(doc, "scheme_aliases_json"), []) if isinstance(a, str) and a] + entry_id = _ensure_doc_catalog_entry( + conn, kind, code, _get(doc, "scheme_name") or code, aliases + ) + else: + report.notes.append(f"{workflow_id}: scheme_code {code!r} is not a valid slug, not linked") + elif code: + report.notes.append(f"{workflow_id}: scheme_code {code!r} on a plain document, not linked") + + created_at = parse_ts(doc["created_at"]) or datetime.now(timezone.utc) + updated_at = parse_ts(doc["updated_at"]) or created_at + conn.execute( + """ + INSERT INTO documents (id, instance, source, document_kind, catalog_entry_id, valid_from, + valid_to, is_disabled, index_status, metadata, uploaded_by_user_id, + created_at, updated_at) + VALUES (%s::uuid, %s, %s, %s, %s::uuid, %s, %s, %s, %s, %s, %s::uuid, %s, %s) + """, + ( + doc_uuid, (_get(doc, "instance") or _default_instance()).strip().lower() or _default_instance(), + Jsonb(source), kind, entry_id, valid_from, valid_to, truthy(_get(doc, "is_disabled")), + Jsonb(_build_index_status(sq, workflow_id)), Jsonb(metadata), uploader, created_at, + updated_at, + ), + ) + report.count("documents", "inserted") + + workflows, job_ids = _build_workflows(sq, doc, uploader, report) + root = next((w for w in workflows if w["type"] in ("pipeline", "ocr_only")), workflows[0]) + for w in workflows: + started_by = w["started_by"] + if w.get("requested_by"): + started_by = _user_for(conn, w["requested_by"][0], w["requested_by"][1], None, None) + conn.execute( + """ + INSERT INTO workflows (id, document_id, workflow_type, temporal_workflow_id, status, + current_stage, stage_timestamps, stop_after_ocr, error_message, + config, started_by_user_id, started_at, completed_at, updated_at) + VALUES (%s::uuid, %s::uuid, %s::workflow_type_enum, %s, %s::workflow_status_enum, + %s::workflow_stage, %s, %s, %s, %s, %s::uuid, %s, %s, %s) + """, + ( + w["id"], doc_uuid, w["type"], w["temporal_id"], w["status"], w["stage"], + Jsonb(w["stamps"]), w["stop_after_ocr"], w["error_message"], Jsonb(w["config"]), + started_by, w["started_at"], w["completed_at"], w["completed_at"] or w["started_at"], + ), + ) + if w["inserted_as"]: + report.count("document_jobs", "inserted") + + for art in _rows(sq, "document_artifacts", "WHERE workflow_id = ? ORDER BY id", (workflow_id,)): + size = art["size_bytes"] + art_metadata = load_json(art["metadata_json"], {}) + conn.execute( + """ + INSERT INTO workflow_artifacts (id, workflow_id, artifact_type, stage, storage_uri, + mime_type, filename, size_bytes, metadata, created_at) + VALUES (%s::uuid, %s::uuid, %s, %s::workflow_stage, %s, %s, %s, %s, %s, %s) + """, + ( + det_id("artifact", f"{workflow_id}:{art['id']}"), + job_ids.get(art["job_id"], root["id"]), + art["artifact_type"], art["stage"] if art["stage"] in _STAGES else None, + art["storage_uri"], art["mime_type"], art["filename"], + size if size is not None and size >= 0 else None, + Jsonb(art_metadata if isinstance(art_metadata, dict) else {}), + parse_ts(art["created_at"]) or created_at, + ), + ) + report.count("document_artifacts", "inserted") + + for page in _rows(sq, "pages", "WHERE workflow_id = ? ORDER BY page_number", (workflow_id,)): + if int(page["page_number"] or 0) < 1: + report.skip("pages", f"{workflow_id}#{page['page_number']}", "page_number < 1") + continue + translations = {} + if any( + _get(page, k) not in (None, "") + for k in ("translated_markdown", "edited_translation", "translation_notes") + ) or truthy(_get(page, "translation_reviewed")): + translations[_get(page, "translation_target_language") or "en"] = PageTranslation( + translated_markdown=_get(page, "translated_markdown"), + edited_translation=_get(page, "edited_translation"), + is_reviewed=truthy(_get(page, "translation_reviewed")), + reviewer_notes=_get(page, "translation_notes"), + provider=_get(page, "translation_provider"), + model=_get(page, "translation_model"), + translated_at=_get(page, "translated_at"), + ).model_dump() + page_meta = PageMetadata( + is_reviewed=truthy(_get(page, "is_reviewed")), + reviewer_notes=_get(page, "reviewer_notes"), + detected_language=_get(page, "detected_language"), + ).model_dump(exclude_none=True) + conn.execute( + """ + INSERT INTO pages (id, document_id, page_number, original_markdown, edited_markdown, + translations, metadata) + VALUES (%s::uuid, %s::uuid, %s, %s, %s, %s, %s) + """, + ( + det_id("page", f"{workflow_id}:{page['page_number']}"), doc_uuid, page["page_number"], + page["original_markdown"], page["edited_markdown"], Jsonb(translations), + Jsonb(page_meta), + ), + ) + report.count("pages", "inserted") + + for chunk in chunk_rows: + if int(chunk["chunk_number"] or 0) < 1: + report.skip("chunks", f"{workflow_id}#{chunk['chunk_number']}", "chunk_number < 1") + continue + page_start = max(int(chunk["page_start"] or 1), 1) + spans = load_json(_get(chunk, "source_spans_json"), []) + conn.execute( + """ + INSERT INTO chunks (id, document_id, chunk_number, original_text, edited_text, + token_count, page_start, page_end, source_spans, section_title, + content_type, is_reference, is_reviewed, is_excluded, reviewer_notes) + VALUES (%s::uuid, %s::uuid, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + """, + ( + det_id("chunk", f"{workflow_id}:{chunk['chunk_number']}"), doc_uuid, + chunk["chunk_number"], chunk["original_text"] or "", chunk["edited_text"], + max(int(chunk["token_count"] or 0), 0), page_start, + max(int(chunk["page_end"] or page_start), page_start), + Jsonb(spans if isinstance(spans, list) else []), chunk["section_title"], + chunk["content_type"], truthy(_get(chunk, "is_reference")), + truthy(_get(chunk, "is_reviewed")), truthy(_get(chunk, "is_excluded")), + chunk["reviewer_notes"], + ), + ) + report.count("chunks", "inserted") + + for audit in audit_by_workflow.pop(workflow_id, []): + insert_audit(conn, audit, doc_uuid, source_instance=(_get(doc, "instance") or _default_instance()), report=report) + return True + + +# ----------------------------------------------------------------------------- +# Audit, settings, OTPs, webhooks +# ----------------------------------------------------------------------------- + +def _legacy_audit_field(field_name: Optional[str], entity: Optional[str]) -> Optional[str]: + if entity == "setting" and field_name in _LEGACY_SETTING_KEYS: + flat = _LEGACY_SETTING_KEYS[field_name] + return f"{db._SETTING_GROUPS[flat][0]}.{flat}" + return field_name + + +def insert_audit(conn, audit: sqlite3.Row, doc_uuid: Optional[str], source_instance: Optional[str], report: Report) -> None: + entity = audit["entity_type"] if audit["entity_type"] in _ENTITY_TYPES else None + workflow_id = audit["workflow_id"] + if audit["document_id"] == "__settings__": + entity, workflow_id = "setting", None + elif workflow_id == "__catalog__": + entity, workflow_id = "catalog", None + metadata = load_json(audit["metadata"], None) + if not isinstance(metadata, dict): + metadata = {"raw": audit["metadata"]} if audit["metadata"] else {} + roles = [r.strip() for r in (_get(audit, "actor_roles") or "").split(",") if r.strip()] + actor_present = any((_get(audit, "actor_user_id"), _get(audit, "actor_username"), _get(audit, "actor_email"), roles)) + actor_uuid = _user_for( + conn, _get(audit, "actor_user_id"), _get(audit, "actor_username"), _get(audit, "actor_email"), roles + ) + inserted = conn.execute( + """ + INSERT INTO audit_logs (id, document_id, workflow_id, instance, action_type, entity_type, + entity_number, field_name, old_value, new_value, metadata, + actor_user_id, actor, created_at) + VALUES (%s::uuid, %s::uuid, %s, %s, %s, %s::audit_entity_type, %s, %s, %s, %s, %s, + %s::uuid, %s, %s) + ON CONFLICT (id) DO NOTHING RETURNING id + """, + ( + det_id("audit", str(audit["id"])), doc_uuid, workflow_id, + (source_instance or "").strip().lower() or None if doc_uuid else None, + audit["action_type"], entity, audit["entity_id"], + _legacy_audit_field(audit["field_name"], entity), audit["old_value"], audit["new_value"], + Jsonb(metadata), actor_uuid, + Jsonb({ + "username": _get(audit, "actor_username") or None, + "email": _get(audit, "actor_email") or None, + "roles": roles, + }) if actor_present else Jsonb({}), + parse_ts(audit["timestamp"]) or datetime.now(timezone.utc), + ), + ).fetchone() + report.count("audit_logs", "inserted" if inserted else "existing") + + +def migrate_settings(sq, conn, report: Report) -> None: + rows = _rows(sq, "settings") + for _ in rows: + report.count("settings", "read") + current = conn.execute( + "SELECT retrieval, candidates, results, query FROM search_settings WHERE singleton" + ).fetchone() + if current is None: + conn.execute("INSERT INTO search_settings DEFAULT VALUES") + current = {"retrieval": {}, "candidates": {}, "results": {}, "query": {}} + if any(current[g] for g in ("retrieval", "candidates", "results", "query")): + for _ in rows: + report.count("settings", "existing") + report.notes.append("search_settings already configured in Postgres, legacy settings not applied") + return + + groups: dict[str, dict[str, Any]] = {} + for row in rows: + flat = _LEGACY_SETTING_KEYS.get(row["key"]) + if flat is None: + report.skip("settings", row["key"], "unknown setting key") + continue + raw = row["value"] + try: + value: Any = ( + raw == "true" if flat in _BOOL_SETTINGS + else int(raw) if flat in _INT_SETTINGS + else float(raw) if flat in _FLOAT_SETTINGS + else raw + ) + except ValueError: + report.skip("settings", row["key"], f"unparseable value {raw!r}") + continue + group, model = db._SETTING_GROUPS[flat] + try: + model.model_validate({flat: value}) + except Exception as exc: + report.skip("settings", row["key"], f"invalid value: {exc}") + continue + groups.setdefault(group, {})[flat] = value + report.count("settings", "inserted") + for group, values in groups.items(): + conn.execute( + f"UPDATE search_settings SET {group} = {group} || %s::jsonb WHERE singleton", (Jsonb(values),) + ) + + +def migrate_email_otps(sq, conn, report: Report) -> None: + now = datetime.now(timezone.utc) + for row in _rows(sq, "email_otps"): + report.count("email_otps", "read") + expires = parse_ts(row["expires_at"]) + if expires is None or expires <= now: + report.count("email_otps", "dropped") + continue + inserted = conn.execute( + """ + INSERT INTO email_otps (email, code_hash, expires_at, attempts, last_sent_at) + VALUES (%s, %s, %s, %s, %s) ON CONFLICT (email) DO NOTHING RETURNING id + """, + (row["email"].strip().lower(), row["code_hash"], expires, max(int(row["attempts"] or 0), 0), + parse_ts(row["last_sent_at"]) or now), + ).fetchone() + report.count("email_otps", "inserted" if inserted else "existing") + + +def migrate_webhook_records(sq, conn, report: Report) -> None: + for row in _rows(sq, "webhook_records"): + report.count("webhook_records", "read") + payload = load_json(row["payload"], None) + if not isinstance(payload, dict): + report.skip("webhook_records", f"{row['source']}/{row['external_id']}", "payload is not a JSON object") + continue + try: + record_id = str(uuid.UUID(row["id"])) + except (ValueError, TypeError): + record_id = det_id("webhook", row["id"] or f"{row['source']}:{row['external_id']}") + published = row["is_published"] + inserted = conn.execute( + """ + INSERT INTO webhook_records (id, source, external_id, title, slug, status, is_published, + category, payload, payload_hash, created_at, updated_at) + VALUES (%s::uuid, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (source, external_id) DO NOTHING RETURNING id + """, + ( + record_id, row["source"], row["external_id"], row["title"], row["slug"], row["status"], + None if published is None else bool(published), row["category"], Jsonb(payload), + row["payload_hash"], parse_ts(row["created_at"]) or datetime.now(timezone.utc), + parse_ts(row["updated_at"]) or datetime.now(timezone.utc), + ), + ).fetchone() + report.count("webhook_records", "inserted" if inserted else "existing") + + +# ----------------------------------------------------------------------------- +# Driver +# ----------------------------------------------------------------------------- + +def run_migration(sqlite_path: str, database_url: str, dry_run: bool = False) -> Report: + report = Report() + sq = sqlite3.connect(f"file:{sqlite_path}?mode=ro", uri=True) + sq.row_factory = sqlite3.Row + try: + if not dry_run: + migrate.apply_migrations(database_url) + elif not migrate.is_current(database_url): + raise SystemExit("Postgres migrations are not applied; apply them before a dry run.") + + with psycopg.connect(database_url, autocommit=not dry_run, row_factory=dict_row) as conn: + conn.execute("SET timezone = 'UTC'") + documents = _rows(sq, "documents", "ORDER BY created_at, workflow_id") + known_workflows = {d["workflow_id"] for d in documents} + for table in ("document_jobs", "document_artifacts", "pages", "chunks"): + if _has_table(sq, table): + total = sq.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] + stray = sq.execute( + f"SELECT COUNT(*) FROM {table} WHERE workflow_id NOT IN " + "(SELECT workflow_id FROM documents)" + ).fetchone()[0] + report.count(table, "read", total) + if stray: + report.count(table, "skipped", stray) + report.notes.append(f"{table}: {stray} rows belong to no document and were not migrated") + + audit_by_workflow: dict[str, list] = {} + orphan_audit: list = [] + for audit in _rows(sq, "audit_logs", "ORDER BY id"): + report.count("audit_logs", "read") + if audit["workflow_id"] in known_workflows and audit["document_id"] != "__settings__": + audit_by_workflow.setdefault(audit["workflow_id"], []).append(audit) + else: + orphan_audit.append(audit) + + with conn.transaction(): + migrate_catalog_registry(sq, conn, report) + + for doc in documents: + report.count("documents", "read") + with conn.transaction(): + migrate_document(sq, conn, doc, report, audit_by_workflow) + + with conn.transaction(): + for audits in audit_by_workflow.values(): + orphan_audit.extend(audits) + for audit in orphan_audit: + insert_audit(conn, audit, None, None, report) + migrate_settings(sq, conn, report) + migrate_email_otps(sq, conn, report) + migrate_webhook_records(sq, conn, report) + migrate_catalog_version(sq, conn, report) + + manifest = _rows(sq, "document_manifest_entries") + if manifest: + report.count("document_manifest_entries", "read", len(manifest)) + report.count("document_manifest_entries", "dropped", len(manifest)) + report.notes.append("document_manifest_entries is not part of the schema and was not migrated") + + if dry_run: + conn.rollback() + finally: + sq.close() + return report + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--sqlite-path", default=os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db")) + parser.add_argument("--database-url", default=os.environ.get("DATABASE_URL", db.DEFAULT_DATABASE_URL)) + parser.add_argument("--dry-run", action="store_true", help="run everything, then roll back") + parser.add_argument("--report", help="write the report as JSON to this path") + args = parser.parse_args() + + if not os.path.exists(args.sqlite_path): + print(f"SQLite database not found at {args.sqlite_path}", file=sys.stderr) + return 2 + + report = run_migration(args.sqlite_path, args.database_url, dry_run=args.dry_run) + print(("DRY RUN - nothing was written\n" if args.dry_run else "") + report.render()) + if args.report: + Path(args.report).write_text(json.dumps(report.as_dict(), indent=2, default=str)) + return 1 if report.mismatches() else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/start_manifest_translation_batch.py b/scripts/start_manifest_translation_batch.py deleted file mode 100644 index 2c9c95c..0000000 --- a/scripts/start_manifest_translation_batch.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python3 -"""Start translation-only workflows for manifest-backed Gujarati/mixed documents in OCR review.""" - -from __future__ import annotations - -import argparse -import json -import os -import sqlite3 -import sys -import time -from datetime import datetime -from pathlib import Path - -from temporalio.client import Client - -PROJECT_ROOT = Path(__file__).resolve().parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -from pipeline import db # noqa: E402 -from pipeline.workflows import TranslationOnlyWorkflow # noqa: E402 - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--temporal-host", default=os.environ.get("TEMPORAL_HOST", "localhost:7233")) - parser.add_argument("--task-queue", default=os.environ.get("TASK_QUEUE", "ocr-pipeline")) - parser.add_argument("--db-path", default=os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db")) - parser.add_argument("--limit", type=int, default=0) - parser.add_argument( - "--selection", - choices=["manifest_gujarati", "all_ocr_review"], - default="manifest_gujarati", - help="Choose whether to start only manifest rows marked Gujarati or any authoritative OCR-review docs.", - ) - parser.add_argument("--report-path", default="/tmp/start_manifest_translation_batch_report.json") - return parser.parse_args() - - -def eligible_documents(db_path: str, selection: str, limit: int = 0) -> list[dict]: - conn = sqlite3.connect(db_path) - conn.row_factory = sqlite3.Row - if selection == "all_ocr_review": - query = """ - SELECT d.workflow_id, d.document_id, d.filename, d.source_manifest_name, m.doc_language - FROM documents d - LEFT JOIN document_manifest_entries m - ON m.manifest_filename = d.source_manifest_name - WHERE d.source_manifest_name IS NOT NULL - AND d.stage = 'ocr_review' - AND d.translation_completed_at IS NULL - ORDER BY d.updated_at DESC, d.workflow_id - """ - else: - query = """ - SELECT d.workflow_id, d.document_id, d.filename, d.source_manifest_name, m.doc_language - FROM documents d - LEFT JOIN document_manifest_entries m - ON m.manifest_filename = d.source_manifest_name - WHERE d.source_manifest_name IS NOT NULL - AND d.stage = 'ocr_review' - AND d.translation_completed_at IS NULL - AND LOWER(COALESCE(m.doc_language, '')) LIKE '%gujarati%' - ORDER BY d.updated_at DESC, d.workflow_id - """ - rows = conn.execute(query).fetchall() - docs = [dict(r) for r in rows] - return docs[:limit] if limit > 0 else docs - - -async def main() -> int: - args = parse_args() - db.init_db() - client = await Client.connect(args.temporal_host) - - docs = eligible_documents(args.db_path, args.selection, args.limit) - report: dict[str, object] = { - "started_at": datetime.utcnow().isoformat(), - "temporal_host": args.temporal_host, - "task_queue": args.task_queue, - "selection": args.selection, - "translation_provider": os.environ.get("TRANSLATION_PROVIDER", "gemma_vllm"), - "translation_model": os.environ.get("TRANSLATION_MODEL", "gemma-4"), - "eligible_documents": len(docs), - "submitted": 0, - "skipped": 0, - "failed": [], - } - - for doc in docs: - temporal_workflow_id = f"{doc['workflow_id']}-translation-{int(time.time())}" - try: - await client.start_workflow( - TranslationOnlyWorkflow.run, - args=[doc["workflow_id"], doc["document_id"], doc["filename"]], - id=temporal_workflow_id, - task_queue=args.task_queue, - ) - job_id = db.create_document_job( - workflow_id=doc["workflow_id"], - job_type="translation_only", - temporal_workflow_id=temporal_workflow_id, - status="running", - current_stage="translation_processing", - config={ - "translation_provider": os.environ.get("TRANSLATION_PROVIDER", "gemma_vllm"), - "translation_model": os.environ.get("TRANSLATION_MODEL", "gemma-4"), - "source": "manifest_ocr_review", - }, - ) - db.update_document_fields(doc["workflow_id"], latest_job_id=job_id) - report["submitted"] = int(report["submitted"]) + 1 - except Exception as exc: - report["failed"].append( - { - "workflow_id": doc["workflow_id"], - "filename": doc["filename"], - "error": str(exc), - } - ) - - report["finished_at"] = datetime.utcnow().isoformat() - Path(args.report_path).write_text(json.dumps(report, indent=2), encoding="utf-8") - print(json.dumps(report, indent=2)) - return 1 if report["failed"] else 0 - - -if __name__ == "__main__": - raise SystemExit(__import__("asyncio").run(main())) diff --git a/scripts/start_ocr_only_batch.py b/scripts/start_ocr_only_batch.py index 70682c0..701b0c0 100644 --- a/scripts/start_ocr_only_batch.py +++ b/scripts/start_ocr_only_batch.py @@ -4,14 +4,16 @@ import argparse import hashlib import json -import os -import sqlite3 import sys import urllib.error import urllib.parse import urllib.request from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from pipeline import db # noqa: E402 + ALLOWED_EXTENSIONS = { ".pdf", ".doc", ".docx", ".ppt", ".pptx", ".xls", ".xlsx", ".csv", @@ -31,31 +33,16 @@ def get_workflow_id(filepath: str) -> str: return f"doc-{hashlib.md5(filepath.encode()).hexdigest()[:12]}" -def already_has_ocr(conn: sqlite3.Connection, filepath: str, fingerprint: str) -> bool: - workflow_id = get_workflow_id(filepath) - row = conn.execute( - """ - SELECT workflow_id, canonical_document_id, ocr_completed_at, stage - FROM documents - WHERE workflow_id = ? - OR canonical_document_id = ? - ORDER BY updated_at DESC, created_at DESC - LIMIT 1 - """, - (workflow_id, fingerprint), - ).fetchone() - if not row: +def already_has_ocr(filepath: str, fingerprint: str) -> bool: + doc = db.get_document(get_workflow_id(filepath)) or db.find_document_by_doc_identifier(fingerprint) + if not doc: return False - if row["ocr_completed_at"]: + if doc["ocr_completed_at"]: return True - if row["stage"] in {"ocr_review", "translation_processing", "translation_review", "chunking", + if doc["stage"] in {"ocr_review", "translation_processing", "translation_review", "chunking", "chunk_review", "ready_for_ingestion", "ingesting", "completed"}: return True - page_row = conn.execute( - "SELECT COUNT(*) AS cnt FROM pages WHERE workflow_id = ?", - (row["workflow_id"],), - ).fetchone() - return bool(page_row and page_row["cnt"] > 0) + return bool(db.get_saved_page_numbers(doc["workflow_id"])) def post_document(api_base: str, filepath: str) -> dict: @@ -76,7 +63,6 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("directory", help="Directory containing source docs") parser.add_argument("--api-base", default="http://localhost:8001", help="Docs-pipeline API base URL") - parser.add_argument("--db-path", default=os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db")) parser.add_argument("--limit", type=int, default=0, help="Optional max files to submit") args = parser.parse_args() @@ -85,9 +71,6 @@ def main() -> int: print(f"Directory not found: {directory}", file=sys.stderr) return 1 - conn = sqlite3.connect(args.db_path) - conn.row_factory = sqlite3.Row - files = sorted( p for p in directory.iterdir() if p.is_file() and p.suffix.lower() in ALLOWED_EXTENSIONS @@ -101,7 +84,7 @@ def main() -> int: for path in files: fingerprint = compute_fingerprint(path) - if already_has_ocr(conn, str(path), fingerprint): + if already_has_ocr(str(path), fingerprint): skipped += 1 print(f"SKIP {path.name}: OCR already present") continue diff --git a/scripts/verify_document_validity_e2e.py b/scripts/verify_document_validity_e2e.py index e2c1b66..f249d58 100644 --- a/scripts/verify_document_validity_e2e.py +++ b/scripts/verify_document_validity_e2e.py @@ -2,11 +2,12 @@ End-to-end check of document validity against a live Qdrant. Run against a local stack: uv run python scripts/verify_document_validity_e2e.py -Needs Qdrant on :6333 and the small embedding model cached; it creates and -deletes its own scratch collection and a throwaway SQLite file, and touches -neither the real index nor the real db. +Needs Qdrant on :6333, the small embedding model cached, and a scratch Postgres +(E2E_DATABASE_URL, migrations are applied to it). It creates and deletes its own +scratch collection and four documents named wf-*, and touches neither the real +index nor the real db. -Real path, no stubs on the parts that matter: real SQLite rows, the real +Real path, no stubs on the parts that matter: real Postgres rows, the real _prepare_records + QdrantVectorStore.upsert the ingest activity calls, real embeddings, a real Qdrant collection, and the real search filter. Only the embedding *model* is swapped for a small cached one so this does not download @@ -15,15 +16,15 @@ import os import sys -import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, REPO) -DB = os.path.join(tempfile.mkdtemp(), "e2e.db") COLLECTION = "e2e_validity_check" -os.environ["DOCUMENT_DB_PATH"] = DB +if not os.environ.get("E2E_DATABASE_URL"): + sys.exit("Set E2E_DATABASE_URL to a scratch Postgres database (it is never the real one).") +os.environ["DATABASE_URL"] = os.environ["E2E_DATABASE_URL"] os.environ["AUTH_DISABLED"] = "true" os.environ["VECTOR_DB_URL"] = "http://localhost:6333" os.environ["VECTOR_DB_COLLECTION_NAME"] = COLLECTION @@ -39,8 +40,10 @@ from pipeline.activities import _prepare_records, _validity_fields_from_doc # noqa: E402 from pipeline.vector_store.qdrant_store import QdrantVectorStore # noqa: E402 -db.DB_PATH = DB db.init_db() +SCRATCH_DOCUMENTS = ("wf-active", "wf-expired", "wf-future", "wf-legacy") +for scratch in SCRATCH_DOCUMENTS: + db.purge_document(scratch, keep_audit=False) PASS, FAIL = [], [] @@ -232,5 +235,7 @@ def search_docs(**kwargs): check("the shortened period takes it out of today's results", search_docs(), ["wf-legacy"]) store.client.delete_collection(COLLECTION) +for scratch in SCRATCH_DOCUMENTS: + db.purge_document(scratch, keep_audit=False) print(f"\n{len(PASS)} passed, {len(FAIL)} failed") sys.exit(1 if FAIL else 0) diff --git a/tests/test_migrate_sqlite_to_postgres.py b/tests/test_migrate_sqlite_to_postgres.py new file mode 100644 index 0000000..e963846 --- /dev/null +++ b/tests/test_migrate_sqlite_to_postgres.py @@ -0,0 +1,393 @@ +"""scripts/migrate_sqlite_to_postgres.py: a legacy SQLite file lands in Postgres. + +The fixture file is built with the legacy schema DDL, the way a production +database looked at cutover. +""" + +import hashlib +import importlib.util +import json +import sqlite3 +from pathlib import Path + +import pytest + +_SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "migrate_sqlite_to_postgres.py" + +LEGACY_DDL = """ +CREATE TABLE documents ( + workflow_id TEXT PRIMARY KEY, document_id TEXT NOT NULL, canonical_document_id TEXT, + filename TEXT NOT NULL, display_name TEXT, source_filename TEXT, source_manifest_name TEXT, + source_file_fingerprint TEXT, filepath TEXT NOT NULL, stage TEXT NOT NULL DEFAULT 'registered', + page_count INTEGER DEFAULT 0, chunk_count INTEGER DEFAULT 0, error_message TEXT, + is_demo INTEGER DEFAULT 0, created_at TEXT, updated_at TEXT, ocr_completed_at TEXT, + translation_completed_at TEXT, chunks_completed_at TEXT, ingested_at TEXT, source_type TEXT, + canonical_input_type TEXT, stop_after_ocr INTEGER DEFAULT 0, reindex_required INTEGER DEFAULT 0, + reindex_reason TEXT, original_artifact_id INTEGER, normalized_artifact_id INTEGER, + latest_job_id INTEGER, is_disabled INTEGER DEFAULT 0, instance TEXT, uploaded_by_user_id TEXT, + uploaded_by_username TEXT, uploaded_by_email TEXT, uploaded_by_roles TEXT, + prod_ready_requested_at TEXT, prod_ready_requested_by_user_id TEXT, + prod_ready_requested_by_username TEXT, valid_from TEXT, valid_to TEXT, + document_kind TEXT DEFAULT 'document', scheme_code TEXT, scheme_name TEXT, + scheme_aliases_json TEXT, tool_routing TEXT, catalog_visible INTEGER DEFAULT 1, + network_visible INTEGER DEFAULT 1 +); +CREATE TABLE audit_logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, workflow_id TEXT NOT NULL, document_id TEXT NOT NULL, + action_type TEXT NOT NULL, entity_type TEXT, entity_id INTEGER, field_name TEXT, old_value TEXT, + new_value TEXT, metadata TEXT, timestamp TEXT NOT NULL, actor_user_id TEXT, actor_username TEXT, + actor_email TEXT, actor_roles TEXT +); +CREATE TABLE pages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, workflow_id TEXT NOT NULL, page_number INTEGER NOT NULL, + original_markdown TEXT, edited_markdown TEXT, is_reviewed INTEGER DEFAULT 0, reviewer_notes TEXT, + detected_language TEXT, translated_markdown TEXT, edited_translation TEXT, + translation_reviewed INTEGER DEFAULT 0, translation_notes TEXT, translation_provider TEXT, + translation_model TEXT, translation_target_language TEXT, translated_at TEXT, + UNIQUE(workflow_id, page_number) +); +CREATE TABLE chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, workflow_id TEXT NOT NULL, chunk_number INTEGER NOT NULL, + original_text TEXT, edited_text TEXT, token_count INTEGER DEFAULT 0, page_start INTEGER DEFAULT 1, + page_end INTEGER DEFAULT 1, source_page_numbers_json TEXT, source_spans_json TEXT, + section_title TEXT, content_type TEXT, is_reference INTEGER DEFAULT 0, chunking_provider TEXT, + chunking_model TEXT, chunking_config_json TEXT, chunking_run_id TEXT, chunk_version INTEGER DEFAULT 1, + is_reviewed INTEGER DEFAULT 0, is_excluded INTEGER DEFAULT 0, reviewer_notes TEXT, + UNIQUE(workflow_id, chunk_number) +); +CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT NOT NULL, description TEXT, updated_at TEXT NOT NULL); +CREATE TABLE document_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, workflow_id TEXT NOT NULL, job_type TEXT NOT NULL, + temporal_workflow_id TEXT, temporal_run_id TEXT, status TEXT NOT NULL DEFAULT 'running', + current_stage TEXT, started_at TEXT NOT NULL, completed_at TEXT, error_message TEXT, config_json TEXT +); +CREATE TABLE document_artifacts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, workflow_id TEXT NOT NULL, job_id INTEGER, + artifact_type TEXT NOT NULL, stage TEXT, storage_uri TEXT NOT NULL, mime_type TEXT, filename TEXT, + size_bytes INTEGER, metadata_json TEXT, created_at TEXT NOT NULL +); +CREATE TABLE document_index_status ( + workflow_id TEXT NOT NULL, index_name TEXT NOT NULL, doc_id TEXT, chunk_count_indexed INTEGER DEFAULT 0, + last_indexed_at TEXT, last_verified_at TEXT, schema_version TEXT, status TEXT NOT NULL DEFAULT 'unknown', + details_json TEXT, PRIMARY KEY (workflow_id, index_name) +); +CREATE TABLE document_manifest_entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, manifest_filename TEXT NOT NULL UNIQUE, title_en TEXT, + imported_at TEXT NOT NULL +); +CREATE TABLE email_otps ( + email TEXT PRIMARY KEY, code_hash TEXT NOT NULL, expires_at TEXT NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL, last_sent_at TEXT NOT NULL +); +CREATE TABLE webhook_records ( + id TEXT PRIMARY KEY, source TEXT NOT NULL DEFAULT 'webhook', external_id TEXT NOT NULL, title TEXT, + slug TEXT, status TEXT, is_published INTEGER, category TEXT, payload TEXT NOT NULL, payload_hash TEXT, + created_at TEXT NOT NULL, updated_at TEXT NOT NULL, UNIQUE (source, external_id) +); +CREATE TABLE scheme_catalog_meta ( + id INTEGER PRIMARY KEY CHECK (id = 1), version INTEGER NOT NULL DEFAULT 0, updated_at TEXT, notes TEXT +); +CREATE TABLE scheme_catalog_entries ( + scheme_code TEXT PRIMARY KEY, scheme_name TEXT, scheme_aliases_json TEXT, instances_json TEXT, + workflow_ids_json TEXT, collection_name TEXT, chunk_count INTEGER DEFAULT 0, + status TEXT NOT NULL DEFAULT 'disabled', network_visible INTEGER DEFAULT 0, promoted_at TEXT, + content_hash TEXT, source TEXT, updated_at TEXT +); +""" + +FP1 = hashlib.md5(b"scheme guideline").hexdigest() +FP2 = hashlib.md5(b"in flight").hexdigest() +WF1 = f"doc-{FP1}" +WF2 = f"doc-{FP2}" +WF3 = "doc-broken" +FUTURE = "2099-01-01T00:00:00" + + +def _load_script(): + spec = importlib.util.spec_from_file_location("migrate_sqlite_to_postgres", _SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _insert(conn, table, **values): + keys = ", ".join(values) + marks = ", ".join("?" for _ in values) + conn.execute(f"INSERT INTO {table} ({keys}) VALUES ({marks})", tuple(values.values())) + + +@pytest.fixture +def legacy_db(tmp_path): + path = tmp_path / "documents.db" + conn = sqlite3.connect(path) + conn.executescript(LEGACY_DDL) + + _insert( + conn, "documents", workflow_id=WF1, document_id=FP1, canonical_document_id=FP1, + filename="guideline.pdf", display_name="Guideline", source_filename="guideline.pdf", + source_file_fingerprint=FP1, filepath="minio://documents/guideline.pdf", stage="completed", + page_count=2, chunk_count=2, created_at="2026-03-01T10:00:00", updated_at="2026-03-05T10:00:00", + ocr_completed_at="2026-03-01T11:00:00", translation_completed_at="2026-03-02T11:00:00", + chunks_completed_at="2026-03-03T11:00:00", ingested_at="2026-03-04T11:00:00", + source_type="document", canonical_input_type="pdf", instance="mh", uploaded_by_user_id="kc-up", + uploaded_by_username="uma", uploaded_by_email="uma@x.in", uploaded_by_roles="contributor,reviewer", + valid_from="2026-03-01", valid_to="2027-03-01", document_kind="scheme", scheme_code="pm-kmy", + scheme_name="PM Kisan Maandhan", scheme_aliases_json=json.dumps(["PMKMY", "kmy"]), + catalog_visible=1, network_visible=0, reindex_required=1, reindex_reason="edit", + ) + _insert( + conn, "documents", workflow_id=WF2, document_id=FP2, filename="inflight.pdf", + filepath="/data/documents/inflight.pdf", stage="approval_for_prod", page_count=1, + created_at="2026-04-01T10:00:00", updated_at="2026-04-01T12:00:00", instance="up", + is_demo=1, prod_ready_requested_at="2026-04-02T09:00:00", + prod_ready_requested_by_user_id="kc-req", prod_ready_requested_by_username="rita", + ocr_completed_at="2026-04-01T11:00:00", + ) + _insert( + conn, "documents", workflow_id=WF3, document_id="not-a-hash", filename="broken.pdf", + filepath="/tmp/broken.pdf", stage="failed", error_message="OCR blew up", + created_at="2026-05-01T10:00:00", updated_at="2026-05-01T10:30:00", instance=" ", + document_kind="document", scheme_code="Bad Code", valid_from="2026-06-01", valid_to="2026-01-01", + ) + + _insert(conn, "document_jobs", id=1, workflow_id=WF1, job_type="pipeline", temporal_workflow_id=WF1, + status="completed", current_stage="completed", started_at="2026-03-01T10:00:00", + completed_at="2026-03-04T11:00:00", config_json=json.dumps({"auto_approve": False}), + temporal_run_id="run-1") + _insert(conn, "document_jobs", id=2, workflow_id=WF1, job_type="reingest", + temporal_workflow_id=f"{WF1}-reingest-1", status="completed", current_stage="ingesting", + started_at="2026-03-10T10:00:00", completed_at="2026-03-10T10:05:00") + _insert(conn, "document_jobs", id=3, workflow_id=WF1, job_type="mystery", + started_at="2026-03-11T10:00:00") + + _insert(conn, "document_artifacts", workflow_id=WF1, job_id=1, artifact_type="original_upload", + stage="registered", storage_uri="minio://documents/guideline.pdf", size_bytes=100, + metadata_json=json.dumps({"uploaded_via": "upload_endpoint"}), created_at="2026-03-01T10:00:01") + _insert(conn, "document_artifacts", workflow_id=WF1, job_id=None, artifact_type="normalized_pdf", + stage="ocr_processing", storage_uri="minio://documents/n.pdf", size_bytes=-1, + created_at="2026-03-01T10:30:00") + _insert(conn, "document_artifacts", workflow_id=WF1, job_id=2, artifact_type="vector_payload_export", + stage="ingesting", storage_uri="minio://documents/v.json", created_at="2026-03-10T10:01:00") + + _insert(conn, "pages", workflow_id=WF1, page_number=1, original_markdown="namaste", is_reviewed=1, + detected_language="hi", translated_markdown="hello", translation_provider="azure", + translation_model="gpt", translation_target_language="en", translated_at="2026-03-02T10:00:00") + _insert(conn, "pages", workflow_id=WF1, page_number=2, original_markdown="english page", + detected_language="en", reviewer_notes="fine") + _insert(conn, "pages", workflow_id="doc-purged", page_number=1, original_markdown="stray") + + _insert(conn, "chunks", workflow_id=WF1, chunk_number=1, original_text="chunk one", token_count=5, + page_start=1, page_end=2, source_spans_json=json.dumps([{"page_number": 1, "start_char": 0, "end_char": 5}]), + chunking_provider="recursive", chunking_model="rs", chunking_config_json='{"size": 450}', + chunk_version=3, is_reviewed=1) + _insert(conn, "chunks", workflow_id=WF1, chunk_number=2, original_text="chunk two", edited_text="two", + page_start=2, page_end=1, is_excluded=1) + + _insert(conn, "document_index_status", workflow_id=WF1, index_name="documents-index", doc_id="abc", + chunk_count_indexed=1, last_indexed_at="2026-03-04T11:00:00", status="indexed", + details_json='{"backend": "qdrant"}') + _insert(conn, "document_index_status", workflow_id=WF1, index_name="schemes-index", doc_id="abc", + chunk_count_indexed=2, last_indexed_at="2026-03-10T10:05:00", status="indexed") + + _insert(conn, "audit_logs", workflow_id=WF1, document_id=FP1, action_type="stage_change", + entity_type="document", field_name="stage", old_value="chunk_review", new_value="completed", + metadata='{"page_count": 2}', timestamp="2026-03-04T11:00:00") + _insert(conn, "audit_logs", workflow_id=WF1, document_id=FP1, action_type="page_edit", + entity_type="page", entity_id=1, metadata="not json", timestamp="2026-03-02T12:00:00", + actor_user_id="kc-rev", actor_username="ravi", actor_email="ravi@x.in", actor_roles="reviewer") + _insert(conn, "audit_logs", workflow_id="doc-purged", document_id="gone", action_type="purge_document", + entity_type="document", timestamp="2026-02-01T00:00:00") + _insert(conn, "audit_logs", workflow_id="__system__", document_id="__settings__", + action_type="settings_change", entity_type="setting", field_name="search_alpha", + old_value="0.7", new_value="0.6", timestamp="2026-01-01T00:00:00") + _insert(conn, "audit_logs", workflow_id="__catalog__", document_id="__catalog__", + action_type="catalog_rebuild", entity_type="catalog", timestamp="2026-01-02T00:00:00") + + for key, value in ( + ("search_alpha", "0.4"), ("search_limit", "20"), ("search_show_highlights", "false"), + ("search_method", "LEXICAL"), ("search_index_name", "documents-index"), ("search_mystery", "x"), + ("search_ef_search", "lots"), + ): + _insert(conn, "settings", key=key, value=value, updated_at="2026-01-01T00:00:00") + + _insert(conn, "email_otps", email="old@x.in", code_hash="h", expires_at="2020-01-01T00:00:00+00:00", + created_at="2020-01-01T00:00:00+00:00", last_sent_at="2020-01-01T00:00:00+00:00") + _insert(conn, "email_otps", email="LIVE@x.in", code_hash="h2", expires_at=FUTURE + "+00:00", + attempts=2, created_at="2026-01-01T00:00:00+00:00", last_sent_at="2026-01-01T00:00:00+00:00") + + _insert(conn, "webhook_records", id="3c8a2f0e-6b1d-4f6e-9d0a-5f8f0b2e7a11", external_id="c-1", + title="T", slug="t", status="published", is_published=1, category="Agri", + payload='{"contentID": "c-1"}', payload_hash="h", created_at="2026-01-01T00:00:00", + updated_at="2026-01-02T00:00:00") + + _insert(conn, "scheme_catalog_meta", id=1, version=5, updated_at="2026-02-01T00:00:00", notes="x") + _insert(conn, "scheme_catalog_entries", scheme_code="cdp", scheme_name="Crop Diversification Programme", + scheme_aliases_json='["CDP"]', status="live", source="bootstrap") + _insert(conn, "scheme_catalog_entries", scheme_code="pm-kmy", scheme_name="Registry name", + scheme_aliases_json='["registry"]', status="live", source="pipeline") + _insert(conn, "document_manifest_entries", manifest_filename="a.csv", imported_at="2026-01-01T00:00:00") + _insert(conn, "document_manifest_entries", manifest_filename="b.csv", imported_at="2026-01-01T00:00:00") + conn.commit() + conn.close() + return str(path) + + +def _counts(db): + with db.get_connection() as conn: + return { + table: conn.execute(f"SELECT count(*) AS n FROM {table}").fetchone()["n"] + for table in ( + "documents", "workflows", "workflow_artifacts", "pages", "chunks", "audit_logs", + "users", "catalog_entries", "email_otps", "webhook_records", + ) + } + + +@pytest.mark.db +@pytest.mark.unit +class TestSqliteToPostgresMigration: + def test_dry_run_writes_nothing_but_reports_everything(self, db_connection, postgres_url, legacy_db): + script = _load_script() + before = _counts(db_connection) + report = script.run_migration(legacy_db, postgres_url, dry_run=True) + assert _counts(db_connection) == before + assert report.tables["documents"]["inserted"] == 3 + assert report.mismatches() == [] + + def test_a_full_run_maps_every_table(self, db_connection, postgres_url, legacy_db): + db = db_connection + report = _load_script().run_migration(legacy_db, postgres_url) + + assert report.mismatches() == [] + tables = report.as_dict()["tables"] + assert tables["documents"]["inserted"] == 3 + assert tables["document_jobs"] == {"read": 3, "inserted": 2, "skipped": 1} + assert tables["pages"] == {"read": 3, "inserted": 2, "skipped": 1} + assert tables["chunks"]["inserted"] == 2 + assert tables["document_manifest_entries"] == {"read": 2, "dropped": 2} + assert tables["email_otps"] == {"read": 2, "inserted": 1, "dropped": 1} + assert tables["settings"]["skipped"] == 2 + + doc = db.get_document(WF1) + assert doc["stage"] == "completed" + assert doc["instance"] == "mh" + assert doc["document_id"] == FP1 + assert doc["display_name"] == "Guideline" + assert (doc["valid_from"], doc["valid_to"]) == ("2026-03-01", "2027-03-01") + assert (doc["scheme_code"], doc["document_kind"]) == ("pm-kmy", "scheme") + assert doc["network_visible"] is False and doc["reindex_required"] is True + assert doc["uploaded_by_username"] == "uma" + assert doc["uploaded_by_roles"] == "contributor,reviewer" + assert doc["created_at"] == "2026-03-01T10:00:00" + assert doc["ocr_completed_at"] == "2026-03-01T11:00:00" + assert doc["chunk_count"] == 2 + assert doc["page_count"] == 2 + assert set(doc["index_status"]) == {"dev"} + assert doc["index_status"]["dev"]["index_name"] == "schemes-index" + + jobs = {j["job_type"]: j for j in db.list_document_jobs(WF1)} + assert set(jobs) == {"pipeline", "reingest"} + assert jobs["pipeline"]["temporal_workflow_id"] == WF1 + assert jobs["pipeline"]["temporal_run_id"] == "run-1" + assert jobs["reingest"]["temporal_workflow_id"] == f"{WF1}-reingest-1" + assert doc["workflow_id"] == WF1 + + artifacts = {a["artifact_type"]: a for a in db.list_document_artifacts(WF1)} + assert artifacts["original_upload"]["job_id"] == jobs["pipeline"]["id"] + assert artifacts["vector_payload_export"]["job_id"] == jobs["reingest"]["id"] + assert artifacts["normalized_pdf"]["job_id"] == jobs["pipeline"]["id"] + assert artifacts["normalized_pdf"]["size_bytes"] is None + assert doc["original_artifact_id"] == artifacts["original_upload"]["id"] + + pages = db.get_pages(WF1) + assert pages[0]["translated_markdown"] == "hello" + assert pages[0]["translation_target_language"] == "en" + assert pages[0]["is_reviewed"] is True and pages[0]["detected_language"] == "hi" + assert pages[1]["translations"] == {} and pages[1]["reviewer_notes"] == "fine" + + chunks = db.get_chunks(WF1, include_excluded=True) + assert [c["chunk_version"] for c in chunks] == [3, 3] + assert chunks[0]["chunking_config_json"] == '{"size": 450}' + assert chunks[1]["page_end"] == 2 and chunks[1]["is_excluded"] is True + + entries = {e["code"]: e for e in db.list_catalog_entries()} + assert entries["cdp"]["source"] == "bootstrap" and entries["cdp"]["status"] == "live" + assert entries["pm-kmy"]["name"] == "Registry name" + assert entries["pm-kmy"]["status"] == "pending_reindex" + assert entries["pm-kmy"]["workflow_ids"] == [WF1] + assert db.get_catalog_meta()["version"] == 6 + + users = {u: db.get_user_by_keycloak_id(u) for u in ("kc-up", "kc-rev", "kc-req")} + assert all(users.values()) + + logs = db.get_audit_logs(WF1) + assert {e["action_type"] for e in logs} == {"stage_change", "page_edit"} + edit = next(e for e in logs if e["action_type"] == "page_edit") + assert edit["actor"] == "ravi@x.in (reviewer)" + assert edit["metadata"] == json.dumps({"raw": "not json"}) + purged = db.get_audit_logs("doc-purged") + assert [e["action_type"] for e in purged] == ["purge_document"] + settings_log = db.get_settings_audit_logs() + assert settings_log[0]["field_name"] == "retrieval.alpha" + + settings = db.get_search_settings() + assert settings["alpha"] == 0.4 and settings["limit"] == 20 + assert settings["showHighlights"] is False and settings["searchMethod"] == "LEXICAL" + assert settings["efSearch"] == 256 + + assert db.get_email_otp("live@x.in")["attempts"] == 2 + assert db.get_email_otp("old@x.in") is None + with db.get_connection() as conn: + hook = conn.execute("SELECT * FROM webhook_records").fetchone() + assert str(hook["id"]) == "3c8a2f0e-6b1d-4f6e-9d0a-5f8f0b2e7a11" + assert hook["is_published"] is True and hook["payload"] == {"contentID": "c-1"} + + def test_in_flight_documents_keep_their_stage_and_prod_request(self, db_connection, postgres_url, legacy_db): + db = db_connection + _load_script().run_migration(legacy_db, postgres_url) + + doc = db.get_document(WF2) + assert doc["stage"] == "approval_for_prod" + assert doc["is_demo"] is True + assert doc["prod_ready_requested_at"] == "2026-04-02T09:00:00" + assert doc["prod_ready_requested_by_username"] == "rita" + assert doc["instance"] == "up" + assert doc["ocr_completed_at"] == "2026-04-01T11:00:00" + types = sorted(j["job_type"] for j in db.list_document_jobs(WF2)) + assert types == ["pipeline", "promote_prod"] + assert db.get_latest_document_job(WF2)["status"] == "waiting_review" + assert db.get_document(doc["id"])["workflow_id"] == WF2 + assert db.get_document(FP2)["id"] == doc["id"] + + def test_unusable_values_are_repaired_and_noted(self, db_connection, postgres_url, legacy_db): + db = db_connection + report = _load_script().run_migration(legacy_db, postgres_url) + + doc = db.get_document(WF3) + assert doc["stage"] == "failed" and doc["error_message"] == "OCR blew up" + assert doc["document_id"] == hashlib.md5(b"not-a-hash").hexdigest() + assert doc["instance"] == "default" + assert doc["scheme_code"] is None + assert doc["valid_from"] == "2026-06-01" and doc["valid_to"] is None + notes = " ".join(report.notes) + assert "valid_to cleared" in notes and "not a valid slug" in notes or "plain document" in notes + + def test_running_it_twice_changes_nothing(self, db_connection, postgres_url, legacy_db): + script = _load_script() + script.run_migration(legacy_db, postgres_url) + first = _counts(db_connection) + + report = script.run_migration(legacy_db, postgres_url) + + assert _counts(db_connection) == first + assert report.tables["documents"] == {"read": 3, "existing": 3} + assert report.tables["audit_logs"]["inserted"] == 0 + assert report.tables["webhook_records"]["existing"] == 1 + assert report.mismatches() == [] + assert db_connection.get_catalog_meta()["version"] == 6 + + def test_legacy_file_is_opened_read_only(self, postgres_url, legacy_db): + script = _load_script() + before = Path(legacy_db).read_bytes() + script.run_migration(legacy_db, postgres_url) + assert Path(legacy_db).read_bytes() == before From fa8b0e88af678577e54cea46c5ab2a77cc97d5d7 Mon Sep 17 00:00:00 2001 From: Naveen Date: Mon, 26 Oct 2026 13:18:48 +0530 Subject: [PATCH 08/53] docs: describe Postgres as the canonical store --- AGENTS.md | 10 ++--- CONTEXT.md | 2 +- README.md | 16 +++---- docs/SYSTEM_DESIGN.md | 68 +++++++++++++++--------------- docs/architecture-pipeline-rbac.md | 20 ++++----- docs/openapi-search.yaml | 8 ++-- 6 files changed, 62 insertions(+), 62 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 764a09e..43e113c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## What this is -A review-driven document ingestion pipeline (product name "Bharat Vistaar Docs Pipeline"): Temporal workflows + FastAPI + SQLite + MinIO + Qdrant. Documents move through explicit stages (OCR → review → translate → review → chunk → review → ingest DEV → Super Admin promote to PROD), with every major output persisted as an inspectable artifact rather than hidden. +A review-driven document ingestion pipeline (product name "Bharat Vistaar Docs Pipeline"): Temporal workflows + FastAPI + Postgres + MinIO + Qdrant. Documents move through explicit stages (OCR → review → translate → review → chunk → review → ingest DEV → Super Admin promote to PROD), with every major output persisted as an inspectable artifact rather than hidden. Full narrative docs (read these before deep architectural changes, don't re-derive): - `README.md` — what the system does, API surface, supported inputs, auth model @@ -58,7 +58,7 @@ Core modules (all under `pipeline/`): - `workflows.py` — Temporal workflow definitions (`DocumentPipelineWorkflow` is the main one; also `ReingestionWorkflow`, `PromoteToProdWorkflow`, `TranslationOnlyWorkflow`, `OcrOnlyWorkflow`, `ChunkingOnlyWorkflow`) - `activities.py` — Temporal activities invoked by the workflows (OCR, translate, chunk, ingest, state updates) - `worker.py` — Temporal worker process; registers workflows/activities on task queue `ocr-pipeline` -- `db.py` — SQLite access layer; SQLite is the **canonical** metadata/review-state store (documents, pages, chunks, jobs, artifacts, audit log), not just a cache +- `db.py` — Postgres access layer (psycopg3 sync connection pool); Postgres is the **canonical** metadata/review-state store (documents, workflows, pages, chunks, artifacts, audit log, settings), not just a cache. Documents and jobs come back as flat dicts keyed by `workflow_id` (the document's main Temporal workflow id); every function also accepts the document UUID. Schema: `migrations/` (applied at api/worker startup by `migrate.py`), described in `docs/POSTGRES_SCHEMA.md` - `document_repository.py` — `DocumentRepository`, a domain layer over `db.py` for document reads; `db.py` stays one-function-per-query with no domain knowledge - `network_constants.py` — fixed values sent on the network (catalog/resource ids, topics, languages, Beckn version). The file v2 edits when the AI layer starts deriving them - `catalog_builder.py` — pure builder mapping a document's knowledge kind (`advisory`/`scheme`) to the single `OnDemand` Beckn catalog announced for that kind; returns `None` for any other kind. No env, no I/O @@ -67,7 +67,7 @@ Core modules (all under `pipeline/`): - `models.py` — Pydantic models, including `DocumentStage` enum and `PIPELINE_STAGES` (the stepper-UI stage list) - `config.py` — `Config` dataclass reading env vars, with defaults - `instances.py` — instance/tenant (state) code ↔ human name mapping; `bv` = Bharat Vistaar platform-wide, others are state codes matching Keycloak group paths (`/states/MH/...` → `mh`) -- `scheme_catalog.py` — SQLite-backed scheme catalog behind the pull-based `/catalog/v1/*` API +- `scheme_catalog.py` — scheme catalog behind the pull-based `/catalog/v1/*` API, served from the `catalog_entries` table and the computed `catalog` view - `ocr/`, `translation/`, `chunking/`, `vector_store/` — pluggable provider packages, each with a `base.py` protocol/ABC and a `service.py` (or equivalent) that dispatches to the configured provider - `auth/` — Keycloak JWT validation, group-based tenancy/permissions, email-OTP flow, admin user management @@ -93,7 +93,7 @@ registered → ocr_processing → ocr_review → translation_processing → tran ### Storage responsibilities (don't blur these) -- **SQLite** — canonical metadata/content/review state (documents, pages, chunks, jobs, artifacts, audit, search settings). Source of truth for anything editable. +- **Postgres** — canonical metadata/content/review state (documents, workflows, pages, chunks, artifacts, audit, search settings). Source of truth for anything editable. - **MinIO** — blob storage for original uploads, normalized files, and stage artifacts (OCR/translation/chunk exports, vector-index payload snapshots). - **Temporal** — orchestration/retries/review-gate signaling only; not a content store. - **Qdrant** — downstream search projections of approved chunk state; never the place to edit content. @@ -108,4 +108,4 @@ React 18 + Vite + Tailwind v4 + Radix UI operator console (not a plain upload fo ### Scripts (`scripts/`) -Operational tools, not one-off hacks — e.g. `list_failed_workflows.py`, `terminate_stuck_workflows.py`, `keycloak_bootstrap_docs_pipeline.py`, `drop_legacy_marqo_columns.py`. Check here before writing a new maintenance script — an equivalent likely exists. +Operational tools, not one-off hacks — e.g. `list_failed_workflows.py`, `terminate_stuck_workflows.py`, `keycloak_bootstrap_docs_pipeline.py`, `migrate_sqlite_to_postgres.py`. Check here before writing a new maintenance script — an equivalent likely exists. diff --git a/CONTEXT.md b/CONTEXT.md index 9da1a46..7b1589d 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -13,7 +13,7 @@ The pipeline stage that runs only after a document's PROD promotion succeeds, an _Avoid_: "publish" alone — ambiguous with Publish to Dev. **Scheme Catalog**: -The SQLite-backed, scheme-only registry that aggregates one or more documents sharing a `scheme_code`. Backs the pull-based `GET /catalog/v1/*` API that other BAPs poll. +The scheme-only registry (`catalog_entries` rows plus the computed `catalog` view) that aggregates one or more documents sharing a `scheme_code`. Backs the pull-based `GET /catalog/v1/*` API that other BAPs poll. _Avoid_: "catalog" alone — ambiguous with the Network Catalog Envelope. **Network Catalog Envelope**: diff --git a/README.md b/README.md index ab9db9c..2266330 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![codecov](https://codecov.io/gh/OpenAgriNet/knowledge-provider/graph/badge.svg?token=2TKM2CR2AC)](https://codecov.io/gh/OpenAgriNet/knowledge-provider) -This repository contains a review-driven document ingestion pipeline built around Temporal workflows, FastAPI, SQLite, MinIO, and Qdrant. It is designed for teams that need to normalize heterogeneous files, extract structured text, review and correct outputs, generate chunks, and publish searchable records into a vector index. +This repository contains a review-driven document ingestion pipeline built around Temporal workflows, FastAPI, Postgres, MinIO, and Qdrant. It is designed for teams that need to normalize heterogeneous files, extract structured text, review and correct outputs, generate chunks, and publish searchable records into a vector index. The system is intentionally operational, not just algorithmic. Documents move through explicit stages, every major output can be persisted as an artifact, and the operator UI is designed to inspect and manage the pipeline rather than hide it. @@ -63,7 +63,7 @@ The platform is composed of six main services: | | | | v v v v +--------+---------+ +--------+--+ +---+---------------+ +------------------+ -| MinIO | | SQLite | | Worker | | Lang Detect | +| MinIO | | Postgres | | Worker | | Lang Detect | | originals, | | canonical | | normalize, OCR, | | language hints | | normalized files, | | metadata, | | translate, chunk, | | before | | artifacts | | pages, | | ingest | | translation | @@ -90,7 +90,7 @@ The document has been accepted into the system and assigned a workflow identifie Typical outputs: -- document row in SQLite +- document row in Postgres - original file reference - initial job record @@ -191,16 +191,16 @@ Document-level view of what has been pushed to the vector index. ## Storage Responsibilities -### SQLite +### Postgres -SQLite is the canonical metadata and review-state store. +Postgres is the canonical metadata and review-state store. Its schema is in `migrations/` and described in `docs/POSTGRES_SCHEMA.md`; the api and worker apply pending migrations at startup. Moving an existing SQLite deployment over is a one-off run of `scripts/migrate_sqlite_to_postgres.py` (see the `migrate-sqlite` compose profile). It owns: - document rows +- workflow rows (one per processing run, shown as jobs) - page rows - chunk rows -- jobs - artifact metadata - audit logs - search settings @@ -320,7 +320,7 @@ Default local ports from `docker-compose.yml`: - Docker and Docker Compose - an OCR provider API key exposed as `MISTRAL_API_KEY` -- enough local disk for SQLite, MinIO artifacts, and Qdrant state +- enough local disk for Postgres, MinIO artifacts, and Qdrant state ### Start @@ -355,7 +355,7 @@ Important runtime variables include: - `MINIO_ACCESS_KEY` - `MINIO_SECRET_KEY` - `MINIO_BUCKET` -- `DOCUMENT_DB_PATH` +- `DATABASE_URL` - `TRANSLATION_PROVIDER` - `TRANSLATION_MODEL` - `TRANSLATION_PAGE_CONCURRENCY` diff --git a/docs/SYSTEM_DESIGN.md b/docs/SYSTEM_DESIGN.md index 7fbaac3..6ccd1f5 100644 --- a/docs/SYSTEM_DESIGN.md +++ b/docs/SYSTEM_DESIGN.md @@ -39,7 +39,7 @@ gate before **PROD**. | Multi-state tenancy | `documents.instance` + JWT groups; lists/search scoped | | DEV before PROD | Stage 9 writes DEV index; stage 10 waits; Super Admin promotes to PROD | | Operability | Runs, queue, audit, reconcile, artifacts UI | -| Durable long work | Temporal retries + SQLite state mirror | +| Durable long work | Temporal retries + Postgres state mirror | ### Non-goals - Real-time chat / low-latency LLM product surface @@ -98,7 +98,7 @@ flowchart TB end subgraph Data - SQLITE[("SQLite WAL
documents.db")] + PGDB[("Postgres
documents, workflows, pages, chunks")] MINIO[("MinIO
objects")] QDEV[("Qdrant DEV index
vector + lexical")] QPROD[("Qdrant PROD
production vectors")] @@ -111,13 +111,13 @@ flowchart TB UI -->|"/api + Bearer JWT"| API UI -.->|OIDC PKCE| KC API -->|JWKS| KC - API --> SQLITE + API --> PGDB API --> MINIO API --> QDEV API -->|start / signal| TEMP TEMP --> TEMPDB TEMP --> WORKER - WORKER --> SQLITE + WORKER --> PGDB WORKER --> MINIO WORKER --> QDEV WORKER -->|promote_document_to_prod_qdrant| QPROD @@ -136,10 +136,10 @@ flowchart TB | **minio** | official | Source PDF + stage artifacts | 9000 | | **qdrant** | official image | DEV search index | 6333 | | **qdrant (prod)** | external / env | PROD vectors after Super Admin promote | `PROD_QDRANT_URL` | -| **sqlite** | volume file | Canonical document/page/chunk/job/audit state | path | +| **postgres** | official image | Canonical document/workflow/page/chunk/audit state | `DATABASE_URL` | | **keycloak** | official | Identity, groups `/states/{ST}/role`, `/global/super-admin` | 8082 | -API and worker share one **SQLite volume** and the same env for providers so +API and worker share one **Postgres database** and the same env for providers so read and write paths stay consistent. --- @@ -161,7 +161,7 @@ read and write paths stay consistent. │ │ ┌───────────────▼──────────────┐ ┌──────────▼────────────────────────────┐ │ ORCHESTRATION LAYER │ │ DOMAIN STATE LAYER │ -│ Temporal workflows │ │ SQLite (SoT for content) │ +│ Temporal workflows │ │ Postgres (SoT for content) │ │ + review signals │ │ MinIO (binaries) │ │ + activity retries │ │ Jobs / audit / index status │ └───────────────┬──────────────┘ └───────────────────────────────────────┘ @@ -245,36 +245,36 @@ Supporting workflows: `OcrOnly`, `TranslationOnly`, `ChunkingOnly`, | Store | System of record for | Not for | |---|---|---| -| **SQLite** | Document stage, pages, chunks, tags, jobs, audit, uploader, `instance` | Binary blobs | +| **Postgres** | Document stage, workflows, pages, chunks, artifacts, audit, uploader, `instance` | Binary blobs | | **MinIO** | PDF originals, exports, payloads | AuthZ decisions | | **Temporal DB** | Workflow history & timers | Business content | | **Qdrant DEV** | Searchable DEV vectors | Authoritative text | | **Qdrant PROD** | Production vectors | Draft/unapproved content | -### 7.2 Key entities (SQLite) +### 7.2 Key entities (Postgres) ```text -documents (workflow_id PK) +documents (id UUID PK) ├── instance # tenant / state code - ├── stage # DocumentStage enum - ├── uploaded_by_* # actor provenance - ├── page_count / chunk_count - ├── latest_job_id - ├── reindex_required - └── is_disabled / is_demo + ├── source (JSONB) # fingerprint, filename, storage_uri, page_count + ├── index_status (JSONB) # dev / prod vector index state + ├── metadata (JSONB) # chunk_count, reindex_required, is_demo, ... + ├── uploaded_by_user_id # → users + └── is_disabled / catalog_entry_id -pages (workflow_id, page_number) - └── OCR + translation fields + review flags +workflows (one per processing run; the latest gives the document's stage) + └── workflow_type, status, current_stage, temporal_workflow_id → Runs UI -chunks (workflow_id, chunk_num) - └── text, lineage, exclusion, provenance - └── chunk_tags[] +workflow_artifacts → files each workflow produced (MinIO) + +pages (document_id, page_number) + └── OCR text + translations (JSONB) + review flags -document_jobs - └── job_type, status, temporal ids → Runs UI +chunks (document_id, chunk_number) + └── text, lineage, exclusion, provenance audit_logs - └── approvals, edits, promote_to_prod + └── approvals, edits, promote_to_prod (outlives the document) ``` ### 7.3 Tenant rule @@ -337,7 +337,7 @@ UI always calls API via `/api` proxy — no direct Temporal/MinIO from browser. | Activity | Purpose | Failure handling | |---|---|---| -| `run_ocr_and_store` | Extract pages → SQLite + MinIO | OCR_RETRY | +| `run_ocr_and_store` | Extract pages → Postgres + MinIO | OCR_RETRY | | `detect_and_translate_pages_from_db` | Lang detect + translate | TRANSLATION_RETRY | | `create_chunks_from_db` | Chunk reviewed text | CHUNK_RETRY | | `auto_tag_chunks_from_db` | Domain tags | CHUNK_RETRY | @@ -353,12 +353,12 @@ Providers are **config-driven** (env URLs/models) so OCR/vLLM backends can chang ```mermaid flowchart LR - SQLITE[(SQLite chunks)] + PGDB[(Postgres chunks)] DEV[(DEV index
Qdrant)] PROD[(PROD Qdrant)] - SQLITE -->|approve_ingestion
ingest_document_from_db| DEV - SQLITE -->|approve_prod Super Admin
promote_document_to_prod_qdrant| PROD + PGDB -->|approve_ingestion
ingest_document_from_db| DEV + PGDB -->|approve_prod Super Admin
promote_document_to_prod_qdrant| PROD DEV -.->|not automatic| PROD ``` @@ -399,8 +399,8 @@ Routes: Dashboard, Documents, New Document, Document Ops, Runs, Search, Users (S ### Runtime properties - API + worker: **same image**, different entrypoint -- Shared SQLite volume (single-writer lock + WAL) -- Horizontal scale: Temporal workers scale out; SQLite remains single-primary (acceptable for ops volume; upgrade path = Postgres if needed) +- Shared Postgres database (pooled connections per process; schema applied from `migrations/` at startup) +- Horizontal scale: Temporal workers and API replicas scale out against the same database --- @@ -408,7 +408,7 @@ Routes: Dashboard, Documents, New Document, Document Ops, Runs, Search, Users (S | Concern | Design | |---|---| -| **Consistency** | SQLite SoT; Temporal for execution; search is eventual projection | +| **Consistency** | Postgres SoT; Temporal for execution; search is eventual projection | | **Idempotency** | Workflow ids; reingest overwrites by document id | | **Observability** | Runs UI, Temporal UI, audit_logs, stage mirror | | **Security** | JWT, least privilege, tenant 404, Super Admin prod gate | @@ -421,7 +421,7 @@ Routes: Dashboard, Documents, New Document, Document Ops, Runs, Search, Users (S ## 15. Threat & trust boundaries ```text -[ Browser ] --JWT--> [ API ] --internal net--> [ Worker / Temporal / MinIO / SQLite ] +[ Browser ] --JWT--> [ API ] --internal net--> [ Worker / Temporal / MinIO / Postgres ] | +--JWKS--> [ Keycloak ] +--HTTP--> [ Qdrant DEV ] @@ -439,7 +439,7 @@ Routes: Dashboard, Documents, New Document, Document Ops, Runs, Search, Users (S | Attribute | Target approach | |---|---| | Availability | Compose restart policies; Temporal retries | -| Durability | Workflow history + SQLite + MinIO objects | +| Durability | Workflow history + Postgres + MinIO objects | | Latency | Search interactive; OCR batch (minutes) | | Isolation | Per-state `instance` + Keycloak groups | | Auditability | append-only audit + Temporal history | @@ -450,7 +450,7 @@ Routes: Dashboard, Documents, New Document, Document Ops, Runs, Search, Users (S ## 17. Evolution / extension points 1. Explicit **“Request PROD approval”** action (today: auto-enter queue after DEV). -2. SQLite → Postgres if multi-writer scale is needed. +2. Moving `chunk_count` / `is_demo` out of `documents.metadata` into columns if list queries slow down. 3. Stricter per-state role on every mutation (not only global permission). 4. Automated quality checks before Super Admin sees queue. diff --git a/docs/architecture-pipeline-rbac.md b/docs/architecture-pipeline-rbac.md index 91e4358..1540cb0 100644 --- a/docs/architecture-pipeline-rbac.md +++ b/docs/architecture-pipeline-rbac.md @@ -43,7 +43,7 @@ flowchart TB end subgraph DataPlane - SQLite["SQLite
documents, pages, chunks,
jobs, audit"] + Postgres["Postgres
documents, pages, chunks,
jobs, audit"] MinIO["MinIO / S3
source PDFs"] QdrantDev["Qdrant DEV
search index"] QdrantProd["Qdrant PROD
search index"] @@ -59,11 +59,11 @@ flowchart TB Browser -->|"OIDC PKCE login"| KC Browser -->|"Bearer JWT + /api/*"| API API -->|"validate JWT / JWKS"| KC - API --> SQLite + API --> Postgres API --> MinIO API -->|"start / signal workflows"| Temporal Temporal --> Worker - Worker --> SQLite + Worker --> Postgres Worker --> MinIO Worker --> OCR Worker --> TR @@ -81,7 +81,7 @@ flowchart TB | **Keycloak** | Source of truth for *who* is Super Admin vs which **state + role** a user has | | **JWT → API** | Map groups/roles → `permissions`, `instances`, `state_roles`; enforce tenancy + capability | | **Temporal workflow** | Durable pipeline with human gates (signals) | -| **SQLite** | System of record for document stage, pages, chunks, uploader, `instance` | +| **Postgres** | System of record for document stage, pages, chunks, uploader, `instance` | | **Qdrant DEV** | Searchable vectors after state-approved ingestion | | **Qdrant PROD** | Production vectors only after Super Admin `approve_prod` | @@ -213,8 +213,8 @@ stateDiagram-v2 | Stage | Who advances it | Effect | |---|---|---| -| Upload → Registered | Contributor / Super Admin (`upload`) | File in MinIO; SQLite row; `instance` set | -| OCR / Translation / Chunking | System (worker activities) | Pages/chunks in SQLite | +| Upload → Registered | Contributor / Super Admin (`upload`) | File in MinIO; Postgres row; `instance` set | +| OCR / Translation / Chunking | System (worker activities) | Pages/chunks in Postgres | | OCR / Translation / Chunk / **Pre-ingestion** approve | State **review** (+ pipeline as needed) or Super Admin | Temporal signal continues workflow | | **Ingesting in Dev** | System after `approve_ingestion` | `ingest_document_from_db` → **Qdrant DEV** | | **Approval for Prod** (queue) | Document waits | Still **DEV-only** in search until promote | @@ -355,14 +355,14 @@ flowchart LR ```mermaid flowchart TB PDF[Source PDF in MinIO] - SQLITE[(SQLite: pages + chunks
+ instance + uploader)] + PGDB[(Postgres: pages + chunks
+ instance + uploader)] DEV[(Qdrant DEV collection)] PROD[(Qdrant PROD collection)] - PDF --> SQLITE - SQLITE -->|"ingest_document_from_db
after approve_ingestion"| DEV + PDF --> PGDB + PGDB -->|"ingest_document_from_db
after approve_ingestion"| DEV DEV -->|"promote_document_to_prod_qdrant
after Super Admin approve_prod"| PROD - SQLITE -.->|"stage mirror"| SQLITE + PGDB -.->|"stage mirror"| PGDB ``` - **DEV ingest** activity: `ingest_document_from_db` (worker). diff --git a/docs/openapi-search.yaml b/docs/openapi-search.yaml index 39e4408..e4dab92 100644 --- a/docs/openapi-search.yaml +++ b/docs/openapi-search.yaml @@ -9,7 +9,7 @@ info: `pipeline/models.py`, and `pipeline/vector_store/qdrant_store.py`. Not the full API surface — only `/search` (vector/hybrid semantic search) - and `/chunks/search` (SQLite keyword search across chunk records). + and `/chunks/search` (keyword search across chunk records). version: "1.0.0" contact: name: knowledge-provider (docs-pipeline) @@ -73,9 +73,9 @@ paths: /chunks/search: get: operationId: searchChunksAcrossDocuments - summary: Keyword search across chunk records (SQLite), for KB maintainer workflows + summary: Keyword search across chunk records (Postgres), for KB maintainer workflows description: | - Distinct from `/search`: this queries chunk rows directly in SQLite by + Distinct from `/search`: this queries chunk rows directly in Postgres by substring/keyword match, not the vector index. Results are scoped to the caller's instance(s) unless the caller is an unrestricted admin. tags: [search] @@ -418,7 +418,7 @@ components: type: array items: type: object - description: Raw chunk record from SQLite (db.search_chunks); shape follows the chunks table, not formally typed here. + description: Raw chunk record from Postgres (db.search_chunks); shape follows the chunks table, not formally typed here. additionalProperties: true total: type: integer From b2d886403e5f5ad02fe0494ae3fd3745de3bb9ed Mon Sep 17 00:00:00 2001 From: Naveen Date: Tue, 27 Oct 2026 16:45:19 +0530 Subject: [PATCH 09/53] feat: index advisory records into Qdrant for advisory search Port of docs-pipeline PR #171 onto the Postgres-backed knowledge-provider. - POST /webhook starts one AdvisoryIndexWorkflow per new or changed record; chunks (document_kind=advisory) go to the Qdrant collection - admin endpoints: /webhook/advisory/{reindex,promote,status} - AdvisoryPromoteWorkflow copies to the PROD collection (PROD_VECTOR_DB_*) - migration 0002 adds webhook_records.indexed_hash / promoted_hash - advisory payload fields on Qdrant points; embedding token counter - ADVISORY_BACKFILL_ON_START queues unindexed rows at worker startup - migration-runner tests use version 0099 as their probe --- ENV.md | 3 + .../0002_webhook_advisory_index_state.sql | 3 + pipeline/activities.py | 49 ++++ pipeline/advisory_chunks.py | 154 +++++++++++++ pipeline/advisory_index.py | 84 +++++++ pipeline/advisory_start.py | 58 +++++ pipeline/api.py | 84 ++++++- pipeline/db.py | 124 +++++++++- pipeline/vector_store/embeddings.py | 12 + pipeline/vector_store/qdrant_store.py | 16 ++ pipeline/worker.py | 13 ++ pipeline/workflows.py | 58 +++++ tests/test_activities.py | 29 +++ tests/test_advisory_activities.py | 126 ++++++++++ tests/test_advisory_api.py | 165 +++++++++++++ tests/test_advisory_chunks.py | 218 ++++++++++++++++++ tests/test_advisory_index.py | 152 ++++++++++++ tests/test_advisory_start.py | 98 ++++++++ tests/test_db.py | 10 +- tests/test_embeddings_count_tokens.py | 24 ++ tests/test_webhook_index_db.py | 107 +++++++++ 21 files changed, 1569 insertions(+), 18 deletions(-) create mode 100644 migrations/0002_webhook_advisory_index_state.sql create mode 100644 pipeline/advisory_chunks.py create mode 100644 pipeline/advisory_index.py create mode 100644 pipeline/advisory_start.py create mode 100644 tests/test_advisory_activities.py create mode 100644 tests/test_advisory_api.py create mode 100644 tests/test_advisory_chunks.py create mode 100644 tests/test_advisory_index.py create mode 100644 tests/test_advisory_start.py create mode 100644 tests/test_embeddings_count_tokens.py create mode 100644 tests/test_webhook_index_db.py diff --git a/ENV.md b/ENV.md index df8b7b7..a073d60 100644 --- a/ENV.md +++ b/ENV.md @@ -88,6 +88,9 @@ Root `.env` (see `.env.example`). Required by FastAPI (`pipeline/api.py`), Tempo |----------|---------|---------| | `WEBHOOK_MAX_RECORDS` | `500` | Max records per `/webhook` request; more returns 413 | | `WEBHOOK_API_KEY` | *(empty — required)* | 32-char hex secret. Callers send it in the `x-api-key` header. Missing/invalid header → 401. Unset/empty → 503 (fails closed, ignores `AUTH_DISABLED`). Generate with `openssl rand -hex 16`. | +| `ADVISORY_BACKFILL_ON_START` | `false` | When `true`, the worker queues an index workflow for every unindexed `webhook_records` row once at startup (backfill of existing records). Leave off normally; `POST /webhook/advisory/reindex` does the same on demand. | + +Each saved record whose `payload_hash` differs from its `indexed_hash` also starts one Temporal workflow `advisory-` that writes advisory chunks (`document_kind=advisory`) to the Qdrant collection `VECTOR_DB_COLLECTION_NAME`. Admin endpoints (Keycloak admin): `POST /webhook/advisory/reindex`, `POST /webhook/advisory/promote` (copies to `PROD_VECTOR_DB_COLLECTION_NAME` (on `PROD_VECTOR_DB_URL`)), `GET /webhook/advisory/status`. Body is a JSON array of objects (max `WEBHOOK_MAX_RECORDS`); stored in the `webhook_records` table, upserted on `(source, external_id)` where `external_id` is `contentID` (else `id`, `slug`, payload hash). diff --git a/migrations/0002_webhook_advisory_index_state.sql b/migrations/0002_webhook_advisory_index_state.sql new file mode 100644 index 0000000..328fc93 --- /dev/null +++ b/migrations/0002_webhook_advisory_index_state.sql @@ -0,0 +1,3 @@ +-- Advisory indexing state: the payload_hash last written to the dev / PROD Qdrant collection. +ALTER TABLE webhook_records ADD COLUMN indexed_hash TEXT; +ALTER TABLE webhook_records ADD COLUMN promoted_hash TEXT; diff --git a/pipeline/activities.py b/pipeline/activities.py index 421013f..8850bd1 100644 --- a/pipeline/activities.py +++ b/pipeline/activities.py @@ -698,6 +698,7 @@ def _prepare_records( "instance": resolved_instance, "instance_name": instance_name, "type": "scheme" if is_scheme else "document", + "document_kind": kind, "source": "docs-pipeline", "filename": external_slug, "name_gu": name_gu, @@ -1647,3 +1648,51 @@ async def detect_and_translate_pages_from_db( }, ) return {"page_count": len(translated), "translated_count": translated_count} + + +async def _run_advisory_activity(record_id: str, promote: bool) -> dict: + """Index (dev) or promote (PROD) one webhook advisory record; shared by both activities.""" + from . import advisory_index + from .vector_store import get_default_index_name, get_vector_store + from .vector_store.embeddings import count_tokens + + also_delete_from = None + if promote: + target = advisory_index.prod_target() + if target is None: + raise RuntimeError("PROD_VECTOR_DB_URL is required to promote advisory to prod") + store, collection = target + mark = "promoted" + else: + store, collection = get_vector_store(), get_default_index_name() + mark = "indexed" + try: + # An unpublished record must also disappear from PROD (deletion only). + also_delete_from = advisory_index.prod_target() + except Exception as exc: + activity.logger.warning("PROD target unavailable for advisory cleanup: %s", exc) + + # Embedding is CPU-bound and synchronous: keep it off the event loop. + result = await asyncio.to_thread( + advisory_index.index_advisory_record, + record_id, + store, + collection, + count_tokens, + mark, + also_delete_from, + ) + advisory_index.log_advisory_audit(record_id, mark, collection, result) + return result + + +@activity.defn +async def index_webhook_advisory(record_id: str) -> dict: + """Write one webhook advisory record's chunks to the dev collection (or delete if unpublished).""" + return await _run_advisory_activity(record_id, promote=False) + + +@activity.defn +async def promote_webhook_advisory(record_id: str) -> dict: + """Copy one webhook advisory record's chunks to the PROD collection (or delete if unpublished).""" + return await _run_advisory_activity(record_id, promote=True) diff --git a/pipeline/advisory_chunks.py b/pipeline/advisory_chunks.py new file mode 100644 index 0000000..2cd248d --- /dev/null +++ b/pipeline/advisory_chunks.py @@ -0,0 +1,154 @@ +"""Turn a published advisory ``webhook_records`` row into Qdrant chunk records. + +One advisory entry (``popList`` item) becomes one chunk when its whole text fits the +embedding model's token limit, otherwise up to three chunks that each carry the advice +(overview + advice, symptoms + advice, conditions + advice). The stored ``text`` is +always the complete chunk text; the embedding model reads only its first tokens. +""" + +from __future__ import annotations + +import hashlib +from typing import Callable + +MAX_EMBEDDING_TOKENS = 512 + +# (section, label) for the question-side fields, in display order. +_FIELD_SECTIONS = ( + ("overview", "Overview"), + ("symptoms", "Symptoms"), + ("conditions", "Conditions"), +) + + +def clean_value(value: object) -> str: + """Return a trimmed string with surrounding quote characters removed ('' for None).""" + if value is None: + return "" + return str(value).strip().strip('"').strip() + + +def _clean_list(value: object) -> str: + """Join the non-empty cleaned items of a list (or a single value) with spaces.""" + if value is None: + return "" + items = value if isinstance(value, (list, tuple)) else [value] + return " ".join(part for part in (clean_value(item) for item in items) if part) + + +def _is_published(row: dict) -> bool: + return row.get("is_published") in (1, True) and clean_value(row.get("status")) == "Published" + + +def _header(title: str, crop: str, problem_type: str, season: str) -> list[str]: + lines = [title] if title else [] + parts = [] + if crop: + parts.append(f"Crop: {crop}") + if problem_type: + parts.append(f"Problem type: {problem_type}") + if season: + parts.append(f"Season: {season}") + if parts: + lines.append(" | ".join(parts)) + return lines + + +def _chunk_id(doc_id: str, pop_id: object, section: str) -> str: + return hashlib.md5(f"{doc_id}:{pop_id}:{section}".encode("utf-8")).hexdigest() + + +def build_chunks( + row: dict, + count_tokens: Callable[[str], int], + max_tokens: int = MAX_EMBEDDING_TOKENS, +) -> list[dict]: + """Build Qdrant records for one ``webhook_records`` row (empty list if unpublished).""" + if not _is_published(row): + return [] + payload = row.get("payload") or {} + pop_list = payload.get("popList") + if not isinstance(pop_list, list): + return [] + + doc_id = str(row["id"]) + title = clean_value(row.get("title")) or clean_value(payload.get("title")) + crop = clean_value(payload.get("commodityName")) + base = { + "doc_id": doc_id, + "workflow_id": "", + "type": "document", + "document_kind": "advisory", + "source": "webhook", + "title_en": title, + "slug": clean_value(row.get("slug")), + "external_id": clean_value(row.get("external_id")), + "crop": crop, + "domain": clean_value(payload.get("domainName")), + "subdomain": clean_value(payload.get("subdomainName")), + "institute": clean_value(payload.get("instituteName")), + "creator": clean_value(payload.get("creatorName")), + "updated_at": clean_value(row.get("updated_at")), + } + + records: list[dict] = [] + + def add(pop: dict, index: int, section: str, text: str, meta: dict) -> None: + pop_id = pop.get("popID") + key = pop_id if pop_id is not None else f"idx{index}" + chunk_id = _chunk_id(doc_id, key, section) + records.append( + { + **base, + **meta, + "_id": chunk_id, + "chunk_id": chunk_id, + "pop_id": pop_id, + "section": section, + "chunk_num": len(records), + "text": text, + "text_for_embedding": text, + } + ) + + for index, pop in enumerate(pop_list): + if not isinstance(pop, dict) or clean_value(pop.get("status")) != "Published": + continue + problem_type = clean_value(pop.get("problemType")) + season = clean_value(pop.get("season")) + meta = { + "problem_type": problem_type, + "problem": clean_value(pop.get("problemDetails")), + "season": season, + "start_period": clean_value(pop.get("startPeriod")), + "end_period": clean_value(pop.get("endPeriod")), + } + header = _header(title, crop, problem_type, season) + fields = { + "overview": clean_value(pop.get("description")), + "symptoms": _clean_list(pop.get("applicableSymptoms")), + "conditions": _clean_list(pop.get("applicableCondition")), + } + advisory = clean_value(pop.get("advisory")) + labelled = [(label, fields[section]) for section, label in _FIELD_SECTIONS if fields[section]] + if advisory: + labelled.append(("Advisory", advisory)) + if not labelled: + continue + + full_text = "\n".join(header + [f"{label}: {body}" for label, body in labelled]) + if count_tokens(full_text) <= max_tokens: + add(pop, index, "full", full_text, meta) + continue + + others = [(section, label) for section, label in _FIELD_SECTIONS if fields[section]] + if not others: + add(pop, index, "advisory", full_text, meta) + continue + for section, label in others: + lines = header + [f"{label}: {fields[section]}"] + if advisory: + lines.append(f"Advisory: {advisory}") + add(pop, index, section, "\n".join(lines), meta) + + return records diff --git a/pipeline/advisory_index.py b/pipeline/advisory_index.py new file mode 100644 index 0000000..c91f2f3 --- /dev/null +++ b/pipeline/advisory_index.py @@ -0,0 +1,84 @@ +"""Index (or remove) one advisory ``webhook_records`` row in a vector store collection.""" + +from __future__ import annotations + +import logging +import os +from typing import Callable, Literal, Optional + +from . import db +from .advisory_chunks import build_chunks + +logger = logging.getLogger(__name__) + + +def _delete_points(store, collection: str, doc_id: str) -> None: + """Delete a record's points; a missing collection is fine, any other failure is not.""" + result = store.delete_by_doc_id(collection, doc_id) or {} + error = str(result.get("error") or "") + if error and "doesn't exist" not in error and "not found" not in error.lower(): + raise RuntimeError(f"Could not delete points for {doc_id} from {collection}: {error}") + + +def index_advisory_record( + record_id: str, + store, + collection: str, + count_tokens: Callable[[str], int], + mark: Literal["indexed", "promoted"] = "indexed", + also_delete_from: Optional[tuple[object, str]] = None, +) -> dict: + """Replace a record's points in ``collection`` with its current chunks. + + Order: read row, delete the record's old points, write the new chunks, then store the + row's payload_hash (``indexed_hash`` or ``promoted_hash``). An unpublished record (no + chunks) only gets its points deleted; ``also_delete_from=(store, collection)`` removes + them from a second collection too (deletion only, never a content push). + """ + row = db.get_webhook_record(record_id) + if row is None: + return {"record_id": record_id, "status": "missing", "chunks": 0} + + doc_id = str(row["id"]) + chunks = build_chunks(row, count_tokens) + _delete_points(store, collection, doc_id) + if chunks: + store.upsert(collection, chunks, batch_size=8) + elif also_delete_from is not None: + other_store, other_collection = also_delete_from + _delete_points(other_store, other_collection, doc_id) + + marker = db.mark_webhook_record_promoted if mark == "promoted" else db.mark_webhook_record_indexed + marker(record_id, row["payload_hash"]) + logger.info("advisory %s record=%s collection=%s chunks=%s", mark, record_id, collection, len(chunks)) + return {"record_id": record_id, "status": "indexed" if chunks else "deleted", "chunks": len(chunks)} + + +def prod_target() -> Optional[tuple[object, str]]: + """The PROD (store, collection) pair from the PROD_VECTOR_DB_* env, or None when not configured.""" + url = (os.environ.get("PROD_VECTOR_DB_URL") or "").strip() + if not url: + return None + from .vector_store.qdrant_store import QdrantVectorStore, get_qdrant_client + + api_key = (os.environ.get("PROD_VECTOR_DB_API_KEY") or "").strip() or None + collection = ( + os.environ.get("PROD_VECTOR_DB_COLLECTION_NAME") + or os.environ.get("VECTOR_DB_COLLECTION_NAME") + or "documents-index" + ).strip() + return QdrantVectorStore(client=get_qdrant_client(url=url, api_key=api_key)), collection + + +def log_advisory_audit(record_id: str, mark: str, collection: str, result: dict) -> None: + """Write one audit_logs row for an indexed / deleted / promoted advisory record.""" + action = "advisory_promoted" if mark == "promoted" else ( + "advisory_deleted" if result.get("status") == "deleted" else "advisory_indexed" + ) + db.log_audit( + workflow_id=f"advisory-{record_id}", + document_id=record_id, + action_type=action, + entity_type="webhook_record", + metadata={"collection": collection, "chunks": result.get("chunks", 0), "status": result.get("status")}, + ) diff --git a/pipeline/advisory_start.py b/pipeline/advisory_start.py new file mode 100644 index 0000000..cc3ef07 --- /dev/null +++ b/pipeline/advisory_start.py @@ -0,0 +1,58 @@ +"""Start advisory index / promote workflows: one Temporal workflow per webhook record.""" + +from __future__ import annotations + +import logging +import os + +from temporalio.exceptions import WorkflowAlreadyStartedError + +from .workflows import AdvisoryIndexWorkflow, AdvisoryPromoteWorkflow + +logger = logging.getLogger(__name__) + +TASK_QUEUE = "ocr-pipeline" + + +async def _start_each(client, workflow_run, id_prefix: str, record_ids: list[str]) -> int: + started = 0 + for record_id in record_ids: + try: + await client.start_workflow( + workflow_run, + args=[record_id], + id=f"{id_prefix}{record_id}", + task_queue=TASK_QUEUE, + ) + started += 1 + except WorkflowAlreadyStartedError: + logger.info("advisory workflow %s%s already running; skipped", id_prefix, record_id) + except Exception: + logger.exception("could not start advisory workflow %s%s", id_prefix, record_id) + return started + + +async def start_advisory_index(client, record_ids: list[str]) -> int: + """Start ``AdvisoryIndexWorkflow`` (id ``advisory-``) for each id; returns how many started.""" + return await _start_each(client, AdvisoryIndexWorkflow.run, "advisory-", record_ids) + + +async def start_advisory_promote(client, record_ids: list[str]) -> int: + """Start ``AdvisoryPromoteWorkflow`` (id ``advisory-promote-``) for each id.""" + return await _start_each(client, AdvisoryPromoteWorkflow.run, "advisory-promote-", record_ids) + + +def _backfill_enabled() -> bool: + return os.environ.get("ADVISORY_BACKFILL_ON_START", "false").strip().lower() in {"1", "true", "yes", "on"} + + +async def backfill_on_start(client) -> int: + """Worker startup hook: when ADVISORY_BACKFILL_ON_START is on (default off), queue every unindexed row.""" + if not _backfill_enabled(): + return 0 + from . import db + + record_ids = db.get_unindexed_webhook_record_ids() + started = await start_advisory_index(client, record_ids) + logger.info("advisory startup backfill: %s unindexed rows, %s workflows started", len(record_ids), started) + return started diff --git a/pipeline/api.py b/pipeline/api.py index b8f0bfd..5b3cf31 100644 --- a/pipeline/api.py +++ b/pipeline/api.py @@ -43,6 +43,7 @@ from temporalio.client import Client, WorkflowFailureError from . import db, document_validity, scheme_catalog +from .advisory_start import start_advisory_index, start_advisory_promote from .auth.config import load_auth_config, validate_auth_config from .auth.deps import ( CurrentUser, @@ -123,6 +124,8 @@ datefmt='%H:%M:%S', ) +logger = logging.getLogger(__name__) + TASK_QUEUE = "ocr-pipeline" _TOKEN_RE = re.compile(r"[\w\-]+", re.UNICODE) @@ -250,6 +253,11 @@ async def lifespan(app: FastAPI): RATE_LIMIT_OTP = os.environ.get("RATE_LIMIT_OTP", "10/minute") _DOCUMENT_PATH_RE = re.compile(r"^/documents/([^/]+)(/.*)?$") +# Only identifier shapes with an indexed lookup are resolved before auth runs: +# the document UUID, the 32-hex fingerprint, or a ``doc-`` workflow id. +_RESOLVABLE_IDENTIFIER_RE = re.compile( + r"^(?:[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}|[0-9a-f]{32}|doc-[0-9a-f]{32}.*)$" +) _static_document_segments: Optional[set[str]] = None @@ -272,10 +280,15 @@ async def canonicalize_document_identifier(request: Request, call_next): Handlers address a document by its Temporal workflow id (they signal and query Temporal with it), so the path is rewritten to that id before routing. - An identifier that matches no document is left alone and 404s in the handler. + An identifier of another shape, or one that matches no document, is left alone and + 404s in the handler; the lookup runs before auth, so it is limited to cheap shapes. """ match = _DOCUMENT_PATH_RE.match(request.scope["path"]) - if match and match.group(1) not in _static_document_path_segments(): + if ( + match + and _RESOLVABLE_IDENTIFIER_RE.match(match.group(1)) + and match.group(1) not in _static_document_path_segments() + ): document = await asyncio.to_thread(db.find_document_by_doc_identifier, match.group(1)) if document and document["workflow_id"] and document["workflow_id"] != match.group(1): new_path = f"/documents/{document['workflow_id']}{match.group(2) or ''}" @@ -4094,6 +4107,7 @@ async def webhook( count = db.upsert_webhook_records("webhook", records) except ValueError as e: raise HTTPException(422, str(e)) + await _start_advisory_index_for(records) return { "success": True, "message": "Webhook processed successfully.", @@ -4101,6 +4115,72 @@ async def webhook( } +async def _start_advisory_index_for(records: list) -> int: + """Start one advisory index workflow per record of this call that is not indexed yet. + + Best effort: the records are already saved, so a Temporal failure never fails the + webhook; the rows stay unindexed and the reindex endpoint (or worker startup backfill) + picks them up. + """ + if temporal_client is None: + logger.warning("Temporal client not ready; advisory indexing skipped for this webhook call") + return 0 + try: + external_ids = [db.webhook_external_id(r) for r in records if isinstance(r, dict)] + record_ids = db.get_unindexed_webhook_record_ids(external_ids=external_ids) + return await start_advisory_index(temporal_client, record_ids) + except Exception: + logger.exception("Could not start advisory indexing for this webhook call") + return 0 + + +class AdvisoryReindexBody(BaseModel): + record_ids: Optional[List[str]] = None + force: bool = False + + +class AdvisoryPromoteBody(BaseModel): + record_ids: Optional[List[str]] = None + + +def _require_temporal() -> Client: + if temporal_client is None: + raise HTTPException(503, "Temporal client is not ready") + return temporal_client + + +@app.post("/webhook/advisory/reindex") +async def advisory_reindex(body: AdvisoryReindexBody, user: RequireAdmin): + """Queue advisory indexing (dev collection): given record ids, else all unindexed (all with force).""" + client = _require_temporal() + if body.record_ids is not None: + record_ids = body.record_ids + elif body.force: + record_ids = db.get_webhook_record_ids() + else: + record_ids = db.get_unindexed_webhook_record_ids() + queued = await start_advisory_index(client, record_ids) + return {"success": True, "queued": queued} + + +@app.post("/webhook/advisory/promote") +async def advisory_promote(body: AdvisoryPromoteBody, user: RequireAdmin): + """Queue promotion to the PROD collection: given record ids, else all published records.""" + client = _require_temporal() + if body.record_ids is not None: + record_ids = body.record_ids + else: + record_ids = db.get_webhook_record_ids(published_only=True) + queued = await start_advisory_promote(client, record_ids) + return {"success": True, "queued": queued} + + +@app.get("/webhook/advisory/status") +async def advisory_status(user: RequireAdmin): + """Counts of webhook advisory records: total, published, indexed, unindexed, promoted.""" + return db.advisory_status_counts() + + # ============================================================================= # Vector index (passage schema) # ============================================================================= diff --git a/pipeline/db.py b/pipeline/db.py index 0799b44..6c5cfe9 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -110,8 +110,9 @@ def check_health() -> dict: with get_connection() as conn: conn.execute("SELECT 1") return {"reachable": True, "migrations_current": migrate.is_current(database_url())} - except Exception as exc: - return {"reachable": False, "migrations_current": False, "error": str(exc)} + except Exception: + logger.exception("database health check failed") + return {"reachable": False, "migrations_current": False} # ============================================================================= @@ -480,6 +481,21 @@ def _parse_timestamp(value: Any) -> Optional[datetime]: return parsed +def _webhook_payload_and_hash(rec: dict) -> tuple[str, str]: + payload = json.dumps(rec, sort_keys=True, ensure_ascii=False, default=str) + return payload, hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def webhook_external_id(rec: dict) -> str: + """The (source, external_id) key of a webhook record: contentID, else id, else slug, else payload hash.""" + _, payload_hash = _webhook_payload_and_hash(rec) + for key in ("contentID", "id", "slug"): + value = rec.get(key) + if value not in (None, ""): + return str(value) + return payload_hash + + def upsert_webhook_records(source: str, records: list) -> int: """Upsert records by (source, external_id) in one transaction; returns len(records). @@ -491,15 +507,8 @@ def upsert_webhook_records(source: str, records: list) -> int: for rec in records: if not isinstance(rec, dict): raise ValueError("Each record must be a JSON object") - payload_text = json.dumps(rec, sort_keys=True, ensure_ascii=False, default=str) - payload_hash = hashlib.sha256(payload_text.encode("utf-8")).hexdigest() - ext = rec.get("contentID") - if ext in (None, ""): - ext = rec.get("id") - if ext in (None, ""): - ext = rec.get("slug") - if ext in (None, ""): - ext = payload_hash + payload_text, payload_hash = _webhook_payload_and_hash(rec) + ext = webhook_external_id(rec) pub = rec.get("isPublished") rows.append(( source, str(ext), rec.get("title"), rec.get("slug"), rec.get("status"), @@ -533,6 +542,99 @@ def upsert_webhook_records(source: str, records: list) -> int: return len(rows) +_WEBHOOK_RECORD_COLUMNS = ( + "id, source, external_id, title, slug, status, is_published, category, payload, " + "payload_hash, created_at, updated_at, indexed_hash, promoted_hash" +) + + +def get_webhook_record(record_id: str) -> Optional[dict]: + """One webhook_records row as a dict (``payload`` already parsed from JSONB); None if missing.""" + if not _is_uuid(record_id): + return None + with get_connection() as conn: + row = conn.execute( + f"SELECT {_WEBHOOK_RECORD_COLUMNS} FROM webhook_records WHERE id = %s", (record_id,) + ).fetchone() + if row is None: + return None + record = dict(row) + record["id"] = str(record["id"]) + if not isinstance(record["payload"], dict): + record["payload"] = {} + return record + + +def get_unindexed_webhook_record_ids( + source: str = "webhook", external_ids: Optional[list] = None +) -> list[str]: + """Ids of rows whose payload changed since they were last indexed (or never indexed). + + ``external_ids`` limits the check to the records of one webhook call. + """ + sql = ( + "SELECT id FROM webhook_records WHERE source = %s " + "AND indexed_hash IS DISTINCT FROM payload_hash" + ) + params: list = [source] + if external_ids is not None: + if not external_ids: + return [] + sql += " AND external_id = ANY(%s)" + params.append([str(e) for e in external_ids]) + with get_connection() as conn: + return [str(r["id"]) for r in conn.execute(sql + " ORDER BY created_at, id", params)] + + +def get_webhook_record_ids(source: str = "webhook", published_only: bool = False) -> list[str]: + """All record ids for a source, optionally only published ones.""" + sql = "SELECT id FROM webhook_records WHERE source = %s" + if published_only: + sql += " AND is_published AND status = 'Published'" + with get_connection() as conn: + return [str(r["id"]) for r in conn.execute(sql + " ORDER BY created_at, id", (source,))] + + +def _mark_webhook_record(column: str, record_id: str, payload_hash: str) -> None: + with get_connection() as conn: + conn.execute(f"UPDATE webhook_records SET {column} = %s WHERE id = %s", (payload_hash, record_id)) + + +def mark_webhook_record_indexed(record_id: str, payload_hash: str) -> None: + """Record the payload_hash that was written to the dev collection.""" + _mark_webhook_record("indexed_hash", record_id, payload_hash) + + +def mark_webhook_record_promoted(record_id: str, payload_hash: str) -> None: + """Record the payload_hash that was written to the PROD collection.""" + _mark_webhook_record("promoted_hash", record_id, payload_hash) + + +def advisory_status_counts(source: str = "webhook") -> dict: + """Counts for the advisory status endpoint.""" + with get_connection() as conn: + row = conn.execute( + """ + SELECT + COUNT(*) AS total, + COUNT(*) FILTER (WHERE is_published AND status = 'Published') AS published, + COUNT(*) FILTER (WHERE indexed_hash = payload_hash) AS indexed, + COUNT(*) FILTER (WHERE promoted_hash = payload_hash) AS promoted + FROM webhook_records WHERE source = %s + """, + (source,), + ).fetchone() + total = int(row["total"]) + indexed = int(row["indexed"]) + return { + "total": total, + "published": int(row["published"]), + "indexed": indexed, + "unindexed": total - indexed, + "promoted": int(row["promoted"]), + } + + # ============================================================================= # Email OTPs # ============================================================================= diff --git a/pipeline/vector_store/embeddings.py b/pipeline/vector_store/embeddings.py index be9a30f..bc9feec 100644 --- a/pipeline/vector_store/embeddings.py +++ b/pipeline/vector_store/embeddings.py @@ -143,3 +143,15 @@ def embed_passages( batch = passages[i : i + batch_size] out.extend(embed_texts(batch, kind="passage", use_e5_prefix=use_e5_prefix)) return out + + +def count_tokens(text: str) -> int: + """Token count of ``text`` as the embedding model sees it (with the passage prefix). + + Uses the local model's tokenizer when embeddings run locally; for a remote + (OpenAI-compatible) provider there is no tokenizer here, so it estimates ~4 chars/token. + """ + prefixed = _prefix_passage(text) + if get_embedding_provider() == "openai_compatible": + return max(1, len(prefixed) // 4) + return len(_get_local_embedder().tokenizer.encode(prefixed)) diff --git a/pipeline/vector_store/qdrant_store.py b/pipeline/vector_store/qdrant_store.py index 844154c..d371353 100644 --- a/pipeline/vector_store/qdrant_store.py +++ b/pipeline/vector_store/qdrant_store.py @@ -53,6 +53,22 @@ "scheme_aliases", "chunk_id", "chunk_index", + # Advisory payload (document_kind=advisory; webhook records and PDF advisory documents) + "document_kind", + "external_id", + "pop_id", + "slug", + "crop", + "domain", + "subdomain", + "institute", + "creator", + "problem_type", + "problem", + "season", + "start_period", + "end_period", + "updated_at", # Document validity - the period this chunk is searchable in. Absent on # points written before validity existed, which search treats as current. "start_date", diff --git a/pipeline/worker.py b/pipeline/worker.py index 697ebe2..f505b4a 100644 --- a/pipeline/worker.py +++ b/pipeline/worker.py @@ -16,17 +16,22 @@ create_chunks_from_db, detect_and_translate_pages, detect_and_translate_pages_from_db, + index_webhook_advisory, ingest_document_from_db, ingest_to_vector_db, persist_document_content, prepare_for_ingestion, promote_document_to_prod_qdrant, + promote_webhook_advisory, publish_catalog_to_network, run_ocr, run_ocr_and_store, update_document_state, ) +from .advisory_start import backfill_on_start from .workflows import ( + AdvisoryIndexWorkflow, + AdvisoryPromoteWorkflow, ChunkingOnlyWorkflow, DocumentPipelineWorkflow, OcrOnlyWorkflow, @@ -81,6 +86,8 @@ async def main(): TranslationOnlyWorkflow, OcrOnlyWorkflow, ChunkingOnlyWorkflow, + AdvisoryIndexWorkflow, + AdvisoryPromoteWorkflow, ], activities=[ run_ocr, @@ -92,6 +99,8 @@ async def main(): ingest_to_vector_db, ingest_document_from_db, promote_document_to_prod_qdrant, + index_webhook_advisory, + promote_webhook_advisory, publish_catalog_to_network, update_document_state, detect_and_translate_pages, @@ -100,6 +109,10 @@ async def main(): ], ) + backfilled = await backfill_on_start(client) + if backfilled: + print(f"Advisory startup backfill queued {backfilled} workflows") + print(f"Worker started on queue: {TASK_QUEUE}") print("Waiting for workflows...") await worker.run() diff --git a/pipeline/workflows.py b/pipeline/workflows.py index d6ab900..4ce63fb 100644 --- a/pipeline/workflows.py +++ b/pipeline/workflows.py @@ -17,8 +17,10 @@ auto_tag_chunks_from_db, create_chunks_from_db, detect_and_translate_pages_from_db, + index_webhook_advisory, ingest_document_from_db, promote_document_to_prod_qdrant, + promote_webhook_advisory, publish_catalog_to_network, run_ocr_and_store, update_document_state, @@ -887,3 +889,59 @@ def get_state(self) -> dict: "chunk_count": self.state.chunk_count, "chunks_completed_at": self.state.chunks_completed_at, } + + +@workflow.defn +class AdvisoryIndexWorkflow: + """Index one webhook advisory record into the dev Qdrant collection.""" + + def __init__(self): + self.state = None + + @workflow.run + async def run(self, record_id: str) -> dict: + self.state = {"record_id": record_id, "stage": "indexing"} + try: + result = await workflow.execute_activity( + index_webhook_advisory, + record_id, + start_to_close_timeout=timedelta(minutes=30), + retry_policy=INGEST_RETRY, + ) + except Exception as e: + self.state.update(stage="failed", error_message=str(e)) + raise + self.state.update(stage="completed", status=result.get("status"), chunks=result.get("chunks")) + return result + + @workflow.query + def get_state(self) -> dict: + return self.state or {} + + +@workflow.defn +class AdvisoryPromoteWorkflow: + """Promote one webhook advisory record into the PROD Qdrant collection.""" + + def __init__(self): + self.state = None + + @workflow.run + async def run(self, record_id: str) -> dict: + self.state = {"record_id": record_id, "stage": "promoting"} + try: + result = await workflow.execute_activity( + promote_webhook_advisory, + record_id, + start_to_close_timeout=timedelta(minutes=30), + retry_policy=INGEST_RETRY, + ) + except Exception as e: + self.state.update(stage="failed", error_message=str(e)) + raise + self.state.update(stage="completed", status=result.get("status"), chunks=result.get("chunks")) + return result + + @workflow.query + def get_state(self) -> dict: + return self.state or {} diff --git a/tests/test_activities.py b/tests/test_activities.py index e17cd92..a752533 100644 --- a/tests/test_activities.py +++ b/tests/test_activities.py @@ -163,6 +163,35 @@ def test_prepare_records_carries_instance_name(self): assert records[0]["instance"] == code assert records[0]["instance_name"] == expected + @pytest.mark.unit + def test_prepare_records_carries_document_kind(self): + """Every record carries its document_kind; only scheme documents get type=scheme.""" + from pipeline.activities import _prepare_records + from pipeline.vector_store.qdrant_store import _record_payload + + chunks = [{"chunk_number": 1, "original_text": "Test chunk", "token_count": 5}] + + advisory = _prepare_records( + document_id="test-doc", filename="test.pdf", chunks=chunks, document_kind="advisory" + ) + assert advisory[0]["document_kind"] == "advisory" + assert advisory[0]["type"] == "document" + assert _record_payload(advisory[0])["document_kind"] == "advisory" + + default = _prepare_records(document_id="test-doc", filename="test.pdf", chunks=chunks) + assert default[0]["document_kind"] == "document" + + scheme = _prepare_records( + document_id="test-doc", + filename="test.pdf", + chunks=chunks, + document_kind="scheme", + scheme_code="pm-kisan", + scheme_name="PM-KISAN", + ) + assert scheme[0]["type"] == "scheme" + assert scheme[0]["document_kind"] == "scheme" + @pytest.mark.unit def test_prepare_records_fills_scheme_aliases(self): """A scheme must never index with scheme_aliases: [].""" diff --git a/tests/test_advisory_activities.py b/tests/test_advisory_activities.py new file mode 100644 index 0000000..0ac0dd2 --- /dev/null +++ b/tests/test_advisory_activities.py @@ -0,0 +1,126 @@ +"""Tests for the advisory index/promote activities, the PROD target and the workflow exports.""" + +import asyncio +import copy +from pathlib import Path + +import pytest + +from pipeline import advisory_index +from tests.test_advisory_chunks import SORGHUM_PAYLOAD +from tests.test_advisory_index import FakeStore + + +def _seed(db): + db.upsert_webhook_records("webhook", [copy.deepcopy(SORGHUM_PAYLOAD)]) + with db.get_connection() as conn: + return str(conn.execute("SELECT id FROM webhook_records").fetchone()["id"]) + + +def _audit_actions(db): + with db.get_connection() as conn: + return [r["action_type"] for r in conn.execute("SELECT action_type FROM audit_logs")] + + +@pytest.fixture +def qdrant_env(monkeypatch): + monkeypatch.setenv("VECTOR_DB_COLLECTION_NAME", "document-data") + monkeypatch.delenv("PROD_VECTOR_DB_URL", raising=False) + monkeypatch.setattr("pipeline.vector_store.embeddings.count_tokens", lambda text: 250) + + +@pytest.mark.unit +@pytest.mark.db +class TestActivities: + def test_index_activity_writes_to_the_dev_collection_and_audits(self, db_connection, qdrant_env, monkeypatch): + from pipeline.activities import index_webhook_advisory + + rid = _seed(db_connection) + store = FakeStore() + monkeypatch.setattr("pipeline.vector_store.get_vector_store", lambda: store) + result = asyncio.run(index_webhook_advisory(rid)) + assert result["status"] == "indexed" + assert store.calls[0][1] == "document-data" + assert db_connection.get_unindexed_webhook_record_ids() == [] + assert "advisory_indexed" in _audit_actions(db_connection) + + def test_promote_activity_writes_to_prod_and_marks_promoted(self, db_connection, qdrant_env, monkeypatch): + from pipeline.activities import promote_webhook_advisory + + rid = _seed(db_connection) + prod = FakeStore() + monkeypatch.setattr(advisory_index, "prod_target", lambda: (prod, "prod-collection")) + result = asyncio.run(promote_webhook_advisory(rid)) + assert result["status"] == "indexed" + assert prod.calls[0][1] == "prod-collection" + assert db_connection.advisory_status_counts()["promoted"] == 1 + assert "advisory_promoted" in _audit_actions(db_connection) + + def test_promote_activity_requires_a_prod_target(self, db_connection, qdrant_env, monkeypatch): + from pipeline.activities import promote_webhook_advisory + + monkeypatch.setattr(advisory_index, "prod_target", lambda: None) + with pytest.raises(RuntimeError, match="PROD_VECTOR_DB_URL"): + asyncio.run(promote_webhook_advisory("any")) + + def test_dev_index_of_unpublished_record_also_clears_prod(self, db_connection, qdrant_env, monkeypatch): + from pipeline.activities import index_webhook_advisory + + payload = copy.deepcopy(SORGHUM_PAYLOAD) + payload["isPublished"] = 0 + db_connection.upsert_webhook_records("webhook", [payload]) + with db_connection.get_connection() as conn: + rid = str(conn.execute("SELECT id FROM webhook_records").fetchone()["id"]) + dev, prod = FakeStore(), FakeStore() + monkeypatch.setattr("pipeline.vector_store.get_vector_store", lambda: dev) + monkeypatch.setattr(advisory_index, "prod_target", lambda: (prod, "prod-collection")) + result = asyncio.run(index_webhook_advisory(rid)) + assert result["status"] == "deleted" + assert prod.calls == [("delete", "prod-collection", rid)] + + def test_dev_index_survives_a_broken_prod_config(self, db_connection, qdrant_env, monkeypatch): + from pipeline.activities import index_webhook_advisory + + rid = _seed(db_connection) + monkeypatch.setattr("pipeline.vector_store.get_vector_store", lambda: FakeStore()) + + def broken(): + raise ValueError("QDRANT_API_KEY is required for remote Qdrant") + + monkeypatch.setattr(advisory_index, "prod_target", broken) + assert asyncio.run(index_webhook_advisory(rid))["status"] == "indexed" + + +class TestProdTarget: + def test_none_without_prod_url(self, monkeypatch): + monkeypatch.delenv("PROD_VECTOR_DB_URL", raising=False) + assert advisory_index.prod_target() is None + + def test_uses_prod_env(self, monkeypatch): + monkeypatch.setenv("PROD_VECTOR_DB_URL", "http://prod:6333") + monkeypatch.setenv("PROD_VECTOR_DB_API_KEY", "k") + monkeypatch.setenv("PROD_VECTOR_DB_COLLECTION_NAME", "prod-docs") + seen = {} + + def fake_client(url=None, api_key=None): + seen.update(url=url, api_key=api_key) + return object() + + monkeypatch.setattr("pipeline.vector_store.qdrant_store.get_qdrant_client", fake_client) + target = advisory_index.prod_target() + assert target is not None and target[1] == "prod-docs" + assert seen == {"url": "http://prod:6333", "api_key": "k"} + + +def test_workflows_are_defined_and_registered(): + from pipeline.workflows import AdvisoryIndexWorkflow, AdvisoryPromoteWorkflow + + assert callable(AdvisoryIndexWorkflow.run) and callable(AdvisoryPromoteWorkflow.run) + source = (Path(__file__).resolve().parent.parent / "pipeline" / "worker.py").read_text() + for name in ( + "AdvisoryIndexWorkflow", + "AdvisoryPromoteWorkflow", + "index_webhook_advisory", + "promote_webhook_advisory", + ): + assert source.count(name) >= 2, name # imported and registered diff --git a/tests/test_advisory_api.py b/tests/test_advisory_api.py new file mode 100644 index 0000000..46cc3ca --- /dev/null +++ b/tests/test_advisory_api.py @@ -0,0 +1,165 @@ +"""API tests: /webhook starts advisory workflows; admin backfill / promote / status endpoints.""" + +import os +from unittest.mock import AsyncMock, patch + +import pytest + +TEST_KEY = "0123456789abcdef0123456789abcdef" +H = {"x-api-key": TEST_KEY} + + +def _rec(content_id, published=1, status="Published"): + return { + "contentID": content_id, + "title": f"Advisory {content_id}", + "status": status, + "isPublished": published, + "popList": [{"popID": 1, "status": "Published"}], + } + + +@pytest.fixture +def client(db_connection, monkeypatch): + from fastapi.testclient import TestClient + + from pipeline import api + + monkeypatch.setattr(api.limiter, "enabled", False) # many posts per test process + with patch.dict(os.environ, {"WEBHOOK_API_KEY": TEST_KEY, "AUTH_DISABLED": "true"}): + yield TestClient(api.app) + + +@pytest.fixture +def temporal(monkeypatch): + from pipeline import api + + mock = AsyncMock() + monkeypatch.setattr(api, "temporal_client", mock) + return mock + + +def _row_ids(db, *content_ids): + with db.get_connection() as conn: + return { + c: str(conn.execute( + "SELECT id FROM webhook_records WHERE external_id = %s", (str(c),) + ).fetchone()["id"]) + for c in content_ids + } + + +def _started(mock): + return sorted(call.kwargs["id"] for call in mock.start_workflow.await_args_list) + + +@pytest.mark.api +@pytest.mark.unit +class TestWebhookStartsIndexing: + def test_one_workflow_per_record_in_the_call(self, client, db_connection, temporal): + r = client.post("/webhook", json=[_rec(1), _rec(2)], headers=H) + assert r.status_code == 200 + assert r.json() == { + "success": True, + "message": "Webhook processed successfully.", + "recordCount": 2, + } + ids = _row_ids(db_connection, 1, 2) + assert _started(temporal) == sorted(f"advisory-{i}" for i in ids.values()) + + def test_only_records_in_the_call_are_started(self, client, db_connection, temporal): + db_connection.upsert_webhook_records("webhook", [_rec(1)]) # existing, unindexed + client.post("/webhook", json=[_rec(2)], headers=H) + ids = _row_ids(db_connection, 1, 2) + assert _started(temporal) == [f"advisory-{ids[2]}"] + + def test_existing_never_indexed_row_is_started_when_resent(self, client, db_connection, temporal): + db_connection.upsert_webhook_records("webhook", [_rec(1)]) + client.post("/webhook", json=[_rec(1)], headers=H) + ids = _row_ids(db_connection, 1) + assert _started(temporal) == [f"advisory-{ids[1]}"] + + def test_nothing_started_when_already_indexed(self, client, db_connection, temporal): + client.post("/webhook", json=[_rec(1)], headers=H) + rid = _row_ids(db_connection, 1)[1] + db_connection.mark_webhook_record_indexed(rid, db_connection.get_webhook_record(rid)["payload_hash"]) + temporal.start_workflow.reset_mock() + r = client.post("/webhook", json=[_rec(1)], headers=H) + assert r.status_code == 200 + temporal.start_workflow.assert_not_called() + + def test_still_200_when_temporal_fails(self, client, db_connection, temporal): + temporal.start_workflow.side_effect = RuntimeError("temporal down") + r = client.post("/webhook", json=[_rec(1)], headers=H) + assert r.status_code == 200 + assert len(db_connection.get_unindexed_webhook_record_ids()) == 1 + + def test_still_200_when_temporal_client_is_none(self, client, db_connection, monkeypatch): + from pipeline import api + + monkeypatch.setattr(api, "temporal_client", None) + assert client.post("/webhook", json=[_rec(1)], headers=H).status_code == 200 + + +@pytest.mark.api +@pytest.mark.unit +class TestAdminEndpoints: + def _seed(self, client, temporal, *records): + client.post("/webhook", json=list(records), headers=H) + temporal.start_workflow.reset_mock() + + def test_reindex_queues_all_unindexed_by_default(self, client, db_connection, temporal): + self._seed(client, temporal, _rec(1), _rec(2)) + r = client.post("/webhook/advisory/reindex", json={}) + assert r.status_code == 200 and r.json() == {"success": True, "queued": 2} + assert len(_started(temporal)) == 2 + + def test_reindex_force_queues_indexed_rows_too(self, client, db_connection, temporal): + self._seed(client, temporal, _rec(1)) + rid = _row_ids(db_connection, 1)[1] + db_connection.mark_webhook_record_indexed(rid, db_connection.get_webhook_record(rid)["payload_hash"]) + assert client.post("/webhook/advisory/reindex", json={}).json()["queued"] == 0 + assert client.post("/webhook/advisory/reindex", json={"force": True}).json()["queued"] == 1 + + def test_reindex_explicit_ids(self, client, db_connection, temporal): + self._seed(client, temporal, _rec(1), _rec(2)) + rid = _row_ids(db_connection, 2)[2] + r = client.post("/webhook/advisory/reindex", json={"record_ids": [rid]}) + assert r.json()["queued"] == 1 + assert _started(temporal) == [f"advisory-{rid}"] + + def test_promote_defaults_to_published_records(self, client, db_connection, temporal): + self._seed(client, temporal, _rec(1), _rec(2, published=0, status="Draft")) + r = client.post("/webhook/advisory/promote", json={}) + assert r.json() == {"success": True, "queued": 1} + published = _row_ids(db_connection, 1)[1] + assert _started(temporal) == [f"advisory-promote-{published}"] + + def test_promote_explicit_ids(self, client, db_connection, temporal): + self._seed(client, temporal, _rec(1), _rec(2)) + rid = _row_ids(db_connection, 2)[2] + r = client.post("/webhook/advisory/promote", json={"record_ids": [rid]}) + assert r.json()["queued"] == 1 + assert _started(temporal) == [f"advisory-promote-{rid}"] + + def test_status_returns_counts(self, client, db_connection, temporal): + self._seed(client, temporal, _rec(1), _rec(2, published=0, status="Draft")) + r = client.get("/webhook/advisory/status") + assert r.status_code == 200 + assert r.json() == {"total": 2, "published": 1, "indexed": 0, "unindexed": 2, "promoted": 0} + + def test_endpoints_require_a_signed_in_admin(self, db_connection, temporal, monkeypatch): + from fastapi.testclient import TestClient + + from pipeline import api + + env = { + "AUTH_DISABLED": "false", + "KEYCLOAK_ISSUER": "https://example.com/realms/test", + "KEYCLOAK_JWKS_URL": "https://example.com/realms/test/protocol/openid-connect/certs", + } + with patch.dict(os.environ, env): + c = TestClient(api.app) + assert c.post("/webhook/advisory/reindex", json={}).status_code == 401 + assert c.post("/webhook/advisory/promote", json={}).status_code == 401 + assert c.get("/webhook/advisory/status").status_code == 401 diff --git a/tests/test_advisory_chunks.py b/tests/test_advisory_chunks.py new file mode 100644 index 0000000..913a805 --- /dev/null +++ b/tests/test_advisory_chunks.py @@ -0,0 +1,218 @@ +"""Tests for the webhook advisory chunk builder.""" + +import copy +import hashlib + +import pytest + +from pipeline.advisory_chunks import build_chunks, clean_value + +HEADER = ( + "Sorghum- Harvesting Management\n" + "Crop: Sorghum | Problem type: Harvesting Management | Season: Rabi" +) +OVERVIEW = ( + "Timely harvesting of AKJ-1 helps preserve grain appearance, processing quality and " + "suitability for flaking. Proper drying and handling after harvest are essential to " + "minimize grain damage and deterioration." +) +SYMPTOMS = ( + "Panicles become mature and dry, grains become hard and attain characteristic colour, " + "and most leaves begin to dry." +) +CONDITIONS = ( + "Crop has reached physiological maturity and panicles are fully developed; grains are " + "firm and suitable for flaking quality." +) +ADVISORY = ( + "Harvest AKJ-1 at physiological maturity when the grains are fully developed and firm. " + "Timely harvesting is important for maintaining grain quality required for flaking. " + "Avoid delayed harvesting, which may increase grain weathering, shattering and mould " + "risk. After harvesting, dry the panicles properly, thresh carefully, clean the grain " + "and reduce moisture to a safe level before storage or processing." +) + +SORGHUM_PAYLOAD = { + "contentID": 987, + "title": "Sorghum- Harvesting Management", + "commodityName": "Sorghum", + "creatorName": "jak", + "domainName": "Agriculture", + "subdomainName": "Cereals", + "instituteName": "ICAR–Directorate of Knowledge Management in Agriculture (DKMA)", + "isPublished": 1, + "status": "Published", + "popList": [ + { + "popID": 819, + "season": "Rabi", + "problemType": '"Harvesting Management"', + "problemDetails": '"Method Of Harvesting"', + "startPeriod": "105", + "endPeriod": "120", + "status": "Published", + "description": f'"{OVERVIEW}"', + "applicableSymptoms": [SYMPTOMS], + "applicableCondition": [CONDITIONS], + "advisory": f'"{ADVISORY}"', + } + ], +} + + +def make_row(payload=None, **overrides): + payload = copy.deepcopy(SORGHUM_PAYLOAD if payload is None else payload) + row = { + "id": "row-1", + "external_id": "987", + "title": payload.get("title", ""), + "slug": "987/sorghum--harvesting-management", + "status": payload.get("status"), + "is_published": payload.get("isPublished"), + "payload": payload, + "payload_hash": "hash-1", + "updated_at": "2026-10-16T09:15:10.000Z", + } + row.update(overrides) + return row + + +def long_when_combined(text: str) -> int: + """Over the limit only when the entry carries more than one question-side field.""" + return 600 if "Overview: " in text and "Symptoms: " in text else 300 + + +def pop_of(row): + return row["payload"]["popList"][0] + + +class TestCleanValue: + def test_strips_wrapping_quotes(self): + assert clean_value('"Aphids"') == "Aphids" + assert clean_value('""') == "" + + def test_handles_none_numbers_and_whitespace(self): + assert clean_value(None) == "" + assert clean_value(819) == "819" + assert clean_value(" x ") == "x" + + +class TestShortEntry: + def test_short_entry_is_one_full_chunk(self): + chunks = build_chunks(make_row(), lambda t: 250) + assert len(chunks) == 1 + assert chunks[0]["section"] == "full" + assert chunks[0]["text"] == "\n".join( + [ + HEADER, + f"Overview: {OVERVIEW}", + f"Symptoms: {SYMPTOMS}", + f"Conditions: {CONDITIONS}", + f"Advisory: {ADVISORY}", + ] + ) + assert chunks[0]["text_for_embedding"] == chunks[0]["text"] + + +class TestLongEntry: + def test_long_entry_splits_into_three_chunks_each_with_advice(self): + chunks = build_chunks(make_row(), long_when_combined) + assert [c["section"] for c in chunks] == ["overview", "symptoms", "conditions"] + for chunk in chunks: + assert chunk["text"].startswith(HEADER) + assert chunk["text"].endswith(f"Advisory: {ADVISORY}") + assert "Symptoms:" not in chunks[0]["text"] + assert f"Symptoms: {SYMPTOMS}" in chunks[1]["text"] + assert [c["chunk_num"] for c in chunks] == [0, 1, 2] + + def test_empty_conditions_gives_two_chunks(self): + row = make_row() + pop_of(row)["applicableCondition"] = [] + chunks = build_chunks(row, long_when_combined) + assert [c["section"] for c in chunks] == ["overview", "symptoms"] + + def test_no_advisory_gives_one_chunk_per_field(self): + row = make_row() + pop_of(row)["advisory"] = None + chunks = build_chunks(row, lambda t: 600) + assert [c["section"] for c in chunks] == ["overview", "symptoms", "conditions"] + for chunk in chunks: + assert "Advisory:" not in chunk["text"] + + def test_only_advisory_is_one_advisory_chunk(self): + row = make_row() + pop = pop_of(row) + pop["description"] = None + pop["applicableSymptoms"] = [] + pop["applicableCondition"] = [] + chunks = build_chunks(row, lambda t: 600) + assert [c["section"] for c in chunks] == ["advisory"] + assert chunks[0]["text"] == f"{HEADER}\nAdvisory: {ADVISORY}" + + +class TestPublishingAndShape: + @pytest.mark.parametrize( + "overrides", + [{"is_published": 0}, {"status": "Draft"}], + ) + def test_unpublished_record_returns_empty(self, overrides): + assert build_chunks(make_row(**overrides), lambda t: 250) == [] + + def test_unpublished_entry_is_skipped(self): + row = make_row() + row["payload"]["popList"].append( + {**copy.deepcopy(pop_of(row)), "popID": 820, "status": "Draft"} + ) + chunks = build_chunks(row, lambda t: 250) + assert [c["pop_id"] for c in chunks] == [819] + + @pytest.mark.parametrize("pop_list", [None, [], "nope"]) + def test_missing_or_empty_poplist_returns_empty(self, pop_list): + row = make_row() + row["payload"]["popList"] = pop_list + assert build_chunks(row, lambda t: 250) == [] + + def test_ids_are_stable_and_unique(self): + first = build_chunks(make_row(), long_when_combined) + second = build_chunks(make_row(), long_when_combined) + assert [c["_id"] for c in first] == [c["_id"] for c in second] + assert len({c["_id"] for c in first}) == 3 + full = build_chunks(make_row(), lambda t: 250)[0] + expected = hashlib.md5(b"row-1:819:full").hexdigest() + assert full["_id"] == expected + assert full["chunk_id"] == expected + + def test_metadata_fields(self): + chunk = build_chunks(make_row(), lambda t: 250)[0] + assert chunk["document_kind"] == "advisory" + assert chunk["type"] == "document" + assert chunk["source"] == "webhook" + assert chunk["workflow_id"] == "" + assert chunk["doc_id"] == "row-1" + assert chunk["external_id"] == "987" + assert chunk["pop_id"] == 819 + assert chunk["title_en"] == "Sorghum- Harvesting Management" + assert chunk["slug"] == "987/sorghum--harvesting-management" + assert chunk["crop"] == "Sorghum" + assert chunk["domain"] == "Agriculture" + assert chunk["subdomain"] == "Cereals" + assert chunk["institute"].startswith("ICAR") + assert chunk["creator"] == "jak" + assert chunk["problem_type"] == "Harvesting Management" + assert chunk["problem"] == "Method Of Harvesting" + assert chunk["season"] == "Rabi" + assert chunk["start_period"] == "105" + assert chunk["end_period"] == "120" + assert chunk["updated_at"] == "2026-10-16T09:15:10.000Z" + + def test_null_values_never_appear_as_text(self): + row = make_row() + pop = pop_of(row) + pop["description"] = None + pop["season"] = None + pop["applicableSymptoms"] = [None, "x"] + chunks = build_chunks(row, lambda t: 250) + assert len(chunks) == 1 + assert "None" not in chunks[0]["text"] + assert "Season:" not in chunks[0]["text"] + assert "Symptoms: x" in chunks[0]["text"] diff --git a/tests/test_advisory_index.py b/tests/test_advisory_index.py new file mode 100644 index 0000000..ab71be5 --- /dev/null +++ b/tests/test_advisory_index.py @@ -0,0 +1,152 @@ +"""Tests for indexing one webhook advisory record into a vector store.""" + +import copy + +import pytest + +from pipeline.advisory_chunks import build_chunks +from pipeline.advisory_index import index_advisory_record +from pipeline.vector_store.qdrant_store import _record_payload +from tests.test_advisory_chunks import SORGHUM_PAYLOAD, make_row + + +class FakeStore: + def __init__(self, delete_result=None, fail_upsert=False): + self.calls = [] + self.records = [] + self._delete_result = delete_result or {"deleted": -1} + self._fail_upsert = fail_upsert + + def delete_by_doc_id(self, name, doc_id): + self.calls.append(("delete", name, doc_id)) + return self._delete_result + + def upsert(self, name, records, batch_size=32): + self.calls.append(("upsert", name, len(records))) + if self._fail_upsert: + raise RuntimeError("qdrant down") + self.records = list(records) + return {"records_ingested": len(records)} + + +def _store_row(db, payload): + db.upsert_webhook_records("webhook", [payload]) + with db.get_connection() as conn: + return str(conn.execute( + "SELECT id FROM webhook_records WHERE external_id = %s", (str(payload["contentID"]),) + ).fetchone()["id"]) + + +def _count(_text): + return 250 + + +@pytest.mark.unit +@pytest.mark.db +class TestIndexAdvisoryRecord: + def test_published_record_is_deleted_then_upserted_then_marked(self, db_connection): + db = db_connection + rid = _store_row(db, copy.deepcopy(SORGHUM_PAYLOAD)) + store = FakeStore() + result = index_advisory_record(rid, store, "document-data", _count) + assert [c[0] for c in store.calls] == ["delete", "upsert"] + assert store.calls[0][1:] == ("document-data", rid) + assert result == {"record_id": rid, "status": "indexed", "chunks": 1} + assert db.get_unindexed_webhook_record_ids() == [] + + def test_unpublished_record_only_deletes(self, db_connection): + db = db_connection + payload = copy.deepcopy(SORGHUM_PAYLOAD) + payload["isPublished"] = 0 + rid = _store_row(db, payload) + store = FakeStore() + result = index_advisory_record(rid, store, "document-data", _count) + assert [c[0] for c in store.calls] == ["delete"] + assert result["status"] == "deleted" and result["chunks"] == 0 + assert db.get_unindexed_webhook_record_ids() == [] + + def test_unpublished_record_also_deletes_from_the_other_collection(self, db_connection): + db = db_connection + payload = copy.deepcopy(SORGHUM_PAYLOAD) + payload["isPublished"] = 0 + rid = _store_row(db, payload) + dev, prod = FakeStore(), FakeStore() + index_advisory_record( + rid, dev, "document-data", _count, also_delete_from=(prod, "prod-collection") + ) + assert prod.calls == [("delete", "prod-collection", rid)] + + def test_published_record_does_not_touch_the_other_collection(self, db_connection): + db = db_connection + rid = _store_row(db, copy.deepcopy(SORGHUM_PAYLOAD)) + dev, prod = FakeStore(), FakeStore() + index_advisory_record( + rid, dev, "document-data", _count, also_delete_from=(prod, "prod-collection") + ) + assert prod.calls == [] + + def test_missing_record_does_nothing(self, db_connection): + store = FakeStore() + result = index_advisory_record("nope", store, "document-data", _count) + assert store.calls == [] + assert result == {"record_id": "nope", "status": "missing", "chunks": 0} + + def test_record_without_poplist_is_marked_indexed(self, db_connection): + db = db_connection + payload = copy.deepcopy(SORGHUM_PAYLOAD) + payload["popList"] = [] + rid = _store_row(db, payload) + store = FakeStore() + result = index_advisory_record(rid, store, "document-data", _count) + assert [c[0] for c in store.calls] == ["delete"] + assert result["status"] == "deleted" + assert db.get_unindexed_webhook_record_ids() == [] + + def test_upsert_failure_leaves_record_unindexed(self, db_connection): + db = db_connection + rid = _store_row(db, copy.deepcopy(SORGHUM_PAYLOAD)) + with pytest.raises(RuntimeError, match="qdrant down"): + index_advisory_record(rid, FakeStore(fail_upsert=True), "document-data", _count) + assert db.get_unindexed_webhook_record_ids() == [rid] + + def test_delete_error_other_than_missing_collection_fails(self, db_connection): + db = db_connection + rid = _store_row(db, copy.deepcopy(SORGHUM_PAYLOAD)) + store = FakeStore(delete_result={"deleted": 0, "error": "connection refused"}) + with pytest.raises(RuntimeError, match="connection refused"): + index_advisory_record(rid, store, "document-data", _count) + assert db.get_unindexed_webhook_record_ids() == [rid] + + def test_missing_collection_on_delete_is_not_an_error(self, db_connection): + db = db_connection + rid = _store_row(db, copy.deepcopy(SORGHUM_PAYLOAD)) + store = FakeStore(delete_result={"deleted": 0, "error": "Collection `x` doesn't exist!"}) + result = index_advisory_record(rid, store, "x", _count) + assert result["status"] == "indexed" + + def test_promote_marks_promoted_not_indexed(self, db_connection): + db = db_connection + rid = _store_row(db, copy.deepcopy(SORGHUM_PAYLOAD)) + index_advisory_record(rid, FakeStore(), "prod", _count, mark="promoted") + assert db.get_unindexed_webhook_record_ids() == [rid] + assert db.advisory_status_counts()["promoted"] == 1 + + def test_reposting_unchanged_record_reuses_point_ids(self, db_connection): + db = db_connection + rid = _store_row(db, copy.deepcopy(SORGHUM_PAYLOAD)) + first, second = FakeStore(), FakeStore() + index_advisory_record(rid, first, "document-data", _count) + index_advisory_record(rid, second, "document-data", _count) + assert [r["_id"] for r in first.records] == [r["_id"] for r in second.records] + + +def test_payload_whitelist_keeps_advisory_fields(): + chunk = build_chunks(make_row(), _count)[0] + payload = _record_payload(chunk) + for key in ( + "document_kind", "external_id", "pop_id", "slug", "crop", "domain", "subdomain", + "institute", "creator", "problem_type", "problem", "season", "start_period", + "end_period", "updated_at", "section", "chunk_id", "text", "doc_id", "title_en", + ): + assert key in payload, key + assert payload["document_kind"] == "advisory" diff --git a/tests/test_advisory_start.py b/tests/test_advisory_start.py new file mode 100644 index 0000000..0c7056e --- /dev/null +++ b/tests/test_advisory_start.py @@ -0,0 +1,98 @@ +"""Tests for starting advisory workflows (one per record).""" + +import asyncio +from unittest.mock import AsyncMock + +import pytest +from temporalio.exceptions import WorkflowAlreadyStartedError + +from pipeline.advisory_start import start_advisory_index, start_advisory_promote +from pipeline.workflows import AdvisoryIndexWorkflow, AdvisoryPromoteWorkflow + + +def _started_ids(client): + return [call.kwargs["id"] for call in client.start_workflow.await_args_list] + + +def test_index_starts_one_workflow_per_record(): + client = AsyncMock() + started = asyncio.run(start_advisory_index(client, ["a", "b"])) + assert started == 2 + assert _started_ids(client) == ["advisory-a", "advisory-b"] + first = client.start_workflow.await_args_list[0] + assert first.args[0] is AdvisoryIndexWorkflow.run + assert first.kwargs["args"] == ["a"] + assert first.kwargs["task_queue"] == "ocr-pipeline" + + +def test_promote_uses_separate_ids(): + client = AsyncMock() + assert asyncio.run(start_advisory_promote(client, ["a"])) == 1 + assert _started_ids(client) == ["advisory-promote-a"] + assert client.start_workflow.await_args_list[0].args[0] is AdvisoryPromoteWorkflow.run + + +def test_already_running_workflow_is_skipped_and_others_still_start(): + client = AsyncMock() + client.start_workflow.side_effect = [ + WorkflowAlreadyStartedError("advisory-a", "AdvisoryIndexWorkflow"), + None, + ] + assert asyncio.run(start_advisory_index(client, ["a", "b"])) == 1 + + +def test_other_errors_are_logged_and_do_not_stop_the_loop(): + client = AsyncMock() + client.start_workflow.side_effect = [RuntimeError("temporal down"), None] + assert asyncio.run(start_advisory_index(client, ["a", "b"])) == 1 + + +def test_no_ids_starts_nothing(): + client = AsyncMock() + assert asyncio.run(start_advisory_index(client, [])) == 0 + client.start_workflow.assert_not_called() + + +@pytest.mark.db +class TestBackfillOnStart: + def _seed(self, db): + db.upsert_webhook_records( + "webhook", + [{"contentID": 1, "status": "Published", "isPublished": 1, "popList": []}], + ) + + def test_off_by_default(self, db_connection, monkeypatch): + from pipeline.advisory_start import backfill_on_start + + monkeypatch.delenv("ADVISORY_BACKFILL_ON_START", raising=False) + self._seed(db_connection) + client = AsyncMock() + assert asyncio.run(backfill_on_start(client)) == 0 + client.start_workflow.assert_not_called() + + @pytest.mark.parametrize("value", ["true", "1", "yes", "on", "TRUE"]) + def test_on_queues_all_unindexed_rows(self, db_connection, monkeypatch, value): + from pipeline.advisory_start import backfill_on_start + + monkeypatch.setenv("ADVISORY_BACKFILL_ON_START", value) + self._seed(db_connection) + client = AsyncMock() + assert asyncio.run(backfill_on_start(client)) == 1 + assert _started_ids(client)[0].startswith("advisory-") + + def test_on_with_nothing_unindexed_starts_nothing(self, db_connection, monkeypatch): + from pipeline.advisory_start import backfill_on_start + + monkeypatch.setenv("ADVISORY_BACKFILL_ON_START", "true") + client = AsyncMock() + assert asyncio.run(backfill_on_start(client)) == 0 + client.start_workflow.assert_not_called() + + def test_off_values(self, db_connection, monkeypatch): + from pipeline.advisory_start import backfill_on_start + + self._seed(db_connection) + for value in ("false", "0", "", "no"): + monkeypatch.setenv("ADVISORY_BACKFILL_ON_START", value) + client = AsyncMock() + assert asyncio.run(backfill_on_start(client)) == 0 diff --git a/tests/test_db.py b/tests/test_db.py index 87e00ea..496010d 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -1230,14 +1230,14 @@ def test_a_new_migration_runs_once(self, db_connection, postgres_url, tmp_path): from pipeline import migrate (tmp_path / "0001_initial_schema.sql").write_text("SELECT 1;") - (tmp_path / "0002_probe.sql").write_text("CREATE TABLE mig_probe (n int);") + (tmp_path / "0099_probe.sql").write_text("CREATE TABLE mig_probe (n int);") try: - assert migrate.apply_migrations(postgres_url, tmp_path) == ["0002"] + assert migrate.apply_migrations(postgres_url, tmp_path) == ["0099"] assert migrate.apply_migrations(postgres_url, tmp_path) == [] finally: with db_connection.get_connection() as conn: conn.execute("DROP TABLE IF EXISTS mig_probe") - conn.execute("DELETE FROM schema_migrations WHERE version = '0002'") + conn.execute("DELETE FROM schema_migrations WHERE version = '0099'") @pytest.mark.db @pytest.mark.unit @@ -1245,13 +1245,13 @@ def test_a_failing_migration_applies_nothing_of_itself(self, db_connection, post from pipeline import migrate (tmp_path / "0001_initial_schema.sql").write_text("SELECT 1;") - (tmp_path / "0002_bad.sql").write_text("CREATE TABLE mig_half (n int); SELECT 1/0;") + (tmp_path / "0099_bad.sql").write_text("CREATE TABLE mig_half (n int); SELECT 1/0;") with pytest.raises(psycopg.errors.DivisionByZero): migrate.apply_migrations(postgres_url, tmp_path) with db_connection.get_connection() as conn: assert conn.execute("SELECT to_regclass('mig_half') AS t").fetchone()["t"] is None assert conn.execute( - "SELECT count(*) AS n FROM schema_migrations WHERE version = '0002'" + "SELECT count(*) AS n FROM schema_migrations WHERE version = '0099'" ).fetchone()["n"] == 0 @pytest.mark.db diff --git a/tests/test_embeddings_count_tokens.py b/tests/test_embeddings_count_tokens.py new file mode 100644 index 0000000..c469f9e --- /dev/null +++ b/tests/test_embeddings_count_tokens.py @@ -0,0 +1,24 @@ +"""Tests for the embedding token counter used to size advisory chunks.""" + +from types import SimpleNamespace + +from pipeline.vector_store import embeddings + + +def test_count_tokens_local_uses_tokenizer(monkeypatch): + monkeypatch.setenv("EMBEDDING_PROVIDER", "sentence_transformers") + fake = SimpleNamespace(tokenizer=SimpleNamespace(encode=lambda text: text.split())) + monkeypatch.setattr(embeddings, "_get_local_embedder", lambda: fake) + # "passage: a b c" -> passage:, a, b, c + assert embeddings.count_tokens("a b c") == 4 + + +def test_count_tokens_remote_estimates_from_length(monkeypatch): + monkeypatch.setenv("EMBEDDING_PROVIDER", "openai_compatible") + text = "x" * 400 + assert embeddings.count_tokens(text) == len("passage: " + text) // 4 + + +def test_count_tokens_remote_is_at_least_one(monkeypatch): + monkeypatch.setenv("EMBEDDING_PROVIDER", "openai_compatible") + assert embeddings.count_tokens("") >= 1 diff --git a/tests/test_webhook_index_db.py b/tests/test_webhook_index_db.py new file mode 100644 index 0000000..0c09b1a --- /dev/null +++ b/tests/test_webhook_index_db.py @@ -0,0 +1,107 @@ +"""Tests for the webhook_records indexing state (indexed_hash / promoted_hash) helpers.""" + + +import pytest + + +def _rec(content_id, title="Wheat Insect Management", published=1, status="Published"): + return { + "contentID": content_id, + "title": title, + "status": status, + "isPublished": published, + "domainName": "Agriculture", + "slug": f"{content_id}/x", + "popList": [{"popID": 1, "status": "Published"}], + } + + +def _row_id(db, content_id): + with db.get_connection() as conn: + return str(conn.execute( + "SELECT id FROM webhook_records WHERE external_id = %s", (str(content_id),) + ).fetchone()["id"]) + + +@pytest.mark.unit +@pytest.mark.db +class TestIndexState: + def test_new_rows_are_unindexed_until_marked(self, db_connection): + db = db_connection + db.upsert_webhook_records("webhook", [_rec(1)]) + rid = _row_id(db, 1) + assert db.get_unindexed_webhook_record_ids() == [rid] + row = db.get_webhook_record(rid) + db.mark_webhook_record_indexed(rid, row["payload_hash"]) + assert db.get_unindexed_webhook_record_ids() == [] + + def test_changed_payload_becomes_unindexed_again(self, db_connection): + db = db_connection + db.upsert_webhook_records("webhook", [_rec(1)]) + rid = _row_id(db, 1) + db.mark_webhook_record_indexed(rid, db.get_webhook_record(rid)["payload_hash"]) + + db.upsert_webhook_records("webhook", [_rec(1)]) # identical + assert db.get_unindexed_webhook_record_ids() == [] + + db.upsert_webhook_records("webhook", [_rec(1, title="Changed")]) + assert db.get_unindexed_webhook_record_ids() == [rid] + + def test_marking_a_stale_hash_leaves_the_row_unindexed(self, db_connection): + db = db_connection + db.upsert_webhook_records("webhook", [_rec(1)]) + rid = _row_id(db, 1) + stale = db.get_webhook_record(rid)["payload_hash"] + db.upsert_webhook_records("webhook", [_rec(1, title="Changed")]) + db.mark_webhook_record_indexed(rid, stale) + assert db.get_unindexed_webhook_record_ids() == [rid] + + def test_unindexed_ids_can_be_scoped_to_external_ids(self, db_connection): + db = db_connection + db.upsert_webhook_records("webhook", [_rec(1), _rec(2), _rec(3)]) + ids = db.get_unindexed_webhook_record_ids(external_ids=["1", "3"]) + assert sorted(ids) == sorted([_row_id(db, 1), _row_id(db, 3)]) + assert db.get_unindexed_webhook_record_ids(external_ids=[]) == [] + + def test_get_webhook_record_parses_payload(self, db_connection): + db = db_connection + db.upsert_webhook_records("webhook", [_rec(101)]) + row = db.get_webhook_record(_row_id(db, 101)) + assert row["payload"]["contentID"] == 101 + assert row["external_id"] == "101" + assert row["is_published"] == 1 + assert row["status"] == "Published" + assert db.get_webhook_record("missing") is None + + def test_webhook_external_id_matches_upsert_rule(self, db_connection): + db = db_connection + assert db.webhook_external_id({"contentID": 7}) == "7" + assert db.webhook_external_id({"id": "abc"}) == "abc" + assert db.webhook_external_id({"slug": "s"}) == "s" + assert len(db.webhook_external_id({"title": "x"})) == 64 # payload hash + + def test_record_id_lists(self, db_connection): + db = db_connection + db.upsert_webhook_records( + "webhook", [_rec(1), _rec(2, published=0, status="Draft")] + ) + assert len(db.get_webhook_record_ids()) == 2 + assert db.get_webhook_record_ids(published_only=True) == [_row_id(db, 1)] + + def test_status_counts(self, db_connection): + db = db_connection + db.upsert_webhook_records( + "webhook", [_rec(1), _rec(2), _rec(3, published=0, status="Draft")] + ) + a, b = _row_id(db, 1), _row_id(db, 2) + db.mark_webhook_record_indexed(a, db.get_webhook_record(a)["payload_hash"]) + db.mark_webhook_record_promoted(a, db.get_webhook_record(a)["payload_hash"]) + counts = db.advisory_status_counts() + assert counts == { + "total": 3, + "published": 2, + "indexed": 1, + "unindexed": 2, + "promoted": 1, + } + assert b # silence unused From dead5f4c92bc0e26eff919ed7883410b65dccfd5 Mon Sep 17 00:00:00 2001 From: Naveen Date: Tue, 27 Oct 2026 16:47:14 +0530 Subject: [PATCH 10/53] fix: bulk advisory promote only pushes records indexed in dev POST /webhook/advisory/promote without record_ids promoted every published record, building chunks from the current payload. A record edited after its last dev index could reach PROD without being indexed in dev. The default now selects only rows where indexed_hash = payload_hash. --- pipeline/api.py | 4 ++-- pipeline/db.py | 13 +++++++++++-- tests/test_advisory_api.py | 16 +++++++++++++--- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/pipeline/api.py b/pipeline/api.py index 5b3cf31..496c118 100644 --- a/pipeline/api.py +++ b/pipeline/api.py @@ -4165,12 +4165,12 @@ async def advisory_reindex(body: AdvisoryReindexBody, user: RequireAdmin): @app.post("/webhook/advisory/promote") async def advisory_promote(body: AdvisoryPromoteBody, user: RequireAdmin): - """Queue promotion to the PROD collection: given record ids, else all published records.""" + """Queue promotion to the PROD collection: given record ids, else all published records already indexed in dev.""" client = _require_temporal() if body.record_ids is not None: record_ids = body.record_ids else: - record_ids = db.get_webhook_record_ids(published_only=True) + record_ids = db.get_webhook_record_ids(published_only=True, indexed_only=True) queued = await start_advisory_promote(client, record_ids) return {"success": True, "queued": queued} diff --git a/pipeline/db.py b/pipeline/db.py index 6c5cfe9..203522f 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -586,11 +586,20 @@ def get_unindexed_webhook_record_ids( return [str(r["id"]) for r in conn.execute(sql + " ORDER BY created_at, id", params)] -def get_webhook_record_ids(source: str = "webhook", published_only: bool = False) -> list[str]: - """All record ids for a source, optionally only published ones.""" +def get_webhook_record_ids( + source: str = "webhook", published_only: bool = False, indexed_only: bool = False +) -> list[str]: + """All record ids for a source, optionally only published and/or dev-indexed ones. + + ``indexed_only`` keeps rows whose current payload is what was last written to the dev + collection (``indexed_hash = payload_hash``), so a bulk promote never pushes a payload + that has not been indexed in dev. + """ sql = "SELECT id FROM webhook_records WHERE source = %s" if published_only: sql += " AND is_published AND status = 'Published'" + if indexed_only: + sql += " AND indexed_hash = payload_hash" with get_connection() as conn: return [str(r["id"]) for r in conn.execute(sql + " ORDER BY created_at, id", (source,))] diff --git a/tests/test_advisory_api.py b/tests/test_advisory_api.py index 46cc3ca..b0b7f8c 100644 --- a/tests/test_advisory_api.py +++ b/tests/test_advisory_api.py @@ -128,12 +128,22 @@ def test_reindex_explicit_ids(self, client, db_connection, temporal): assert r.json()["queued"] == 1 assert _started(temporal) == [f"advisory-{rid}"] - def test_promote_defaults_to_published_records(self, client, db_connection, temporal): + def test_promote_defaults_to_published_records_indexed_in_dev(self, client, db_connection, temporal): self._seed(client, temporal, _rec(1), _rec(2, published=0, status="Draft")) + ids = _row_ids(db_connection, 1, 2) + for rid in ids.values(): + db_connection.mark_webhook_record_indexed(rid, db_connection.get_webhook_record(rid)["payload_hash"]) + temporal.start_workflow.reset_mock() r = client.post("/webhook/advisory/promote", json={}) assert r.json() == {"success": True, "queued": 1} - published = _row_ids(db_connection, 1)[1] - assert _started(temporal) == [f"advisory-promote-{published}"] + assert _started(temporal) == [f"advisory-promote-{ids[1]}"] + + def test_default_promote_skips_records_not_indexed_in_dev(self, client, db_connection, temporal): + self._seed(client, temporal, _rec(1)) + temporal.start_workflow.reset_mock() + r = client.post("/webhook/advisory/promote", json={}) + assert r.json() == {"success": True, "queued": 0} + assert _started(temporal) == [] def test_promote_explicit_ids(self, client, db_connection, temporal): self._seed(client, temporal, _rec(1), _rec(2)) From 6b4a3d49febfee6b6bf0af9107d314b8d3dda221 Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 11:07:16 +0530 Subject: [PATCH 11/53] feat(db): Postgres schema v2 with groups, catalogs and code-level documents Rewrite migration 0001 in place (no Postgres holds v1 yet): - add groups (tenant node, keycloak id filled on login sync) and users.group_id - replace catalog_entries with catalogs, one row per (group, schema type) - documents carry catalog_id, code, name and aliases; period, uploader, index_status and flags live in metadata; counts are derived from pages/chunks - audit_logs snapshot group_id instead of instance - enums become plain TEXT, validated in pipeline/models.py - catalog view becomes catalog_status (one row per document) db.py gets group/catalog get-or-create, find_document_by_code, reset_document_for_reupload and DuplicateDocumentCode. The flat document dict and db params are renamed to group/kind/code/name/aliases. instances.py becomes group_names.py. Scheme/advisory map to schema types in network_constants.py (placeholder values). Bootstrap catalog seeding is removed. --- migrations/0001_initial_schema.sql | 186 +++--- pipeline/catalog_builder.py | 12 +- pipeline/db.py | 747 ++++++++++++---------- pipeline/document_repository.py | 2 +- pipeline/{instances.py => group_names.py} | 26 +- pipeline/models.py | 96 ++- pipeline/network_constants.py | 30 + pipeline/scheme_catalog.py | 154 ++--- 8 files changed, 682 insertions(+), 571 deletions(-) rename pipeline/{instances.py => group_names.py} (80%) diff --git a/migrations/0001_initial_schema.sql b/migrations/0001_initial_schema.sql index 3fc75d0..6ea17de 100644 --- a/migrations/0001_initial_schema.sql +++ b/migrations/0001_initial_schema.sql @@ -6,26 +6,26 @@ BEGIN RETURN NEW; END $$ LANGUAGE plpgsql; -CREATE TYPE workflow_stage AS ENUM ( - 'registered', 'ocr_processing', 'ocr_review', 'translation_processing', - 'translation_review', 'chunking', 'chunk_review', 'ready_for_ingestion', - 'ingesting', 'publishing_to_network', 'approval_for_prod', 'ingesting_prod', - 'completed', 'failed' -); -CREATE TYPE workflow_type_enum AS ENUM ( - 'pipeline', 'ocr_only', 'ocr_retry', 'translation_retry', 'chunking_retry', - 'reingest', 'promote_prod' +-- 1. groups: one row per tenant node (provider), mirrored from Keycloak +CREATE TABLE groups ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + keycloak_group_id TEXT UNIQUE, + name TEXT NOT NULL, + path TEXT NOT NULL UNIQUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() ); -CREATE TYPE workflow_status_enum AS ENUM ('running', 'waiting_review', 'completed', 'failed'); -CREATE TYPE audit_entity_type AS ENUM ('document', 'page', 'chunk', 'setting', 'catalog'); +CREATE TRIGGER trg_groups_updated_at BEFORE UPDATE ON groups + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); --- 1. users +-- 2. users CREATE TABLE users ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), keycloak_user_id TEXT NOT NULL UNIQUE, username TEXT, email TEXT, roles TEXT[] NOT NULL DEFAULT '{}', + group_id UUID REFERENCES groups(id) ON DELETE SET NULL, first_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(), last_seen_at TIMESTAMPTZ NOT NULL DEFAULT now() ); @@ -34,61 +34,51 @@ CREATE UNIQUE INDEX users_email_lower_key ON users (lower(email)) WHERE email IS INSERT INTO users (keycloak_user_id, username, roles) VALUES ('local-admin', 'local-admin', '{superadmin}'); --- 2. catalog_entries -CREATE TABLE catalog_entries ( +-- 3. catalogs: one row per provider group and resource type +CREATE TABLE catalogs ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - type TEXT NOT NULL CHECK (type ~ '^[a-z0-9]+([_-][a-z0-9]+)*$'), - code TEXT NOT NULL CHECK (code ~ '^[a-z0-9]+(-[a-z0-9]+)*$'), + group_id UUID NOT NULL REFERENCES groups(id) ON DELETE RESTRICT, + type TEXT NOT NULL CHECK (type ~ '^[a-z0-9]+:[A-Za-z0-9]+$'), name TEXT NOT NULL, - aliases TEXT[] NOT NULL DEFAULT '{}' CHECK (array_position(aliases, '') IS NULL), - metadata JSONB NOT NULL DEFAULT '{"source": "pipeline"}' CHECK (jsonb_typeof(metadata) = 'object'), + metadata JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(metadata) = 'object'), created_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), - UNIQUE (type, code), - UNIQUE (id, type) + UNIQUE (group_id, type) ); -CREATE TRIGGER trg_catalog_entries_updated_at BEFORE UPDATE ON catalog_entries +CREATE TRIGGER trg_catalogs_updated_at BEFORE UPDATE ON catalogs FOR EACH ROW EXECUTE FUNCTION set_updated_at(); --- 3. documents +-- 4. documents CREATE TABLE documents ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - instance TEXT NOT NULL, - source JSONB NOT NULL - CHECK (jsonb_typeof(source) = 'object') - CHECK (source->>'fingerprint' ~ '^[0-9a-f]{32}$'), - document_kind TEXT NOT NULL DEFAULT 'document' CHECK (document_kind ~ '^[a-z0-9]+([_-][a-z0-9]+)*$'), - catalog_entry_id UUID, - valid_from DATE, - valid_to DATE CHECK (valid_from <= valid_to), - is_disabled BOOLEAN NOT NULL DEFAULT FALSE, - index_status JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(index_status) = 'object'), - metadata JSONB NOT NULL - DEFAULT '{"catalog_visible": true, "network_visible": true, "is_demo": false, "chunk_count": 0, "reindex_required": false}' - CHECK (jsonb_typeof(metadata) = 'object') - CHECK ((metadata->>'chunk_count')::int >= 0), - uploaded_by_user_id UUID REFERENCES users(id), - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), - FOREIGN KEY (catalog_entry_id, document_kind) REFERENCES catalog_entries (id, type) - ON UPDATE CASCADE ON DELETE RESTRICT + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + catalog_id UUID NOT NULL REFERENCES catalogs(id) ON DELETE RESTRICT, + code TEXT NOT NULL CHECK (code ~ '^[a-z0-9]+(-[a-z0-9]+)*$'), + name TEXT NOT NULL, + aliases TEXT[] NOT NULL DEFAULT '{}' CHECK (array_position(aliases, '') IS NULL), + source JSONB NOT NULL + CHECK (jsonb_typeof(source) = 'object') + CHECK (source->>'fingerprint' ~ '^[0-9a-f]{32}$'), + is_disabled BOOLEAN NOT NULL DEFAULT FALSE, + metadata JSONB NOT NULL + DEFAULT '{"index_status": {}, "catalog_visible": true, "network_visible": true, "reindex_required": false}' + CHECK (jsonb_typeof(metadata) = 'object'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (catalog_id, code) ); -CREATE INDEX documents_instance_idx ON documents (instance); CREATE INDEX documents_created_at_idx ON documents (created_at DESC); -CREATE INDEX documents_catalog_entry_idx ON documents (catalog_entry_id) WHERE catalog_entry_id IS NOT NULL; -CREATE INDEX documents_fingerprint_idx ON documents (instance, (source->>'fingerprint')); -CREATE INDEX documents_is_demo_idx ON documents (((metadata->>'is_demo')::boolean)); +CREATE INDEX documents_fingerprint_idx ON documents (catalog_id, (source->>'fingerprint')); CREATE TRIGGER trg_documents_updated_at BEFORE UPDATE ON documents FOR EACH ROW EXECUTE FUNCTION set_updated_at(); --- 4. workflows +-- 5. workflows CREATE TABLE workflows ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE, - workflow_type workflow_type_enum NOT NULL, + workflow_type TEXT NOT NULL, temporal_workflow_id TEXT NOT NULL, - status workflow_status_enum NOT NULL DEFAULT 'running', - current_stage workflow_stage NOT NULL DEFAULT 'registered', + status TEXT NOT NULL DEFAULT 'running', + current_stage TEXT NOT NULL DEFAULT 'registered', stage_timestamps JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(stage_timestamps) = 'object' AND (stage_timestamps - ARRAY['ocr', 'translation', 'chunks']) = '{}'::jsonb), @@ -111,12 +101,12 @@ SELECT DISTINCT ON (document_id) * FROM workflows ORDER BY document_id, started_at DESC, id DESC; --- 5. workflow_artifacts +-- 6. workflow_artifacts CREATE TABLE workflow_artifacts ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), workflow_id UUID NOT NULL REFERENCES workflows(id) ON DELETE CASCADE, artifact_type TEXT NOT NULL CHECK (btrim(artifact_type) <> ''), - stage workflow_stage, + stage TEXT, storage_uri TEXT NOT NULL, mime_type TEXT, filename TEXT, @@ -127,7 +117,7 @@ CREATE TABLE workflow_artifacts ( CREATE INDEX workflow_artifacts_workflow_idx ON workflow_artifacts (workflow_id, created_at DESC); CREATE INDEX workflow_artifacts_type_idx ON workflow_artifacts (workflow_id, artifact_type, created_at DESC); --- 6. pages +-- 7. pages CREATE TABLE pages ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE, @@ -143,7 +133,7 @@ CREATE TABLE pages ( CREATE TRIGGER trg_pages_updated_at BEFORE UPDATE ON pages FOR EACH ROW EXECUTE FUNCTION set_updated_at(); --- 7. chunks +-- 8. chunks CREATE TABLE chunks ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE, @@ -170,14 +160,14 @@ CREATE INDEX chunks_text_trgm_idx ON chunks CREATE TRIGGER trg_chunks_updated_at BEFORE UPDATE ON chunks FOR EACH ROW EXECUTE FUNCTION set_updated_at(); --- 8. audit_logs +-- 9. audit_logs CREATE TABLE audit_logs ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), document_id UUID REFERENCES documents(id) ON DELETE SET NULL, workflow_id TEXT, - instance TEXT, + group_id UUID REFERENCES groups(id) ON DELETE SET NULL, action_type TEXT NOT NULL CHECK (btrim(action_type) <> ''), - entity_type audit_entity_type, + entity_type TEXT, entity_number INTEGER, field_name TEXT, old_value TEXT, @@ -189,11 +179,12 @@ CREATE TABLE audit_logs ( CHECK (entity_type IN ('setting', 'catalog') OR workflow_id IS NOT NULL) ); CREATE INDEX audit_logs_document_idx ON audit_logs (document_id, created_at DESC); +CREATE INDEX audit_logs_group_idx ON audit_logs (group_id, created_at DESC); CREATE INDEX audit_logs_created_idx ON audit_logs (created_at DESC); CREATE INDEX audit_logs_action_idx ON audit_logs (action_type, created_at DESC); CREATE INDEX audit_logs_settings_idx ON audit_logs (created_at DESC) WHERE entity_type = 'setting'; --- 9. search_settings +-- 10. search_settings CREATE TABLE search_settings ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), singleton BOOLEAN NOT NULL DEFAULT TRUE UNIQUE CHECK (singleton), @@ -208,7 +199,7 @@ CREATE TRIGGER trg_search_settings_updated_at BEFORE UPDATE ON search_settings FOR EACH ROW EXECUTE FUNCTION set_updated_at(); INSERT INTO search_settings DEFAULT VALUES; --- 10. email_otps +-- 11. email_otps CREATE TABLE email_otps ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), email TEXT NOT NULL UNIQUE CHECK (email = lower(email)), @@ -237,56 +228,35 @@ CREATE TABLE webhook_records ( ); CREATE INDEX webhook_records_source_status_idx ON webhook_records (source, status); --- catalog view -CREATE VIEW catalog AS +-- catalog_status view: one row per document, computed from catalogs, documents and workflows +CREATE VIEW catalog_status AS SELECT - e.id AS entry_id, - e.type, - e.code, - e.name, - e.aliases, - e.metadata->>'source' AS source, + d.id AS document_id, + c.id AS catalog_id, + c.group_id, + g.name AS group_name, + c.type, + d.code, + d.name, + d.aliases, CASE - WHEN COALESCE(a.has_pending_reindex, FALSE) THEN 'pending_reindex' - WHEN COALESCE(a.live_count, 0) > 0 THEN 'live' - WHEN COALESCE(a.pending_prod_count, 0) > 0 THEN 'pending_prod' - WHEN COALESCE(a.doc_count, 0) = 0 AND e.metadata->>'source' = 'bootstrap' THEN 'live' + WHEN x.is_live AND COALESCE((d.metadata->>'reindex_required')::boolean, FALSE) THEN 'pending_reindex' + WHEN x.is_live THEN 'live' + WHEN cw.current_stage = 'approval_for_prod' AND NOT d.is_disabled THEN 'pending_prod' ELSE 'disabled' END AS status, - CASE WHEN COALESCE(cardinality(a.instances), 0) = 0 THEN ARRAY['default']::TEXT[] ELSE a.instances END AS instances, - COALESCE(a.workflow_ids, '{}'::TEXT[]) AS workflow_ids, - COALESCE(a.chunk_count, 0)::INTEGER AS chunk_count, - CASE - WHEN COALESCE(a.doc_count, 0) = 0 THEN COALESCE(e.metadata->>'source' = 'bootstrap', FALSE) - ELSE COALESCE(a.network_visible, FALSE) - END AS network_visible, - a.promoted_at -FROM catalog_entries e -LEFT JOIN LATERAL ( - SELECT - COUNT(*) AS doc_count, - COUNT(*) FILTER (WHERE x.is_live) AS live_count, - BOOL_OR(x.is_live AND COALESCE((x.metadata->>'reindex_required')::boolean, FALSE)) AS has_pending_reindex, - COUNT(*) FILTER (WHERE x.current_stage = 'approval_for_prod' AND NOT x.is_disabled) AS pending_prod_count, - ARRAY(SELECT DISTINCT d2.instance FROM documents d2 - WHERE d2.catalog_entry_id = e.id ORDER BY d2.instance) AS instances, - ARRAY(SELECT w.temporal_workflow_id FROM documents d3 - JOIN LATERAL ( - SELECT w2.temporal_workflow_id FROM workflows w2 - WHERE w2.document_id = d3.id AND w2.workflow_type = 'pipeline' - ORDER BY w2.started_at ASC, w2.id ASC LIMIT 1 - ) w ON TRUE - WHERE d3.catalog_entry_id = e.id - ORDER BY d3.created_at, d3.id) AS workflow_ids, - COALESCE(SUM((x.metadata->>'chunk_count')::int), 0) AS chunk_count, - BOOL_OR(COALESCE((x.metadata->>'network_visible')::boolean, TRUE)) AS network_visible, - MAX((x.index_status->'prod'->>'last_indexed_at')::timestamptz) FILTER (WHERE x.is_live) AS promoted_at - FROM ( - SELECT d.*, cw.current_stage, - (cw.current_stage = 'completed' AND NOT d.is_disabled - AND COALESCE((d.metadata->>'catalog_visible')::boolean, TRUE)) AS is_live - FROM documents d - LEFT JOIN document_current_workflow cw ON cw.document_id = d.id - WHERE d.catalog_entry_id = e.id - ) x -) a ON TRUE; + (SELECT w.temporal_workflow_id FROM workflows w + WHERE w.document_id = d.id AND w.workflow_type = 'pipeline' + ORDER BY w.started_at ASC, w.id ASC LIMIT 1) AS workflow_id, + (SELECT count(*) FROM chunks ch WHERE ch.document_id = d.id)::INTEGER AS chunk_count, + COALESCE((d.metadata->>'network_visible')::boolean, TRUE) AS network_visible, + CASE WHEN x.is_live + THEN (d.metadata->'index_status'->'prod'->>'last_indexed_at')::timestamptz END AS promoted_at +FROM documents d +JOIN catalogs c ON c.id = d.catalog_id +JOIN groups g ON g.id = c.group_id +LEFT JOIN document_current_workflow cw ON cw.document_id = d.id +CROSS JOIN LATERAL ( + SELECT (cw.current_stage = 'completed' AND NOT d.is_disabled + AND COALESCE((d.metadata->>'catalog_visible')::boolean, TRUE)) AS is_live +) x; diff --git a/pipeline/catalog_builder.py b/pipeline/catalog_builder.py index 7b7da27..0e3fa37 100644 --- a/pipeline/catalog_builder.py +++ b/pipeline/catalog_builder.py @@ -75,19 +75,19 @@ PUBLISHABLE_DOCUMENT_KINDS = tuple(_KNOWLEDGE_KIND_CATALOGS) -def normalize_document_kind(document_kind: Optional[str]) -> str: - """Coerce a stored `documents.document_kind` to its lookup form.""" - return (document_kind or DEFAULT_DOCUMENT_KIND).strip().lower() +def normalize_document_kind(kind: Optional[str]) -> str: + """Coerce a document's knowledge kind (from its catalog type) to its lookup form.""" + return (kind or DEFAULT_DOCUMENT_KIND).strip().lower() -def build_catalog(document_kind: Optional[str]) -> Optional[dict]: - """Return the catalog to announce for `document_kind`, or None. +def build_catalog(kind: Optional[str]) -> Optional[dict]: + """Return the catalog to announce for `kind`, or None. None means this kind has nothing to announce - `document`, `video` and any operator-entered custom slug. Callers must skip the publish rather than send an empty `catalogs` array, which the spec rejects (`minItems: 1`). """ - spec = _KNOWLEDGE_KIND_CATALOGS.get(normalize_document_kind(document_kind)) + spec = _KNOWLEDGE_KIND_CATALOGS.get(normalize_document_kind(kind)) if spec is None: return None diff --git a/pipeline/db.py b/pipeline/db.py index 203522f..c90ed21 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -41,6 +41,7 @@ ResultSettings, RetrievalSettings, ) +from .network_constants import catalog_type_for_kind, kind_for_catalog_type logger = logging.getLogger(__name__) @@ -136,13 +137,13 @@ def check_health() -> dict: DEV_INDEX_KEY = "dev" -def _default_instance() -> str: +def _default_group() -> str: return (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" -def _normalize_instances(instances: list[str]) -> list[str]: - default = _default_instance() - return sorted({(i or "").strip().lower() or default for i in instances}) +def _normalize_groups(groups: list[str]) -> list[str]: + default = _default_group() + return sorted({(i or "").strip().lower() or default for i in groups}) def _iso_now() -> str: @@ -245,7 +246,8 @@ def _resolve_document_uuid(conn, identifier: Optional[str]) -> Optional[str]: """ SELECT id FROM documents d WHERE EXISTS ( - SELECT 1 FROM jsonb_each(d.index_status) e WHERE e.value->>'doc_id' = %s + SELECT 1 FROM jsonb_each(COALESCE(d.metadata->'index_status', '{}'::jsonb)) e + WHERE e.value->>'doc_id' = %s ) OR md5(d.source->>'fingerprint') = %s ORDER BY created_at DESC LIMIT 1 """, @@ -267,15 +269,16 @@ def _require_document_uuid(conn, identifier: str) -> str: _DOC_SELECT = """ SELECT - d.id, d.instance, d.source, d.document_kind, d.catalog_entry_id, - d.valid_from, d.valid_to, d.is_disabled, d.index_status, d.metadata, + d.id, d.catalog_id, d.code, d.name, d.aliases, d.source, d.is_disabled, d.metadata, d.created_at, d.updated_at, + c.type AS catalog_type, g.name AS group_name, + (SELECT count(*) FROM pages p WHERE p.document_id = d.id) AS page_total, + (SELECT count(*) FROM chunks ch WHERE ch.document_id = d.id) AS chunk_total, root.temporal_workflow_id AS root_workflow_id, root.stop_after_ocr AS root_stop_after_ocr, cw.id AS current_workflow_pk, cw.current_stage AS current_stage, cw.error_message AS current_error_message, - ce.code AS entry_code, ce.name AS entry_name, ce.aliases AS entry_aliases, u.keycloak_user_id AS up_user_id, u.username AS up_username, u.email AS up_email, u.roles AS up_roles, ts.ocr_completed_at, ts.translation_completed_at, ts.chunks_completed_at, @@ -286,6 +289,8 @@ def _require_document_uuid(conn, identifier: str) -> str: oa.id AS original_artifact_id, na.id AS normalized_artifact_id FROM documents d + JOIN catalogs c ON c.id = d.catalog_id + JOIN groups g ON g.id = c.group_id LEFT JOIN document_current_workflow cw ON cw.document_id = d.id LEFT JOIN LATERAL ( SELECT w.temporal_workflow_id, w.stop_after_ocr FROM workflows w @@ -293,8 +298,7 @@ def _require_document_uuid(conn, identifier: str) -> str: ORDER BY (w.workflow_type IN ('pipeline', 'ocr_only')) DESC, w.started_at ASC, w.id ASC LIMIT 1 ) root ON TRUE - LEFT JOIN catalog_entries ce ON ce.id = d.catalog_entry_id - LEFT JOIN users u ON u.id = d.uploaded_by_user_id + LEFT JOIN users u ON u.id = NULLIF(d.metadata->>'uploaded_by_user_id', '')::uuid LEFT JOIN LATERAL ( SELECT min(w.stage_timestamps->>'ocr') AS ocr_completed_at, min(w.stage_timestamps->>'translation') AS translation_completed_at, @@ -324,6 +328,16 @@ def _require_document_uuid(conn, identifier: str) -> str: """ +def _kind_of(catalog_type: Optional[str]) -> Optional[str]: + """Knowledge kind a catalog type stands for; an unmapped type is returned as is.""" + if not catalog_type: + return None + try: + return kind_for_catalog_type(catalog_type) + except ValueError: + return catalog_type + + def _document_from_row(row: Optional[dict]) -> Optional[dict]: if row is None: return None @@ -331,39 +345,37 @@ def _document_from_row(row: Optional[dict]) -> Optional[dict]: meta = row["metadata"] or {} fingerprint = source.get("fingerprint") roles = row["up_roles"] or [] - entry_aliases = row["entry_aliases"] + entry_aliases = row["aliases"] doc = { "id": row["id"], "workflow_id": row["root_workflow_id"], "document_id": fingerprint, "canonical_document_id": fingerprint, "filename": source.get("original_filename") or "", - "display_name": meta.get("display_name"), + "name": row["name"], "source_filename": source.get("original_filename"), "source_manifest_name": None, "source_file_fingerprint": fingerprint, "filepath": source.get("storage_uri") or "", - "instance": row["instance"], + "group": (row["group_name"] or "").lower(), "stage": row["current_stage"] or "registered", - "page_count": source.get("page_count") or 0, - "chunk_count": meta.get("chunk_count") or 0, + "page_count": int(row["page_total"] or 0), + "chunk_count": int(row["chunk_total"] or 0), "error_message": row["current_error_message"], - "is_demo": bool(meta.get("is_demo")), "is_disabled": bool(row["is_disabled"]), "source_type": source.get("source_type"), "canonical_input_type": source.get("canonical_input_type"), "stop_after_ocr": bool(row["root_stop_after_ocr"]), "reindex_required": bool(meta.get("reindex_required")), "reindex_reason": meta.get("reindex_reason"), - "document_kind": row["document_kind"], - "scheme_code": row["entry_code"], - "scheme_name": row["entry_name"], - "scheme_aliases_json": json.dumps(list(entry_aliases)) if entry_aliases is not None else None, - "tool_routing": "qdrant" if row["entry_code"] else None, + "kind": _kind_of(row["catalog_type"]), + "code": row["code"], + "aliases": list(entry_aliases or []), + "tool_routing": "qdrant", "catalog_visible": bool(meta.get("catalog_visible", True)), "network_visible": bool(meta.get("network_visible", True)), - "valid_from": row["valid_from"], - "valid_to": row["valid_to"], + "valid_from": meta.get("valid_from"), + "valid_to": meta.get("valid_to"), "uploaded_by_user_id": row["up_user_id"], "uploaded_by_username": row["up_username"], "uploaded_by_email": row["up_email"], @@ -378,8 +390,8 @@ def _document_from_row(row: Optional[dict]) -> Optional[dict]: "original_artifact_id": row["original_artifact_id"], "normalized_artifact_id": row["normalized_artifact_id"], "latest_job_id": row["current_workflow_pk"], - "catalog_entry_id": row["catalog_entry_id"], - "index_status": row["index_status"] or {}, + "catalog_id": row["catalog_id"], + "index_status": meta.get("index_status") or {}, "created_at": row["created_at"], "updated_at": row["updated_at"], } @@ -394,23 +406,80 @@ def _fetch_document(conn, doc_uuid: Optional[str]) -> Optional[dict]: def _document_filters( - include_demo: bool, include_disabled: bool, - instances: Optional[list[str]], + groups: Optional[list[str]], alias: str = "d", + group_alias: str = "g", ) -> tuple[list[str], list]: + """WHERE clauses for document lists. The query must join ``groups`` as ``group_alias``.""" clauses: list[str] = [] params: list = [] - if not include_demo: - clauses.append(f"NOT COALESCE(({alias}.metadata->>'is_demo')::boolean, FALSE)") if not include_disabled: clauses.append(f"NOT {alias}.is_disabled") - if instances is not None: - clauses.append(f"lower({alias}.instance) = ANY(%s)") - params.append(_normalize_instances(instances)) + if groups is not None: + clauses.append(f"lower({group_alias}.name) = ANY(%s)") + params.append(_normalize_groups(groups)) return clauses, params +# ============================================================================= +# Groups +# ============================================================================= + +def _tenant_path(group: str) -> str: + """Keycloak group path of a tenant code: ``bv`` is ``/global``, others ``/states/``.""" + return "/global" if group == "bv" else f"/states/{group.upper()}" + + +def _tenant_code_from_path(path: str) -> str: + return "bv" if path == "/global" else path.rstrip("/").rsplit("/", 1)[-1].lower() + + +def upsert_group(path: str, keycloak_group_id: Optional[str] = None, conn=None) -> str: + """Create or refresh the ``groups`` row for a Keycloak group path; returns its UUID. + + Rows made by the SQLite move have no Keycloak id yet; the first sync that + sees the path fills it in. + """ + if conn is None: + with get_connection() as own: + return upsert_group(path, keycloak_group_id, conn=own) + row = conn.execute( + """ + INSERT INTO groups (keycloak_group_id, name, path) VALUES (%s, %s, %s) + ON CONFLICT (path) DO UPDATE SET + keycloak_group_id = COALESCE(EXCLUDED.keycloak_group_id, groups.keycloak_group_id) + RETURNING id + """, + (keycloak_group_id, _tenant_code_from_path(path), path), + ).fetchone() + return str(row["id"]) + + +def get_or_create_group(group: Optional[str], conn=None) -> str: + """UUID of the group for a tenant code (``bv``, ``mh``, ...), created when missing.""" + code = (group or _default_group()).strip().lower() or _default_group() + return upsert_group(_tenant_path(code), conn=conn) + + +def get_or_create_catalog(group_id: str, kind: str, conn=None) -> dict: + """The ``(group, kind)`` catalog row, created with a default name when missing.""" + if conn is None: + with get_connection() as own: + return get_or_create_catalog(group_id, kind, conn=own) + catalog_type = catalog_type_for_kind(kind) + row = conn.execute( + """ + INSERT INTO catalogs (group_id, type, name) + SELECT g.id, %s, g.name || ' ' || %s FROM groups g WHERE g.id = %s::uuid + ON CONFLICT (group_id, type) DO UPDATE SET type = EXCLUDED.type + RETURNING id, group_id, type, name, metadata + """, + (catalog_type, kind, group_id), + ).fetchone() + return _clean_row(row) + + # ============================================================================= # Users # ============================================================================= @@ -421,16 +490,29 @@ def upsert_user( email: Optional[str] = None, roles: Any = None, conn=None, + tenants: Optional[list[str]] = None, ) -> Optional[str]: - """Create or refresh the ``users`` row for a Keycloak subject; returns its UUID.""" + """Create or refresh the ``users`` row for a Keycloak subject; returns its UUID. + + ``tenants`` are the tenant codes in the user's token. A user has one + ``group_id``, so the first code (sorted) wins and a warning is logged when + there are several. Authorization still reads the token, not this column. + """ keycloak_user_id = (keycloak_user_id or "").strip() if not keycloak_user_id: return None if conn is None: with get_connection() as own: - return upsert_user(keycloak_user_id, username, email, roles, conn=own) + return upsert_user(keycloak_user_id, username, email, roles, conn=own, tenants=tenants) role_list = _csv_roles(roles) + tenant_codes = sorted({(t or "").strip().lower() for t in (tenants or []) if (t or "").strip()}) + if len(tenant_codes) > 1: + logger.warning( + "user %s has several tenants %s; users.group_id uses %s", + keycloak_user_id, tenant_codes, tenant_codes[0], + ) + group_uuid = get_or_create_group(tenant_codes[0], conn=conn) if tenant_codes else None email = (email or "").strip() or None username = (username or "").strip() or None for candidate_email in ([email, None] if email else [None]): @@ -438,17 +520,18 @@ def upsert_user( with conn.transaction(): row = conn.execute( """ - INSERT INTO users (keycloak_user_id, username, email, roles) - VALUES (%s, %s, %s, %s) + INSERT INTO users (keycloak_user_id, username, email, roles, group_id) + VALUES (%s, %s, %s, %s, %s::uuid) ON CONFLICT (keycloak_user_id) DO UPDATE SET username = COALESCE(EXCLUDED.username, users.username), email = COALESCE(EXCLUDED.email, users.email), roles = CASE WHEN cardinality(EXCLUDED.roles) > 0 THEN EXCLUDED.roles ELSE users.roles END, + group_id = COALESCE(EXCLUDED.group_id, users.group_id), last_seen_at = now() RETURNING id """, - (keycloak_user_id, username, candidate_email, role_list), + (keycloak_user_id, username, candidate_email, role_list, group_uuid), ).fetchone() return str(row["id"]) except psycopg.errors.UniqueViolation: @@ -742,100 +825,65 @@ def bump_catalog_version(reason: str = "") -> int: return int(row["version"]) -def upsert_catalog_entry( - *, - type: str, - code: str, - name: Optional[str] = None, - aliases: Optional[list[str]] = None, - source: Optional[str] = None, - conn=None, -) -> dict: - """Insert or update one ``catalog_entries`` row. ``None`` fields keep stored values.""" - if conn is None: - with get_connection() as own: - return upsert_catalog_entry( - type=type, code=code, name=name, aliases=aliases, source=source, conn=own - ) - alias_list = [a for a in (aliases or []) if a] if aliases is not None else None - row = conn.execute( - """ - INSERT INTO catalog_entries (type, code, name, aliases, metadata) - VALUES (%s, %s, COALESCE(%s, %s), COALESCE(%s, '{}'::text[]), %s) - ON CONFLICT (type, code) DO UPDATE SET - name = COALESCE(%s, catalog_entries.name), - aliases = COALESCE(%s, catalog_entries.aliases), - metadata = CASE WHEN %s::text IS NULL THEN catalog_entries.metadata - ELSE catalog_entries.metadata || jsonb_build_object('source', %s::text) END - RETURNING id, type, code, name, aliases, metadata - """, - ( - type, code, name, code, alias_list, Jsonb({"source": source or "pipeline"}), - name, alias_list, source, source, - ), - ).fetchone() - return _clean_row(row) - - -def list_catalog_entry_codes(type: str) -> set[str]: +def list_document_codes(kind: str) -> set[str]: + """Every document code in catalogs of one knowledge kind.""" with get_connection() as conn: - rows = conn.execute("SELECT code FROM catalog_entries WHERE type = %s", (type,)).fetchall() + rows = conn.execute( + "SELECT d.code FROM documents d JOIN catalogs c ON c.id = d.catalog_id WHERE c.type = %s", + (catalog_type_for_kind(kind),), + ).fetchall() return {r["code"] for r in rows} def _catalog_row(row: dict) -> dict: cleaned = _clean_row(row) + cleaned.pop("group_name", None) cleaned["aliases"] = list(row["aliases"] or []) - cleaned["instances"] = list(row["instances"] or []) - cleaned["workflow_ids"] = list(row["workflow_ids"] or []) + cleaned["groups"] = [(row["group_name"] or "").lower()] + cleaned["workflow_ids"] = [row["workflow_id"]] if row["workflow_id"] else [] + cleaned["source"] = "pipeline" return cleaned -def list_catalog_entries(status: Optional[str] = None, type: str = "scheme") -> list[dict]: - """Rows of the ``catalog`` view for one entry type.""" +def list_catalog_entries(status: Optional[str] = None, kind: str = "scheme") -> list[dict]: + """Rows of the ``catalog_status`` view (one per document) for one knowledge kind.""" + catalog_type = catalog_type_for_kind(kind) with get_connection() as conn: if status: rows = conn.execute( - "SELECT * FROM catalog WHERE type = %s AND status = %s ORDER BY code", - (type, status), + "SELECT * FROM catalog_status WHERE type = %s AND status = %s ORDER BY code", + (catalog_type, status), ).fetchall() else: rows = conn.execute( - "SELECT * FROM catalog WHERE type = %s ORDER BY code", (type,) + "SELECT * FROM catalog_status WHERE type = %s ORDER BY code", (catalog_type,) ).fetchall() return [_catalog_row(r) for r in rows] -def get_catalog_entry(code: str, type: str = "scheme") -> Optional[dict]: +def get_catalog_entry(code: str, kind: str = "scheme") -> Optional[dict]: with get_connection() as conn: row = conn.execute( - "SELECT * FROM catalog WHERE type = %s AND code = %s", (type, code) + "SELECT * FROM catalog_status WHERE type = %s AND code = %s LIMIT 1", + (catalog_type_for_kind(kind), code), ).fetchone() return _catalog_row(row) if row else None -def list_used_scheme_codes(exclude_workflow_id: Optional[str] = None) -> set[str]: - """Every scheme code already taken - the collision domain for auto-generated codes. +def list_used_scheme_codes(exclude_workflow_id: Optional[str] = None, kind: str = "scheme") -> set[str]: + """Every document code already taken in a kind - the collision domain for auto-generated codes. - The entry the excluded document alone holds is not counted, so re-deriving a - document's own code does not collide with itself. + The excluded document's own code is not counted, so re-deriving a + document's code does not collide with itself. """ with get_connection() as conn: doc_uuid = _resolve_document_uuid(conn, exclude_workflow_id) if exclude_workflow_id else None rows = conn.execute( """ - SELECT e.code FROM catalog_entries e - WHERE e.type = 'scheme' - AND NOT ( - %s::uuid IS NOT NULL - AND e.metadata->>'source' <> 'bootstrap' - AND EXISTS (SELECT 1 FROM documents d - WHERE d.catalog_entry_id = e.id AND d.id = %s::uuid) - AND NOT EXISTS (SELECT 1 FROM documents d - WHERE d.catalog_entry_id = e.id AND d.id <> %s::uuid) - ) + SELECT d.code FROM documents d JOIN catalogs c ON c.id = d.catalog_id + WHERE c.type = %s AND (%s::uuid IS NULL OR d.id <> %s::uuid) """, - (doc_uuid, doc_uuid, doc_uuid), + (catalog_type_for_kind(kind), doc_uuid, doc_uuid), ).fetchall() return {r["code"] for r in rows} @@ -844,59 +892,30 @@ def list_scheme_catalog_entries_for_workflow(workflow_id: str) -> list[dict]: """Catalog rows whose documents still include this workflow.""" with get_connection() as conn: rows = conn.execute( - "SELECT code, workflow_ids FROM catalog WHERE type = 'scheme' AND %s = ANY(workflow_ids)", - (workflow_id,), + "SELECT code, workflow_id FROM catalog_status WHERE type = %s AND workflow_id = %s", + (catalog_type_for_kind("scheme"), workflow_id), ).fetchall() - return [{"scheme_code": r["code"], "workflow_ids": list(r["workflow_ids"])} for r in rows] + return [{"code": r["code"], "workflow_ids": [r["workflow_id"]]} for r in rows] -def _link_catalog_entry( - conn, - doc_uuid: str, - updates: dict, -) -> dict[str, Any]: - """Resolve document_kind / scheme_* updates into documents column changes. +def find_document_by_code(group: str, kind: str, code: str) -> Optional[dict]: + """The document holding ``code`` in the group's ``kind`` catalog, if any.""" + tenant = (group or _default_group()).strip().lower() or _default_group() + with get_connection() as conn: + row = conn.execute( + """ + SELECT d.id FROM documents d + JOIN catalogs c ON c.id = d.catalog_id + JOIN groups g ON g.id = c.group_id + WHERE lower(g.name) = %s AND c.type = %s AND d.code = %s + """, + (tenant, catalog_type_for_kind(kind), code), + ).fetchone() + return _fetch_document(conn, str(row["id"])) if row else None - Returns the extra ``documents`` assignments (``document_kind``, - ``catalog_entry_id``) so the kind and its entry change in one statement, - which the composite foreign key requires. - """ - current = conn.execute( - """ - SELECT d.document_kind, d.catalog_entry_id, ce.type AS entry_type, - ce.code AS entry_code, ce.name AS entry_name, ce.aliases AS entry_aliases - FROM documents d LEFT JOIN catalog_entries ce ON ce.id = d.catalog_entry_id - WHERE d.id = %s::uuid - """, - (doc_uuid,), - ).fetchone() - kind = updates.get("document_kind") or current["document_kind"] - code = updates["scheme_code"] if "scheme_code" in updates else current["entry_code"] - name = updates["scheme_name"] if "scheme_name" in updates else None - aliases = None - if "scheme_aliases_json" in updates: - raw = updates["scheme_aliases_json"] - aliases = json.loads(raw) if isinstance(raw, str) else list(raw or []) - - changes: dict[str, Any] = {} - if "document_kind" in updates: - changes["document_kind"] = kind - - current_entry_id = str(current["catalog_entry_id"]) if current["catalog_entry_id"] else None - new_entry_id = None - if kind != "document" and code: - moved = current["entry_type"] != kind or current["entry_code"] != code - if moved: - if name is None: - name = current["entry_name"] - if aliases is None and current["entry_aliases"] is not None: - aliases = list(current["entry_aliases"]) - new_entry_id = upsert_catalog_entry( - type=kind, code=code, name=name, aliases=aliases, source="pipeline", conn=conn - )["id"] - if new_entry_id != current_entry_id: - changes["catalog_entry_id"] = new_entry_id - return changes + +class DuplicateDocumentCode(ValueError): + """The ``(catalog, code)`` pair is already held by another document.""" # ============================================================================= @@ -909,7 +928,7 @@ def upsert_document( filename: str, filepath: str, canonical_document_id: Optional[str] = None, - display_name: Optional[str] = None, + name: Optional[str] = None, source_filename: Optional[str] = None, source_manifest_name: Optional[str] = None, source_file_fingerprint: Optional[str] = None, @@ -927,7 +946,7 @@ def upsert_document( original_artifact_id: Optional[str] = None, normalized_artifact_id: Optional[str] = None, latest_job_id: Optional[str] = None, - instance: Optional[str] = None, + group: Optional[str] = None, uploaded_by_user_id: Optional[str] = None, uploaded_by_username: Optional[str] = None, uploaded_by_email: Optional[str] = None, @@ -935,11 +954,20 @@ def upsert_document( valid_from: Optional[str] = None, valid_to: Optional[str] = None, with_workflow: bool = True, + kind: Optional[str] = None, + code: Optional[str] = None, + aliases: Optional[list[str]] = None, ) -> str: """Insert or update a document record; returns the document UUID. - ``instance`` is applied on INSERT only. Updates leave the existing tenant - stamp alone so a re-upload / restart cannot silently reassign tenants. + A new document needs ``kind`` (``scheme`` / ``advisory``) and ``code``: + ``group`` (a tenant code) and the kind pick its ``(group, kind)`` catalog, + which is created when missing. ``name`` becomes ``documents.name`` (the + code when absent). ``DuplicateDocumentCode`` is raised when another + document already holds ``code`` in that catalog. + + ``group`` and the catalog are applied on INSERT only. Updates leave them + alone so a re-upload / restart cannot silently reassign tenants. Uploader identity is set on INSERT, and filled on UPDATE only when currently null. ``valid_from``/``valid_to`` are the document's validity period, stored as @@ -948,12 +976,15 @@ def upsert_document( to is the caller's call (see ``pipeline/document_validity.py``); omitting both stores NULL, which search reads as "no stated period". + ``page_count`` and ``chunk_count`` are accepted for call-site compatibility + and ignored: both are counted from ``pages`` and ``chunks``. + A new document also gets its first workflow row (``temporal_workflow_id`` = ``workflow_id``) unless ``with_workflow`` is False, for callers that create it themselves with ``create_document_job`` (addressing the document by the returned UUID, since no workflow id resolves to it yet). """ - instance_value = (instance or _default_instance()).strip().lower() or _default_instance() + group_value = (group or _default_group()).strip().lower() or _default_group() fingerprint = _fingerprint_for( source_file_fingerprint or canonical_document_id, document_id, workflow_id ) @@ -963,30 +994,29 @@ def upsert_document( "storage_uri": filepath, "source_type": source_type, "canonical_input_type": canonical_input_type, - "page_count": page_count, } with get_connection() as conn: doc_uuid = _resolve_document_uuid(conn, workflow_id) if workflow_id else None if doc_uuid: patch = {k: v for k, v in source_patch.items() if v is not None} DocumentSource.model_validate({**_load(conn, doc_uuid, "source"), **patch}) - metadata_patch: dict[str, Any] = {"chunk_count": chunk_count} - if display_name: - metadata_patch["display_name"] = display_name uploader_uuid = ( upsert_user(uploaded_by_user_id, uploaded_by_username, uploaded_by_email, - uploaded_by_roles, conn=conn) + uploaded_by_roles, conn=conn, tenants=[group_value]) if uploaded_by_user_id else None ) conn.execute( """ UPDATE documents SET source = source || %s::jsonb, - metadata = metadata || %s::jsonb, - uploaded_by_user_id = COALESCE(uploaded_by_user_id, %s::uuid) + name = COALESCE(%s, name), + metadata = CASE + WHEN %s::text IS NULL OR metadata ? 'uploaded_by_user_id' THEN metadata + ELSE metadata || jsonb_build_object('uploaded_by_user_id', %s::text) + END WHERE id = %s::uuid """, - (Jsonb(patch), Jsonb(metadata_patch), uploader_uuid, doc_uuid), + (Jsonb(patch), name or None, uploader_uuid, uploader_uuid, doc_uuid), ) cw = conn.execute( "SELECT id FROM document_current_workflow WHERE document_id = %s::uuid", (doc_uuid,) @@ -994,7 +1024,7 @@ def upsert_document( if cw: conn.execute( """ - UPDATE workflows SET current_stage = %s::workflow_stage, error_message = %s, + UPDATE workflows SET current_stage = %s, error_message = %s, stop_after_ocr = %s WHERE id = %s::uuid """, @@ -1002,32 +1032,46 @@ def upsert_document( ) return doc_uuid + if not kind or not code: + raise ValueError("A new document needs kind and code") source = DocumentSource.model_validate( {k: v for k, v in source_patch.items() if v is not None} ).model_dump(exclude_none=True) - metadata = DocumentMetadata( - display_name=display_name, chunk_count=chunk_count - ).model_dump(exclude_none=True) user_uuid = ( upsert_user(uploaded_by_user_id, uploaded_by_username, uploaded_by_email, - uploaded_by_roles, conn=conn) + uploaded_by_roles, conn=conn, tenants=[group_value]) if uploaded_by_user_id else None ) - row = conn.execute( - """ - INSERT INTO documents (instance, source, metadata, valid_from, valid_to, uploaded_by_user_id) - VALUES (%s, %s, %s, %s::date, %s::date, %s::uuid) - RETURNING id - """, - (instance_value, Jsonb(source), Jsonb(metadata), valid_from, valid_to, user_uuid), - ).fetchone() + metadata = DocumentMetadata( + valid_from=valid_from, valid_to=valid_to, uploaded_by_user_id=user_uuid + ).model_dump(exclude_none=True) + catalog = get_or_create_catalog(get_or_create_group(group_value, conn=conn), kind, conn=conn) + try: + with conn.transaction(): + row = conn.execute( + """ + INSERT INTO documents (catalog_id, code, name, aliases, source, metadata) + VALUES (%s::uuid, %s, %s, %s, %s, %s) + RETURNING id + """, + ( + catalog["id"], code, name or code, + [a for a in (aliases or []) if a], Jsonb(source), Jsonb(metadata), + ), + ).fetchone() + except psycopg.errors.UniqueViolation as exc: + if exc.diag.constraint_name == "documents_catalog_id_code_key": + raise DuplicateDocumentCode( + f"Code '{code}' is already used in the {kind} catalog of {group_value}" + ) from exc + raise if with_workflow: conn.execute( """ INSERT INTO workflows (document_id, workflow_type, temporal_workflow_id, current_stage, error_message, stop_after_ocr, started_by_user_id) - VALUES (%s::uuid, %s::workflow_type_enum, %s, %s::workflow_stage, %s, %s, %s::uuid) + VALUES (%s::uuid, %s, %s, %s, %s, %s, %s::uuid) """, ( str(row["id"]), @@ -1038,6 +1082,43 @@ def upsert_document( return str(row["id"]) +def reset_document_for_reupload( + doc_uuid: str, + *, + filename: str, + filepath: str, + fingerprint: str, + source_type: Optional[str] = None, + canonical_input_type: Optional[str] = None, +) -> None: + """Point an existing document at a re-uploaded file. + + ``source`` is replaced, pages and chunks are cleared for the new run, and + the document is flagged ``reindex_required`` because its vector index is now + stale. Earlier workflows and artifacts stay as history. + """ + source = DocumentSource.model_validate({ + "fingerprint": fingerprint, + "original_filename": filename, + "storage_uri": filepath, + "source_type": source_type, + "canonical_input_type": canonical_input_type, + }).model_dump(exclude_none=True) + with get_connection() as conn: + conn.execute("DELETE FROM pages WHERE document_id = %s::uuid", (doc_uuid,)) + conn.execute("DELETE FROM chunks WHERE document_id = %s::uuid", (doc_uuid,)) + conn.execute( + """ + UPDATE documents SET + source = %s::jsonb, + metadata = (metadata - 'chunking') + || jsonb_build_object('reindex_required', TRUE, 'reindex_reason', 'source_replaced') + WHERE id = %s::uuid + """, + (Jsonb(source), doc_uuid), + ) + + def _load(conn, doc_uuid: str, column: str) -> dict: row = conn.execute(f"SELECT {column} FROM documents WHERE id = %s::uuid", (doc_uuid,)).fetchone() return dict(row[column]) if row else {} @@ -1050,7 +1131,11 @@ def update_document_stage( chunk_count: Optional[int] = None, error_message: Optional[str] = None ): - """Update the stage (of the document's current workflow) and counts.""" + """Update the stage (of the document's current workflow). + + ``page_count`` and ``chunk_count`` are accepted for call-site compatibility; + both are counted from ``pages`` and ``chunks``, so only the audit entry keeps them. + """ old_stage = None audit_workflow_id = workflow_id with get_connection() as conn: @@ -1065,7 +1150,7 @@ def update_document_stage( return old_stage = cw["current_stage"] - assignments = ["current_stage = %s::workflow_stage"] + assignments = ["current_stage = %s"] params: list = [stage] if error_message is not None: assignments.append("error_message = %s") @@ -1087,18 +1172,7 @@ def update_document_stage( f"UPDATE workflows SET {', '.join(assignments)} WHERE id = %s::uuid", params ) - doc_assignments = ["updated_at = now()"] - doc_params: list = [] - if page_count is not None: - doc_assignments.append("source = source || jsonb_build_object('page_count', %s::int)") - doc_params.append(page_count) - if chunk_count is not None: - doc_assignments.append("metadata = metadata || jsonb_build_object('chunk_count', %s::int)") - doc_params.append(chunk_count) - doc_params.append(doc_uuid) - conn.execute( - f"UPDATE documents SET {', '.join(doc_assignments)} WHERE id = %s::uuid", doc_params - ) + conn.execute("UPDATE documents SET updated_at = now() WHERE id = %s::uuid", (doc_uuid,)) audit_workflow_id = _root_workflow_id(conn, doc_uuid) or workflow_id if old_stage and old_stage != stage: @@ -1250,25 +1324,23 @@ def list_documents( stage: Optional[str] = None, limit: int = 100, offset: int = 0, - include_demo: bool = False, include_disabled: bool = False, - instances: Optional[list[str]] = None, + groups: Optional[list[str]] = None, ) -> list[dict]: - """List documents with optional stage / instance filters. + """List documents with optional stage / group filters. Args: stage: Filter by document stage limit: Max documents to return offset: Pagination offset - include_demo: If False (default), excludes demo documents from results include_disabled: If False (default), excludes soft-deleted documents from results - instances: If provided, only return docs whose instance is in this list + groups: If provided, only return docs whose group is in this list """ - if instances is not None and not _normalize_instances(instances): + if groups is not None and not _normalize_groups(groups): return [] - clauses, params = _document_filters(include_demo, include_disabled, instances) + clauses, params = _document_filters(include_disabled, groups) if stage: - clauses.append("cw.current_stage = %s::workflow_stage") + clauses.append("cw.current_stage = %s") params.append(stage) where = f"WHERE {' AND '.join(clauses)}" if clauses else "" with get_connection() as conn: @@ -1300,14 +1372,13 @@ def list_documents( def get_document_summary_counts( - include_demo: bool = False, include_disabled: bool = False, - instances: Optional[list[str]] = None, + groups: Optional[list[str]] = None, ) -> dict: """Return aggregate document counts for dashboard and migration planning.""" - if instances is not None and not _normalize_instances(instances): + if groups is not None and not _normalize_groups(groups): return {key: 0 for key in _SUMMARY_KEYS} - clauses, params = _document_filters(include_demo, include_disabled, instances) + clauses, params = _document_filters(include_disabled, groups) where = f"WHERE {' AND '.join(clauses)}" if clauses else "" with get_connection() as conn: row = conn.execute( @@ -1330,6 +1401,8 @@ def get_document_summary_counts( COALESCE(SUM((SELECT COUNT(*) FROM workflows j WHERE j.document_id = d.id AND j.status = 'running')), 0) AS running_jobs FROM documents d + JOIN catalogs c ON c.id = d.catalog_id + JOIN groups g ON g.id = c.group_id LEFT JOIN document_current_workflow cw ON cw.document_id = d.id {where} """, @@ -1338,37 +1411,38 @@ def get_document_summary_counts( return {key: int(row[key] or 0) for key in _SUMMARY_KEYS} -def get_document_counts_by_instance( - include_demo: bool = False, +def get_document_counts_by_group( include_disabled: bool = False, - instances: Optional[list[str]] = None, + groups: Optional[list[str]] = None, ) -> list[dict]: - """Return document counts grouped by instance (state), for dashboard breakdowns.""" - if instances is not None and not _normalize_instances(instances): + """Return document counts grouped by group (state), for dashboard breakdowns.""" + if groups is not None and not _normalize_groups(groups): return [] - clauses, params = _document_filters(include_demo, include_disabled, instances) + clauses, params = _document_filters(include_disabled, groups) where = f"WHERE {' AND '.join(clauses)}" if clauses else "" with get_connection() as conn: rows = conn.execute( f""" SELECT - lower(d.instance) AS instance, + lower(g.name) AS "group", COUNT(*) AS count, COUNT(*) FILTER (WHERE cw.current_stage = 'completed') AS success, COUNT(*) FILTER (WHERE cw.current_stage = 'failed') AS failed, COUNT(*) FILTER (WHERE cw.current_stage IN ('ocr_review', 'translation_review', 'chunk_review', 'approval_for_prod')) AS dev_approval FROM documents d + JOIN catalogs c ON c.id = d.catalog_id + JOIN groups g ON g.id = c.group_id LEFT JOIN document_current_workflow cw ON cw.document_id = d.id {where} - GROUP BY lower(d.instance) - ORDER BY count DESC, instance ASC + GROUP BY lower(g.name) + ORDER BY count DESC, "group" ASC """, params, ).fetchall() return [ { - "instance": row["instance"], + "group": row["group"], "count": int(row["count"] or 0), "success": int(row["success"] or 0), "failed": int(row["failed"] or 0), @@ -1378,18 +1452,6 @@ def get_document_counts_by_instance( ] -def set_document_demo(workflow_id: str, is_demo: bool = True): - """Mark a document as demo (filtered from UI by default).""" - with get_connection() as conn: - doc_uuid = _resolve_document_uuid(conn, workflow_id) - if doc_uuid: - conn.execute( - "UPDATE documents SET metadata = metadata || jsonb_build_object('is_demo', %s::boolean) " - "WHERE id = %s::uuid", - (bool(is_demo), doc_uuid), - ) - - def set_document_disabled(workflow_id: str, is_disabled: bool = True): """Soft delete a document (filtered from all queries by default). @@ -1421,7 +1483,7 @@ def purge_document(workflow_id: str, keep_audit: bool = True) -> dict: Audit rows are kept by default: they are the record that the deletion happened, not part of the document itself. Their ``document_id`` becomes - NULL; the ``workflow_id`` and ``instance`` snapshots stay. + NULL; the ``workflow_id`` and ``group`` snapshots stay. Returns a per-table count of deleted rows. """ @@ -1467,7 +1529,7 @@ def get_document_count(stage: Optional[str] = None) -> int: """ SELECT COUNT(*) AS cnt FROM documents d JOIN document_current_workflow cw ON cw.document_id = d.id - WHERE cw.current_stage = %s::workflow_stage + WHERE cw.current_stage = %s """, (stage,), ).fetchone() @@ -1479,9 +1541,10 @@ def get_document_count(stage: Optional[str] = None) -> int: def update_document_fields(workflow_id: str, **updates: object) -> Optional[dict]: """Update selected document fields and return the document. - Legacy flat names are mapped onto the tables: ``stage``/``error_message`` act - on the current workflow, ``scheme_*`` on the catalog entry, counts and - flags on the JSONB columns. Names derived from other rows (artifact and job + Flat names are mapped onto the tables: ``stage``/``error_message`` act on the + current workflow, ``code``/``name``/``aliases`` on the document columns, + ``kind`` moves the document to the group's catalog of that kind, and flags, + period and uploader go into ``metadata``. Names derived from other rows (counts, artifact and job pointers, completion timestamps) are accepted and ignored. """ with get_connection() as conn: @@ -1501,15 +1564,11 @@ def update_document_fields(workflow_id: str, **updates: object) -> Optional[dict for key, value in updates.items(): if key == "stage": - wf_sets.append("current_stage = %s::workflow_stage") + wf_sets.append("current_stage = %s") wf_params.append(value) elif key == "error_message": wf_sets.append("error_message = %s") wf_params.append(value) - elif key == "page_count": - source_patch["page_count"] = value - elif key == "chunk_count": - meta_patch["chunk_count"] = value elif key == "source_type": source_patch["source_type"] = value elif key == "canonical_input_type": @@ -1522,11 +1581,17 @@ def update_document_fields(workflow_id: str, **updates: object) -> Optional[dict elif key in ("document_id", "canonical_document_id", "source_file_fingerprint"): if isinstance(value, str) and _FINGERPRINT_RE.match(value): source_patch["fingerprint"] = value - elif key == "display_name": + elif key == "name": if value: - meta_patch["display_name"] = value - else: - meta_remove.append("display_name") + doc_sets.append("name = %s") + doc_params.append(value) + elif key == "code": + if value: + doc_sets.append("code = %s") + doc_params.append(value) + elif key == "aliases": + doc_sets.append("aliases = %s") + doc_params.append([a for a in (value or []) if a]) elif key in ("reindex_required", "catalog_visible", "network_visible"): meta_patch[key] = bool(value) elif key == "reindex_reason": @@ -1542,24 +1607,33 @@ def update_document_fields(workflow_id: str, **updates: object) -> Optional[dict (bool(value), doc_uuid), ) elif key in ("valid_from", "valid_to"): - doc_sets.append(f"{key} = %s::date") - doc_params.append(value) + meta_patch[key] = value.isoformat() if isinstance(value, date) else value elif key == "uploaded_by_user_id" and value: - doc_sets.append("uploaded_by_user_id = COALESCE(uploaded_by_user_id, %s::uuid)") - doc_params.append( - upsert_user( + has_uploader = conn.execute( + "SELECT metadata ? 'uploaded_by_user_id' AS has FROM documents WHERE id = %s::uuid", + (doc_uuid,), + ).fetchone()["has"] + if not has_uploader: + meta_patch["uploaded_by_user_id"] = upsert_user( str(value), updates.get("uploaded_by_username"), updates.get("uploaded_by_email"), updates.get("uploaded_by_roles"), conn=conn, ) - ) - if any(k in updates for k in ("document_kind", "scheme_code", "scheme_name", "scheme_aliases_json")): - for column, new_value in _link_catalog_entry(conn, doc_uuid, updates).items(): - doc_sets.append(f"{column} = %s" + ("::uuid" if column == "catalog_entry_id" else "")) - doc_params.append(new_value) + if updates.get("kind"): + current = conn.execute( + "SELECT c.group_id, c.type FROM documents d JOIN catalogs c ON c.id = d.catalog_id " + "WHERE d.id = %s::uuid", + (doc_uuid,), + ).fetchone() + if current["type"] != catalog_type_for_kind(str(updates["kind"])): + target = get_or_create_catalog( + str(current["group_id"]), str(updates["kind"]), conn=conn + ) + doc_sets.append("catalog_id = %s::uuid") + doc_params.append(target["id"]) if source_patch: doc_sets.append("source = source || %s::jsonb") @@ -1580,10 +1654,18 @@ def update_document_fields(workflow_id: str, **updates: object) -> Optional[dict (*wf_params, doc_uuid), ) if doc_sets: - conn.execute( - f"UPDATE documents SET {', '.join(doc_sets)} WHERE id = %s::uuid", - (*doc_params, doc_uuid), - ) + try: + with conn.transaction(): + conn.execute( + f"UPDATE documents SET {', '.join(doc_sets)} WHERE id = %s::uuid", + (*doc_params, doc_uuid), + ) + except psycopg.errors.UniqueViolation as exc: + if exc.diag.constraint_name == "documents_catalog_id_code_key": + raise DuplicateDocumentCode( + "Another document in the target catalog already uses this code" + ) from exc + raise elif wf_sets: conn.execute("UPDATE documents SET updated_at = now() WHERE id = %s::uuid", (doc_uuid,)) return _fetch_document(conn, doc_uuid) @@ -1732,8 +1814,8 @@ def create_document_job( if job_type == "promote_prod": adopted = conn.execute( """ - UPDATE workflows SET temporal_workflow_id = %s, status = %s::workflow_status_enum, - current_stage = COALESCE(%s::workflow_stage, current_stage), + UPDATE workflows SET temporal_workflow_id = %s, status = %s, + current_stage = COALESCE(%s, current_stage), config = config || %s::jsonb WHERE id = ( SELECT id FROM workflows @@ -1751,8 +1833,8 @@ def create_document_job( """ INSERT INTO workflows (document_id, workflow_type, temporal_workflow_id, status, current_stage, stop_after_ocr, config, started_by_user_id) - VALUES (%s::uuid, %s::workflow_type_enum, %s, %s::workflow_status_enum, - COALESCE(%s, 'registered')::workflow_stage, %s, %s, %s::uuid) + VALUES (%s::uuid, %s, %s, %s, + COALESCE(%s, 'registered'), %s, %s, %s::uuid) RETURNING id """, ( @@ -1778,9 +1860,9 @@ def update_document_job(job_id: str, **updates: object) -> Optional[dict]: if key not in allowed_fields: continue if key == "status": - set_clauses.append("status = %s::workflow_status_enum") + set_clauses.append("status = %s") elif key == "current_stage": - set_clauses.append("current_stage = %s::workflow_stage") + set_clauses.append("current_stage = %s") elif key == "completed_at": set_clauses.append("completed_at = %s::timestamptz") elif key == "temporal_run_id": @@ -1838,20 +1920,19 @@ def list_document_jobs(workflow_id: str, limit: int = 50) -> list[dict]: def list_operations_queue( limit: int = 100, offset: int = 0, - include_demo: bool = False, include_disabled: bool = False, - instances: Optional[list[str]] = None, + groups: Optional[list[str]] = None, ) -> tuple[list[dict], int]: - """List documents needing action, optionally scoped to tenant instances. + """List documents needing action, optionally scoped to tenant groups. - ``instances``: + ``groups``: - ``None`` - unrestricted (super admin / bypass) - ``[]`` - restricted user with no states -> empty result - - non-empty - only those document.instance values + - non-empty - only those document.group values """ - if instances is not None and not _normalize_instances(instances): + if groups is not None and not _normalize_groups(groups): return [], 0 - clauses, params = _document_filters(include_demo, include_disabled, instances) + clauses, params = _document_filters(include_disabled, groups) clauses.append( "(cw.current_stage IN ('ocr_review', 'translation_review', 'chunk_review', " "'ready_for_ingestion', 'approval_for_prod', 'failed') " @@ -1861,6 +1942,8 @@ def list_operations_queue( where = f"WHERE {' AND '.join(clauses)}" from_sql = """ FROM documents d + JOIN catalogs c ON c.id = d.catalog_id + JOIN groups g ON g.id = c.group_id JOIN document_current_workflow cw ON cw.document_id = d.id """ with get_connection() as conn: @@ -1874,7 +1957,7 @@ def list_operations_queue( LIMIT 1) AS workflow_id, d.source->>'original_filename' AS filename, cw.current_stage AS stage, - d.instance, + lower(g.name) AS "group", cw.error_message, COALESCE((d.metadata->>'reindex_required')::boolean, FALSE) AS reindex_required, d.metadata->>'reindex_reason' AS reindex_reason, @@ -1896,24 +1979,24 @@ def list_runs( limit: int = 100, offset: int = 0, status: Optional[str] = None, - instances: Optional[list[str]] = None, + groups: Optional[list[str]] = None, ) -> list[dict]: """List workflows with document title fields for the Runs UI. Args: - instances: If provided, only return runs whose document instance is in + groups: If provided, only return runs whose document group is in this list. Empty list returns no rows. None = unrestricted. """ where_parts: list[str] = [] params: list = [] if status: - where_parts.append("w.status = %s::workflow_status_enum") + where_parts.append("w.status = %s") params.append(status) - if instances is not None: - normalized = _normalize_instances(instances) + if groups is not None: + normalized = _normalize_groups(groups) if not normalized: return [] - where_parts.append("lower(d.instance) = ANY(%s)") + where_parts.append("lower(g.name) = ANY(%s)") params.append(normalized) where_sql = f"WHERE {' AND '.join(where_parts)}" if where_parts else "" with get_connection() as conn: @@ -1925,12 +2008,14 @@ def list_runs( ORDER BY (r.workflow_type IN ('pipeline', 'ocr_only')) DESC, r.started_at ASC, r.id ASC LIMIT 1) AS doc_workflow_id, d.source->>'original_filename' AS filename, - d.metadata->>'display_name' AS display_name, + d.name AS name, d.source->>'original_filename' AS source_filename, cw.current_stage AS document_stage, - d.instance AS instance + lower(g.name) AS "group" FROM workflows w JOIN documents d ON d.id = w.document_id + JOIN catalogs c ON c.id = d.catalog_id + JOIN groups g ON g.id = c.group_id LEFT JOIN document_current_workflow cw ON cw.document_id = d.id {where_sql} ORDER BY w.started_at DESC, w.id DESC @@ -1942,7 +2027,7 @@ def list_runs( for row in rows: job = _job_from_row(row) job["document_uuid"] = str(row["document_id"]) - for extra in ("filename", "display_name", "source_filename", "document_stage", "instance"): + for extra in ("filename", "name", "source_filename", "document_stage", "group"): job[extra] = row[extra] result.append(job) return result @@ -2024,7 +2109,7 @@ def add_document_artifact( """ INSERT INTO workflow_artifacts (workflow_id, artifact_type, stage, storage_uri, mime_type, filename, size_bytes, metadata) - VALUES (%s::uuid, %s, %s::workflow_stage, %s, %s, %s, %s, %s) + VALUES (%s::uuid, %s, %s, %s, %s, %s, %s, %s) RETURNING id """, ( @@ -2063,7 +2148,7 @@ def get_document_artifact(workflow_id: str, artifact_id: str) -> Optional[dict]: # ============================================================================= -# Vector index state (documents.index_status) +# Vector index state (documents.metadata.index_status) # ============================================================================= def _index_entry_to_legacy(workflow_id: str, key: str, entry: dict) -> dict: @@ -2092,7 +2177,7 @@ def upsert_document_index_status( status: str = "unknown", details: Optional[dict] = None, ) -> dict: - """Merge one index's state into ``documents.index_status``. + """Merge one index's state into ``documents.metadata.index_status``. The key is ``prod`` for the PROD collection names and ``dev`` otherwise; the collection name is kept inside the entry. ``None`` arguments leave the @@ -2102,7 +2187,7 @@ def upsert_document_index_status( with get_connection() as conn: doc_uuid = _require_document_uuid(conn, workflow_id) row = conn.execute( - "SELECT index_status->%s::text AS entry FROM documents WHERE id = %s::uuid FOR UPDATE", + "SELECT metadata->'index_status'->%s::text AS entry FROM documents WHERE id = %s::uuid FOR UPDATE", (key, doc_uuid), ).fetchone() existing = dict(row["entry"] or {}) @@ -2118,7 +2203,9 @@ def upsert_document_index_status( merged = {**existing, **{k: v for k, v in incoming.items() if v is not None}, "status": status} validated = IndexState.model_validate(merged).model_dump() conn.execute( - "UPDATE documents SET index_status = index_status || jsonb_build_object(%s::text, %s::jsonb) " + "UPDATE documents SET metadata = jsonb_set(" + "metadata || jsonb_build_object('index_status', COALESCE(metadata->'index_status', '{}'::jsonb)), " + "ARRAY['index_status', %s::text], %s::jsonb) " "WHERE id = %s::uuid", (key, Jsonb(validated), doc_uuid), ) @@ -2147,8 +2234,8 @@ def get_document_index_status(workflow_id: str, index_name: str) -> Optional[dic return None -def list_index_summaries(include_demo: bool = False, include_disabled: bool = False) -> list[dict]: - clauses, params = _document_filters(include_demo, include_disabled, None) +def list_index_summaries(include_disabled: bool = False) -> list[dict]: + clauses, params = _document_filters(include_disabled, None) where = f"AND {' AND '.join(clauses)}" if clauses else "" with get_connection() as conn: rows = conn.execute( @@ -2161,7 +2248,7 @@ def list_index_summaries(include_demo: bool = False, include_disabled: bool = Fa THEN 1 ELSE 0 END) AS stale_documents, MAX(e.value->>'last_indexed_at') AS last_indexed_at, array_agg(DISTINCT COALESCE(e.value->>'status', 'unknown')) AS statuses - FROM documents d, jsonb_each(d.index_status) e + FROM documents d, jsonb_each(COALESCE(d.metadata->'index_status', '{{}}'::jsonb)) e WHERE TRUE {where} GROUP BY COALESCE(e.value->>'index_name', e.key) ORDER BY index_name @@ -2212,13 +2299,13 @@ def resolve_chunk_provenance( excerpt = text[:320] + ("..." if len(text) > 320 else "") page_start = int(chunk.get("page_start") or 1) page_end = int(chunk.get("page_end") or page_start) - display_name = doc.get("display_name") or doc.get("filename") or workflow_id + doc_name = doc.get("name") or doc.get("filename") or workflow_id return { "workflow_id": workflow_id, "doc_id": doc.get("document_id"), "filename": workflow_id, - "doc_name": display_name, + "doc_name": doc_name, "chunk_num": int(chunk_num), "page_start": page_start, "page_end": page_end, @@ -2275,13 +2362,15 @@ def log_audit( roles = _csv_roles(actor_roles) with get_connection() as conn: doc_uuid = None if system_entry else _resolve_document_uuid(conn, workflow_id) - instance = None + group_uuid = None stored_workflow_id = None if system_entry else workflow_id if doc_uuid: doc_row = conn.execute( - "SELECT instance FROM documents WHERE id = %s::uuid", (doc_uuid,) + "SELECT c.group_id FROM documents d JOIN catalogs c ON c.id = d.catalog_id " + "WHERE d.id = %s::uuid", + (doc_uuid,), ).fetchone() - instance = doc_row["instance"] + group_uuid = str(doc_row["group_id"]) if _is_uuid(workflow_id): stored_workflow_id = _root_workflow_id(conn, doc_uuid) or workflow_id if stored_workflow_id is None and entity not in ("setting", "catalog"): @@ -2292,14 +2381,14 @@ def log_audit( ) row = conn.execute( """ - INSERT INTO audit_logs (document_id, workflow_id, instance, action_type, entity_type, + INSERT INTO audit_logs (document_id, workflow_id, group_id, action_type, entity_type, entity_number, field_name, old_value, new_value, metadata, actor_user_id, actor) - VALUES (%s::uuid, %s, %s, %s, %s::audit_entity_type, %s, %s, %s, %s, %s, %s::uuid, %s) + VALUES (%s::uuid, %s, %s::uuid, %s, %s, %s, %s, %s, %s, %s, %s::uuid, %s) RETURNING id """, ( - doc_uuid, stored_workflow_id, instance, action_type, entity, entity_id, + doc_uuid, stored_workflow_id, group_uuid, action_type, entity, entity_id, field_name, old_value, new_value, Jsonb(metadata or {}), actor_uuid, Jsonb({"username": actor_username or None, "email": actor_email or None, "roles": roles}) if (actor_user_id or actor_username or actor_email or roles) else Jsonb({}), @@ -2371,17 +2460,20 @@ def _normalize_audit_row(row) -> dict: _AUDIT_SELECT = """ SELECT a.id, a.workflow_id, a.action_type, a.entity_type, a.entity_number, a.field_name, - a.old_value, a.new_value, a.metadata, a.created_at, a.actor, a.instance AS audit_instance, + a.old_value, a.new_value, a.metadata, a.created_at, a.actor, ag.name AS audit_group, au.keycloak_user_id AS actor_keycloak_id, d.source->>'fingerprint' AS doc_fingerprint, d.source->>'original_filename' AS filename, - d.metadata->>'display_name' AS display_name, - d.instance AS doc_instance, + d.name AS document_name, + dg.name AS doc_group, uu.username AS uploaded_by_username, uu.email AS uploaded_by_email FROM audit_logs a LEFT JOIN documents d ON d.id = a.document_id + LEFT JOIN groups ag ON ag.id = a.group_id + LEFT JOIN catalogs dc ON dc.id = d.catalog_id + LEFT JOIN groups dg ON dg.id = dc.group_id LEFT JOIN users au ON au.id = a.actor_user_id - LEFT JOIN users uu ON uu.id = d.uploaded_by_user_id + LEFT JOIN users uu ON uu.id = NULLIF(d.metadata->>'uploaded_by_user_id', '')::uuid """ @@ -2414,8 +2506,8 @@ def _audit_row_to_legacy(row: dict) -> dict: "actor_email": actor.get("email"), "actor_roles": ",".join(roles) if roles else None, "filename": row["filename"], - "display_name": row["display_name"], - "instance": row["doc_instance"] or row["audit_instance"], + "name": row["document_name"], + "group": row["doc_group"] or row["audit_group"], "uploaded_by_username": row["uploaded_by_username"], "uploaded_by_email": row["uploaded_by_email"], } @@ -2472,16 +2564,16 @@ def get_audit_log_count(workflow_id: str, action_type: Optional[str] = None) -> def _global_audit_filters( - action_type: Optional[str], instances: Optional[list[str]] + action_type: Optional[str], groups: Optional[list[str]] ) -> tuple[list[str], list]: where_parts: list[str] = ["a.action_type <> %s"] params: list = [CATALOG_VERSION_ACTION] if action_type: where_parts.append("a.action_type = %s") params.append(action_type) - if instances is not None: - where_parts.append("lower(COALESCE(a.instance, d.instance, %s)) = ANY(%s)") - params.extend([_default_instance(), _normalize_instances(instances)]) + if groups is not None: + where_parts.append("lower(COALESCE(ag.name, dg.name, %s)) = ANY(%s)") + params.extend([_default_group(), _normalize_groups(groups)]) return where_parts, params @@ -2489,7 +2581,7 @@ def get_all_audit_logs( action_type: Optional[str] = None, limit: int = 100, offset: int = 0, - instances: Optional[list[str]] = None, + groups: Optional[list[str]] = None, ) -> list[dict]: """ Get all audit logs across all documents. @@ -2498,14 +2590,14 @@ def get_all_audit_logs( action_type: Optional filter by action type limit: Maximum number of entries to return offset: Offset for pagination - instances: Tenant scope (None = all, [] = none) + groups: Tenant scope (None = all, [] = none) Returns: List of audit log entries as dicts, with document filename included """ - if instances is not None and not _normalize_instances(instances): + if groups is not None and not _normalize_groups(groups): return [] - where_parts, params = _global_audit_filters(action_type, instances) + where_parts, params = _global_audit_filters(action_type, groups) with get_connection() as conn: rows = conn.execute( f"{_AUDIT_SELECT} WHERE {' AND '.join(where_parts)} " @@ -2517,15 +2609,19 @@ def get_all_audit_logs( def get_all_audit_log_count( action_type: Optional[str] = None, - instances: Optional[list[str]] = None, + groups: Optional[list[str]] = None, ) -> int: """Get total count of all audit logs (optionally tenant-scoped).""" - if instances is not None and not _normalize_instances(instances): + if groups is not None and not _normalize_groups(groups): return 0 - where_parts, params = _global_audit_filters(action_type, instances) + where_parts, params = _global_audit_filters(action_type, groups) with get_connection() as conn: row = conn.execute( - f"SELECT COUNT(*) AS cnt FROM audit_logs a LEFT JOIN documents d ON d.id = a.document_id " + "SELECT COUNT(*) AS cnt FROM audit_logs a " + "LEFT JOIN documents d ON d.id = a.document_id " + "LEFT JOIN groups ag ON ag.id = a.group_id " + "LEFT JOIN catalogs dc ON dc.id = d.catalog_id " + "LEFT JOIN groups dg ON dg.id = dc.group_id " f"WHERE {' AND '.join(where_parts)}", params, ).fetchone() @@ -2876,8 +2972,8 @@ def save_chunks(workflow_id: str, chunks: list[dict]): Save all chunks for a document. Re-chunking runs in one transaction: the document's chunks are deleted, the - new ones inserted, and ``metadata.chunk_count`` and ``metadata.chunking`` - (with ``run_number`` incremented) updated together. + new ones inserted, and ``metadata.chunking`` (with ``run_number`` + incremented) updated together. """ with get_connection() as conn: doc_uuid = _require_document_uuid(conn, workflow_id) @@ -2921,7 +3017,7 @@ def save_chunks(workflow_id: str, chunks: list[dict]): chunk.get("reviewer_notes"), ), ) - meta_patch: dict[str, Any] = {"chunk_count": len(chunks)} + meta_patch: dict[str, Any] = {} if chunks: first = chunks[0] meta_patch["chunking"] = ChunkingRun( @@ -2931,10 +3027,13 @@ def save_chunks(workflow_id: str, chunks: list[dict]): config=_load_json_value(first.get("chunking_config_json"), {}), created_at=_iso_now(), ).model_dump() - conn.execute( - "UPDATE documents SET metadata = metadata || %s::jsonb WHERE id = %s::uuid", - (Jsonb(meta_patch), doc_uuid), - ) + if meta_patch: + conn.execute( + "UPDATE documents SET metadata = metadata || %s::jsonb WHERE id = %s::uuid", + (Jsonb(meta_patch), doc_uuid), + ) + else: + conn.execute("UPDATE documents SET updated_at = now() WHERE id = %s::uuid", (doc_uuid,)) def get_chunks(workflow_id: str, include_excluded: bool = False) -> list[dict]: @@ -2959,12 +3058,12 @@ def search_chunks( offset: int = 0, include_excluded: bool = False, stage: Optional[str] = None, - instances: Optional[list[str]] = None, + groups: Optional[list[str]] = None, ) -> tuple[list[dict], int]: """Search chunks across documents for maintainer workflows. - instances: If provided, only return chunks whose owning document's instance - is in this list (tenant scoping). ``None`` means no instance restriction. + groups: If provided, only return chunks whose owning document's group + is in this list (tenant scoping). ``None`` means no group restriction. """ where_clauses = ["TRUE"] params: list = [] @@ -2972,14 +3071,14 @@ def search_chunks( if not include_excluded: where_clauses.append("NOT c.is_excluded") if stage: - where_clauses.append("cw.current_stage = %s::workflow_stage") + where_clauses.append("cw.current_stage = %s") params.append(stage) - if instances is not None: - normalized = _normalize_instances(instances) + if groups is not None: + normalized = _normalize_groups(groups) if not normalized: where_clauses.append("FALSE") else: - where_clauses.append("lower(d.instance) = ANY(%s)") + where_clauses.append("lower(g.name) = ANY(%s)") params.append(normalized) if query and query.strip(): where_clauses.append("lower(coalesce(c.edited_text, c.original_text)) LIKE %s") @@ -2988,6 +3087,8 @@ def search_chunks( base_from_sql = f""" FROM chunks c JOIN documents d ON d.id = c.document_id + JOIN catalogs cat ON cat.id = d.catalog_id + JOIN groups g ON g.id = cat.group_id LEFT JOIN document_current_workflow cw ON cw.document_id = d.id WHERE {' AND '.join(where_clauses)} """ @@ -3003,7 +3104,7 @@ def search_chunks( d.metadata->'chunking' AS chunking, d.source->>'fingerprint' AS document_id, d.source->>'original_filename' AS filename, - d.metadata->>'display_name' AS display_name, + d.name AS name, cw.current_stage AS stage, d.updated_at AS document_updated_at, (SELECT r.temporal_workflow_id FROM workflows r WHERE r.document_id = d.id @@ -3020,7 +3121,7 @@ def search_chunks( for row in rows: row = dict(row) chunk = _chunk_from_row(row, row["root_workflow_id"]) - for extra in ("document_id", "filename", "display_name", "stage", "document_updated_at"): + for extra in ("document_id", "filename", "name", "stage", "document_updated_at"): chunk[extra] = _clean(row[extra]) chunks.append(chunk) return chunks, total diff --git a/pipeline/document_repository.py b/pipeline/document_repository.py index 9f2e2c3..61312bd 100644 --- a/pipeline/document_repository.py +++ b/pipeline/document_repository.py @@ -27,4 +27,4 @@ def get_document_kind(self, workflow_id: str) -> Optional[str]: doc = self._db.get_document(workflow_id) if not doc: return None - return doc.get("document_kind") + return doc.get("kind") diff --git a/pipeline/instances.py b/pipeline/group_names.py similarity index 80% rename from pipeline/instances.py rename to pipeline/group_names.py index c5486a7..2711d60 100644 --- a/pipeline/instances.py +++ b/pipeline/group_names.py @@ -1,11 +1,13 @@ -"""Instance (state / tenant) code → human-readable name. +"""Group (state / tenant) code → human-readable name. -``documents.instance`` stores a short lowercase code — ``bv`` for the Bharat +A group code is a short lowercase ``groups.name`` — ``bv`` for the Bharat Vistaar portal, otherwise the state code matching the Keycloak group path -(``/states/MH/...`` → ``mh``). That code is what the pipeline filters on, but -it is not what a reader wants to see: consumers of the vector payload and the -master catalog get ``instance_name`` alongside it so they can show "Maharashtra" -without shipping their own lookup table. +(``/states/MH/...`` → ``mh``). A document's group is derived through its +catalog (``documents.catalog_id`` → ``catalogs.group_id``). That code is what +the pipeline filters on, but it is not what a reader wants to see: consumers of +the vector payload and the master catalog get ``instance_name`` (their field +name, unchanged) alongside the code so they can show "Maharashtra" without +shipping their own lookup table. Codes follow the standard two-letter Indian state/UT abbreviations. Unknown codes fall back to a title-cased version of the code itself rather than raising @@ -17,7 +19,7 @@ import os # Portal / platform-wide documents (not a state). -PORTAL_INSTANCE = "bv" +PORTAL_GROUP = "bv" def prod_stage_disabled() -> bool: @@ -48,7 +50,7 @@ def publish_to_network_enabled() -> bool: in {"true", "1", "yes"}) -INSTANCE_NAMES: dict[str, str] = { +GROUP_NAMES: dict[str, str] = { # Platform "bv": "Bharat Vistaar", "default": "Default", @@ -96,15 +98,15 @@ def publish_to_network_enabled() -> bool: } -def instance_display_name(code: str | None) -> str: - """Human-readable name for an instance code (``mh`` → ``Maharashtra``). +def group_display_name(code: str | None) -> str: + """Human-readable name for a group code (``mh`` → ``Maharashtra``). Unknown codes are title-cased so the field is never empty. """ normalized = (code or "").strip().lower() if not normalized: - return INSTANCE_NAMES["default"] - known = INSTANCE_NAMES.get(normalized) + return GROUP_NAMES["default"] + known = GROUP_NAMES.get(normalized) if known: return known # Unknown code: make it presentable rather than failing the ingest. diff --git a/pipeline/models.py b/pipeline/models.py index 88e4e86..cf22315 100644 --- a/pipeline/models.py +++ b/pipeline/models.py @@ -7,7 +7,7 @@ from enum import Enum from typing import Any, Optional -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator class DocumentStage(str, Enum): @@ -28,6 +28,34 @@ class DocumentStage(str, Enum): FAILED = "failed" +class WorkflowType(str, Enum): + """``workflows.workflow_type``: kind of processing run. Stored as TEXT.""" + PIPELINE = "pipeline" + OCR_ONLY = "ocr_only" + OCR_RETRY = "ocr_retry" + TRANSLATION_RETRY = "translation_retry" + CHUNKING_RETRY = "chunking_retry" + REINGEST = "reingest" + PROMOTE_PROD = "promote_prod" + + +class WorkflowStatus(str, Enum): + """``workflows.status``. Stored as TEXT.""" + RUNNING = "running" + WAITING_REVIEW = "waiting_review" + COMPLETED = "completed" + FAILED = "failed" + + +class AuditEntityType(str, Enum): + """``audit_logs.entity_type``. Stored as TEXT.""" + DOCUMENT = "document" + PAGE = "page" + CHUNK = "chunk" + SETTING = "setting" + CATALOG = "catalog" + + # Pipeline stage order for stepper UI PIPELINE_STAGES = [ ("registered", "Registered", "Document uploaded"), @@ -185,12 +213,12 @@ class DocumentSummary(BaseModel): canonical_document_id: Optional[str] = None workflow_id: str # The Temporal workflow ID (use this for API calls) filename: str - display_name: Optional[str] = None + name: Optional[str] = None # documents.name source_filename: Optional[str] = None source_manifest_name: Optional[str] = None source_file_fingerprint: Optional[str] = None authoritative: bool = False - instance: str = "default" + group: str = "default" # tenant code (groups.name) of the document's catalog stage: DocumentStage page_count: int chunk_count: int @@ -204,11 +232,11 @@ class DocumentSummary(BaseModel): uploaded_by_username: Optional[str] = None uploaded_by_email: Optional[str] = None uploaded_by_roles: list[str] = [] - # Scheme catalog metadata (document_kind scheme → Master Catalog) - document_kind: str = "document" - scheme_code: Optional[str] = None - scheme_name: Optional[str] = None - scheme_aliases: list[str] = [] + # Catalog placement: kind comes from the catalog type, code/name/aliases are + # the document's own (a scheme's code feeds the Master Catalog). + kind: Optional[str] = None # scheme | advisory + code: Optional[str] = None + aliases: list[str] = [] tool_routing: Optional[str] = None catalog_visible: bool = True network_visible: bool = True @@ -231,10 +259,10 @@ class SchemeMetadataUpdate(BaseModel): `document_validity.parse_period` so a bad period comes back as the same 400 a bad kind does, not a Pydantic 422. """ - document_kind: Optional[str] = None # document | scheme - scheme_code: Optional[str] = None - scheme_name: Optional[str] = None - scheme_aliases: Optional[list[str]] = None + kind: Optional[str] = None # scheme | advisory + code: Optional[str] = None + name: Optional[str] = None + aliases: Optional[list[str]] = None tool_routing: Optional[str] = None # qdrant | legacy | both catalog_visible: Optional[bool] = None network_visible: Optional[bool] = None @@ -312,8 +340,8 @@ class DocumentGraph(BaseModel): runtime: dict[str, Any] -class InstanceDocumentCount(BaseModel): - instance: str +class GroupDocumentCount(BaseModel): + group: str count: int success: int = 0 failed: int = 0 @@ -328,7 +356,7 @@ class DocumentCohortsResponse(BaseModel): review_queue: int failed_documents: int by_stage: dict[str, int] - by_instance: list[InstanceDocumentCount] = [] + by_group: list[GroupDocumentCount] = [] needs_reindex: int running_jobs: int = 0 @@ -404,8 +432,8 @@ class AuditLogEntry(BaseModel): # Document context — joined from documents. Without these declared, FastAPI # strips them from the response and the UI can only show a raw doc_id hash. filename: Optional[str] = None - display_name: Optional[str] = None - instance: Optional[str] = None + name: Optional[str] = None + group: Optional[str] = None uploaded_by_username: Optional[str] = None uploaded_by_email: Optional[str] = None @@ -493,7 +521,6 @@ class DocumentSource(BaseModel): storage_uri: Optional[str] = None source_type: Optional[str] = None # spreadsheet | document | unknown canonical_input_type: Optional[str] = None # spreadsheet | pdf - page_count: Optional[int] = Field(default=None, ge=0) class IndexState(BaseModel): @@ -521,18 +548,43 @@ class ChunkingRun(BaseModel): class DocumentMetadata(BaseModel): - """``documents.metadata``: display and flag fields.""" + """``documents.metadata``: period, uploader, index state and flag fields.""" model_config = ConfigDict(extra="forbid") - display_name: Optional[str] = None + valid_from: Optional[str] = None # YYYY-MM-DD + valid_to: Optional[str] = None # YYYY-MM-DD, None = no end + uploaded_by_user_id: Optional[str] = None # users.id (UUID string) + index_status: dict[str, IndexState] = Field(default_factory=dict) # keys dev / prod catalog_visible: bool = True network_visible: bool = True - is_demo: bool = False - chunk_count: int = Field(default=0, ge=0) reindex_required: bool = False reindex_reason: Optional[str] = None chunking: Optional[ChunkingRun] = None + @model_validator(mode="after") + def _period_is_ordered(self) -> "DocumentMetadata": + if self.valid_from and self.valid_to and self.valid_from > self.valid_to: + raise ValueError("valid_from must not be after valid_to") + return self + + +class CatalogNetworkState(BaseModel): + """``catalogs.metadata.network``: set once the catalog is published.""" + model_config = ConfigDict(extra="forbid") + + catalog_id: Optional[str] = None + last_published_at: Optional[str] = None + + +class CatalogMetadata(BaseModel): + """``catalogs.metadata``: Beckn publish information.""" + model_config = ConfigDict(extra="forbid") + + publish_directives: dict[str, Any] = Field(default_factory=dict) + visible_networks: list[str] = Field(default_factory=list) + criteria: dict[str, Any] = Field(default_factory=dict) + network: Optional[CatalogNetworkState] = None + class PageMetadata(BaseModel): """``pages.metadata``: OCR review and language fields.""" diff --git a/pipeline/network_constants.py b/pipeline/network_constants.py index 422a946..5254ac5 100644 --- a/pipeline/network_constants.py +++ b/pipeline/network_constants.py @@ -33,6 +33,36 @@ DEFAULT_DOCUMENT_KIND = "document" +# Knowledge kinds the pipeline serves, and the network schema each maps to. +# ``catalogs.type`` stores the schema name; code asks this map what a type means +# instead of comparing ``openagrinet:*`` strings. The scheme mapping is a +# placeholder (ADR 0003 types schemes as KnowledgeAdvisory) - change it here only. +KIND_SCHEME = "scheme" +KIND_ADVISORY = "advisory" +KIND_TO_CATALOG_TYPE = { + KIND_SCHEME: "openagrinet:AgricultureResource", + KIND_ADVISORY: "openagrinet:KnowledgeAdvisory", +} +CATALOG_TYPE_TO_KIND = {schema: kind for kind, schema in KIND_TO_CATALOG_TYPE.items()} + + +def catalog_type_for_kind(kind: str) -> str: + """``catalogs.type`` for a knowledge kind; ValueError for an unknown kind.""" + try: + return KIND_TO_CATALOG_TYPE[(kind or "").strip().lower()] + except KeyError: + raise ValueError( + f"Unknown document kind {kind!r}. Use one of: {', '.join(KIND_TO_CATALOG_TYPE)}" + ) from None + + +def kind_for_catalog_type(catalog_type: str) -> str: + """Knowledge kind a ``catalogs.type`` stands for; ValueError for an unknown type.""" + try: + return CATALOG_TYPE_TO_KIND[catalog_type] + except KeyError: + raise ValueError(f"Unknown catalog type {catalog_type!r}") from None + # Same provider on every catalog - this pipeline publishes as one provider # regardless of knowledge kind. CATALOG_PROVIDER = { diff --git a/pipeline/scheme_catalog.py b/pipeline/scheme_catalog.py index cdf2bb7..a6a6eb5 100644 --- a/pipeline/scheme_catalog.py +++ b/pipeline/scheme_catalog.py @@ -15,14 +15,10 @@ from typing import Any, Optional from . import db, document_validity +from .network_constants import catalog_type_for_kind SCHEME_CODE_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") -# document_kind is a free-form slug (scheme, advisory, video, or a custom -# type entered in the UI) — same character rules as scheme_code but allows -# underscores too since kind names read more naturally that way (e.g. "how_to"). -DOCUMENT_KIND_RE = re.compile(r"^[a-z0-9]+(?:[_-][a-z0-9]+)*$") -SUGGESTED_DOCUMENT_KINDS = ("document", "scheme", "advisory", "video") _SCHEME_CODE_STOPWORDS = { "a", "an", "the", "of", "on", "for", "and", "or", "to", "in", "at", "by", @@ -191,14 +187,14 @@ def utc_now_iso() -> str: return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") -def central_instances() -> set[str]: +def central_groups() -> set[str]: raw = (os.environ.get("CENTRAL_INSTANCES") or "default").strip() return {p.strip().lower() for p in raw.split(",") if p.strip()} or {"default"} -def default_network_visible(instance: str | None) -> bool: - inst = (instance or "default").strip().lower() or "default" - return inst in central_instances() +def default_network_visible(group: str | None) -> bool: + code = (group or "default").strip().lower() or "default" + return code in central_groups() def scheme_collection_name() -> str: @@ -233,19 +229,21 @@ def _acronym_from_title(title: str) -> str: return letters -def derive_scheme_code(title: str, exclude_workflow_id: Optional[str] = None) -> str: +def derive_scheme_code( + title: str, exclude_workflow_id: Optional[str] = None, kind: str = "scheme" +) -> str: """Auto-generate a scheme_code from a document title: acronym of significant words, falling back to a truncated slug if the title has too few words to produce a useful acronym (e.g. a single-word title). Appends -2, -3, ... on - collision against every scheme_code already in use (documents table + - scheme_catalog_entries registry) so two schemes never share a code. + collision against every code already in use for the kind (``documents.code``) + so two documents of a kind never share a code across tenants. """ base = _acronym_from_title(title) if len(base) < 2: slug = re.sub(r"[^a-z0-9]+", "", (title or "").strip().lower()) base = slug[:8] or "scheme" - used = db.list_used_scheme_codes(exclude_workflow_id=exclude_workflow_id) + used = db.list_used_scheme_codes(exclude_workflow_id=exclude_workflow_id, kind=kind) if base not in used: return base n = 2 @@ -344,8 +342,8 @@ def _dedupe_aliases(aliases: list[str]) -> list[str]: def resolve_scheme_aliases( - scheme_code: str | None, - scheme_name: str | None, + code: str | None, + name: str | None, explicit: Any = None, ) -> list[str]: """Aliases to index for a scheme, in precedence order. @@ -357,60 +355,26 @@ def resolve_scheme_aliases( All three are merged rather than short-circuited, so a curator adding one alias does not silently discard the bootstrap and derived ones. """ - code = (scheme_code or "").strip().lower() + code = (code or "").strip().lower() merged: list[str] = list(parse_aliases(explicit)) bootstrap = BOOTSTRAP_VECTOR_SCHEMES.get(code) if bootstrap: merged.extend(bootstrap.get("scheme_aliases") or []) - if not (scheme_name or "").strip(): - scheme_name = bootstrap.get("scheme_name") + if not (name or "").strip(): + name = bootstrap.get("scheme_name") - merged.extend(_derive_aliases(code, scheme_name or "")) + merged.extend(_derive_aliases(code, name or "")) return _dedupe_aliases(merged) -def _seed_bootstrap_entry(code: str, definition: dict[str, Any]) -> None: - db.upsert_catalog_entry( - type="scheme", - code=code, - name=definition["scheme_name"], - aliases=list(definition.get("scheme_aliases") or []), - source="bootstrap", - ) - - -def bootstrap_catalog_if_empty(*, force: bool = False) -> dict[str, Any]: - """Seed bootstrap vector schemes that have no catalog entry yet (all of them if force).""" - existing = db.list_catalog_entry_codes("scheme") - missing = {c: d for c, d in BOOTSTRAP_VECTOR_SCHEMES.items() if force or c not in existing} - if not missing: - meta = db.get_catalog_meta() - return {"bootstrapped": False, "version": meta.get("version", 0), "entry_count": len(existing)} - - for code, definition in missing.items(): - _seed_bootstrap_entry(code, definition) - version = db.bump_catalog_version("bootstrap_vector_schemes") - return { - "bootstrapped": True, - "version": version, - "vector_count": len(BOOTSTRAP_VECTOR_SCHEMES), - "legacy_count": len(BOOTSTRAP_LEGACY_SCHEMES), - } - - def rebuild_catalog_from_documents(*, reason: str = "rebuild") -> int: """ - Re-seed bootstrap codes that have no entry, then bump the catalog version. + Bump the catalog version. - Status, instances, chunk counts and the rest are computed by the ``catalog`` - view, so there is nothing to recompute here. Bootstrap-only codes with no - documents stay live. + Status, groups, chunk counts and the rest are computed by the + ``catalog_status`` view, so there is nothing to recompute here. """ - existing = db.list_catalog_entry_codes("scheme") - for code, definition in BOOTSTRAP_VECTOR_SCHEMES.items(): - if code not in existing: - _seed_bootstrap_entry(code, definition) return db.bump_catalog_version(reason) @@ -419,7 +383,7 @@ def on_scheme_document_promoted(workflow_id: str) -> Optional[int]: doc = db.get_document(workflow_id) if not doc: return None - if (doc.get("document_kind") or "document") != "scheme": + if doc.get("kind") != "scheme": return None # Clear reindex flag on successful promote (new code is live) if int(doc.get("reindex_required") or 0): @@ -431,7 +395,7 @@ def on_scheme_document_promoted(workflow_id: str) -> Optional[int]: return rebuild_catalog_from_documents(reason=f"promote_scheme:{workflow_id}") -def on_scheme_document_disabled(workflow_id: str, scheme_code: Optional[str] = None) -> Optional[int]: +def on_scheme_document_disabled(workflow_id: str, code: Optional[str] = None) -> Optional[int]: return rebuild_catalog_from_documents(reason=f"disable_scheme:{workflow_id}") @@ -490,10 +454,10 @@ def build_snapshot( # Live snapshot for warmers: only status=live (pending excluded unless include_pending) if not include_pending and st != "live": continue - instances = e["instances"] + groups = e["groups"] if instance: inst = instance.strip().lower() - if inst not in {i.lower() for i in instances} and instances: + if inst not in {g.lower() for g in groups} and groups: continue nv = bool(e.get("network_visible")) if network_visible is not None and nv != network_visible: @@ -505,12 +469,12 @@ def build_snapshot( "scheme_aliases": e["aliases"], "tool": "search_schemes", "tool_routing": "qdrant", - "instances": instances or ["default"], + "instances": groups or ["default"], "network_visible": nv, "chunk_count": int(e.get("chunk_count") or 0), "workflow_ids": e["workflow_ids"], "promoted_at": e.get("promoted_at"), - "source": e.get("source") or "bootstrap", + "source": e.get("source") or "pipeline", "status": st, } ) @@ -550,17 +514,17 @@ def build_snapshot( def apply_scheme_metadata( workflow_id: str, *, - document_kind: Optional[str] = None, - scheme_code: Optional[str] = None, - scheme_name: Optional[str] = None, - scheme_aliases: Optional[list[str]] = None, + kind: Optional[str] = None, + code: Optional[str] = None, + name: Optional[str] = None, + aliases: Optional[list[str]] = None, tool_routing: Optional[str] = None, catalog_visible: Optional[bool] = None, network_visible: Optional[bool] = None, valid_from: Optional[str] = None, valid_to: Optional[str] = None, ) -> dict[str, Any]: - """Update document scheme fields and rebuild catalog when needed. + """Update document catalog fields and rebuild catalog when needed. `valid_from`/`valid_to` set the document's validity period (see `pipeline/document_validity.py`). Either end alone is enough - the other is @@ -573,60 +537,56 @@ def apply_scheme_metadata( raise LookupError(f"Document not found: {workflow_id}") updates: dict[str, Any] = {} - old_code = (doc.get("scheme_code") or "").strip().lower() or None - old_kind = (doc.get("document_kind") or "document").strip().lower() + old_code = (doc.get("code") or "").strip().lower() or None + old_kind = (doc.get("kind") or "").strip().lower() code_renamed = False pending_reindex = False - if document_kind is not None: - kind = document_kind.strip().lower().replace(" ", "-") - if not kind or not DOCUMENT_KIND_RE.match(kind): - raise ValueError( - f"Invalid document_kind '{document_kind}'. Use a lowercase slug, " - f"e.g. one of {SUGGESTED_DOCUMENT_KINDS} or a custom type." - ) - updates["document_kind"] = kind + if kind is not None: + new_kind = kind.strip().lower() + catalog_type_for_kind(new_kind) # ValueError for a kind with no catalog type + updates["kind"] = new_kind else: - kind = old_kind + new_kind = old_kind new_code = old_code - if scheme_code is not None: - new_code = validate_scheme_code(scheme_code) if scheme_code else None - updates["scheme_code"] = new_code + if code is not None: + new_code = validate_scheme_code(code) if code else None + updates["code"] = new_code if old_code and new_code and old_code != new_code: code_renamed = True - if scheme_name is not None: - updates["scheme_name"] = scheme_name.strip() if scheme_name else None + if name is not None: + updates["name"] = name.strip() if name else None - # Auto-derive scheme_code from the title when the caller set a name but no + # Auto-derive the code from the title when the caller set a name but no # explicit code (the UI form only collects a name and previews the code; # this is the authoritative, collision-checked source of truth for it). - final_name = (updates["scheme_name"] if "scheme_name" in updates else doc.get("scheme_name")) or "" - if scheme_code is None and not new_code and kind == "scheme" and final_name.strip(): - new_code = derive_scheme_code(final_name, exclude_workflow_id=workflow_id) - updates["scheme_code"] = new_code + final_name = (updates["name"] if "name" in updates else doc.get("name")) or "" + if code is None and not new_code and new_kind == "scheme" and final_name.strip(): + new_code = derive_scheme_code(final_name, exclude_workflow_id=workflow_id, kind=new_kind) + updates["code"] = new_code - if scheme_aliases is not None: - updates["scheme_aliases_json"] = json.dumps(parse_aliases(scheme_aliases)) + if aliases is not None: + updates["aliases"] = parse_aliases(aliases) if tool_routing is not None: tr = tool_routing.strip().lower() if tr not in ("qdrant", "legacy", "both"): raise ValueError("tool_routing must be qdrant|legacy|both") updates["tool_routing"] = tr - elif kind == "scheme" and not doc.get("tool_routing"): + elif new_kind == "scheme" and not doc.get("tool_routing"): updates["tool_routing"] = "qdrant" if catalog_visible is not None: updates["catalog_visible"] = 1 if catalog_visible else 0 - elif kind == "scheme" and doc.get("catalog_visible") is None: + elif new_kind == "scheme" and doc.get("catalog_visible") is None: updates["catalog_visible"] = 1 if network_visible is not None: updates["network_visible"] = 1 if network_visible else 0 - elif kind == "scheme" and doc.get("network_visible") is None: - updates["network_visible"] = 1 if default_network_visible(doc.get("instance")) else 0 + elif new_kind == "scheme" and doc.get("network_visible") is None: + updates["network_visible"] = 1 if default_network_visible(doc.get("group")) else 0 if valid_from is not None or valid_to is not None: # Validated here rather than at the SQL layer, and against the stored @@ -639,10 +599,6 @@ def apply_scheme_metadata( updates["valid_from"] = period.start_date updates["valid_to"] = period.end_date - if kind == "scheme" and not (new_code or updates.get("scheme_code") or old_code): - # Allow setting kind first without code only if not completing as scheme - pass - if code_renamed and (doc.get("stage") == "completed") and not int(doc.get("is_disabled") or 0): # Rule (b): hide new code until reindex updates["reindex_required"] = 1 @@ -654,7 +610,7 @@ def apply_scheme_metadata( # Rebuild catalog if scheme-related final = db.get_document(workflow_id) or doc - final_kind = (final.get("document_kind") or "document").strip().lower() + final_kind = (final.get("kind") or "").strip().lower() catalog_version = None if final_kind == "scheme" or old_kind == "scheme" or code_renamed: reason = "scheme_metadata_patch" From ad6f794fa1d1878b76778f85c1b4cef75f565984 Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 11:07:17 +0530 Subject: [PATCH 12/53] feat(api): require document kind at upload and rename tenancy to group - /upload, /documents and /documents/batch take kind (scheme|advisory) and an optional code; an existing code updates that document with the new file - group, kind, code, name and aliases replace instance, document_kind, scheme_code, scheme_name and scheme_aliases in requests and responses - tenancy helpers renamed to group; users.group_id set from the token tenants - demo flag, X-Include-Demo, POST /catalog/v1/bootstrap and the never-classified reingest guard are removed - /catalog/v1 keys, Qdrant payload keys and Temporal arguments keep their names --- pipeline/activities.py | 85 +++--- pipeline/api.py | 417 ++++++++++++++------------ pipeline/auth/deps.py | 5 +- pipeline/auth/groups.py | 10 +- pipeline/auth/keycloak_admin.py | 10 +- pipeline/auth/models.py | 28 +- pipeline/auth/tenancy.py | 70 ++--- pipeline/discovery_publish_service.py | 10 +- 8 files changed, 327 insertions(+), 308 deletions(-) diff --git a/pipeline/activities.py b/pipeline/activities.py index 8850bd1..a0d3643 100644 --- a/pipeline/activities.py +++ b/pipeline/activities.py @@ -28,7 +28,7 @@ from .chunking import chunk_pages, load_chunking_config from .discovery_publish_service import DiscoveryPublishService from .document_repository import DocumentRepository -from .instances import instance_display_name, publish_to_network_enabled +from .group_names import group_display_name, publish_to_network_enabled from .network_constants import RESULT_REJECTED from .ocr import ocr_pdf as run_ocr_pdf from .ocr import ocr_pdf_in_segments as run_ocr_pdf_in_segments @@ -619,8 +619,8 @@ def clean_text_for_ingestion(text: str) -> str: return result.strip() -def _normalize_instance(value: str | None) -> str: - """Ingest-side instance normalizer (mirrors auth.tenancy without importing FastAPI).""" +def _normalize_group(value: str | None) -> str: + """Ingest-side group normalizer (mirrors auth.tenancy without importing FastAPI).""" text = (value or "").strip().lower() if text: return text @@ -636,16 +636,16 @@ def _prepare_records( name_en: str | None = None, description: str | None = None, include_e5_prefix_field: bool = True, - instance: str | None = None, - document_kind: str = "document", - scheme_code: str | None = None, - scheme_name: str | None = None, - scheme_aliases: list[str] | None = None, + group: str | None = None, + kind: str = "document", + code: str | None = None, + name: str | None = None, + aliases: list[str] | None = None, valid_from: str | None = None, valid_to: str | None = None, ) -> list[dict]: metadata = _get_doc_metadata(filename) - resolved_instance = _normalize_instance(instance) + resolved_group = _normalize_group(group) llm_doc_description = _get_doc_description(filename) doc_hash = hashlib.md5(document_id.encode()).hexdigest() @@ -661,19 +661,17 @@ def _prepare_records( short_description = metadata.get("doc_short_description", "") effective_description = description or llm_doc_description or short_description - kind = (document_kind or "document").strip().lower() - is_scheme = kind == "scheme" and bool((scheme_code or "").strip()) - resolved_scheme_name = scheme_name or name_en or default_name + kind = (kind or "document").strip().lower() + is_scheme = kind == "scheme" and bool((code or "").strip()) + resolved_name = name or name_en or default_name # Merge curator-supplied, bootstrap and derived aliases so a scheme is never # indexed with an empty alias list (it would then only match its exact name). - aliases = ( - scheme_catalog.resolve_scheme_aliases( - scheme_code, resolved_scheme_name, scheme_aliases - ) + resolved_aliases = ( + scheme_catalog.resolve_scheme_aliases(code, resolved_name, aliases) if is_scheme else [] ) - instance_name = instance_display_name(resolved_instance) + group_name = group_display_name(resolved_group) # One period for the whole document, resolved before the loop: every chunk # of a document expires together, and re-deriving it per chunk would let a # midnight boundary split one document across two periods. @@ -695,8 +693,8 @@ def _prepare_records( "_id": point_id, "doc_id": document_id, "workflow_id": workflow_id or "", - "instance": resolved_instance, - "instance_name": instance_name, + "instance": resolved_group, # vector payload key, kept for consumers + "instance_name": group_name, "type": "scheme" if is_scheme else "document", "document_kind": kind, "source": "docs-pipeline", @@ -731,9 +729,9 @@ def _prepare_records( if validity.end_date is not None: record["end_date"] = validity.end_date if is_scheme: - record["scheme_code"] = (scheme_code or "").strip().lower() - record["scheme_name"] = resolved_scheme_name - record["scheme_aliases"] = aliases + record["scheme_code"] = (code or "").strip().lower() + record["scheme_name"] = resolved_name + record["scheme_aliases"] = resolved_aliases if include_e5_prefix_field: record["text_for_embedding"] = f"passage: {text}" if text else "passage:" records.append(record) @@ -742,26 +740,13 @@ def _prepare_records( def _scheme_fields_from_doc(doc: dict | None) -> dict: - """Extract scheme kwargs for _prepare_records from a documents row.""" - import json as _json - + """Extract kind/code/name/aliases kwargs for _prepare_records from a documents row.""" doc = doc or {} - aliases_raw = doc.get("scheme_aliases_json") - aliases: list[str] = [] - if isinstance(aliases_raw, list): - aliases = [str(a) for a in aliases_raw] - elif isinstance(aliases_raw, str) and aliases_raw.strip(): - try: - parsed = _json.loads(aliases_raw) - if isinstance(parsed, list): - aliases = [str(a) for a in parsed] - except Exception: - aliases = [] return { - "document_kind": (doc.get("document_kind") or "document"), - "scheme_code": doc.get("scheme_code"), - "scheme_name": doc.get("scheme_name"), - "scheme_aliases": aliases, + "kind": (doc.get("kind") or "document"), + "code": doc.get("code"), + "name": doc.get("name"), + "aliases": [str(a) for a in (doc.get("aliases") or [])], } @@ -790,7 +775,7 @@ def prepare_ingestion_records( name_gu: str | None = None, name_en: str | None = None, description: str | None = None, - instance: str | None = None, + group: str | None = None, ) -> list[dict]: """Public helper used by tests and scripts when preparing vector-index payloads.""" return _prepare_records( @@ -801,7 +786,7 @@ def prepare_ingestion_records( name_gu=name_gu, name_en=name_en, description=description, - instance=instance, + group=group, ) @@ -1230,7 +1215,7 @@ async def prepare_for_ingestion( name_gu=name_gu, name_en=name_en, description=description, - instance=(doc or {}).get("instance"), + group=(doc or {}).get("group"), **_validity_fields_from_doc(doc), ) activity.logger.info(f"Prepared {len(records)} records") @@ -1304,8 +1289,8 @@ async def promote_document_to_prod_qdrant( chunks = db.get_chunks(workflow_id, include_excluded=True) doc = db.get_document(workflow_id) or {} scheme_kwargs = _scheme_fields_from_doc(doc) - kind = (scheme_kwargs.get("document_kind") or "document").strip().lower() - is_scheme = kind == "scheme" and bool((scheme_kwargs.get("scheme_code") or "").strip()) + kind = (scheme_kwargs.get("kind") or "document").strip().lower() + is_scheme = kind == "scheme" and bool((scheme_kwargs.get("code") or "").strip()) if is_scheme: prod_url = ( @@ -1339,7 +1324,7 @@ async def promote_document_to_prod_qdrant( filename, chunks, workflow_id=workflow_id, - instance=doc.get("instance"), + group=doc.get("group"), **scheme_kwargs, **_validity_fields_from_doc(doc), ) @@ -1363,8 +1348,8 @@ async def promote_document_to_prod_qdrant( "collection": prod_collection, "records_ingested": result.get("records_ingested", len(records)), "prod_qdrant_url": prod_url, - "document_kind": kind, - "scheme_code": scheme_kwargs.get("scheme_code"), + "kind": kind, + "code": scheme_kwargs.get("code"), }, ) catalog_version = None @@ -1402,7 +1387,7 @@ async def ingest_document_from_db( filename, chunks, workflow_id=workflow_id, - instance=doc.get("instance"), + group=doc.get("group"), **_scheme_fields_from_doc(doc), **_validity_fields_from_doc(doc), ) @@ -1477,7 +1462,7 @@ async def publish_catalog_to_network(workflow_id: str, transaction_id: str) -> d filename = "network_publish_skipped.json" exchange = { "skipped": True, - "document_kind": document_kind, + "kind": document_kind, "reason": skip_reason, } else: diff --git a/pipeline/api.py b/pipeline/api.py index 496c118..8f3f6ed 100644 --- a/pipeline/api.py +++ b/pipeline/api.py @@ -71,15 +71,15 @@ from .auth.models import AuthUser from .auth.permissions import Permission from .auth.tenancy import ( - PORTAL_INSTANCE, - allowed_instances, - assert_document_instance_access, - normalize_instance, - resolve_create_instance, + PORTAL_GROUP, + allowed_groups, + assert_document_group_access, + normalize_group, + resolve_create_group, unrestricted, - user_can_access_instance, + user_can_access_group, ) -from .instances import prod_stage_disabled, publish_to_network_enabled +from .group_names import prod_stage_disabled, publish_to_network_enabled from .models import ( PIPELINE_STAGES, PROD_ONLY_STAGES, @@ -104,6 +104,7 @@ SearchSettingsUpdate, SettingsAuditResponse, ) +from .network_constants import catalog_type_for_kind from .workflows import ( ChunkingOnlyWorkflow, DocumentPipelineWorkflow, @@ -145,7 +146,7 @@ async def lifespan(app: FastAPI): if auth_cfg.disabled: logging.warning( "WARNING: AUTH_DISABLED=true — every caller is treated as synthetic " - "master_admin with unrestricted instance access. Do not expose this " + "master_admin with unrestricted group access. Do not expose this " "API beyond the internal network or set AUTH_DISABLED=false until " "the maintainer UI sends Bearer tokens." ) @@ -372,15 +373,15 @@ def _compute_file_fingerprint(filepath: Path) -> str: return md5.hexdigest() -def _instance_scope_for_user(user: AuthUser) -> Optional[list[str]]: - """None = unrestricted; otherwise only these instance ids.""" - allowed = allowed_instances(user) +def _group_scope_for_user(user: AuthUser) -> Optional[list[str]]: + """None = unrestricted; otherwise only these group codes.""" + allowed = allowed_groups(user) if allowed is None: return None return sorted(allowed) -def _document_cohorts_payload(summary: dict, by_instance: Optional[list[dict]] = None) -> dict: +def _document_cohorts_payload(summary: dict, by_group: Optional[list[dict]] = None) -> dict: """Shape Postgres aggregate counts for DocumentCohortsResponse (all ints).""" def _i(key: str, default: int = 0) -> int: val = summary.get(key, default) @@ -404,24 +405,61 @@ def _i(key: str, default: int = 0) -> int: "ready_for_ingestion": _i("ready_for_ingestion_documents"), "failed": _i("failed_documents"), }, - "by_instance": by_instance or [], + "by_group": by_group or [], } -def _resolve_create_instance(user: AuthUser, requested: Optional[str] = None) -> str: - """Plan 2: stamp document.instance from Keycloak-allowed states / portal.""" - return resolve_create_instance(user, requested) +def _resolve_create_group(user: AuthUser, requested: Optional[str] = None) -> str: + """Pick the document's group from Keycloak-allowed states / portal.""" + return resolve_create_group(user, requested) + + +def _document_title(filename: str) -> str: + """Human title for a new document: the file name without extension, separators as spaces.""" + stem = Path(filename or "").stem + return re.sub(r"[_\-\s]+", " ", stem).strip() or "Untitled document" + + +def _resolve_document_identity( + user: AuthUser, + group: str, + kind: str, + code: str, + filename: str, +) -> tuple[str, str, Optional[dict]]: + """Kind, code and (when ``code`` already exists) the document an upload replaces. + + An explicit ``code`` that is already held in the tenant's catalog of that + kind means "update this document": the caller reuses it instead of inserting. + Without a code one is derived from the file name. + """ + kind = (kind or "").strip().lower() + try: + catalog_type_for_kind(kind) + except ValueError as exc: + raise HTTPException(400, str(exc)) from exc + code = (code or "").strip() + if code: + try: + code = scheme_catalog.validate_scheme_code(code) + except ValueError as exc: + raise HTTPException(400, str(exc)) from exc + existing = db.find_document_by_code(group, kind, code) + if existing: + existing = assert_document_group_access(user, existing) + return kind, code, existing + return kind, scheme_catalog.derive_scheme_code(_document_title(filename), kind=kind), None def _require_document_for_user(workflow_id: str, user: AuthUser) -> dict: - """Load a document or 404 if missing / outside the caller's instance scope.""" - return assert_document_instance_access(user, db.get_document(workflow_id)) + """Load a document or 404 if missing / outside the caller's group scope.""" + return assert_document_group_access(user, db.get_document(workflow_id)) def _document_for_user_or_none(workflow_id: str, user: AuthUser) -> Optional[dict]: """Like _require_document_for_user but returns None instead of raising (bulk paths).""" doc = db.get_document(workflow_id) - if not doc or not user_can_access_instance(user, doc.get("instance")): + if not doc or not user_can_access_group(user, doc.get("group")): return None return doc @@ -471,12 +509,12 @@ def _document_summary_from_row(doc: dict, current_job: Optional[dict] = None) -> canonical_document_id=doc.get("canonical_document_id"), workflow_id=doc["workflow_id"], filename=doc["filename"], - display_name=doc.get("display_name"), + name=doc.get("name"), source_filename=doc.get("source_filename"), source_manifest_name=doc.get("source_manifest_name"), source_file_fingerprint=doc.get("source_file_fingerprint"), authoritative=bool(doc.get("source_manifest_name")), - instance=normalize_instance(doc.get("instance")), + group=normalize_group(doc.get("group")), stage=DocumentStage(doc["stage"]), page_count=doc.get("page_count") or 0, chunk_count=doc.get("chunk_count") or 0, @@ -493,10 +531,9 @@ def _document_summary_from_row(doc: dict, current_job: Optional[dict] = None) -> uploaded_by_username=doc.get("uploaded_by_username"), uploaded_by_email=doc.get("uploaded_by_email"), uploaded_by_roles=_parse_roles_field(doc.get("uploaded_by_roles")), - document_kind=(doc.get("document_kind") or "document"), - scheme_code=doc.get("scheme_code"), - scheme_name=doc.get("scheme_name"), - scheme_aliases=_parse_scheme_aliases(doc), + kind=doc.get("kind"), + code=doc.get("code"), + aliases=list(doc.get("aliases") or []), tool_routing=doc.get("tool_routing"), catalog_visible=bool(int(doc["catalog_visible"])) if doc.get("catalog_visible") is not None else True, network_visible=bool(int(doc["network_visible"])) if doc.get("network_visible") is not None else True, @@ -873,8 +910,8 @@ async def auth_me(user: CurrentUser): "groups": list(user.groups or []), "state_roles": dict(user.state_roles or {}), "is_super_admin": bool(user.is_superadmin), - # Plan 2: portal instance code for BV / platform docs (not a state). - "portal_instance": PORTAL_INSTANCE, + # Portal group code for BV / platform docs (not a state). + "portal_group": PORTAL_GROUP, "auth_disabled": user.token_disabled_mode, } @@ -979,11 +1016,17 @@ async def start_document_workflow( min_tokens: int = 100, index_name: str = "documents-index", stop_after_ocr: bool = False, - instance: str = "", + group: str = "", + kind: str = Query(..., description="scheme or advisory"), + code: str = "", ): """ Start a new document processing workflow. + ``kind`` picks the tenant's catalog (scheme or advisory). ``code`` + names the document in it; when that code already exists the existing + document is updated with the new file instead of a second one being created. + The workflow will: 1. Run OCR 2. Wait for approval (unless auto_approve=True) @@ -995,10 +1038,13 @@ async def start_document_workflow( Rate limited to 10 requests/minute per IP. Requires permission: upload (no-op while AUTH_DISABLED=true). """ - create_instance = _resolve_create_instance(user, instance) + create_group = _resolve_create_group(user, group) # Validate file path to prevent path traversal attacks filepath = validate_file_path(data.filepath) source_filename = get_filename_from_path(filepath) + doc_kind, doc_code, replace_doc = _resolve_document_identity( + user, create_group, kind, code, source_filename + ) source_file_fingerprint = _compute_file_fingerprint(filepath) canonical_document_id = source_file_fingerprint @@ -1010,7 +1056,7 @@ async def start_document_workflow( existing_doc = db.get_document(workflow_id) if existing_doc: # Same fingerprint/path must not leak or restart another tenant's doc. - existing_doc = assert_document_instance_access(user, existing_doc) + existing_doc = assert_document_group_access(user, existing_doc) try: handle = temporal_client.get_workflow_handle(workflow_id) state = await handle.query("get_state") @@ -1023,7 +1069,7 @@ async def start_document_workflow( source_filename=source_filename, source_file_fingerprint=source_file_fingerprint, authoritative=bool(existing_doc.get("source_manifest_name")) if existing_doc else False, - instance=normalize_instance(existing_doc.get("instance")), + group=normalize_group(existing_doc.get("group")), stage=DocumentStage(state.get("stage", "registered")), page_count=state.get("page_count", 0), chunk_count=state.get("chunk_count", 0), @@ -1061,25 +1107,40 @@ async def start_document_workflow( # that knows the upload day. A reviewer confirms or moves it later, on the # form that sets the document type. validity = document_validity.default_period() - document_uuid = db.upsert_document( - workflow_id=workflow_id, - document_id=document_id, - canonical_document_id=canonical_document_id, - filename=source_filename, - source_filename=source_filename, - source_file_fingerprint=source_file_fingerprint, - filepath=str(filepath), - stage="registered", - stop_after_ocr=stop_after_ocr, - instance=create_instance, - uploaded_by_user_id=actor.get("user_id") or None, - uploaded_by_username=actor.get("username") or None, - uploaded_by_email=actor.get("email") or None, - uploaded_by_roles=actor.get("roles_csv") or None, - valid_from=validity.start_date, - valid_to=validity.end_date, - with_workflow=False, - ) + if replace_doc: + document_uuid = replace_doc["id"] + db.reset_document_for_reupload( + document_uuid, + filename=source_filename, + filepath=str(filepath), + fingerprint=source_file_fingerprint, + ) + else: + try: + document_uuid = db.upsert_document( + workflow_id=workflow_id, + document_id=document_id, + canonical_document_id=canonical_document_id, + filename=source_filename, + source_filename=source_filename, + source_file_fingerprint=source_file_fingerprint, + filepath=str(filepath), + stage="registered", + stop_after_ocr=stop_after_ocr, + group=create_group, + uploaded_by_user_id=actor.get("user_id") or None, + uploaded_by_username=actor.get("username") or None, + uploaded_by_email=actor.get("email") or None, + uploaded_by_roles=actor.get("roles_csv") or None, + valid_from=validity.start_date, + valid_to=validity.end_date, + with_workflow=False, + kind=doc_kind, + code=doc_code, + name=_document_title(source_filename), + ) + except db.DuplicateDocumentCode as exc: + raise HTTPException(409, str(exc)) from exc db.create_document_job( workflow_id=document_uuid, job_type="ocr_only" if stop_after_ocr else "pipeline", @@ -1103,7 +1164,13 @@ async def start_document_workflow( entity_type="document", field_name="registered", new_value=source_filename, - metadata={"source": "register_endpoint", "instance": create_instance}, + metadata={ + "source": "register_endpoint", + "group": create_group, + "kind": doc_kind, + "code": doc_code, + "replaced_existing": bool(replace_doc), + }, user=user, ) @@ -1115,7 +1182,7 @@ async def start_document_workflow( source_filename=source_filename, source_file_fingerprint=source_file_fingerprint, authoritative=False, - instance=create_instance, + group=create_group, stage=DocumentStage.REGISTERED, page_count=0, chunk_count=0, @@ -1138,17 +1205,26 @@ async def upload_and_process( min_tokens: int = 100, index_name: str = "documents-index", stop_after_ocr: bool = False, - instance: str = "", + group: str = "", + kind: str = Query(..., description="scheme or advisory"), + code: str = "", ): """ Upload a supported file and start processing workflow. + ``kind`` picks the tenant's catalog (scheme or advisory). ``code`` + names the document in it; when that code already exists the existing + document is updated with the new file instead of a second one being created. + The file is stored in MinIO and then processed through the pipeline. Validates both file extension and PDF magic bytes for security. Rate limited to 10 requests/minute per IP. Requires permission: upload (no-op while AUTH_DISABLED=true). """ - create_instance = _resolve_create_instance(user, instance) + create_group = _resolve_create_group(user, group) + doc_kind, doc_code, replace_doc = _resolve_document_identity( + user, create_group, kind, code, file.filename or "" + ) suffix = Path(file.filename or "").suffix.lower() if suffix not in ALLOWED_EXTENSIONS: raise HTTPException(400, f"Unsupported file type: {suffix}") @@ -1188,7 +1264,7 @@ async def upload_and_process( # If Postgres was purged, avoid returning stale Temporal state and create a fresh run ID. existing_doc = db.get_document(workflow_id) if existing_doc: - existing_doc = assert_document_instance_access(user, existing_doc) + existing_doc = assert_document_group_access(user, existing_doc) try: handle = temporal_client.get_workflow_handle(workflow_id) state = await handle.query("get_state") @@ -1201,7 +1277,7 @@ async def upload_and_process( source_filename=file.filename, source_file_fingerprint=file_hash, authoritative=bool(existing_doc.get("source_manifest_name")) if existing_doc else False, - instance=normalize_instance(existing_doc.get("instance")), + group=normalize_group(existing_doc.get("group")), stage=DocumentStage(state.get("stage", "registered")), page_count=state.get("page_count", 0), chunk_count=state.get("chunk_count", 0), @@ -1237,25 +1313,40 @@ async def upload_and_process( actor = _actor_from_user(user) # See the manifest upload path above: the period starts at the upload. validity = document_validity.default_period() - document_uuid = db.upsert_document( - workflow_id=workflow_id, - document_id=document_id, - canonical_document_id=canonical_document_id, - filename=file.filename, - source_filename=file.filename, - source_file_fingerprint=file_hash, - filepath=minio_path, - stage="registered", - stop_after_ocr=stop_after_ocr, - instance=create_instance, - uploaded_by_user_id=actor.get("user_id") or None, - uploaded_by_username=actor.get("username") or None, - uploaded_by_email=actor.get("email") or None, - uploaded_by_roles=actor.get("roles_csv") or None, - valid_from=validity.start_date, - valid_to=validity.end_date, - with_workflow=False, - ) + if replace_doc: + document_uuid = replace_doc["id"] + db.reset_document_for_reupload( + document_uuid, + filename=file.filename, + filepath=minio_path, + fingerprint=file_hash, + ) + else: + try: + document_uuid = db.upsert_document( + workflow_id=workflow_id, + document_id=document_id, + canonical_document_id=canonical_document_id, + filename=file.filename, + source_filename=file.filename, + source_file_fingerprint=file_hash, + filepath=minio_path, + stage="registered", + stop_after_ocr=stop_after_ocr, + group=create_group, + uploaded_by_user_id=actor.get("user_id") or None, + uploaded_by_username=actor.get("username") or None, + uploaded_by_email=actor.get("email") or None, + uploaded_by_roles=actor.get("roles_csv") or None, + valid_from=validity.start_date, + valid_to=validity.end_date, + with_workflow=False, + kind=doc_kind, + code=doc_code, + name=_document_title(file.filename or ""), + ) + except db.DuplicateDocumentCode as exc: + raise HTTPException(409, str(exc)) from exc job_id = db.create_document_job( workflow_id=document_uuid, job_type="ocr_only" if stop_after_ocr else "pipeline", @@ -1292,7 +1383,10 @@ async def upload_and_process( new_value=file.filename, metadata={ "source": "upload_endpoint", - "instance": create_instance, + "group": create_group, + "kind": doc_kind, + "code": doc_code, + "replaced_existing": bool(replace_doc), "size_bytes": file_size, "auto_approve": auto_approve, }, @@ -1315,7 +1409,7 @@ async def upload_and_process( source_filename=file.filename, source_file_fingerprint=file_hash, authoritative=False, - instance=create_instance, + group=create_group, stage=DocumentStage.REGISTERED, page_count=0, chunk_count=0, @@ -1337,10 +1431,11 @@ async def start_batch_workflows( chunk_overlap: int = 128, min_tokens: int = 100, stop_after_ocr: bool = False, - instance: str = "", + group: str = "", + kind: str = Query(..., description="scheme or advisory"), ): """Start workflows for all supported documents in a directory.""" - create_instance = _resolve_create_instance(user, instance) + create_group = _resolve_create_group(user, group) directory = Path(data.directory) if not directory.exists(): raise HTTPException(404, f"Directory not found: {data.directory}") @@ -1361,7 +1456,9 @@ async def start_batch_workflows( chunk_overlap=chunk_overlap, min_tokens=min_tokens, stop_after_ocr=stop_after_ocr, - instance=create_instance, + group=create_group, + kind=kind, + code="", ) results.append(result) except Exception as e: @@ -1387,23 +1484,20 @@ async def list_documents( stage: Optional[DocumentStage] = None, limit: int = Query(100, le=500), offset: int = Query(0, ge=0), - x_include_demo: Optional[str] = Header(None, alias="X-Include-Demo"), x_include_disabled: Optional[str] = Header(None, alias="X-Include-Disabled") ): """ List all document workflows. Uses Postgres for fast listing (no Temporal queries for performance). - Demo documents are excluded by default - use X-Include-Demo: true header to show them. Disabled (soft-deleted) documents are excluded by default - use X-Include-Disabled: true to show them. - Results are limited to instances the caller can access. + Results are limited to groups the caller can access. Pagination: - limit: Max documents to return (default 100, max 500) - offset: Skip first N documents (default 0) """ stage_filter = stage.value if stage else None - include_demo = x_include_demo and x_include_demo.lower() == "true" include_disabled = x_include_disabled and x_include_disabled.lower() == "true" # Use Postgres only for fast listing - no Temporal queries @@ -1411,9 +1505,8 @@ async def list_documents( stage=stage_filter, limit=limit, offset=offset, - include_demo=include_demo, include_disabled=include_disabled, - instances=_instance_scope_for_user(user), + groups=_group_scope_for_user(user), ) return [_document_summary_from_row(doc) for doc in docs] @@ -1422,47 +1515,39 @@ async def list_documents( @app.get("/documents/summary", response_model=DocumentCohortsResponse) async def get_documents_summary( user: CurrentUser, - x_include_demo: Optional[str] = Header(None, alias="X-Include-Demo"), x_include_disabled: Optional[str] = Header(None, alias="X-Include-Disabled") ): """Return aggregate Postgres counts for dashboard totals and migration planning.""" - include_demo = x_include_demo and x_include_demo.lower() == "true" include_disabled = x_include_disabled and x_include_disabled.lower() == "true" - instance_scope = _instance_scope_for_user(user) + group_scope = _group_scope_for_user(user) summary = db.get_document_summary_counts( - include_demo=include_demo, include_disabled=include_disabled, - instances=instance_scope, + groups=group_scope, ) - by_instance = db.get_document_counts_by_instance( - include_demo=include_demo, + by_group = db.get_document_counts_by_group( include_disabled=include_disabled, - instances=instance_scope, + groups=group_scope, ) - return _document_cohorts_payload(summary, by_instance) + return _document_cohorts_payload(summary, by_group) @app.get("/documents/cohorts", response_model=DocumentCohortsResponse) async def get_document_cohorts( user: CurrentUser, - x_include_demo: Optional[str] = Header(None, alias="X-Include-Demo"), x_include_disabled: Optional[str] = Header(None, alias="X-Include-Disabled") ): """Return machine-friendly cohort counts for queueing and orchestration.""" - include_demo = x_include_demo and x_include_demo.lower() == "true" include_disabled = x_include_disabled and x_include_disabled.lower() == "true" - instance_scope = _instance_scope_for_user(user) + group_scope = _group_scope_for_user(user) summary = db.get_document_summary_counts( - include_demo=include_demo, include_disabled=include_disabled, - instances=instance_scope, + groups=group_scope, ) - by_instance = db.get_document_counts_by_instance( - include_demo=include_demo, + by_group = db.get_document_counts_by_group( include_disabled=include_disabled, - instances=instance_scope, + groups=group_scope, ) - return _document_cohorts_payload(summary, by_instance) + return _document_cohorts_payload(summary, by_group) @app.get("/operations/queue", response_model=OperationQueueResponse) @@ -1470,22 +1555,19 @@ async def get_operations_queue( user: RequireSearch, limit: int = Query(100, le=500), offset: int = Query(0, ge=0), - x_include_demo: Optional[str] = Header(None, alias="X-Include-Demo"), x_include_disabled: Optional[str] = Header(None, alias="X-Include-Disabled"), ): """Return documents that currently need operator or agent action. - Results are limited to instances the caller can access (state users never + Results are limited to groups the caller can access (state users never see portal ``bv`` or other states' documents). """ - include_demo = x_include_demo and x_include_demo.lower() == "true" include_disabled = x_include_disabled and x_include_disabled.lower() == "true" rows, total = db.list_operations_queue( limit=limit, offset=offset, - include_demo=include_demo, include_disabled=include_disabled, - instances=_instance_scope_for_user(user), + groups=_group_scope_for_user(user), ) items = [ OperationQueueEntry( @@ -1513,14 +1595,14 @@ async def list_runs( ): """List recent document jobs visible to the caller (tenant/role scoped). - Restricted users only see jobs whose document ``instance`` is in their + Restricted users only see jobs whose document ``group`` is in their allowed states. Super-admin / bypass users see all tenants. """ return db.list_runs( limit=limit, offset=offset, status=status, - instances=_instance_scope_for_user(user), + groups=_group_scope_for_user(user), ) @@ -1530,7 +1612,7 @@ async def get_run(job_id: str, user: RequireSearch): run = db.get_document_job(job_id) if not run: raise HTTPException(404, f"Run not found: {job_id}") - # Enforce same instance scope as list: hide other tenants' runs as 404. + # Enforce same group scope as list: hide other tenants' runs as 404. workflow_id = run.get("workflow_id") if workflow_id: _require_document_for_user(str(workflow_id), user) @@ -1539,10 +1621,6 @@ async def get_run(job_id: str, user: RequireSearch): return run -def _parse_scheme_aliases(doc: dict) -> list[str]: - return scheme_catalog.parse_aliases(doc.get("scheme_aliases_json")) - - def _build_document_detail(doc: dict) -> DocumentDetail: workflow_id = doc["workflow_id"] current_job = db.get_latest_document_job(workflow_id) @@ -1552,12 +1630,12 @@ def _build_document_detail(doc: dict) -> DocumentDetail: canonical_document_id=doc.get("canonical_document_id"), workflow_id=workflow_id, filename=doc["filename"], - display_name=doc.get("display_name"), + name=doc.get("name"), source_filename=doc.get("source_filename"), source_manifest_name=doc.get("source_manifest_name"), source_file_fingerprint=doc.get("source_file_fingerprint"), authoritative=bool(doc.get("source_manifest_name")), - instance=normalize_instance(doc.get("instance")), + group=normalize_group(doc.get("group")), filepath=doc["filepath"], stage=DocumentStage(doc["stage"]), page_count=doc.get("page_count", 0), @@ -1586,10 +1664,9 @@ def _build_document_detail(doc: dict) -> DocumentDetail: uploaded_by_username=doc.get("uploaded_by_username"), uploaded_by_email=doc.get("uploaded_by_email"), uploaded_by_roles=_parse_roles_field(doc.get("uploaded_by_roles")), - document_kind=(doc.get("document_kind") or "document"), - scheme_code=doc.get("scheme_code"), - scheme_name=doc.get("scheme_name"), - scheme_aliases=_parse_scheme_aliases(doc), + kind=doc.get("kind"), + code=doc.get("code"), + aliases=list(doc.get("aliases") or []), tool_routing=doc.get("tool_routing"), catalog_visible=bool(int(doc["catalog_visible"])) if doc.get("catalog_visible") is not None else True, network_visible=bool(int(doc["network_visible"])) if doc.get("network_visible") is not None else True, @@ -2053,7 +2130,7 @@ async def disable_document( doc = _require_document_for_user(workflow_id, user) if not user.has_permission(Permission.ADMIN): - if not user.has_permission_for_instance(Permission.DELETE_OWN, doc.get("instance")): + if not user.has_permission_for_group(Permission.DELETE_OWN, doc.get("group")): raise HTTPException(403, "Missing permission: delete_own") if purge: raise HTTPException( @@ -2110,13 +2187,13 @@ async def disable_document( result["vector_error"] = vector_result["error"] # PROD Qdrant cleanup, if this document was ever promoted - kind = (doc.get("document_kind") or "document").strip().lower() - if kind == "scheme" or doc.get("scheme_code"): + kind = (doc.get("kind") or "").strip().lower() + if kind == "scheme": q_result = _delete_scheme_points_from_prod(doc) result["scheme_qdrant_deleted"] = q_result try: result["catalog_version"] = scheme_catalog.on_scheme_document_disabled( - workflow_id, scheme_code=doc.get("scheme_code") + workflow_id, code=doc.get("code") ) except Exception as exc: result["catalog_error"] = str(exc) @@ -2138,7 +2215,7 @@ async def disable_document( entity_type="document", metadata={ "filename": doc.get("filename"), - "scheme_code": doc.get("scheme_code"), + "code": doc.get("code"), "stage": doc.get("stage"), "minio_artifacts_deleted": result["minio_artifacts_deleted"], "keep_audit": keep_audit, @@ -2159,7 +2236,7 @@ async def disable_document( "vector_deleted": result["vector_deleted"], "scheme_qdrant_deleted": result.get("scheme_qdrant_deleted"), "prod_qdrant_deleted": result.get("prod_qdrant_deleted"), - "scheme_code": doc.get("scheme_code"), + "code": doc.get("code"), "catalog_version": result.get("catalog_version"), }, user=user, @@ -2175,19 +2252,19 @@ async def patch_scheme_metadata( user: RequireReview, ): """ - Set document_kind / scheme fields used by Master Catalog and Qdrant scheme promote. + Set kind / code / name / aliases used by Master Catalog and Qdrant scheme promote. - On completed docs, renaming scheme_code sets pending_reindex (excluded from live + On completed docs, renaming the code sets pending_reindex (excluded from live vector_schemes until re-promote). Other metadata edits bump catalog version only. """ _require_document_for_user(workflow_id, user) try: result = scheme_catalog.apply_scheme_metadata( workflow_id, - document_kind=body.document_kind, - scheme_code=body.scheme_code, - scheme_name=body.scheme_name, - scheme_aliases=body.scheme_aliases, + kind=body.kind, + code=body.code, + name=body.name, + aliases=body.aliases, tool_routing=body.tool_routing, catalog_visible=body.catalog_visible, network_visible=body.network_visible, @@ -2213,10 +2290,10 @@ async def patch_scheme_metadata( doc = result["document"] return { "workflow_id": workflow_id, - "document_kind": doc.get("document_kind"), - "scheme_code": doc.get("scheme_code"), - "scheme_name": doc.get("scheme_name"), - "scheme_aliases": scheme_catalog.parse_aliases(doc.get("scheme_aliases_json")), + "kind": doc.get("kind"), + "code": doc.get("code"), + "name": doc.get("name"), + "aliases": list(doc.get("aliases") or []), "tool_routing": doc.get("tool_routing"), "catalog_visible": bool(int(doc["catalog_visible"])) if doc.get("catalog_visible") is not None else True, "network_visible": bool(int(doc["network_visible"])) if doc.get("network_visible") is not None else True, @@ -2238,7 +2315,6 @@ async def patch_scheme_metadata( async def catalog_version(request: Request, user: CurrentUser): """Cheap poll for cache warmers: { version, updated_at }.""" _require_catalog_read(request, user) - scheme_catalog.bootstrap_catalog_if_empty() meta = db.get_catalog_meta() return { "version": int(meta.get("version") or 0), @@ -2258,7 +2334,6 @@ async def catalog_schemes( ): """List schemes from the master catalog (vector-indexed + filters).""" _require_catalog_read(request, user) - scheme_catalog.bootstrap_catalog_if_empty() snap = scheme_catalog.build_snapshot( instance=instance, status=status, @@ -2290,7 +2365,6 @@ async def catalog_snapshot( collections metadata, and monotonic catalog_version. """ _require_catalog_read(request, user) - scheme_catalog.bootstrap_catalog_if_empty() return scheme_catalog.build_snapshot( instance=instance, status=status, @@ -2303,7 +2377,6 @@ async def catalog_snapshot( async def catalog_tool_prompt(request: Request, user: CurrentUser): """Prompt fragments only (for AI system prompt assembly).""" _require_catalog_read(request, user) - scheme_catalog.bootstrap_catalog_if_empty() snap = scheme_catalog.build_snapshot() return { "catalog_version": snap["catalog_version"], @@ -2317,14 +2390,12 @@ async def catalog_tool_prompt(request: Request, user: CurrentUser): @app.post("/catalog/v1/rebuild") async def catalog_rebuild(request: Request, user: RequireAdmin): - """Admin: recompute catalog entries from documents + preserve bootstrap codes.""" + """Admin: bump the catalog version (entries are computed from documents).""" from .auth.permissions import Permission # Admin JWT always OK; service key also accepted for automation if not _service_key_ok(request) and not user.has_permission(Permission.ADMIN): raise HTTPException(403, "Rebuild requires admin") - # Ensure bootstrap seeds exist before rebuild merge - scheme_catalog.bootstrap_catalog_if_empty() version = scheme_catalog.rebuild_catalog_from_documents(reason="admin_rebuild") snap = scheme_catalog.build_snapshot() _log_audit( @@ -2342,13 +2413,6 @@ async def catalog_rebuild(request: Request, user: RequireAdmin): } -@app.post("/catalog/v1/bootstrap") -async def catalog_bootstrap(request: Request, user: RequireAdmin, force: bool = Query(False)): - """Admin: seed 13 vector schemes if empty (or force=true).""" - result = scheme_catalog.bootstrap_catalog_if_empty(force=force) - return result - - @app.post("/documents/{workflow_id}/restore") async def restore_document(workflow_id: str, user: RequireAdmin): """ @@ -2392,24 +2456,6 @@ async def reingest_document( # Get document from Postgres doc = _require_document_for_user(workflow_id, user) - # Reingest skips the entire review pipeline (including ready_for_ingestion, - # where document type / scheme name get set) and jumps straight to pushing - # chunks to Qdrant. That's fine for re-pushing already-classified content - # (e.g. after an embedding/index change), but a trap if it's used as a - # substitute for the real approval flow on a never-classified document — - # it would silently ingest with document_kind='document' and no scheme - # metadata, the exact raw-hash/filename-fallback catalog rows we've - # had to clean up by hand. 'document' is never offered as a choice in the - # classification panel, so seeing it here means classification was never done. - if (doc.get("document_kind") or "document") == "document" and not doc.get("scheme_code") and not doc.get("scheme_name"): - raise HTTPException( - 400, - "This document has never been classified (document type / scheme name not set). " - "Set it via PATCH /documents/{workflow_id}/scheme-metadata (or the classification " - "panel at the ready_for_ingestion stage) before reingesting — reingest skips that " - "review stage entirely and would otherwise ingest with no scheme metadata.", - ) - # Get chunks from Postgres chunks = db.get_chunks(workflow_id, include_excluded=False) if not chunks: @@ -2636,19 +2682,6 @@ async def clear_reindex_required(workflow_id: str, user: RequirePipeline): } -@app.post("/documents/{workflow_id}/demo") -async def set_document_demo(workflow_id: str, user: RequireAdmin, is_demo: bool = Query(True)): - """ - Mark a document as demo. - - Demo documents are excluded from the UI by default but always available - for API testing via include_demo=true parameter. - """ - _require_document_for_user(workflow_id, user) - db.set_document_demo(workflow_id, is_demo) - return {"workflow_id": workflow_id, "is_demo": is_demo} - - async def _reconcile_single_document(doc: dict) -> dict: workflow_id = doc.get("workflow_id") current_stage = doc.get("stage") @@ -3225,14 +3258,14 @@ async def get_all_audit_logs( Each entry includes the document filename for context. """ - scope = _instance_scope_for_user(user) + scope = _group_scope_for_user(user) logs = db.get_all_audit_logs( action_type=action_type, limit=limit, offset=offset, - instances=scope, + groups=scope, ) - total = db.get_all_audit_log_count(action_type, instances=scope) + total = db.get_all_audit_log_count(action_type, groups=scope) return AuditLogResponse( logs=logs, @@ -3466,14 +3499,14 @@ async def search_chunks_across_documents( stage: Optional[DocumentStage] = Query(None, description="Optional document stage filter"), ): """Search chunks across all documents for KB maintainer workflows.""" - # Tenant-scope via the owning document's instance (None = unrestricted admin). + # Tenant-scope via the owning document's group (None = unrestricted admin). chunks, total = db.search_chunks( query=q, limit=limit, offset=offset, include_excluded=include_excluded, stage=stage.value if stage else None, - instances=_instance_scope_for_user(user), + groups=_group_scope_for_user(user), ) return { "items": chunks, @@ -3896,16 +3929,13 @@ async def get_vector_index_stats(index_name: str, user: RequireSearch): @app.get("/indexes/summary") async def get_vector_indexes_summary( user: RequireSearch, - x_include_demo: Optional[str] = Header(None, alias="X-Include-Demo"), x_include_disabled: Optional[str] = Header(None, alias="X-Include-Disabled"), ): """Summarize index coverage from Postgres-backed index status plus live vector-store stats.""" from .vector_store import get_vector_store - include_demo = x_include_demo and x_include_demo.lower() == "true" include_disabled = x_include_disabled and x_include_disabled.lower() == "true" summaries = db.list_index_summaries( - include_demo=include_demo, include_disabled=include_disabled, ) if not summaries: @@ -4295,12 +4325,11 @@ async def reconcile_document_states(user: RequirePipeline): 'ready_for_ingestion', 'ingesting' ] - # Scope to caller's instances (None = unrestricted bypass / all tenants). + # Scope to caller's groups (None = unrestricted bypass / all tenants). docs = db.list_documents( limit=1000, - include_demo=True, include_disabled=True, - instances=_instance_scope_for_user(user), + groups=_group_scope_for_user(user), ) active_docs = [d for d in docs if d.get('stage') in active_stages] diff --git a/pipeline/auth/deps.py b/pipeline/auth/deps.py index 4b192f3..5a3af81 100644 --- a/pipeline/auth/deps.py +++ b/pipeline/auth/deps.py @@ -87,7 +87,10 @@ def _sync_user_row(user: AuthUser) -> None: if last is not None and now - last < _USER_SYNC_INTERVAL_SECONDS: return try: - db.upsert_user(user.user_id, user.username, user.email, user.roles) + db.upsert_user( + user.user_id, user.username, user.email, user.roles, + tenants=list(user.instances or []), + ) except Exception: logging.warning("users upsert failed for %s", user.user_id, exc_info=True) return diff --git a/pipeline/auth/groups.py b/pipeline/auth/groups.py index f907c02..4104b6e 100644 --- a/pipeline/auth/groups.py +++ b/pipeline/auth/groups.py @@ -217,17 +217,17 @@ def extract_groups_claim(claims: dict[str, Any]) -> list[str]: return [] -def role_for_instance(access: GroupAccess, instance: str | None) -> str | None: - """Return the user's role for a state instance, or SUPER_ADMIN when global. +def role_for_group(access: GroupAccess, group: str | None) -> str | None: + """Return the user's role for a state group, or SUPER_ADMIN when global. A ``bh_viewer`` has no per-state group, but reads every state — so any - instance resolves to STATE_VIEW unless a stronger state role is held. + group resolves to STATE_VIEW unless a stronger state role is held. """ if access.is_super_admin: return ROLE_SUPER_ADMIN - if not instance: + if not group: return None - role = access.state_roles.get(normalize_state_code(instance)) + role = access.state_roles.get(normalize_state_code(group)) if role: return role return ROLE_STATE_VIEW if access.is_bh_viewer else None diff --git a/pipeline/auth/keycloak_admin.py b/pipeline/auth/keycloak_admin.py index 0b67380..f4b1a2c 100644 --- a/pipeline/auth/keycloak_admin.py +++ b/pipeline/auth/keycloak_admin.py @@ -28,7 +28,7 @@ group_leaf_for_role, normalize_role, ) -from .tenancy import PORTAL_INSTANCE +from .tenancy import PORTAL_GROUP # Realm-role descriptions written when a role is auto-created. STATE_ROLE_DESCRIPTIONS: dict[str, str] = { @@ -506,7 +506,7 @@ def provision_user( "roles": [r.get("name") for r in (roles or []) if isinstance(r, dict)], "app_url": ui_url, "login_url": login_url, - "portal_instance": PORTAL_INSTANCE, + "portal_group": PORTAL_GROUP, "share_message": "\n".join(share_lines), } @@ -517,7 +517,7 @@ def list_access_options() -> dict[str, Any]: return { "keycloak_admin_configured": cfg.configured, "realm": cfg.realm, - "portal_instance": PORTAL_INSTANCE, + "portal_group": PORTAL_GROUP, "access_types": [ { "id": "super_admin", @@ -579,9 +579,9 @@ def list_access_options() -> dict[str, Any]: "notes": [ "Users sign in with Google SSO using the same email you enter.", "No app password is required for normal SSO login.", - "State codes must match document instance tags (mh, up, …).", + "State codes must match document group codes (mh, up, …).", "A centre is modelled as just another state code.", - "Super admin (BV) documents can use portal instance 'bv'.", + "Super admin (BV) documents can use portal group 'bv'.", "Access comes from group membership only — no direct role assignment.", "Keycloak groups: /global/super-admin, /global/bh-viewer, " "/states/{STATE}/{admin|approver|contributor|view}", diff --git a/pipeline/auth/models.py b/pipeline/auth/models.py index dd7fcdd..6a3ab40 100644 --- a/pipeline/auth/models.py +++ b/pipeline/auth/models.py @@ -29,7 +29,7 @@ # Roles not limited by the JWT ``instances`` claim. # State-level roles are restricted to their claimed instances (tenants/states). -INSTANCE_UNRESTRICTED_ROLES = SUPERADMIN_ROLES | GLOBAL_READ_ROLES +GROUP_UNRESTRICTED_ROLES = SUPERADMIN_ROLES | GLOBAL_READ_ROLES @dataclass @@ -73,44 +73,44 @@ def is_admin(self) -> bool: """ return self.is_superadmin - def is_instance_unrestricted(self) -> bool: - """True when the caller may access every instance (all tenants/states). + def is_group_unrestricted(self) -> bool: + """True when the caller may access every group (all tenants/states). Two cases: local bypass mode with no scoped claim, or platform superadmin even when the token carries a narrow ``instances`` claim. """ if self.token_disabled_mode and not self.instances: return True - return bool(INSTANCE_UNRESTRICTED_ROLES & self._role_keys) + return bool(GROUP_UNRESTRICTED_ROLES & self._role_keys) - def has_instance(self, instance: str) -> bool: + def has_group(self, group: str) -> bool: if not self.instances: - # Empty instance list means "no tenant restriction yet" only in disabled mode. + # Empty list means "no tenant restriction yet" only in disabled mode. return self.token_disabled_mode - return instance.strip().lower() in {i.lower() for i in self.instances} + return group.strip().lower() in {i.lower() for i in self.instances} def has_env(self, env: str) -> bool: if not self.envs: return self.token_disabled_mode return env.strip().lower() in {e.lower() for e in self.envs} - def role_for_instance(self, instance: str | None) -> str | None: + def role_for_group(self, group: str | None) -> str | None: """Role in a given state; super_admin always returns super_admin. - A ``bh_viewer`` reads every state, so any instance falls back to + A ``bh_viewer`` reads every state, so any group falls back to STATE_VIEW when no stronger per-state role is held. """ if self.is_superadmin: return ROLE_SUPER_ADMIN - if not instance: + if not group: return None - role = self.state_roles.get(normalize_state_code(instance)) + role = self.state_roles.get(normalize_state_code(group)) if role: return role return ROLE_STATE_VIEW if self.is_bh_viewer else None - def has_permission_for_instance( - self, permission: Permission | str, instance: str | None + def has_permission_for_group( + self, permission: Permission | str, group: str | None ) -> bool: """Permission check scoped to a state when per-state roles are present. @@ -126,7 +126,7 @@ def has_permission_for_instance( if not self.state_roles: return needed in self.permissions - role = self.role_for_instance(instance) + role = self.role_for_group(group) if not role: return False from .permissions import permissions_for_roles diff --git a/pipeline/auth/tenancy.py b/pipeline/auth/tenancy.py index 7b1d23a..f6826e3 100644 --- a/pipeline/auth/tenancy.py +++ b/pipeline/auth/tenancy.py @@ -1,12 +1,14 @@ -"""Instance (tenant / state) access helpers — Plan 2 (Keycloak + document.instance). +"""Group (tenant / state) access helpers — Keycloak groups + document groups. -Access who-sees-what-state comes from the JWT (Keycloak groups). The database -only stores which state a *document* belongs to (``documents.instance``). +Who-sees-what-state comes from the JWT (Keycloak groups). The database only +stores which group a *document* belongs to (``groups`` via +``documents.catalog_id -> catalogs.group_id``); the document dict carries its +tenant code as ``group``. -Plan 2 conventions: +Conventions: - State codes: lowercase (``mh``, ``up``, …) matching Keycloak ``/states/MH/...`` - Portal / BV platform docs: ``bv`` (Bharat Vistaar) - - Super admin (``/global/super-admin``): unrestricted instances + - Super admin (``/global/super-admin``): unrestricted groups """ from __future__ import annotations @@ -18,52 +20,52 @@ from .models import AuthUser # Bharat Vistaar portal / platform-wide documents (not a state). -PORTAL_INSTANCE = "bv" +PORTAL_GROUP = "bv" -def default_instance() -> str: - """Fallback instance id for new docs and legacy rows without a value.""" +def default_group() -> str: + """Fallback group code for new docs and legacy rows without a value.""" return (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" -def normalize_instance(value: str | None) -> str: +def normalize_group(value: str | None) -> str: text = (value or "").strip().lower() - return text or default_instance() + return text or default_group() def unrestricted(user: AuthUser) -> bool: - """True when the caller may see all instances (super admin / local bypass).""" - return user.is_instance_unrestricted() + """True when the caller may see all groups (super admin / local bypass).""" + return user.is_group_unrestricted() -def allowed_instances(user: AuthUser) -> set[str] | None: +def allowed_groups(user: AuthUser) -> set[str] | None: """ - Return the set of instance ids the user may access, or None if unrestricted. + Return the set of group codes the user may access, or None if unrestricted. """ if unrestricted(user): return None - return {normalize_instance(i) for i in user.instances if str(i).strip()} + return {normalize_group(i) for i in user.instances if str(i).strip()} -def user_can_access_instance(user: AuthUser, instance: str | None) -> bool: - allowed = allowed_instances(user) +def user_can_access_group(user: AuthUser, group: str | None) -> bool: + allowed = allowed_groups(user) if allowed is None: return True if not allowed: return False - return normalize_instance(instance) in allowed + return normalize_group(group) in allowed -def assert_instance_access(user: AuthUser, instance: str | None) -> str: - """Raise 403 if user cannot access instance; return normalized instance id.""" - normalized = normalize_instance(instance) - if not user_can_access_instance(user, normalized): - raise HTTPException(403, f"No access to instance: {normalized}") +def assert_group_access(user: AuthUser, group: str | None) -> str: + """Raise 403 if user cannot access group; return normalized group code.""" + normalized = normalize_group(group) + if not user_can_access_group(user, normalized): + raise HTTPException(403, f"No access to group: {normalized}") return normalized -def resolve_create_instance(user: AuthUser, requested: str | None = None) -> str: - """Pick document ``instance`` at create/upload time (Plan 2). +def resolve_create_group(user: AuthUser, requested: str | None = None) -> str: + """Pick the document's ``group`` at create/upload time. - Super admin / bypass: use requested, else portal ``bv``, else DEFAULT_INSTANCE. - Single-state user: that state (requested must match if provided). @@ -74,12 +76,12 @@ def resolve_create_instance(user: AuthUser, requested: str | None = None) -> str if unrestricted(user): if requested_norm: - return normalize_instance(requested_norm) + return normalize_group(requested_norm) # Prefer portal tag for platform operators when nothing chosen. - portal = (os.environ.get("PORTAL_INSTANCE") or PORTAL_INSTANCE).strip().lower() - return normalize_instance(portal or default_instance()) + portal = (os.environ.get("PORTAL_INSTANCE") or PORTAL_GROUP).strip().lower() + return normalize_group(portal or default_group()) - allowed = allowed_instances(user) or set() + allowed = allowed_groups(user) or set() if not allowed: raise HTTPException( 403, @@ -88,7 +90,7 @@ def resolve_create_instance(user: AuthUser, requested: str | None = None) -> str if requested_norm: if requested_norm not in allowed: - raise HTTPException(403, f"No access to instance: {requested_norm}") + raise HTTPException(403, f"No access to group: {requested_norm}") return requested_norm if len(allowed) == 1: @@ -96,18 +98,18 @@ def resolve_create_instance(user: AuthUser, requested: str | None = None) -> str raise HTTPException( 400, - "instance is required when you have multiple states; " + "group is required when you have multiple states; " f"choose one of: {', '.join(sorted(allowed))}", ) -def assert_document_instance_access(user: AuthUser, doc: dict | None) -> dict: +def assert_document_group_access(user: AuthUser, doc: dict | None) -> dict: """ - Ensure the document exists and the user may access its instance. + Ensure the document exists and the user may access its group. Missing / forbidden both return 404 to avoid leaking other tenants' ids. """ if not doc: raise HTTPException(404, "Document not found") - if not user_can_access_instance(user, doc.get("instance")): + if not user_can_access_group(user, doc.get("group")): raise HTTPException(404, "Document not found") return doc diff --git a/pipeline/discovery_publish_service.py b/pipeline/discovery_publish_service.py index d34f2b8..99c947c 100644 --- a/pipeline/discovery_publish_service.py +++ b/pipeline/discovery_publish_service.py @@ -45,11 +45,11 @@ def __init__( def publish( self, transaction_id: str, - document_kind: Optional[str], + kind: Optional[str], workflow_id: Optional[str] = None, ) -> dict: """ - POST a catalog/publish envelope for `document_kind`. + POST a catalog/publish envelope for `kind`. `transaction_id` is supplied by the caller (generated once per publish, reused across retries) - only `messageId` is generated fresh here, per @@ -59,12 +59,12 @@ def publish( with `skipped=True`: the spec declares `message.catalogs` as `minItems: 1`, so there is no valid "publish nothing" request to send. """ - catalog = build_catalog(document_kind) + catalog = build_catalog(kind) if catalog is None: logger.info( - "workflow_id=%s document_kind=%s network_publish_skipped=True", + "workflow_id=%s kind=%s network_publish_skipped=True", workflow_id, - document_kind, + kind, ) return { "skipped": True, From da238f214b712daf3ed6e8771496cad914cc3f98 Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 11:07:18 +0530 Subject: [PATCH 13/53] feat(scripts): move SQLite data into the v2 schema Build groups and catalogs from legacy instance codes, fold period, uploader and index state into documents.metadata, skip kind=document and demo documents with counts in the report. Deploy migration plan is still to be written. --- scripts/migrate_sqlite_to_postgres.py | 232 +++++++++++++++++--------- 1 file changed, 155 insertions(+), 77 deletions(-) diff --git a/scripts/migrate_sqlite_to_postgres.py b/scripts/migrate_sqlite_to_postgres.py index dc583c0..a37214b 100644 --- a/scripts/migrate_sqlite_to_postgres.py +++ b/scripts/migrate_sqlite_to_postgres.py @@ -12,10 +12,18 @@ second run finds the rows already there and changes nothing. A document is migrated in one transaction. ``--dry-run`` runs everything and rolls it back. -Mapping: one legacy ``documents`` row becomes one document plus its main -workflow (``temporal_workflow_id`` = the legacy ``workflow_id``); each -``document_jobs`` row becomes a workflow; ``scheme_catalog_*`` becomes -``catalog_entries``; ``document_manifest_entries`` is dropped. +Mapping (Schema v2): one legacy ``documents`` row becomes one document plus +its main workflow (``temporal_workflow_id`` = the legacy ``workflow_id``); each +``document_jobs`` row becomes a workflow. The legacy ``instance`` becomes a +``groups`` row (``bv`` is ``/global``, others ``/states/``, Keycloak id +filled in by the first login sync) and the group's ``catalogs`` row for the +document's kind. ``scheme_code`` / ``scheme_name`` / ``scheme_aliases_json`` +(or the ``scheme_catalog_entries`` registry entry for the code) become the +document's ``code`` / ``name`` / ``aliases``; the validity period, uploader and +vector index state move into ``documents.metadata``. Not migrated, and listed in +the report: documents of kind ``document`` (no catalog), demo documents +(``is_demo``), registry entries no document uses (bootstrap-only), and +``document_manifest_entries``. """ import argparse @@ -45,11 +53,11 @@ PageMetadata, PageTranslation, ) +from pipeline.network_constants import KIND_TO_CATALOG_TYPE, catalog_type_for_kind # noqa: E402 _NAMESPACE = uuid.UUID("3f2b6c0e-5d1a-4a53-9a77-0c6a5c1d7e10") _FINGERPRINT_RE = re.compile(r"^[0-9a-f]{32}$") _CODE_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") -_KIND_RE = re.compile(r"^[a-z0-9]+([_-][a-z0-9]+)*$") _STAGES = { "registered", "ocr_processing", "ocr_review", "translation_processing", "translation_review", "chunking", "chunk_review", "ready_for_ingestion", @@ -221,7 +229,10 @@ def _get(row: sqlite3.Row, key: str, default: Any = None) -> Any: # Catalog # ----------------------------------------------------------------------------- -def migrate_catalog_registry(sq, conn, report: Report) -> None: +def load_catalog_registry(sq, report: Report) -> dict[str, dict[str, Any]]: + """Legacy ``scheme_catalog_entries`` by code. Their name and aliases are merged + into the documents holding the code; see ``settle_catalog_registry``.""" + registry: dict[str, dict[str, Any]] = {} for row in _rows(sq, "scheme_catalog_entries"): report.count("scheme_catalog_entries", "read") code = (row["scheme_code"] or "").strip().lower() @@ -229,17 +240,24 @@ def migrate_catalog_registry(sq, conn, report: Report) -> None: report.skip("scheme_catalog_entries", str(row["scheme_code"]), "code is not a valid slug") continue aliases = [a for a in load_json(_get(row, "scheme_aliases_json"), []) if isinstance(a, str) and a] - source = _get(row, "source") if _get(row, "source") in ("bootstrap", "pipeline", "import") else "pipeline" - inserted = conn.execute( - """ - INSERT INTO catalog_entries (id, type, code, name, aliases, metadata) - VALUES (%s::uuid, 'scheme', %s, %s, %s, %s) - ON CONFLICT (type, code) DO NOTHING RETURNING id - """, - (det_id("entry", f"scheme:{code}"), code, row["scheme_name"] or code, aliases, - Jsonb({"source": source})), - ).fetchone() - report.count("scheme_catalog_entries", "inserted" if inserted else "existing") + registry[code] = {"name": row["scheme_name"] or code, "aliases": aliases} + return registry + + +def settle_catalog_registry(registry: dict[str, dict[str, Any]], consumed: set[str], report: Report) -> None: + """Registry entries a migrated document used count as inserted (merged into the + document); the rest have no document and are dropped.""" + unused = [] + for code in sorted(registry): + if code in consumed: + report.count("scheme_catalog_entries", "inserted") + else: + report.count("scheme_catalog_entries", "dropped") + unused.append(code) + if unused: + report.notes.append( + f"scheme_catalog_entries without a document were not migrated: {', '.join(unused)}" + ) def migrate_catalog_version(sq, conn, report: Report) -> None: @@ -267,20 +285,49 @@ def migrate_catalog_version(sq, conn, report: Report) -> None: ) -def _ensure_doc_catalog_entry(conn, kind: str, code: str, name: str, aliases: list[str]) -> str: - conn.execute( +def _instance_of(doc: sqlite3.Row) -> str: + return (_get(doc, "instance") or _default_instance()).strip().lower() or _default_instance() + + +def _group_for(conn, instance: str) -> str: + """The ``groups`` row of a tenant code, created with a deterministic id.""" + path = db._tenant_path(instance) + row = conn.execute( """ - INSERT INTO catalog_entries (id, type, code, name, aliases, metadata) - VALUES (%s::uuid, %s, %s, %s, %s, %s) ON CONFLICT (type, code) DO NOTHING + INSERT INTO groups (id, name, path) VALUES (%s::uuid, %s, %s) + ON CONFLICT (path) DO UPDATE SET name = groups.name RETURNING id """, - (det_id("entry", f"{kind}:{code}"), kind, code, name or code, aliases, - Jsonb({"source": "pipeline"})), - ) - return str( - conn.execute( - "SELECT id FROM catalog_entries WHERE type = %s AND code = %s", (kind, code) - ).fetchone()["id"] - ) + (det_id("group", path), db._tenant_code_from_path(path), path), + ).fetchone() + return str(row["id"]) + + +def _catalog_for(conn, group_id: str, kind: str) -> str: + """The ``(group, kind)`` catalog, created with a deterministic id.""" + catalog_type = catalog_type_for_kind(kind) + row = conn.execute( + """ + INSERT INTO catalogs (id, group_id, type, name) + SELECT %s::uuid, g.id, %s, g.name || ' ' || %s FROM groups g WHERE g.id = %s::uuid + ON CONFLICT (group_id, type) DO UPDATE SET type = EXCLUDED.type RETURNING id + """, + (det_id("catalog", f"{group_id}:{catalog_type}"), catalog_type, kind, group_id), + ).fetchone() + return str(row["id"]) + + +def _slug(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", (text or "").lower()).strip("-")[:40].strip("-") + + +def _unique_code(conn, catalog_id: str, base: str) -> str: + code, n = base, 1 + while conn.execute( + "SELECT 1 FROM documents WHERE catalog_id = %s::uuid AND code = %s", (catalog_id, code) + ).fetchone(): + n += 1 + code = f"{base}-{n}" + return code # ----------------------------------------------------------------------------- @@ -393,7 +440,23 @@ def _build_workflows( return workflows, job_ids -def migrate_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workflow: dict[str, list]) -> bool: +def _skip_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workflow: dict[str, list], reason: str) -> None: + """Leave a document behind. Its history rows are counted as skipped; its audit + entries are kept, tied to the tenant's group but to no document.""" + workflow_id = doc["workflow_id"] + report.skip("documents", workflow_id, reason) + for table in ("document_jobs", "document_artifacts", "pages", "chunks"): + report.count(table, "skipped", len(_rows(sq, table, "WHERE workflow_id = ?", (workflow_id,)))) + for audit in audit_by_workflow.pop(workflow_id, []): + insert_audit(conn, audit, None, _instance_of(doc), report) + + +def migrate_document( + sq, conn, doc: sqlite3.Row, report: Report, audit_by_workflow: dict[str, list], + registry: Optional[dict[str, dict[str, Any]]] = None, consumed: Optional[set[str]] = None, +) -> bool: + registry = registry if registry is not None else {} + consumed = consumed if consumed is not None else set() workflow_id = doc["workflow_id"] doc_uuid = det_id("document", workflow_id) exists = conn.execute("SELECT 1 FROM documents WHERE id = %s::uuid", (doc_uuid,)).fetchone() @@ -406,6 +469,14 @@ def migrate_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workfl report.count(label, "existing", len(_rows(sq, table, "WHERE workflow_id = ?", (workflow_id,)))) return False + kind = (_get(doc, "document_kind") or "document").strip().lower() + if truthy(_get(doc, "is_demo")): + _skip_document(sq, conn, doc, report, audit_by_workflow, "demo document (is_demo)") + return False + if kind not in KIND_TO_CATALOG_TYPE: + _skip_document(sq, conn, doc, report, audit_by_workflow, f"document_kind {kind!r} has no catalog") + return False + fingerprint = next( ( c for c in ( @@ -420,12 +491,7 @@ def migrate_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workfl fingerprint = hashlib.md5((doc["document_id"] or workflow_id).encode()).hexdigest() report.notes.append(f"{workflow_id}: no hex fingerprint, derived md5 of document_id") - page_count = max(int(doc["page_count"] or 0), 0) chunk_rows = _rows(sq, "chunks", "WHERE workflow_id = ? ORDER BY chunk_number", (workflow_id,)) - kind = (_get(doc, "document_kind") or "document").strip().lower() - if not _KIND_RE.match(kind): - report.notes.append(f"{workflow_id}: invalid document_kind {kind!r}, stored as 'document'") - kind = "document" chunking = None if chunk_rows: @@ -436,12 +502,24 @@ def migrate_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workfl model=_get(first, "chunking_model"), config=load_json(_get(first, "chunking_config_json"), {}), ) + + valid_from, valid_to = parse_date(_get(doc, "valid_from")), parse_date(_get(doc, "valid_to")) + if valid_from and valid_to and valid_from > valid_to: + report.notes.append(f"{workflow_id}: valid_from after valid_to, valid_to cleared") + valid_to = None + + uploader = _user_for( + conn, _get(doc, "uploaded_by_user_id"), _get(doc, "uploaded_by_username"), + _get(doc, "uploaded_by_email"), _get(doc, "uploaded_by_roles"), + ) + metadata = DocumentMetadata( - display_name=_get(doc, "display_name") or None, + valid_from=valid_from.isoformat() if valid_from else None, + valid_to=valid_to.isoformat() if valid_to else None, + uploaded_by_user_id=uploader, + index_status=_build_index_status(sq, workflow_id), catalog_visible=truthy(_get(doc, "catalog_visible"), True), network_visible=truthy(_get(doc, "network_visible"), True), - is_demo=truthy(_get(doc, "is_demo")), - chunk_count=len(chunk_rows) if chunk_rows else max(int(doc["chunk_count"] or 0), 0), reindex_required=truthy(_get(doc, "reindex_required")), reindex_reason=_get(doc, "reindex_reason") or None, chunking=chunking, @@ -452,46 +530,41 @@ def migrate_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workfl storage_uri=doc["filepath"], source_type=_get(doc, "source_type"), canonical_input_type=_get(doc, "canonical_input_type"), - page_count=page_count, ).model_dump(exclude_none=True) - valid_from, valid_to = parse_date(_get(doc, "valid_from")), parse_date(_get(doc, "valid_to")) - if valid_from and valid_to and valid_from > valid_to: - report.notes.append(f"{workflow_id}: valid_from after valid_to, valid_to cleared") - valid_to = None - - uploader = _user_for( - conn, _get(doc, "uploaded_by_user_id"), _get(doc, "uploaded_by_username"), - _get(doc, "uploaded_by_email"), _get(doc, "uploaded_by_roles"), - ) - - entry_id = None + catalog_id = _catalog_for(conn, _group_for(conn, _instance_of(doc)), kind) code = (_get(doc, "scheme_code") or "").strip().lower() - if code and kind != "document": - if _CODE_RE.match(code): - aliases = [a for a in load_json(_get(doc, "scheme_aliases_json"), []) if isinstance(a, str) and a] - entry_id = _ensure_doc_catalog_entry( - conn, kind, code, _get(doc, "scheme_name") or code, aliases - ) - else: - report.notes.append(f"{workflow_id}: scheme_code {code!r} is not a valid slug, not linked") - elif code: - report.notes.append(f"{workflow_id}: scheme_code {code!r} on a plain document, not linked") + entry = registry.get(code) if kind == "scheme" and code else None + if entry is not None: + consumed.add(code) + name = (entry or {}).get("name") or _get(doc, "scheme_name") or _get(doc, "display_name") or None + aliases = (entry or {}).get("aliases") or [ + a for a in load_json(_get(doc, "scheme_aliases_json"), []) if isinstance(a, str) and a + ] + if code and not _CODE_RE.match(code): + report.notes.append(f"{workflow_id}: scheme_code {code!r} is not a valid slug, code derived") + code = "" + if not code: + code = _slug(name or "") or _slug(Path(doc["filename"] or "").stem) + if not _CODE_RE.match(code): + code = f"doc-{fingerprint[:8]}" + report.notes.append(f"{workflow_id}: no usable scheme_code, derived {code!r}") + unique = _unique_code(conn, catalog_id, code) + if unique != code: + report.notes.append(f"{workflow_id}: code {code!r} already used in its catalog, stored as {unique!r}") + code = unique created_at = parse_ts(doc["created_at"]) or datetime.now(timezone.utc) updated_at = parse_ts(doc["updated_at"]) or created_at conn.execute( """ - INSERT INTO documents (id, instance, source, document_kind, catalog_entry_id, valid_from, - valid_to, is_disabled, index_status, metadata, uploaded_by_user_id, + INSERT INTO documents (id, catalog_id, code, name, aliases, source, is_disabled, metadata, created_at, updated_at) - VALUES (%s::uuid, %s, %s, %s, %s::uuid, %s, %s, %s, %s, %s, %s::uuid, %s, %s) + VALUES (%s::uuid, %s::uuid, %s, %s, %s, %s, %s, %s, %s, %s) """, ( - doc_uuid, (_get(doc, "instance") or _default_instance()).strip().lower() or _default_instance(), - Jsonb(source), kind, entry_id, valid_from, valid_to, truthy(_get(doc, "is_disabled")), - Jsonb(_build_index_status(sq, workflow_id)), Jsonb(metadata), uploader, created_at, - updated_at, + doc_uuid, catalog_id, code, name or code, aliases, Jsonb(source), + truthy(_get(doc, "is_disabled")), Jsonb(metadata), created_at, updated_at, ), ) report.count("documents", "inserted") @@ -507,8 +580,8 @@ def migrate_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workfl INSERT INTO workflows (id, document_id, workflow_type, temporal_workflow_id, status, current_stage, stage_timestamps, stop_after_ocr, error_message, config, started_by_user_id, started_at, completed_at, updated_at) - VALUES (%s::uuid, %s::uuid, %s::workflow_type_enum, %s, %s::workflow_status_enum, - %s::workflow_stage, %s, %s, %s, %s, %s::uuid, %s, %s, %s) + VALUES (%s::uuid, %s::uuid, %s, %s, %s, + %s, %s, %s, %s, %s, %s::uuid, %s, %s, %s) """, ( w["id"], doc_uuid, w["type"], w["temporal_id"], w["status"], w["stage"], @@ -526,7 +599,7 @@ def migrate_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workfl """ INSERT INTO workflow_artifacts (id, workflow_id, artifact_type, stage, storage_uri, mime_type, filename, size_bytes, metadata, created_at) - VALUES (%s::uuid, %s::uuid, %s, %s::workflow_stage, %s, %s, %s, %s, %s, %s) + VALUES (%s::uuid, %s::uuid, %s, %s, %s, %s, %s, %s, %s, %s) """, ( det_id("artifact", f"{workflow_id}:{art['id']}"), @@ -604,7 +677,7 @@ def migrate_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workfl report.count("chunks", "inserted") for audit in audit_by_workflow.pop(workflow_id, []): - insert_audit(conn, audit, doc_uuid, source_instance=(_get(doc, "instance") or _default_instance()), report=report) + insert_audit(conn, audit, doc_uuid, source_instance=_instance_of(doc), report=report) return True @@ -636,16 +709,16 @@ def insert_audit(conn, audit: sqlite3.Row, doc_uuid: Optional[str], source_insta ) inserted = conn.execute( """ - INSERT INTO audit_logs (id, document_id, workflow_id, instance, action_type, entity_type, + INSERT INTO audit_logs (id, document_id, workflow_id, group_id, action_type, entity_type, entity_number, field_name, old_value, new_value, metadata, actor_user_id, actor, created_at) - VALUES (%s::uuid, %s::uuid, %s, %s, %s, %s::audit_entity_type, %s, %s, %s, %s, %s, + VALUES (%s::uuid, %s::uuid, %s, %s::uuid, %s, %s, %s, %s, %s, %s, %s, %s::uuid, %s, %s) ON CONFLICT (id) DO NOTHING RETURNING id """, ( det_id("audit", str(audit["id"])), doc_uuid, workflow_id, - (source_instance or "").strip().lower() or None if doc_uuid else None, + _group_for(conn, source_instance.strip().lower()) if (source_instance or "").strip() else None, audit["action_type"], entity, audit["entity_id"], _legacy_audit_field(audit["field_name"], entity), audit["old_value"], audit["new_value"], Jsonb(metadata), actor_uuid, @@ -794,13 +867,18 @@ def run_migration(sqlite_path: str, database_url: str, dry_run: bool = False) -> else: orphan_audit.append(audit) - with conn.transaction(): - migrate_catalog_registry(sq, conn, report) + registry = load_catalog_registry(sq, report) + consumed: set[str] = set() for doc in documents: report.count("documents", "read") with conn.transaction(): - migrate_document(sq, conn, doc, report, audit_by_workflow) + migrate_document(sq, conn, doc, report, audit_by_workflow, registry, consumed) + settle_catalog_registry(registry, consumed, report) + + skipped = Counter(i["reason"] for i in report.skip_reasons if i["table"] == "documents") + for reason, n in sorted(skipped.items()): + report.notes.append(f"documents not migrated: {n} x {reason}") with conn.transaction(): for audits in audit_by_workflow.values(): From 9482c7210ed36bf0edac6888291324c520c4de48 Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 11:07:18 +0530 Subject: [PATCH 14/53] test: update for schema v2 and add constraint tests Rename to group/kind/code/name/aliases, cover upload with kind, re-upload of an existing code, catalog_status, group snapshots in audit_logs and the new unique/foreign-key constraints in tests/test_schema_v2.py. --- tests/conftest.py | 6 +- tests/test_activities.py | 65 +- tests/test_api.py | 255 +++++++- tests/test_auth.py | 2 +- tests/test_db.py | 192 +++--- tests/test_discovery_publish_service.py | 32 +- tests/test_document_repository.py | 42 +- ...{test_instances.py => test_group_names.py} | 32 +- tests/test_groups.py | 46 +- tests/test_migrate_sqlite_to_postgres.py | 79 ++- tests/test_provenance.py | 16 +- tests/test_schema_v2.py | 598 ++++++++++++++++++ tests/test_scheme_catalog.py | 301 ++++----- tests/test_tenancy.py | 308 ++++++--- 14 files changed, 1504 insertions(+), 470 deletions(-) rename tests/{test_instances.py => test_group_names.py} (81%) create mode 100644 tests/test_schema_v2.py diff --git a/tests/conftest.py b/tests/conftest.py index 49b2aa2..ccea78e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,7 +23,7 @@ _TRUNCATE_SQL = """ - TRUNCATE documents, users, catalog_entries, audit_logs, email_otps, + TRUNCATE documents, users, groups, catalogs, audit_logs, email_otps, webhook_records, search_settings CASCADE """ @@ -202,7 +202,9 @@ def sample_document(db_connection): document_id=document_id, filename="test.pdf", filepath="/app/books/test.pdf", - stage="registered" + stage="registered", + kind="scheme", + code="test-scheme", ) return { diff --git a/tests/test_activities.py b/tests/test_activities.py index a752533..191e76a 100644 --- a/tests/test_activities.py +++ b/tests/test_activities.py @@ -10,6 +10,7 @@ """ import asyncio +import itertools import os from unittest.mock import MagicMock @@ -21,6 +22,14 @@ os.environ["TRANSLATION_VLLM_BASE_URL"] = "http://localhost:8000/v1" +_code_seq = itertools.count(1) + + +def _next_code() -> str: + """A unique document code for fixtures (documents need one in v2).""" + return f"doc-{next(_code_seq)}" + + class TestChunkingActivity: """Tests for the chunking activity.""" @@ -158,7 +167,7 @@ def test_prepare_records_carries_instance_name(self): document_id="test-doc", filename="test.pdf", chunks=chunks, - instance=code, + group=code, ) assert records[0]["instance"] == code assert records[0]["instance_name"] == expected @@ -172,7 +181,7 @@ def test_prepare_records_carries_document_kind(self): chunks = [{"chunk_number": 1, "original_text": "Test chunk", "token_count": 5}] advisory = _prepare_records( - document_id="test-doc", filename="test.pdf", chunks=chunks, document_kind="advisory" + document_id="test-doc", filename="test.pdf", chunks=chunks, kind="advisory" ) assert advisory[0]["document_kind"] == "advisory" assert advisory[0]["type"] == "document" @@ -185,9 +194,9 @@ def test_prepare_records_carries_document_kind(self): document_id="test-doc", filename="test.pdf", chunks=chunks, - document_kind="scheme", - scheme_code="pm-kisan", - scheme_name="PM-KISAN", + kind="scheme", + code="pm-kisan", + name="PM-KISAN", ) assert scheme[0]["type"] == "scheme" assert scheme[0]["document_kind"] == "scheme" @@ -203,11 +212,11 @@ def test_prepare_records_fills_scheme_aliases(self): document_id="test-doc", filename="NMEO-OPGUIEDELINES.pdf", chunks=chunks, - instance="bv", - document_kind="scheme", - scheme_code="nmeoop", - scheme_name="National Mission on Edible Oils - Oil Palm", - scheme_aliases=None, + group="bv", + kind="scheme", + code="nmeoop", + name="National Mission on Edible Oils - Oil Palm", + aliases=None, ) record = records[0] @@ -229,8 +238,8 @@ def test_prepare_records_non_scheme_has_no_aliases(self): document_id="test-doc", filename="test.pdf", chunks=chunks, - instance="mh", - document_kind="document", + group="mh", + kind="document", ) assert "scheme_aliases" not in records[0] @@ -429,6 +438,8 @@ def test_update_state(self, db_connection): workflow_id = "state-update-test" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-state", filename="test.pdf", @@ -436,6 +447,12 @@ def test_update_state(self, db_connection): stage="registered" ) + # Page and chunk counts are derived from the pages table, not stored. + db_connection.save_pages( + workflow_id, + [{"page_number": n, "original_markdown": f"page {n}"} for n in range(1, 6)], + ) + asyncio.run(update_document_state( workflow_id=workflow_id, stage="ocr_processing", @@ -548,7 +565,7 @@ def __init__(self): "_upload_file_to_minio", lambda *a, **k: ("minio://documents/network_publish_skipped.json", 45, "application/json"), ) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "advisory"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory"}) monkeypatch.setattr(db, "get_latest_document_job", lambda workflow_id: None) recorded = {} @@ -591,9 +608,9 @@ class FakeService: def __init__(self): pass - def publish(self, transaction_id, document_kind, workflow_id=None): + def publish(self, transaction_id, kind, workflow_id=None): assert transaction_id == "txn-1" - assert document_kind == "advisory" + assert kind == "advisory" assert workflow_id == "wf-1" return fake_result @@ -603,7 +620,7 @@ def publish(self, transaction_id, document_kind, workflow_id=None): "_upload_file_to_minio", lambda *a, **k: ("minio://documents/network_publish.json", 123, "application/json"), ) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "advisory"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory"}) monkeypatch.setattr(db, "get_latest_document_job", lambda workflow_id: {"id": 42}) recorded = {} @@ -639,11 +656,11 @@ async def test_publish_failure_propagates_and_records_nothing(self, monkeypatch) import pipeline.db as db class FakeService: - def publish(self, transaction_id, document_kind, workflow_id=None): + def publish(self, transaction_id, kind, workflow_id=None): raise RuntimeError("discovery service unreachable") monkeypatch.setattr(activities, "DiscoveryPublishService", FakeService) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "advisory"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory"}) add_artifact_calls = [] monkeypatch.setattr(db, "add_document_artifact", lambda **kwargs: add_artifact_calls.append(kwargs)) @@ -661,7 +678,7 @@ async def test_unmapped_kind_records_a_skipped_artifact(self, monkeypatch): import pipeline.db as db class FakeService: - def publish(self, transaction_id, document_kind, workflow_id=None): + def publish(self, transaction_id, kind, workflow_id=None): return { "skipped": True, "envelope": None, @@ -677,7 +694,7 @@ def publish(self, transaction_id, document_kind, workflow_id=None): "_upload_file_to_minio", lambda *a, **k: ("minio://documents/network_publish_skipped.json", 45, "application/json"), ) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "document"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "document"}) monkeypatch.setattr(db, "get_latest_document_job", lambda workflow_id: None) recorded = {} @@ -706,7 +723,7 @@ async def test_non_rejected_results_complete_the_stage(self, monkeypatch, result import pipeline.db as db class FakeService: - def publish(self, transaction_id, document_kind, workflow_id=None): + def publish(self, transaction_id, kind, workflow_id=None): return { "skipped": False, "envelope": {"context": {"messageId": "msg-3"}}, @@ -722,7 +739,7 @@ def publish(self, transaction_id, document_kind, workflow_id=None): "_upload_file_to_minio", lambda *a, **k: ("minio://documents/network_publish.json", 12, "application/json"), ) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "advisory"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory"}) monkeypatch.setattr(db, "get_latest_document_job", lambda workflow_id: None) monkeypatch.setattr(db, "add_document_artifact", lambda **kwargs: 1) @@ -740,7 +757,7 @@ async def test_rejected_result_records_the_errors_then_fails(self, monkeypatch): errors = [{"code": "SCH_SCHEMA_VALIDATION_FAILED", "message": "topics is required"}] class FakeService: - def publish(self, transaction_id, document_kind, workflow_id=None): + def publish(self, transaction_id, kind, workflow_id=None): return { "skipped": False, "envelope": {"context": {"messageId": "msg-4"}}, @@ -756,7 +773,7 @@ def publish(self, transaction_id, document_kind, workflow_id=None): "_upload_file_to_minio", lambda *a, **k: ("minio://documents/network_publish.json", 12, "application/json"), ) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "advisory"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory"}) monkeypatch.setattr(db, "get_latest_document_job", lambda workflow_id: None) recorded = {} diff --git a/tests/test_api.py b/tests/test_api.py index 3e319c0..113b0dc 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -11,6 +11,7 @@ """ import hashlib +import itertools import os import pytest @@ -21,6 +22,14 @@ os.environ["ALLOWED_FILE_PATHS"] = "/app/books,/tmp" +_code_seq = itertools.count(1) + + +def _next_code() -> str: + """A unique document code for fixtures (documents need one in v2).""" + return f"doc-{next(_code_seq)}" + + class TestHealthEndpoints: """Tests for health check endpoints.""" @@ -68,6 +77,8 @@ def test_list_documents(self, test_client, db_connection): """Test listing documents.""" # Create a test document db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id="api-test-001", document_id="doc-001", filename="test.pdf", @@ -85,6 +96,8 @@ def test_list_documents(self, test_client, db_connection): def test_list_documents_with_stage_filter(self, test_client, db_connection): """Test filtering documents by stage.""" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id="filter-api-001", document_id="doc-filter-001", filename="completed.pdf", @@ -104,6 +117,8 @@ def test_get_document_from_db(self, test_client, db_connection): """Test getting document details from the database.""" workflow_id = "get-test-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-get-001", filename="test.pdf", @@ -132,6 +147,8 @@ class TestDocumentIdentifiers: def _document(self, db_connection): db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=self.WORKFLOW_ID, document_id=self.FINGERPRINT, filename="identifiers.pdf", @@ -185,6 +202,8 @@ def test_the_uuid_is_rewritten_to_the_temporal_id_before_handlers_run( @pytest.mark.unit def test_static_document_routes_are_not_mistaken_for_identifiers(self, test_client, db_connection): db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id="summary", document_id=hashlib.md5(b"summary").hexdigest(), filename="summary", filepath="/tmp/s.pdf", ) @@ -203,6 +222,8 @@ class TestArtifactAndJobIdentifiers: def _document(self, db_connection): db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id="wf-art-api", document_id=hashlib.md5(b"art").hexdigest(), filename="a.pdf", filepath="minio://documents/a.pdf", stage="ocr_review", ) @@ -271,6 +292,8 @@ class TestProdApproval: def _document_at_the_gate(self, db_connection, workflow_id): db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id=f"doc-{workflow_id}", filename="test.pdf", @@ -334,6 +357,8 @@ def test_the_document_carries_no_validity_window(self, test_client, db_connectio def test_the_wrong_stage_is_still_refused(self, test_client, db_connection): workflow_id = "prod-approval-005" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id=f"doc-{workflow_id}", filename="test.pdf", @@ -356,6 +381,8 @@ class TestDocumentValidityEndpoints: def _uploaded_document(self, db_connection, workflow_id): db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id=f"doc-{workflow_id}", filename="test.pdf", @@ -375,6 +402,7 @@ def test_upload_starts_today_and_never_expires(self, test_client, sample_pdf_con response = test_client.post( "/upload", + params={"kind": "scheme"}, files={"file": ("validity-upload.pdf", sample_pdf_content, "application/pdf")}, ) @@ -393,13 +421,13 @@ def test_an_end_date_can_be_set_and_then_cleared(self, test_client, db_connectio set_response = test_client.patch( f"/documents/{workflow_id}/scheme-metadata", - json={"document_kind": "advisory", "valid_from": "2026-01-01", "valid_to": "2026-06-30"}, + json={"kind": "advisory", "valid_from": "2026-01-01", "valid_to": "2026-06-30"}, ) assert set_response.json()["valid_to"] == "2026-06-30" cleared = test_client.patch( f"/documents/{workflow_id}/scheme-metadata", - json={"document_kind": "advisory", "valid_to": ""}, + json={"kind": "advisory", "valid_to": ""}, ) assert cleared.status_code == 200 @@ -429,7 +457,7 @@ def test_stores_the_reviewers_edit(self, test_client, db_connection): response = test_client.patch( f"/documents/{workflow_id}/scheme-metadata", json={ - "document_kind": "advisory", + "kind": "advisory", "valid_from": "2026-10-01", "valid_to": "2026-12-31", }, @@ -451,7 +479,7 @@ def test_moving_only_the_end_keeps_the_stored_start(self, test_client, db_connec response = test_client.patch( f"/documents/{workflow_id}/scheme-metadata", - json={"document_kind": "advisory", "valid_to": "2027-06-30"}, + json={"kind": "advisory", "valid_to": "2027-06-30"}, ) assert response.status_code == 200 @@ -467,7 +495,7 @@ def test_classifying_without_dates_leaves_the_period_alone(self, test_client, db test_client.patch( f"/documents/{workflow_id}/scheme-metadata", - json={"document_kind": "advisory"}, + json={"kind": "advisory"}, ) stored = db_connection.get_document(workflow_id) @@ -483,7 +511,7 @@ def test_rejects_an_end_before_the_start(self, test_client, db_connection): response = test_client.patch( f"/documents/{workflow_id}/scheme-metadata", json={ - "document_kind": "advisory", + "kind": "advisory", "valid_from": "2026-12-31", "valid_to": "2026-01-01", }, @@ -500,7 +528,7 @@ def test_rejects_a_malformed_date(self, test_client, db_connection): response = test_client.patch( f"/documents/{workflow_id}/scheme-metadata", - json={"document_kind": "advisory", "valid_to": "31-12-2026"}, + json={"kind": "advisory", "valid_to": "31-12-2026"}, ) assert response.status_code == 400 @@ -517,13 +545,13 @@ def test_a_rejected_period_stores_nothing(self, test_client, db_connection): test_client.patch( f"/documents/{workflow_id}/scheme-metadata", - json={"document_kind": "advisory", "valid_to": "not-a-date"}, + json={"kind": "advisory", "valid_to": "not-a-date"}, ) stored = db_connection.get_document(workflow_id) assert stored["valid_from"] == "2026-01-01" assert stored["valid_to"] == "2026-06-30" - assert (stored["document_kind"] or "document") == "document" + assert stored["kind"] == "scheme" @pytest.mark.api @pytest.mark.unit @@ -663,6 +691,8 @@ def test_list_pages(self, test_client, db_connection): """Test listing pages for a document.""" workflow_id = "pages-test-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-pages", filename="test.pdf", @@ -689,6 +719,8 @@ def test_get_page(self, test_client, db_connection): """Test getting a specific page.""" workflow_id = "page-get-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-page", filename="test.pdf", @@ -712,6 +744,8 @@ def test_get_page_invalid_number(self, test_client, db_connection): """Test validation rejects invalid page numbers.""" workflow_id = "page-invalid-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-invalid", filename="test.pdf", @@ -733,6 +767,8 @@ def test_update_page(self, test_client, db_connection): """Test updating a page.""" workflow_id = "page-update-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-update", filename="test.pdf", @@ -763,6 +799,8 @@ def test_list_chunks(self, test_client, db_connection): """Test listing chunks for a document.""" workflow_id = "chunks-test-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-chunks", filename="test.pdf", @@ -789,6 +827,8 @@ def test_list_chunks_include_excluded(self, test_client, db_connection): """Test including excluded chunks in listing.""" workflow_id = "chunks-excluded-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-excluded", filename="test.pdf", @@ -819,6 +859,8 @@ def test_get_chunk_invalid_number(self, test_client, db_connection): """Test validation rejects invalid chunk numbers.""" workflow_id = "chunk-invalid-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-chunk-invalid", filename="test.pdf", @@ -839,6 +881,8 @@ def test_get_document_audit(self, test_client, db_connection): """Test getting audit logs for a document.""" workflow_id = "audit-test-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-audit", filename="test.pdf", @@ -933,6 +977,8 @@ def test_page_not_found(self, test_client, db_connection): """Test 404 for missing page.""" workflow_id = "page-404-test" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-404", filename="test.pdf", @@ -966,3 +1012,194 @@ def test_inline_content_disposition_strips_crlf(self): assert header.count(":") >= 1 assert "\r\nX-Injected" not in header assert header.startswith('inline; filename="') + + +class TestUploadCatalogIdentity: + """/upload picks the tenant's catalog from ``kind`` and names the document by ``code``.""" + + @pytest.fixture(autouse=True) + def _fresh_rate_limit(self): + # /upload is limited per IP; these tests post more often than one minute allows. + import pipeline.api as api + + api.limiter.reset() + yield + api.limiter.reset() + + @staticmethod + def _pdf(sample_pdf_content: bytes, tag: str) -> bytes: + # A different trailer gives a different fingerprint, hence a new workflow id. + return sample_pdf_content + f"\n% {tag}".encode() + + @pytest.mark.api + @pytest.mark.unit + def test_kind_is_required(self, test_client, sample_pdf_content): + response = test_client.post( + "/upload", files={"file": ("a.pdf", sample_pdf_content, "application/pdf")} + ) + + assert response.status_code == 422 + + @pytest.mark.api + @pytest.mark.unit + @pytest.mark.parametrize("kind", ["document", "video", "", "Scheme-X"]) + def test_an_unknown_kind_is_rejected(self, test_client, sample_pdf_content, kind): + response = test_client.post( + "/upload", + params={"kind": kind}, + files={"file": ("a.pdf", sample_pdf_content, "application/pdf")}, + ) + + assert response.status_code == 400 + assert "scheme" in response.json()["detail"] + + @pytest.mark.api + @pytest.mark.unit + def test_an_invalid_code_is_rejected(self, test_client, sample_pdf_content): + response = test_client.post( + "/upload", + params={"kind": "scheme", "code": "Not A Slug"}, + files={"file": ("a.pdf", sample_pdf_content, "application/pdf")}, + ) + + assert response.status_code == 400 + + @pytest.mark.api + @pytest.mark.unit + def test_upload_creates_the_catalog_and_derives_the_code_from_the_filename( + self, test_client, db_connection, sample_pdf_content + ): + response = test_client.post( + "/upload", + params={"kind": "scheme", "group": "mh"}, + files={"file": ("National Mission on Natural Farming.pdf", sample_pdf_content, "application/pdf")}, + ) + + assert response.status_code == 200 + doc = test_client.get(f"/documents/{response.json()['workflow_id']}").json() + assert doc["kind"] == "scheme" + assert doc["group"] == "mh" + assert doc["code"] == "nmnf" + assert doc["name"] == "National Mission on Natural Farming" + with db_connection.get_connection() as conn: + catalog = conn.execute( + "SELECT c.type, g.name, g.path FROM catalogs c JOIN groups g ON g.id = c.group_id" + ).fetchall() + assert [(r["type"], r["name"], r["path"]) for r in catalog] == [ + ("openagrinet:AgricultureResource", "mh", "/states/MH") + ] + + @pytest.mark.api + @pytest.mark.unit + def test_a_second_upload_reuses_the_catalog_and_gets_a_fresh_code( + self, test_client, db_connection, sample_pdf_content + ): + first = test_client.post( + "/upload", + params={"kind": "advisory", "group": "mh"}, + files={"file": ("Crop Advisory.pdf", self._pdf(sample_pdf_content, "one"), "application/pdf")}, + ) + second = test_client.post( + "/upload", + params={"kind": "advisory", "group": "mh"}, + files={"file": ("Crop Advisory.pdf", self._pdf(sample_pdf_content, "two"), "application/pdf")}, + ) + + assert first.status_code == second.status_code == 200 + codes = { + test_client.get(f"/documents/{r.json()['workflow_id']}").json()["code"] + for r in (first, second) + } + assert codes == {"ca", "ca-2"} + with db_connection.get_connection() as conn: + assert conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] == 1 + + @pytest.mark.api + @pytest.mark.unit + def test_an_existing_code_updates_the_same_document( + self, test_client, db_connection, sample_pdf_content + ): + first = test_client.post( + "/upload", + params={"kind": "scheme", "group": "mh", "code": "pm-kisan"}, + files={"file": ("pmkisan.pdf", self._pdf(sample_pdf_content, "v1"), "application/pdf")}, + ) + assert first.status_code == 200 + before = test_client.get(f"/documents/{first.json()['workflow_id']}").json() + db_connection.save_pages( + before["id"], [{"page_number": 1, "original_markdown": "old text"}] + ) + db_connection.save_chunks(before["id"], [{"chunk_number": 1, "original_text": "old"}]) + + second = test_client.post( + "/upload", + params={"kind": "scheme", "group": "mh", "code": "pm-kisan"}, + files={"file": ("pmkisan-2026.pdf", self._pdf(sample_pdf_content, "v2"), "application/pdf")}, + ) + + assert second.status_code == 200 + assert second.json()["workflow_id"] != first.json()["workflow_id"] + after = test_client.get(f"/documents/{before['id']}").json() + assert after["id"] == before["id"] + assert after["filename"] == "pmkisan-2026.pdf" + assert after["source_file_fingerprint"] != before["source_file_fingerprint"] + assert after["reindex_required"] is True + assert after["reindex_reason"] == "source_replaced" + assert (after["page_count"], after["chunk_count"]) == (0, 0) + with db_connection.get_connection() as conn: + assert conn.execute("SELECT count(*) AS n FROM documents").fetchone()["n"] == 1 + assert conn.execute("SELECT count(*) AS n FROM workflows").fetchone()["n"] == 2 + + @pytest.mark.api + @pytest.mark.unit + def test_the_same_code_in_another_tenant_is_a_different_document( + self, test_client, db_connection, sample_pdf_content + ): + for group, tag in (("mh", "a"), ("gj", "b")): + response = test_client.post( + "/upload", + params={"kind": "scheme", "group": group, "code": "pm-kisan"}, + files={"file": ("p.pdf", self._pdf(sample_pdf_content, tag), "application/pdf")}, + ) + assert response.status_code == 200 + + with db_connection.get_connection() as conn: + assert conn.execute("SELECT count(*) AS n FROM documents").fetchone()["n"] == 2 + assert conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] == 2 + + @pytest.mark.api + @pytest.mark.unit + def test_replacing_a_document_needs_access_to_its_tenant(self, db_connection): + from fastapi import HTTPException + + import pipeline.api as api + from pipeline.auth.jwt import claims_to_user + + db_connection.upsert_document( + workflow_id="wf-mh-doc", + document_id="d-mh", + filename="a.pdf", + filepath="/tmp/a.pdf", + group="mh", + kind="scheme", + code="pm-kisan", + ) + outsider = claims_to_user( + {"sub": "u1", "realm_access": {"roles": ["content_curator"]}, "instances": ["gj"]} + ) + insider = claims_to_user( + {"sub": "u2", "realm_access": {"roles": ["content_curator"]}, "instances": ["mh"]} + ) + + with pytest.raises(HTTPException) as exc: + api._resolve_document_identity(outsider, "mh", "scheme", "pm-kisan", "a.pdf") + assert exc.value.status_code == 404 + + kind, code, existing = api._resolve_document_identity(insider, "mh", "scheme", "pm-kisan", "a.pdf") + assert (kind, code, existing["workflow_id"]) == ("scheme", "pm-kisan", "wf-mh-doc") + + @pytest.mark.api + @pytest.mark.unit + def test_the_demo_and_bootstrap_routes_are_gone(self, test_client): + assert test_client.post("/documents/anything/demo").status_code in (404, 405) + assert test_client.post("/catalog/v1/bootstrap").status_code in (404, 405) diff --git a/tests/test_auth.py b/tests/test_auth.py index d34b534..b4578f7 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -84,7 +84,7 @@ def test_claims_to_user_maps_keycloak_shape(): assert Permission.UPLOAD in user.permissions assert user.instances == ["tenant-a", "tenant-b"] assert user.envs == ["dev"] - assert user.has_instance("Tenant-A") + assert user.has_group("Tenant-A") assert not user.has_env("prod") diff --git a/tests/test_db.py b/tests/test_db.py index 496010d..2d2d5b6 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -12,6 +12,7 @@ """ import hashlib +import re import psycopg import pytest @@ -21,6 +22,30 @@ def _md5(text): return hashlib.md5(text.encode()).hexdigest() +def _code_for(workflow_id): + return re.sub(r"[^a-z0-9]+", "-", (workflow_id or "doc").lower()).strip("-") or "doc" + + +@pytest.fixture(autouse=True) +def _new_documents_default_to_a_scheme(monkeypatch): + """A new v2 document needs a kind and a code; these tests are about everything else. + + The requirement itself is pinned in test_schema_v2.py. Here a new document is a + scheme whose code is its workflow id, unless a test says otherwise. + """ + from pipeline import db + + real = db.upsert_document + + def with_defaults(*args, **kwargs): + kwargs.setdefault("kind", "scheme") + if not kwargs.get("code"): + kwargs["code"] = _code_for(kwargs.get("workflow_id")) + return real(*args, **kwargs) + + monkeypatch.setattr(db, "upsert_document", with_defaults) + + class TestDocumentOperations: """Tests for document CRUD operations.""" @@ -133,34 +158,17 @@ def test_update_document_stage(self, db_connection): stage="registered" ) - db_connection.update_document_stage( - workflow_id=workflow_id, - stage="ocr_review", - page_count=5, - chunk_count=10 - ) + db_connection.update_document_stage(workflow_id=workflow_id, stage="ocr_review") doc = db_connection.get_document(workflow_id) assert doc["stage"] == "ocr_review" - assert doc["page_count"] == 5 - assert doc["chunk_count"] == 10 - - @pytest.mark.db - @pytest.mark.unit - def test_set_document_demo(self, db_connection): - """Test marking document as demo.""" - workflow_id = "demo-test" - db_connection.upsert_document( - workflow_id=workflow_id, - document_id="doc-demo", - filename="test.pdf", - filepath="/app/books/test.pdf", - stage="registered" - ) - - db_connection.set_document_demo(workflow_id, True) + # Counts are derived from pages and chunks, not stored. + assert (doc["page_count"], doc["chunk_count"]) == (0, 0) + db_connection.save_pages(workflow_id, [{"page_number": 1, "original_markdown": "a"}]) + db_connection.save_chunks(workflow_id, [{"chunk_number": 1, "original_text": "x"}, + {"chunk_number": 2, "original_text": "y"}]) doc = db_connection.get_document(workflow_id) - assert doc["is_demo"] == 1 + assert (doc["page_count"], doc["chunk_count"]) == (1, 2) @pytest.mark.db @pytest.mark.unit @@ -606,26 +614,31 @@ def _new_doc(db, workflow_id, **kwargs): class TestSchemaConstraints: """Rules the database itself enforces.""" + @staticmethod + def _catalog_id(db): + with db.get_connection() as conn: + return db.get_or_create_catalog(db.get_or_create_group("mh", conn=conn), "scheme", conn=conn)["id"] + @pytest.mark.db @pytest.mark.unit - def test_document_kind_must_match_its_catalog_entry_type(self, db_connection): + def test_document_code_must_be_a_slug(self, db_connection): + catalog_id = self._catalog_id(db_connection) with db_connection.get_connection() as conn: - entry = db_connection.upsert_catalog_entry(type="scheme", code="x-scheme", name="X", conn=conn) - with pytest.raises(psycopg.errors.ForeignKeyViolation): + with pytest.raises(psycopg.errors.CheckViolation): conn.execute( - "INSERT INTO documents (instance, source, document_kind, catalog_entry_id) " - "VALUES ('default', %s, 'advisory', %s::uuid)", - (psycopg.types.json.Jsonb({"fingerprint": _md5("k")}), entry["id"]), + "INSERT INTO documents (catalog_id, code, name, source) VALUES (%s::uuid, 'Bad Code', 'N', %s)", + (catalog_id, psycopg.types.json.Jsonb({"fingerprint": _md5("k")})), ) @pytest.mark.db @pytest.mark.unit def test_source_requires_a_hex_fingerprint(self, db_connection): + catalog_id = self._catalog_id(db_connection) with db_connection.get_connection() as conn: with pytest.raises(psycopg.errors.CheckViolation): conn.execute( - "INSERT INTO documents (instance, source) VALUES ('default', %s)", - (psycopg.types.json.Jsonb({"fingerprint": "not-hex"}),), + "INSERT INTO documents (catalog_id, code, name, source) VALUES (%s::uuid, 'c', 'C', %s)", + (catalog_id, psycopg.types.json.Jsonb({"fingerprint": "not-hex"})), ) @pytest.mark.db @@ -642,7 +655,8 @@ def test_stage_timestamps_only_allow_known_keys(self, db_connection): @pytest.mark.db @pytest.mark.unit def test_validity_period_must_not_end_before_it_starts(self, db_connection): - with pytest.raises(psycopg.errors.CheckViolation): + # The period lives in documents.metadata, so Pydantic enforces the order. + with pytest.raises(ValueError): _new_doc(db_connection, "wf-bad-period", valid_from="2026-02-01", valid_to="2026-01-01") @pytest.mark.db @@ -889,14 +903,14 @@ def test_source_spans_round_trip_and_derive_page_numbers(self, db_connection): @pytest.mark.db @pytest.mark.unit def test_search_chunks_matches_edited_text_first(self, db_connection): - _new_doc(db_connection, "wf-search", instance="mh") + _new_doc(db_connection, "wf-search", group="mh") db_connection.save_chunks("wf-search", [ {"chunk_number": 1, "original_text": "alpha beta", "edited_text": "gamma delta"}, ]) hits, total = db_connection.search_chunks(query="gamma") assert total == 1 and hits[0]["workflow_id"] == "wf-search" assert db_connection.search_chunks(query="alpha")[1] == 0 - assert db_connection.search_chunks(query="gamma", instances=["up"])[1] == 0 + assert db_connection.search_chunks(query="gamma", groups=["up"])[1] == 0 class TestIndexStatus: @@ -952,7 +966,7 @@ class TestAuditTrail: @pytest.mark.db @pytest.mark.unit def test_audit_outlives_the_document_and_keeps_tenant_scope(self, db_connection): - _new_doc(db_connection, "wf-aud", instance="mh") + _new_doc(db_connection, "wf-aud", group="mh") db_connection.log_audit( workflow_id="wf-aud", document_id="x", action_type="purge_document", entity_type="document", actor_user_id="kc-9", actor_username="boss", @@ -964,8 +978,8 @@ def test_audit_outlives_the_document_and_keeps_tenant_scope(self, db_connection) assert [entry["action_type"] for entry in logs] == ["purge_document"] assert logs[0]["actor"] == "b@x.in (superadmin)" assert logs[0]["actor_user_id"] == "kc-9" - assert db_connection.get_all_audit_log_count(instances=["mh"]) == 1 - assert db_connection.get_all_audit_log_count(instances=["up"]) == 0 + assert db_connection.get_all_audit_log_count(groups=["mh"]) == 1 + assert db_connection.get_all_audit_log_count(groups=["up"]) == 0 @pytest.mark.db @pytest.mark.unit @@ -1017,10 +1031,10 @@ def test_a_partial_update_keeps_the_rest_of_its_group(self, db_connection): class TestCatalogView: @staticmethod - def _scheme_doc(db, workflow_id, code, stage, **kwargs): - _new_doc(db, workflow_id, stage=stage, **kwargs) - db.update_document_fields( - workflow_id, document_kind="scheme", scheme_code=code, scheme_name=code.upper(), + def _scheme_doc(db, workflow_id, code, stage, kind="scheme", **kwargs): + _new_doc( + db, workflow_id, stage=stage, kind=kind, code=code, + name=code.upper(), **kwargs, ) @pytest.mark.db @@ -1034,7 +1048,6 @@ def test_status_rules(self, db_connection): self._scheme_doc(db, "wf-wip", "wip-one", "chunk_review") self._scheme_doc(db, "wf-off", "off-one", "completed") db.set_document_disabled("wf-off", True) - db.upsert_catalog_entry(type="scheme", code="boot-one", name="Boot", source="bootstrap") status = {e["code"]: e["status"] for e in db.list_catalog_entries()} assert status == { @@ -1043,62 +1056,71 @@ def test_status_rules(self, db_connection): "gate-one": "pending_prod", "wip-one": "disabled", "off-one": "disabled", - "boot-one": "live", } @pytest.mark.db @pytest.mark.unit - def test_computed_columns(self, db_connection): + def test_computed_columns(self, db_connection, monkeypatch): + monkeypatch.setenv("PROD_VECTOR_DB_COLLECTION_NAME", "prod-docs") db = db_connection - self._scheme_doc(db, "wf-a", "multi", "completed", instance="mh") - self._scheme_doc(db, "wf-b", "multi", "completed", instance="up") + self._scheme_doc(db, "wf-a", "scheme-a", "completed", group="mh") + self._scheme_doc(db, "wf-b", "scheme-b", "completed", group="up") db.save_chunks("wf-a", [{"chunk_number": 1, "original_text": "x"}]) db.save_chunks("wf-b", [{"chunk_number": 1, "original_text": "y"}, {"chunk_number": 2, "original_text": "z"}]) db.update_document_fields("wf-b", network_visible=False) - db.upsert_document_index_status("wf-a", "documents-index", last_indexed_at="2026-05-01T00:00:00") + db.upsert_document_index_status("wf-a", "documents-index", last_indexed_at="2026-04-01T00:00:00") + db.upsert_document_index_status("wf-a", "prod-docs", last_indexed_at="2026-05-01T00:00:00") - entry = db.get_catalog_entry("multi") - assert entry["instances"] == ["mh", "up"] - assert sorted(entry["workflow_ids"]) == ["wf-a", "wf-b"] - assert entry["chunk_count"] == 3 - assert entry["network_visible"] is True - assert entry["promoted_at"] is None + a = db.get_catalog_entry("scheme-a") + b = db.get_catalog_entry("scheme-b") + assert (a["groups"], a["workflow_ids"], a["chunk_count"]) == (["mh"], ["wf-a"], 1) + assert (b["groups"], b["workflow_ids"], b["chunk_count"]) == (["up"], ["wf-b"], 2) + assert a["network_visible"] is True and b["network_visible"] is False + assert a["promoted_at"].startswith("2026-05-01T00:00:00") + assert b["promoted_at"] is None @pytest.mark.db @pytest.mark.unit - def test_entries_without_documents_default_to_the_default_instance(self, db_connection): - db_connection.upsert_catalog_entry(type="scheme", code="lonely", name="Lonely", source="import") - entry = db_connection.get_catalog_entry("lonely") - assert entry["instances"] == ["default"] - assert entry["status"] == "disabled" - assert entry["network_visible"] is False + def test_catalogs_without_documents_are_not_listed(self, db_connection): + db = db_connection + with db.get_connection() as conn: + db.get_or_create_catalog(db.get_or_create_group("mh", conn=conn), "scheme", conn=conn) + assert db.list_catalog_entries() == [] + assert db.get_catalog_entry("anything") is None @pytest.mark.db @pytest.mark.unit - def test_other_entry_types_stay_out_of_the_scheme_listing(self, db_connection): - _new_doc(db_connection, "wf-adv", stage="completed") - db_connection.update_document_fields( - "wf-adv", document_kind="advisory", scheme_code="kharif", scheme_name="Kharif", - ) + def test_other_kinds_stay_out_of_the_scheme_listing(self, db_connection): + self._scheme_doc(db_connection, "wf-adv", "kharif", "completed", kind="advisory") assert db_connection.list_catalog_entries() == [] - assert [e["code"] for e in db_connection.list_catalog_entries(type="advisory")] == ["kharif"] + assert [e["code"] for e in db_connection.list_catalog_entries(kind="advisory")] == ["kharif"] + assert db_connection.list_document_codes("advisory") == {"kharif"} + assert db_connection.list_document_codes("scheme") == set() @pytest.mark.db @pytest.mark.unit - def test_changing_the_kind_moves_or_unlinks_the_entry(self, db_connection): - _new_doc(db_connection, "wf-move") - db_connection.update_document_fields( - "wf-move", document_kind="scheme", scheme_code="movable", scheme_name="Movable", - ) - db_connection.update_document_fields("wf-move", document_kind="advisory") + def test_changing_the_kind_moves_the_document_and_rejects_unknown_kinds(self, db_connection): + _new_doc(db_connection, "wf-move", kind="scheme", code="movable", name="Movable") + before = db_connection.get_document("wf-move")["catalog_id"] + db_connection.update_document_fields("wf-move", kind="advisory") doc = db_connection.get_document("wf-move") - assert (doc["document_kind"], doc["scheme_code"], doc["scheme_name"]) == ( + assert (doc["kind"], doc["code"], doc["name"]) == ( "advisory", "movable", "Movable", ) - db_connection.update_document_fields("wf-move", document_kind="document") - doc = db_connection.get_document("wf-move") - assert doc["scheme_code"] is None and doc["catalog_entry_id"] is None + assert doc["catalog_id"] != before + with pytest.raises(ValueError): + db_connection.update_document_fields("wf-move", kind="document") + assert db_connection.get_document("wf-move")["kind"] == "advisory" + + @pytest.mark.db + @pytest.mark.unit + def test_used_codes_exclude_the_document_itself(self, db_connection): + self._scheme_doc(db_connection, "wf-u1", "taken", "completed") + self._scheme_doc(db_connection, "wf-u2", "other", "completed") + assert db_connection.list_used_scheme_codes() == {"taken", "other"} + assert db_connection.list_used_scheme_codes(exclude_workflow_id="wf-u1") == {"other"} + assert db_connection.list_used_scheme_codes(kind="advisory") == set() @pytest.mark.db @pytest.mark.unit @@ -1157,34 +1179,32 @@ class TestOperationsViews: @pytest.mark.db @pytest.mark.unit def test_summary_counts_and_queue_use_the_current_workflow(self, db_connection): - _new_doc(db_connection, "wf-q1", stage="ocr_review", instance="mh") - _new_doc(db_connection, "wf-q2", stage="completed", instance="up") + _new_doc(db_connection, "wf-q1", stage="ocr_review", group="mh") + _new_doc(db_connection, "wf-q2", stage="completed", group="up") db_connection.create_document_job("wf-q2", "reingest", current_stage="ingesting") counts = db_connection.get_document_summary_counts() assert counts["total_documents"] == 2 assert counts["ocr_review_documents"] == 1 assert counts["completed_documents"] == 0 assert counts["running_jobs"] == 3 - assert db_connection.get_document_summary_counts(instances=[])["total_documents"] == 0 - by_instance = {r["instance"]: r for r in db_connection.get_document_counts_by_instance()} - assert by_instance["mh"]["dev_approval"] == 1 + assert db_connection.get_document_summary_counts(groups=[])["total_documents"] == 0 + by_group = {r["group"]: r for r in db_connection.get_document_counts_by_group()} + assert by_group["mh"]["dev_approval"] == 1 rows, total = db_connection.list_operations_queue() assert total == 2 assert {r["workflow_id"]: r["job_type"] for r in rows} == { "wf-q1": "pipeline", "wf-q2": "reingest", } - assert db_connection.list_operations_queue(instances=["up"])[1] == 1 + assert db_connection.list_operations_queue(groups=["up"])[1] == 1 @pytest.mark.db @pytest.mark.unit - def test_demo_and_disabled_documents_are_hidden_by_default(self, db_connection): - _new_doc(db_connection, "wf-demo") + def test_disabled_documents_are_hidden_by_default(self, db_connection): _new_doc(db_connection, "wf-off") - db_connection.set_document_demo("wf-demo", True) db_connection.set_document_disabled("wf-off", True) assert db_connection.list_documents() == [] - assert len(db_connection.list_documents(include_demo=True, include_disabled=True)) == 2 + assert len(db_connection.list_documents(include_disabled=True)) == 1 @pytest.mark.db @pytest.mark.unit diff --git a/tests/test_discovery_publish_service.py b/tests/test_discovery_publish_service.py index 30052a9..66e2198 100644 --- a/tests/test_discovery_publish_service.py +++ b/tests/test_discovery_publish_service.py @@ -101,7 +101,7 @@ def test_publish_success_envelope_shape(self, monkeypatch): mock_client.post.return_value = mock_response with patch("pipeline.discovery_publish_service.httpx.Client", return_value=mock_client): - result = service.publish(transaction_id="txn-123", document_kind="advisory") + result = service.publish(transaction_id="txn-123", kind="advisory") mock_client.post.assert_called_once() call_args = mock_client.post.call_args @@ -156,8 +156,8 @@ def test_publish_generates_fresh_message_id_per_call(self, monkeypatch): mock_client.post.return_value = mock_response with patch("pipeline.discovery_publish_service.httpx.Client", return_value=mock_client): - first = service.publish(transaction_id="txn-shared", document_kind="advisory") - second = service.publish(transaction_id="txn-shared", document_kind="advisory") + first = service.publish(transaction_id="txn-shared", kind="advisory") + second = service.publish(transaction_id="txn-shared", kind="advisory") assert first["envelope"]["context"]["transactionId"] == "txn-shared" assert second["envelope"]["context"]["transactionId"] == "txn-shared" @@ -183,7 +183,7 @@ def test_publish_raises_on_http_error(self, monkeypatch): with patch("pipeline.discovery_publish_service.httpx.Client", return_value=mock_client): with pytest.raises(httpx.HTTPStatusError): - service.publish(transaction_id="txn-123", document_kind="advisory") + service.publish(transaction_id="txn-123", kind="advisory") @pytest.mark.unit def test_publish_raises_on_connection_error(self, monkeypatch): @@ -200,7 +200,7 @@ def test_publish_raises_on_connection_error(self, monkeypatch): with patch("pipeline.discovery_publish_service.httpx.Client", return_value=mock_client): with pytest.raises(httpx.ConnectError): - service.publish(transaction_id="txn-123", document_kind="advisory") + service.publish(transaction_id="txn-123", kind="advisory") class TestDiscoveryPublishServiceSchemeCatalog: @@ -223,7 +223,7 @@ def test_scheme_kind_publishes_the_schemes_catalog(self, monkeypatch): mock_client.post.return_value = mock_response with patch("pipeline.discovery_publish_service.httpx.Client", return_value=mock_client): - result = service.publish(transaction_id="txn-9", document_kind="scheme") + result = service.publish(transaction_id="txn-9", kind="scheme") envelope = mock_client.post.call_args.kwargs["json"] catalog = envelope["message"]["catalogs"][0] @@ -247,7 +247,7 @@ def test_no_http_call_is_made(self, monkeypatch, kind): mock_client = MagicMock() with patch("pipeline.discovery_publish_service.httpx.Client", return_value=mock_client): - result = service.publish(transaction_id="txn-skip", document_kind=kind) + result = service.publish(transaction_id="txn-skip", kind=kind) mock_client.post.assert_not_called() assert result["skipped"] is True @@ -268,7 +268,7 @@ def test_reports_the_result_status_for_our_catalog(self, monkeypatch, status): "pipeline.discovery_publish_service.httpx.Client", return_value=_client_returning(response), ): - result = service.publish(transaction_id="txn-v", document_kind="advisory") + result = service.publish(transaction_id="txn-v", kind="advisory") assert result["result_status"] == status @@ -284,7 +284,7 @@ def test_carries_the_errors_array_through(self, monkeypatch): "pipeline.discovery_publish_service.httpx.Client", return_value=_client_returning(response), ): - result = service.publish(transaction_id="txn-v", document_kind="advisory") + result = service.publish(transaction_id="txn-v", kind="advisory") assert result["errors"] == errors @@ -309,7 +309,7 @@ def test_picks_our_catalog_out_of_several_results(self, monkeypatch): "pipeline.discovery_publish_service.httpx.Client", return_value=_client_returning(response), ): - result = service.publish(transaction_id="txn-v", document_kind="advisory") + result = service.publish(transaction_id="txn-v", kind="advisory") assert result["result_status"] == "ACCEPTED" @@ -336,7 +336,7 @@ def test_unreadable_result_is_unknown_not_a_failure(self, monkeypatch, body): "pipeline.discovery_publish_service.httpx.Client", return_value=_client_returning(response), ): - result = service.publish(transaction_id="txn-v", document_kind="advisory") + result = service.publish(transaction_id="txn-v", kind="advisory") assert result["result_status"] is None assert result["errors"] == [] @@ -354,7 +354,7 @@ def test_non_json_body_is_unknown_not_a_failure(self, monkeypatch): "pipeline.discovery_publish_service.httpx.Client", return_value=_client_returning(response), ): - result = service.publish(transaction_id="txn-v", document_kind="advisory") + result = service.publish(transaction_id="txn-v", kind="advisory") assert result["result_status"] is None assert result["errors"] == [] @@ -364,7 +364,7 @@ def test_skipped_publish_has_no_result_status(self, monkeypatch): service = _service(monkeypatch) with patch("pipeline.discovery_publish_service.httpx.Client", return_value=MagicMock()): - result = service.publish(transaction_id="txn-v", document_kind="video") + result = service.publish(transaction_id="txn-v", kind="video") assert result["skipped"] is True assert result["result_status"] is None @@ -387,7 +387,7 @@ def test_logs_the_request_and_the_response_status(self, monkeypatch, caplog): ) with patch("pipeline.discovery_publish_service.httpx.Client", return_value=ok): service.publish( - transaction_id="txn-log", document_kind="advisory", workflow_id="wf-log" + transaction_id="txn-log", kind="advisory", workflow_id="wf-log" ) messages = [r.getMessage() for r in caplog.records] @@ -413,7 +413,7 @@ def test_logs_an_error_when_the_call_fails(self, monkeypatch, caplog): with patch("pipeline.discovery_publish_service.httpx.Client", return_value=client): with pytest.raises(httpx.ConnectError): service.publish( - transaction_id="txn-log", document_kind="advisory", workflow_id="wf-log" + transaction_id="txn-log", kind="advisory", workflow_id="wf-log" ) errors = [r.getMessage() for r in caplog.records if r.levelno == logging.ERROR] @@ -433,7 +433,7 @@ def test_skipped_publish_logs_no_request(self, monkeypatch, caplog): "pipeline.discovery_publish_service.httpx.Client", return_value=MagicMock() ): service.publish( - transaction_id="txn-log", document_kind="video", workflow_id="wf-log" + transaction_id="txn-log", kind="video", workflow_id="wf-log" ) messages = [r.getMessage() for r in caplog.records] diff --git a/tests/test_document_repository.py b/tests/test_document_repository.py index 85d00d8..caa5d2f 100644 --- a/tests/test_document_repository.py +++ b/tests/test_document_repository.py @@ -24,7 +24,7 @@ def get_document(self, workflow_id): class TestGetDocumentKind: @pytest.mark.unit def test_returns_the_stored_kind(self): - repo = DocumentRepository(FakeDb(document={"document_kind": "advisory"})) + repo = DocumentRepository(FakeDb(document={"kind": "advisory"})) assert repo.get_document_kind("wf-1") == "advisory" @@ -35,42 +35,56 @@ def test_returns_none_for_a_missing_document(self): assert repo.get_document_kind("wf-nope") is None @pytest.mark.unit - def test_returns_none_when_the_column_is_null(self): - # Old rows predate the column default, so it can be NULL. - repo = DocumentRepository(FakeDb(document={"document_kind": None})) + def test_returns_none_when_the_kind_is_missing(self): + # A catalog type the pipeline has no kind for reads as None. + repo = DocumentRepository(FakeDb(document={"kind": None})) assert repo.get_document_kind("wf-1") is None @pytest.mark.unit def test_does_not_normalise_the_value(self): - repo = DocumentRepository(FakeDb(document={"document_kind": " Advisory "})) + repo = DocumentRepository(FakeDb(document={"kind": " Advisory "})) assert repo.get_document_kind("wf-1") == " Advisory " @pytest.mark.unit def test_passes_the_workflow_id_through(self): - fake = FakeDb(document={"document_kind": "scheme"}) + fake = FakeDb(document={"kind": "scheme"}) DocumentRepository(fake).get_document_kind("wf-42") assert fake.get_document_called_with == "wf-42" -class TestAgainstRealSqlite: +@pytest.fixture +def scheme_document(db_connection): + """A document registered the v2 way (it needs a kind and a code).""" + db_connection.upsert_document( + workflow_id="repo-wf-1", + document_id="repo-doc-1", + filename="repo.pdf", + filepath="/app/books/repo.pdf", + stage="registered", + kind="scheme", + code="repo-scheme", + ) + return "repo-wf-1" + + +class TestAgainstRealPostgres: """Guards against db.py drifting out from under the repository.""" @pytest.mark.unit @pytest.mark.db - def test_reads_a_kind_written_through_the_db_layer(self, db_connection, sample_document): - workflow_id = sample_document["workflow_id"] - db_connection.update_document_fields(workflow_id, document_kind="advisory") - - assert DocumentRepository().get_document_kind(workflow_id) == "advisory" + def test_reads_the_kind_of_the_documents_catalog(self, db_connection, scheme_document): + assert DocumentRepository().get_document_kind(scheme_document) == "scheme" @pytest.mark.unit @pytest.mark.db - def test_unclassified_document_reads_as_the_column_default(self, db_connection, sample_document): - assert DocumentRepository().get_document_kind(sample_document["workflow_id"]) == "document" + def test_reads_a_kind_written_through_the_db_layer(self, db_connection, scheme_document): + db_connection.update_document_fields(scheme_document, kind="advisory") + + assert DocumentRepository().get_document_kind(scheme_document) == "advisory" @pytest.mark.unit @pytest.mark.db diff --git a/tests/test_instances.py b/tests/test_group_names.py similarity index 81% rename from tests/test_instances.py rename to tests/test_group_names.py index 8e7c043..449afdb 100644 --- a/tests/test_instances.py +++ b/tests/test_group_names.py @@ -1,9 +1,9 @@ -"""Tests for instance code → state name resolution.""" +"""Tests for group (tenant) code → state name resolution.""" import pytest -class TestInstanceDisplayName: +class TestGroupDisplayName: @pytest.mark.unit @pytest.mark.parametrize( "code,expected", @@ -22,30 +22,30 @@ class TestInstanceDisplayName: ], ) def test_known_codes(self, code, expected): - from pipeline.instances import instance_display_name + from pipeline.group_names import group_display_name - assert instance_display_name(code) == expected + assert group_display_name(code) == expected @pytest.mark.unit @pytest.mark.parametrize("code,expected", [("or", "Odisha"), ("od", "Odisha"), ("ts", "Telangana"), ("tg", "Telangana")]) def test_legacy_code_aliases(self, code, expected): - from pipeline.instances import instance_display_name + from pipeline.group_names import group_display_name - assert instance_display_name(code) == expected + assert group_display_name(code) == expected @pytest.mark.unit def test_blank_falls_back_to_default(self): - from pipeline.instances import instance_display_name + from pipeline.group_names import group_display_name - assert instance_display_name("") == "Default" - assert instance_display_name(None) == "Default" + assert group_display_name("") == "Default" + assert group_display_name(None) == "Default" @pytest.mark.unit def test_unknown_code_is_made_presentable(self): """A new state must degrade to a readable label, never break ingestion.""" - from pipeline.instances import instance_display_name + from pipeline.group_names import group_display_name - assert instance_display_name("new-state") == "New State" + assert group_display_name("new-state") == "New State" class TestProdStageDisabled: @@ -54,7 +54,7 @@ class TestProdStageDisabled: @pytest.mark.unit @pytest.mark.parametrize("value", ["true", "TRUE", "True", "1", "yes", "YES"]) def test_truthy_values_disable_prod(self, monkeypatch, value): - from pipeline.instances import prod_stage_disabled + from pipeline.group_names import prod_stage_disabled monkeypatch.setenv("DISABLE_PROD_SETTING", value) assert prod_stage_disabled() is True @@ -62,7 +62,7 @@ def test_truthy_values_disable_prod(self, monkeypatch, value): @pytest.mark.unit @pytest.mark.parametrize("value", ["false", "FALSE", "0", "no", "", "anything-else"]) def test_other_values_keep_prod_enabled(self, monkeypatch, value): - from pipeline.instances import prod_stage_disabled + from pipeline.group_names import prod_stage_disabled monkeypatch.setenv("DISABLE_PROD_SETTING", value) assert prod_stage_disabled() is False @@ -70,14 +70,14 @@ def test_other_values_keep_prod_enabled(self, monkeypatch, value): @pytest.mark.unit def test_defaults_to_enabled_when_unset(self, monkeypatch): """Absent flag must keep the full pipeline — never silently skip PROD.""" - from pipeline.instances import prod_stage_disabled + from pipeline.group_names import prod_stage_disabled monkeypatch.delenv("DISABLE_PROD_SETTING", raising=False) assert prod_stage_disabled() is False @pytest.mark.unit def test_whitespace_is_tolerated(self, monkeypatch): - from pipeline.instances import prod_stage_disabled + from pipeline.group_names import prod_stage_disabled monkeypatch.setenv("DISABLE_PROD_SETTING", " true ") assert prod_stage_disabled() is True @@ -121,7 +121,7 @@ def test_publishing_to_network_is_prod_only(self): ("true", True), ("TRUE", True), (" 1 ", True), ("yes", True)], ) def test_publish_to_network_enabled(monkeypatch, value, expected): - from pipeline.instances import publish_to_network_enabled + from pipeline.group_names import publish_to_network_enabled if value is None: monkeypatch.delenv("PUBLISH_TO_NETWORK", raising=False) diff --git a/tests/test_groups.py b/tests/test_groups.py index 96bcb23..f90d1f7 100644 --- a/tests/test_groups.py +++ b/tests/test_groups.py @@ -9,11 +9,11 @@ ROLE_STATE_VIEW, ROLE_SUPER_ADMIN, parse_group_paths, - role_for_instance, + role_for_group, ) from pipeline.auth.jwt import claims_to_user from pipeline.auth.permissions import Permission, UserRole, permissions_for_roles -from pipeline.auth.tenancy import allowed_instances, user_can_access_instance +from pipeline.auth.tenancy import allowed_groups, user_can_access_group def test_parse_global_super_admin(): @@ -33,9 +33,9 @@ def test_parse_multi_state_roles(): assert access.is_super_admin is False assert access.state_roles == {"mh": ROLE_STATE_ADMIN, "up": ROLE_STATE_VIEW} assert set(access.instances) == {"mh", "up"} - assert role_for_instance(access, "MH") == ROLE_STATE_ADMIN - assert role_for_instance(access, "up") == ROLE_STATE_VIEW - assert role_for_instance(access, "bh") is None + assert role_for_group(access, "MH") == ROLE_STATE_ADMIN + assert role_for_group(access, "up") == ROLE_STATE_VIEW + assert role_for_group(access, "bh") is None def test_approver_and_contributor_paths(): @@ -61,17 +61,17 @@ def test_contributor_approves_stages_but_cannot_publish_to_dev(): # OCR / translation / chunk approvals all ride on REVIEW. assert user.has_permission(Permission.REVIEW) - assert user.has_permission_for_instance(Permission.REVIEW, "up") + assert user.has_permission_for_group(Permission.REVIEW, "up") # …but publishing to DEV is its own permission. assert not user.has_permission(Permission.APPROVE_INGESTION) - assert not user.has_permission_for_instance(Permission.APPROVE_INGESTION, "up") + assert not user.has_permission_for_group(Permission.APPROVE_INGESTION, "up") assert not user.has_permission(Permission.DELETE_OWN) # Approver and admin both clear the DEV publish gate. for group, role in (("approver", "state_approver"), ("admin", "state_admin")): stronger = claims_to_user({"sub": role, "groups": [f"/states/UP/{group}"]}) assert stronger.has_permission(Permission.APPROVE_INGESTION), role - assert stronger.has_permission_for_instance(Permission.APPROVE_INGESTION, "up"), role + assert stronger.has_permission_for_group(Permission.APPROVE_INGESTION, "up"), role # Only the admin may delete. approver = claims_to_user({"sub": "a", "groups": ["/states/UP/approver"]}) @@ -109,14 +109,14 @@ def test_bh_viewer_reads_every_state_without_write(): assert access.is_super_admin is False assert access.state_roles == {} # No per-state group, but any state resolves to view. - assert role_for_instance(access, "mh") == ROLE_STATE_VIEW - assert role_for_instance(access, "wb") == ROLE_STATE_VIEW + assert role_for_group(access, "mh") == ROLE_STATE_VIEW + assert role_for_group(access, "wb") == ROLE_STATE_VIEW def test_bh_viewer_keeps_stronger_state_role(): access = parse_group_paths(["/global/bh-viewer", "/states/MH/admin"]) - assert role_for_instance(access, "mh") == ROLE_STATE_ADMIN - assert role_for_instance(access, "up") == ROLE_STATE_VIEW + assert role_for_group(access, "mh") == ROLE_STATE_ADMIN + assert role_for_group(access, "up") == ROLE_STATE_VIEW def test_higher_role_wins_same_state(): @@ -172,17 +172,17 @@ def test_claims_to_user_from_groups_only(): assert user.state_roles["mh"] == "state_admin" assert user.state_roles["up"] == "state_view" assert user.is_superadmin is False - assert allowed_instances(user) == {"mh", "up"} - assert user_can_access_instance(user, "mh") - assert not user_can_access_instance(user, "bh") + assert allowed_groups(user) == {"mh", "up"} + assert user_can_access_group(user, "mh") + assert not user_can_access_group(user, "bh") # Union of state_admin + state_view permissions assert Permission.UPLOAD in user.permissions assert Permission.SEARCH in user.permissions assert Permission.MANAGE_USERS not in user.permissions - assert user.has_permission_for_instance(Permission.UPLOAD, "mh") is True - assert user.has_permission_for_instance(Permission.UPLOAD, "up") is False - assert user.has_permission_for_instance(Permission.SEARCH, "up") is True - assert user.has_permission_for_instance(Permission.REVIEW, "up") is False + assert user.has_permission_for_group(Permission.UPLOAD, "mh") is True + assert user.has_permission_for_group(Permission.UPLOAD, "up") is False + assert user.has_permission_for_group(Permission.SEARCH, "up") is True + assert user.has_permission_for_group(Permission.REVIEW, "up") is False def test_claims_super_admin_unrestricted(): @@ -194,8 +194,8 @@ def test_claims_super_admin_unrestricted(): } ) assert user.is_superadmin is True - assert allowed_instances(user) is None - assert user_can_access_instance(user, "mh") + assert allowed_groups(user) is None + assert user_can_access_group(user, "mh") assert Permission.MANAGE_USERS in user.permissions @@ -210,5 +210,5 @@ def test_legacy_instances_claim_still_works(): ) assert user.instances == ["tenant-a"] assert Permission.UPLOAD in user.permissions - assert user_can_access_instance(user, "tenant-a") - assert not user_can_access_instance(user, "mh") + assert user_can_access_group(user, "tenant-a") + assert not user_can_access_group(user, "mh") diff --git a/tests/test_migrate_sqlite_to_postgres.py b/tests/test_migrate_sqlite_to_postgres.py index e963846..99b2884 100644 --- a/tests/test_migrate_sqlite_to_postgres.py +++ b/tests/test_migrate_sqlite_to_postgres.py @@ -100,6 +100,8 @@ WF1 = f"doc-{FP1}" WF2 = f"doc-{FP2}" WF3 = "doc-broken" +WF4 = "doc-plain" +WF5 = "doc-demo" FUTURE = "2099-01-01T00:00:00" @@ -139,7 +141,7 @@ def legacy_db(tmp_path): conn, "documents", workflow_id=WF2, document_id=FP2, filename="inflight.pdf", filepath="/data/documents/inflight.pdf", stage="approval_for_prod", page_count=1, created_at="2026-04-01T10:00:00", updated_at="2026-04-01T12:00:00", instance="up", - is_demo=1, prod_ready_requested_at="2026-04-02T09:00:00", + document_kind="advisory", prod_ready_requested_at="2026-04-02T09:00:00", prod_ready_requested_by_user_id="kc-req", prod_ready_requested_by_username="rita", ocr_completed_at="2026-04-01T11:00:00", ) @@ -147,7 +149,18 @@ def legacy_db(tmp_path): conn, "documents", workflow_id=WF3, document_id="not-a-hash", filename="broken.pdf", filepath="/tmp/broken.pdf", stage="failed", error_message="OCR blew up", created_at="2026-05-01T10:00:00", updated_at="2026-05-01T10:30:00", instance=" ", - document_kind="document", scheme_code="Bad Code", valid_from="2026-06-01", valid_to="2026-01-01", + document_kind="advisory", scheme_code="Bad Code", valid_from="2026-06-01", valid_to="2026-01-01", + ) + _insert( + conn, "documents", workflow_id=WF4, document_id="plain", filename="plain.pdf", + filepath="/tmp/plain.pdf", stage="registered", created_at="2026-05-02T10:00:00", + updated_at="2026-05-02T10:00:00", instance="mh", document_kind="document", + ) + _insert( + conn, "documents", workflow_id=WF5, document_id="demo", filename="demo.pdf", + filepath="/tmp/demo.pdf", stage="completed", created_at="2026-05-03T10:00:00", + updated_at="2026-05-03T10:00:00", instance="mh", document_kind="scheme", scheme_code="demo-s", + is_demo=1, ) _insert(conn, "document_jobs", id=1, workflow_id=WF1, job_type="pipeline", temporal_workflow_id=WF1, @@ -175,6 +188,7 @@ def legacy_db(tmp_path): _insert(conn, "pages", workflow_id=WF1, page_number=2, original_markdown="english page", detected_language="en", reviewer_notes="fine") _insert(conn, "pages", workflow_id="doc-purged", page_number=1, original_markdown="stray") + _insert(conn, "pages", workflow_id=WF5, page_number=1, original_markdown="demo page") _insert(conn, "chunks", workflow_id=WF1, chunk_number=1, original_text="chunk one", token_count=5, page_start=1, page_end=2, source_spans_json=json.dumps([{"page_number": 1, "start_char": 0, "end_char": 5}]), @@ -195,6 +209,8 @@ def legacy_db(tmp_path): _insert(conn, "audit_logs", workflow_id=WF1, document_id=FP1, action_type="page_edit", entity_type="page", entity_id=1, metadata="not json", timestamp="2026-03-02T12:00:00", actor_user_id="kc-rev", actor_username="ravi", actor_email="ravi@x.in", actor_roles="reviewer") + _insert(conn, "audit_logs", workflow_id=WF5, document_id="demo", action_type="stage_change", + entity_type="document", timestamp="2026-05-03T11:00:00") _insert(conn, "audit_logs", workflow_id="doc-purged", document_id="gone", action_type="purge_document", entity_type="document", timestamp="2026-02-01T00:00:00") _insert(conn, "audit_logs", workflow_id="__system__", document_id="__settings__", @@ -238,7 +254,7 @@ def _counts(db): table: conn.execute(f"SELECT count(*) AS n FROM {table}").fetchone()["n"] for table in ( "documents", "workflows", "workflow_artifacts", "pages", "chunks", "audit_logs", - "users", "catalog_entries", "email_otps", "webhook_records", + "users", "groups", "catalogs", "email_otps", "webhook_records", ) } @@ -252,6 +268,7 @@ def test_dry_run_writes_nothing_but_reports_everything(self, db_connection, post report = script.run_migration(legacy_db, postgres_url, dry_run=True) assert _counts(db_connection) == before assert report.tables["documents"]["inserted"] == 3 + assert report.tables["documents"]["skipped"] == 2 assert report.mismatches() == [] def test_a_full_run_maps_every_table(self, db_connection, postgres_url, legacy_db): @@ -260,21 +277,28 @@ def test_a_full_run_maps_every_table(self, db_connection, postgres_url, legacy_d assert report.mismatches() == [] tables = report.as_dict()["tables"] - assert tables["documents"]["inserted"] == 3 + assert tables["documents"] == {"read": 5, "inserted": 3, "skipped": 2} assert tables["document_jobs"] == {"read": 3, "inserted": 2, "skipped": 1} - assert tables["pages"] == {"read": 3, "inserted": 2, "skipped": 1} + assert tables["pages"] == {"read": 4, "inserted": 2, "skipped": 2} assert tables["chunks"]["inserted"] == 2 assert tables["document_manifest_entries"] == {"read": 2, "dropped": 2} assert tables["email_otps"] == {"read": 2, "inserted": 1, "dropped": 1} assert tables["settings"]["skipped"] == 2 + assert tables["scheme_catalog_entries"] == {"read": 2, "inserted": 1, "dropped": 1} + skipped = {i["key"]: i["reason"] for i in report.skip_reasons if i["table"] == "documents"} + assert skipped == { + WF4: "document_kind 'document' has no catalog", + WF5: "demo document (is_demo)", + } + assert db.get_document(WF4) is None and db.get_document(WF5) is None doc = db.get_document(WF1) assert doc["stage"] == "completed" - assert doc["instance"] == "mh" + assert doc["group"] == "mh" assert doc["document_id"] == FP1 - assert doc["display_name"] == "Guideline" + assert doc["name"] == "Registry name" # one name per document in v2 assert (doc["valid_from"], doc["valid_to"]) == ("2026-03-01", "2027-03-01") - assert (doc["scheme_code"], doc["document_kind"]) == ("pm-kmy", "scheme") + assert (doc["code"], doc["kind"]) == ("pm-kmy", "scheme") assert doc["network_visible"] is False and doc["reindex_required"] is True assert doc["uploaded_by_username"] == "uma" assert doc["uploaded_by_roles"] == "contributor,reviewer" @@ -282,6 +306,8 @@ def test_a_full_run_maps_every_table(self, db_connection, postgres_url, legacy_d assert doc["ocr_completed_at"] == "2026-03-01T11:00:00" assert doc["chunk_count"] == 2 assert doc["page_count"] == 2 + assert doc["catalog_id"] + assert (doc["name"], doc["aliases"]) == ("Registry name", ["registry"]) assert set(doc["index_status"]) == {"dev"} assert doc["index_status"]["dev"]["index_name"] == "schemes-index" @@ -311,12 +337,32 @@ def test_a_full_run_maps_every_table(self, db_connection, postgres_url, legacy_d assert chunks[1]["page_end"] == 2 and chunks[1]["is_excluded"] is True entries = {e["code"]: e for e in db.list_catalog_entries()} - assert entries["cdp"]["source"] == "bootstrap" and entries["cdp"]["status"] == "live" + assert set(entries) == {"pm-kmy"} # the bootstrap-only "cdp" entry has no document assert entries["pm-kmy"]["name"] == "Registry name" assert entries["pm-kmy"]["status"] == "pending_reindex" assert entries["pm-kmy"]["workflow_ids"] == [WF1] assert db.get_catalog_meta()["version"] == 6 + with db.get_connection() as conn: + groups = {g["path"]: g for g in conn.execute("SELECT * FROM groups").fetchall()} + catalogs = conn.execute( + "SELECT g.path, c.type FROM catalogs c JOIN groups g ON g.id = c.group_id ORDER BY 1, 2" + ).fetchall() + meta = conn.execute( + "SELECT metadata FROM documents WHERE id = %s::uuid", (doc["id"],) + ).fetchone()["metadata"] + assert set(groups) == {"/states/MH", "/states/UP", "/states/DEFAULT"} + assert all(g["keycloak_group_id"] is None for g in groups.values()) + assert [(c["path"], c["type"]) for c in catalogs] == [ + ("/states/DEFAULT", "openagrinet:KnowledgeAdvisory"), + ("/states/MH", "openagrinet:AgricultureResource"), + ("/states/UP", "openagrinet:KnowledgeAdvisory"), + ] + assert (meta["valid_from"], meta["valid_to"]) == ("2026-03-01", "2027-03-01") + assert meta["uploaded_by_user_id"] == db.get_user_by_keycloak_id("kc-up")["id"] + assert meta["index_status"]["dev"]["index_name"] == "schemes-index" + assert not {"is_demo", "chunk_count", "display_name"} & set(meta) + users = {u: db.get_user_by_keycloak_id(u) for u in ("kc-up", "kc-rev", "kc-req")} assert all(users.values()) @@ -327,6 +373,9 @@ def test_a_full_run_maps_every_table(self, db_connection, postgres_url, legacy_d assert edit["metadata"] == json.dumps({"raw": "not json"}) purged = db.get_audit_logs("doc-purged") assert [e["action_type"] for e in purged] == ["purge_document"] + demo_trail = db.get_audit_logs(WF5) # kept for the skipped demo document, tied to its group + assert [e["action_type"] for e in demo_trail] == ["stage_change"] + assert demo_trail[0]["group"] == "mh" settings_log = db.get_settings_audit_logs() assert settings_log[0]["field_name"] == "retrieval.alpha" @@ -348,10 +397,10 @@ def test_in_flight_documents_keep_their_stage_and_prod_request(self, db_connecti doc = db.get_document(WF2) assert doc["stage"] == "approval_for_prod" - assert doc["is_demo"] is True + assert doc["kind"] == "advisory" and doc["code"] == "inflight" assert doc["prod_ready_requested_at"] == "2026-04-02T09:00:00" assert doc["prod_ready_requested_by_username"] == "rita" - assert doc["instance"] == "up" + assert doc["group"] == "up" assert doc["ocr_completed_at"] == "2026-04-01T11:00:00" types = sorted(j["job_type"] for j in db.list_document_jobs(WF2)) assert types == ["pipeline", "promote_prod"] @@ -366,11 +415,11 @@ def test_unusable_values_are_repaired_and_noted(self, db_connection, postgres_ur doc = db.get_document(WF3) assert doc["stage"] == "failed" and doc["error_message"] == "OCR blew up" assert doc["document_id"] == hashlib.md5(b"not-a-hash").hexdigest() - assert doc["instance"] == "default" - assert doc["scheme_code"] is None + assert doc["group"] == "default" + assert doc["kind"] == "advisory" and doc["code"] == "broken" assert doc["valid_from"] == "2026-06-01" and doc["valid_to"] is None notes = " ".join(report.notes) - assert "valid_to cleared" in notes and "not a valid slug" in notes or "plain document" in notes + assert "valid_to cleared" in notes and "not a valid slug, code derived" in notes def test_running_it_twice_changes_nothing(self, db_connection, postgres_url, legacy_db): script = _load_script() @@ -380,7 +429,7 @@ def test_running_it_twice_changes_nothing(self, db_connection, postgres_url, leg report = script.run_migration(legacy_db, postgres_url) assert _counts(db_connection) == first - assert report.tables["documents"] == {"read": 3, "existing": 3} + assert report.tables["documents"] == {"read": 5, "existing": 3, "skipped": 2} assert report.tables["audit_logs"]["inserted"] == 0 assert report.tables["webhook_records"]["existing"] == 1 assert report.mismatches() == [] diff --git a/tests/test_provenance.py b/tests/test_provenance.py index f7a1794..5b52532 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -1,6 +1,7 @@ """Tests for vector-store provenance ingest fields and /provenance/chunk resolver.""" import hashlib +import itertools import pytest from fastapi.testclient import TestClient @@ -8,6 +9,13 @@ from pipeline.activities import prepare_ingestion_records from pipeline.api import app +_code_seq = itertools.count(1) + + +def _next_code() -> str: + """A unique document code for fixtures (documents need one in v2).""" + return f"doc-{next(_code_seq)}" + @pytest.mark.unit def test_prepare_records_includes_provenance_fields(): @@ -52,12 +60,14 @@ def test_find_document_by_legacy_doc_id_hash(db_connection): document_id = "legacy-doc-id-001" workflow_id = "doc-legacy000001" db.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id=document_id, filename="legacy.pdf", filepath="/tmp/legacy.pdf", stage="completed", - display_name="Legacy PDF", + name="Legacy PDF", ) db.save_chunks( workflow_id, @@ -97,12 +107,14 @@ def test_provenance_chunk_endpoint(db_connection): workflow_id = "doc-provenance01" document_id = hashlib.md5(b"prov-doc-id-xyz").hexdigest() db.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id=document_id, filename="prov.pdf", filepath="/tmp/prov.pdf", stage="completed", - display_name="Provenance PDF", + name="Provenance PDF", ) db.save_chunks( workflow_id, diff --git a/tests/test_schema_v2.py b/tests/test_schema_v2.py new file mode 100644 index 0000000..df13c54 --- /dev/null +++ b/tests/test_schema_v2.py @@ -0,0 +1,598 @@ +""" +Schema v2 rules: groups, catalogs, documents keyed by (catalog, code), the +group snapshot on audit_logs and the catalog_status view. + +Everything runs against the real Postgres from conftest.py. +""" + +import hashlib +import logging + +import psycopg +import pytest +from psycopg.types.json import Jsonb + + +def _md5(text): + return hashlib.md5(text.encode()).hexdigest() + + +def _doc(db, workflow_id, **kwargs): + kwargs.setdefault("document_id", _md5(workflow_id)) + kwargs.setdefault("filename", f"{workflow_id}.pdf") + kwargs.setdefault("filepath", f"minio://documents/{workflow_id}.pdf") + kwargs.setdefault("kind", "scheme") + kwargs.setdefault("code", workflow_id) + return db.upsert_document(workflow_id=workflow_id, **kwargs) + + +def _group(db, group="mh"): + return db.get_or_create_group(group) + + +class TestGroups: + @pytest.mark.db + @pytest.mark.unit + def test_get_or_create_group_is_idempotent_and_maps_tenant_codes_to_paths(self, db_connection): + db = db_connection + mh = _group(db, "mh") + assert _group(db, "MH") == mh + assert _group(db, "up") != mh + bv = _group(db, "bv") + with db.get_connection() as conn: + rows = {r["name"]: r["path"] for r in conn.execute("SELECT name, path FROM groups").fetchall()} + assert rows == {"mh": "/states/MH", "up": "/states/UP", "bv": "/global"} + assert _group(db, "bv") == bv + + @pytest.mark.db + @pytest.mark.unit + def test_upsert_group_fills_the_keycloak_id_of_an_existing_path(self, db_connection): + db = db_connection + first = db.upsert_group("/states/MH") # as created by the SQLite move: no Keycloak id yet + again = db.upsert_group("/states/MH", keycloak_group_id="kc-group-1") + assert again == first + with db.get_connection() as conn: + row = conn.execute("SELECT keycloak_group_id, name FROM groups WHERE id = %s::uuid", (first,)).fetchone() + assert (row["keycloak_group_id"], row["name"]) == ("kc-group-1", "mh") + # A later sync that does not know the id must not erase it. + db.upsert_group("/states/MH") + with db.get_connection() as conn: + row = conn.execute("SELECT keycloak_group_id FROM groups WHERE id = %s::uuid", (first,)).fetchone() + assert row["keycloak_group_id"] == "kc-group-1" + + @pytest.mark.db + @pytest.mark.unit + def test_keycloak_group_id_is_unique_but_nullable(self, db_connection): + with db_connection.get_connection() as conn: + conn.execute("INSERT INTO groups (name, path) VALUES ('a', '/states/A')") + conn.execute("INSERT INTO groups (name, path) VALUES ('b', '/states/B')") # two NULLs are fine + conn.execute("INSERT INTO groups (keycloak_group_id, name, path) VALUES ('k', 'c', '/states/C')") + with pytest.raises(psycopg.errors.UniqueViolation): + conn.execute("INSERT INTO groups (keycloak_group_id, name, path) VALUES ('k', 'd', '/states/D')") + + @pytest.mark.db + @pytest.mark.unit + def test_a_group_with_catalogs_cannot_be_deleted(self, db_connection): + db = db_connection + group_id = _group(db) + db.get_or_create_catalog(group_id, "scheme") + with db.get_connection() as conn: + with pytest.raises(psycopg.errors.ForeignKeyViolation): + conn.execute("DELETE FROM groups WHERE id = %s::uuid", (group_id,)) + + @pytest.mark.db + @pytest.mark.unit + def test_deleting_a_group_clears_users_and_audit_snapshots(self, db_connection): + db = db_connection + group_id = _group(db, "zz") + db.upsert_user("kc-zz", "zed", None, [], tenants=["zz"]) + with db.get_connection() as conn: + conn.execute( + "INSERT INTO audit_logs (action_type, entity_type, group_id) VALUES ('x', 'setting', %s::uuid)", + (group_id,), + ) + conn.execute("DELETE FROM groups WHERE id = %s::uuid", (group_id,)) + assert conn.execute("SELECT group_id FROM audit_logs WHERE action_type = 'x'").fetchone()["group_id"] is None + assert db.get_user_by_keycloak_id("kc-zz")["group_id"] is None + + +class TestCatalogs: + @pytest.mark.db + @pytest.mark.unit + def test_get_or_create_catalog_is_idempotent_per_group_and_kind(self, db_connection): + db = db_connection + mh = _group(db, "mh") + scheme = db.get_or_create_catalog(mh, "scheme") + assert db.get_or_create_catalog(mh, "scheme")["id"] == scheme["id"] + assert scheme["type"] == "openagrinet:AgricultureResource" + assert scheme["name"] == "mh scheme" + advisory = db.get_or_create_catalog(mh, "advisory") + assert advisory["id"] != scheme["id"] + assert advisory["type"] == "openagrinet:KnowledgeAdvisory" + assert db.get_or_create_catalog(_group(db, "up"), "scheme")["id"] != scheme["id"] + + @pytest.mark.db + @pytest.mark.unit + def test_a_kind_without_a_catalog_type_is_rejected(self, db_connection): + with pytest.raises(ValueError): + db_connection.get_or_create_catalog(_group(db_connection), "video") + + @pytest.mark.db + @pytest.mark.unit + def test_one_catalog_per_group_and_type(self, db_connection): + db = db_connection + group_id = _group(db) + with db.get_connection() as conn: + conn.execute( + "INSERT INTO catalogs (group_id, type, name) VALUES (%s::uuid, 'openagrinet:MandiPrice', 'A')", + (group_id,), + ) + with pytest.raises(psycopg.errors.UniqueViolation): + conn.execute( + "INSERT INTO catalogs (group_id, type, name) VALUES (%s::uuid, 'openagrinet:MandiPrice', 'B')", + (group_id,), + ) + + @pytest.mark.db + @pytest.mark.unit + @pytest.mark.parametrize("bad_type", ["scheme", "openagrinet:", ":Thing", "open agrinet:X", "a:b:c", ""]) + def test_type_must_look_like_a_schema_name(self, db_connection, bad_type): + group_id = _group(db_connection) + with db_connection.get_connection() as conn: + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute( + "INSERT INTO catalogs (group_id, type, name) VALUES (%s::uuid, %s, 'N')", + (group_id, bad_type), + ) + + @pytest.mark.db + @pytest.mark.unit + def test_catalog_metadata_must_be_an_object(self, db_connection): + group_id = _group(db_connection) + with db_connection.get_connection() as conn: + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute( + "INSERT INTO catalogs (group_id, type, name, metadata) " + "VALUES (%s::uuid, 'openagrinet:MandiPrice', 'N', '[]'::jsonb)", + (group_id,), + ) + + @pytest.mark.db + @pytest.mark.unit + def test_a_catalog_with_documents_cannot_be_deleted(self, db_connection): + db = db_connection + _doc(db, "wf-held") + with db.get_connection() as conn: + with pytest.raises(psycopg.errors.ForeignKeyViolation): + conn.execute("DELETE FROM catalogs") + + +class TestDocumentIdentity: + @pytest.mark.db + @pytest.mark.unit + def test_a_new_document_needs_a_kind_and_a_code(self, db_connection): + db = db_connection + common = dict(workflow_id="wf-x", document_id=_md5("x"), filename="x.pdf", filepath="/x.pdf") + with pytest.raises(ValueError): + db.upsert_document(**common, code="x") + with pytest.raises(ValueError): + db.upsert_document(**common, kind="scheme") + assert db.get_document("wf-x") is None + + @pytest.mark.db + @pytest.mark.unit + def test_a_new_document_lands_in_its_tenants_catalog_of_that_kind(self, db_connection): + db = db_connection + _doc(db, "wf-land", group="mh", kind="advisory", code="kharif", name="Kharif advice") + doc = db.get_document("wf-land") + assert (doc["group"], doc["kind"], doc["code"], doc["name"]) == ( + "mh", "advisory", "kharif", "Kharif advice", + ) + expected = db.get_or_create_catalog(_group(db, "mh"), "advisory")["id"] + assert doc["catalog_id"] == expected + + @pytest.mark.db + @pytest.mark.unit + def test_the_name_defaults_to_the_code(self, db_connection): + _doc(db_connection, "wf-name", code="pm-kisan") + assert db_connection.get_document("wf-name")["name"] == "pm-kisan" + + @pytest.mark.db + @pytest.mark.unit + def test_a_code_is_unique_within_its_catalog(self, db_connection): + db = db_connection + _doc(db, "wf-one", group="mh", code="pm-kisan") + with pytest.raises(db.DuplicateDocumentCode): + _doc(db, "wf-two", group="mh", code="pm-kisan") + assert db.get_document("wf-two") is None + + @pytest.mark.db + @pytest.mark.unit + def test_the_same_code_is_fine_in_another_group_or_kind(self, db_connection): + db = db_connection + _doc(db, "wf-mh", group="mh", code="pm-kisan") + _doc(db, "wf-up", group="up", code="pm-kisan") + _doc(db, "wf-adv", group="mh", code="pm-kisan", kind="advisory") + assert {d["workflow_id"] for d in db.list_documents()} == {"wf-mh", "wf-up", "wf-adv"} + + @pytest.mark.db + @pytest.mark.unit + def test_a_duplicate_code_leaves_no_half_written_rows(self, db_connection): + db = db_connection + _doc(db, "wf-first", code="dup") + with pytest.raises(db.DuplicateDocumentCode): + _doc(db, "wf-second", code="dup") + with db.get_connection() as conn: + assert conn.execute("SELECT count(*) AS n FROM documents").fetchone()["n"] == 1 + assert conn.execute("SELECT count(*) AS n FROM workflows").fetchone()["n"] == 1 + + @pytest.mark.db + @pytest.mark.unit + def test_find_document_by_code(self, db_connection): + db = db_connection + _doc(db, "wf-find", group="mh", code="pm-kisan") + found = db.find_document_by_code("mh", "scheme", "pm-kisan") + assert found and found["workflow_id"] == "wf-find" + assert db.find_document_by_code("MH", "scheme", "pm-kisan")["id"] == found["id"] + assert db.find_document_by_code("up", "scheme", "pm-kisan") is None + assert db.find_document_by_code("mh", "advisory", "pm-kisan") is None + assert db.find_document_by_code("mh", "scheme", "other") is None + + @pytest.mark.db + @pytest.mark.unit + def test_changing_the_kind_moves_the_document_within_its_group(self, db_connection): + db = db_connection + _doc(db, "wf-kind", group="mh", code="movable") + db.update_document_fields("wf-kind", kind="advisory") + doc = db.get_document("wf-kind") + assert doc["kind"] == "advisory" + with db.get_connection() as conn: + row = conn.execute( + "SELECT c.group_id, c.type FROM catalogs c WHERE c.id = %s::uuid", (doc["catalog_id"],) + ).fetchone() + assert str(row["group_id"]) == _group(db, "mh") + assert row["type"] == "openagrinet:KnowledgeAdvisory" + + @pytest.mark.db + @pytest.mark.unit + def test_moving_to_a_kind_that_already_holds_the_code_is_a_duplicate(self, db_connection): + db = db_connection + _doc(db, "wf-s", group="mh", code="same") + _doc(db, "wf-a", group="mh", code="same", kind="advisory") + with pytest.raises(db.DuplicateDocumentCode): + db.update_document_fields("wf-s", kind="advisory") + assert db.get_document("wf-s")["kind"] == "scheme" + + +class TestReupload: + @pytest.mark.db + @pytest.mark.unit + def test_reset_for_reupload_replaces_the_source_and_clears_the_content(self, db_connection): + db = db_connection + doc_uuid = _doc(db, "wf-re", filename="old.pdf", valid_from="2026-01-01", valid_to="2026-12-31") + db.save_pages("wf-re", [{"page_number": 1, "original_markdown": "old"}]) + db.save_chunks("wf-re", [{"chunk_number": 1, "original_text": "old", "chunking_provider": "p"}]) + db.upsert_document_index_status("wf-re", "documents-index", status="indexed") + artifact_id = db.add_document_artifact("wf-re", "original_upload", "minio://documents/old.pdf") + new_fingerprint = _md5("new content") + + db.reset_document_for_reupload( + doc_uuid, filename="new.pdf", filepath="minio://documents/new.pdf", fingerprint=new_fingerprint, + ) + + doc = db.get_document("wf-re") + assert doc["id"] == doc_uuid + assert doc["source_file_fingerprint"] == new_fingerprint + assert (doc["filename"], doc["filepath"]) == ("new.pdf", "minio://documents/new.pdf") + assert db.get_pages("wf-re") == [] + assert db.get_chunks("wf-re") == [] + assert (doc["page_count"], doc["chunk_count"]) == (0, 0) + assert doc["reindex_required"] is True + assert doc["reindex_reason"] == "source_replaced" + + # What is not content survives: workflows, artifacts, period, index state, code. + assert [j["job_type"] for j in db.list_document_jobs("wf-re")] == ["pipeline"] + assert db.get_document_artifact("wf-re", artifact_id) is not None + assert (doc["valid_from"], doc["valid_to"]) == ("2026-01-01", "2026-12-31") + assert "dev" in doc["index_status"] + assert doc["code"] == "wf-re" + with db.get_connection() as conn: + assert "chunking" not in conn.execute( + "SELECT metadata FROM documents WHERE id = %s::uuid", (doc_uuid,) + ).fetchone()["metadata"] + + @pytest.mark.db + @pytest.mark.unit + def test_a_rechunk_after_reupload_starts_a_fresh_run_number(self, db_connection): + db = db_connection + doc_uuid = _doc(db, "wf-run") + db.save_chunks("wf-run", [{"chunk_number": 1, "original_text": "a", "chunking_provider": "p"}]) + db.reset_document_for_reupload( + doc_uuid, filename="n.pdf", filepath="/n.pdf", fingerprint=_md5("n"), + ) + db.save_chunks("wf-run", [{"chunk_number": 1, "original_text": "b", "chunking_provider": "p"}]) + assert db.get_chunk("wf-run", 1)["chunk_version"] == 1 + + @pytest.mark.db + @pytest.mark.unit + def test_a_fingerprint_must_be_hex(self, db_connection): + doc_uuid = _doc(db_connection, "wf-bad") + with pytest.raises(ValueError): + db_connection.reset_document_for_reupload( + doc_uuid, filename="n.pdf", filepath="/n.pdf", fingerprint="not-hex", + ) + + +class TestUserGroup: + @pytest.mark.db + @pytest.mark.unit + def test_a_single_tenant_sets_the_group_without_a_warning(self, db_connection, caplog): + db = db_connection + with caplog.at_level(logging.WARNING, logger="pipeline.db"): + db.upsert_user("kc-one", "one", None, [], tenants=["MH"]) + assert db.get_user_by_keycloak_id("kc-one")["group_id"] == _group(db, "mh") + assert not [r for r in caplog.records if "tenants" in r.getMessage()] + + @pytest.mark.db + @pytest.mark.unit + def test_several_tenants_use_the_first_sorted_one_and_warn(self, db_connection, caplog): + db = db_connection + with caplog.at_level(logging.WARNING, logger="pipeline.db"): + db.upsert_user("kc-many", "many", None, [], tenants=["up", "mh", "up"]) + assert db.get_user_by_keycloak_id("kc-many")["group_id"] == _group(db, "mh") + warnings = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] + assert any("kc-many" in w and "mh" in w for w in warnings) + + @pytest.mark.db + @pytest.mark.unit + def test_a_login_without_tenants_keeps_the_stored_group(self, db_connection): + db = db_connection + db.upsert_user("kc-keep", "keep", None, [], tenants=["up"]) + db.upsert_user("kc-keep", "keep", None, []) + db.upsert_user("kc-keep", "keep", None, [], tenants=[]) + assert db.get_user_by_keycloak_id("kc-keep")["group_id"] == _group(db, "up") + + @pytest.mark.db + @pytest.mark.unit + def test_a_new_login_moves_the_user_to_the_new_first_tenant(self, db_connection): + db = db_connection + db.upsert_user("kc-move", "move", None, [], tenants=["up"]) + db.upsert_user("kc-move", "move", None, [], tenants=["mh"]) + assert db.get_user_by_keycloak_id("kc-move")["group_id"] == _group(db, "mh") + + @pytest.mark.db + @pytest.mark.unit + def test_users_without_tenants_have_no_group(self, db_connection): + db_connection.upsert_user("kc-none", "none", None, []) + assert db_connection.get_user_by_keycloak_id("kc-none")["group_id"] is None + + @pytest.mark.db + @pytest.mark.unit + def test_the_uploader_lands_in_the_documents_metadata_and_gets_the_tenant_group(self, db_connection): + db = db_connection + _doc(db, "wf-up", group="mh", uploaded_by_user_id="kc-up", uploaded_by_username="uma") + doc = db.get_document("wf-up") + user = db.get_user_by_keycloak_id("kc-up") + assert doc["uploaded_by_user_id"] == "kc-up" + assert user["group_id"] == _group(db, "mh") + with db.get_connection() as conn: + meta = conn.execute("SELECT metadata FROM documents").fetchone()["metadata"] + assert meta["uploaded_by_user_id"] == user["id"] + + +class TestAuditGroupSnapshot: + @pytest.mark.db + @pytest.mark.unit + def test_an_entry_snapshots_the_documents_group(self, db_connection): + db = db_connection + _doc(db, "wf-aud", group="mh") + db.log_audit(workflow_id="wf-aud", document_id="x", action_type="page_edit", entity_type="page") + with db.get_connection() as conn: + row = conn.execute("SELECT group_id FROM audit_logs WHERE action_type = 'page_edit'").fetchone() + assert str(row["group_id"]) == _group(db, "mh") + + @pytest.mark.db + @pytest.mark.unit + def test_the_snapshot_survives_a_purge_and_keeps_tenant_scope(self, db_connection): + db = db_connection + _doc(db, "wf-gone", group="mh") + _doc(db, "wf-other", group="up", code="other") + for wf in ("wf-gone", "wf-other"): + db.log_audit(workflow_id=wf, document_id="x", action_type="purge_document", entity_type="document") + db.purge_document("wf-gone") + db.purge_document("wf-other") + + with db.get_connection() as conn: + rows = conn.execute( + "SELECT document_id, group_id, workflow_id FROM audit_logs WHERE action_type = 'purge_document'" + ).fetchall() + assert {r["workflow_id"] for r in rows} == {"wf-gone", "wf-other"} + assert all(r["document_id"] is None for r in rows) + assert {str(r["group_id"]) for r in rows} == {_group(db, "mh"), _group(db, "up")} + assert db.get_all_audit_log_count(groups=["mh"]) == 1 + assert db.get_all_audit_log_count(groups=["up"]) == 1 + assert db.get_all_audit_log_count(groups=["gj"]) == 0 + assert [e["workflow_id"] for e in db.get_all_audit_logs(groups=["mh"])] == ["wf-gone"] + assert db.get_all_audit_logs(groups=["mh"])[0]["group"] == "mh" + + @pytest.mark.db + @pytest.mark.unit + def test_system_entries_have_no_group(self, db_connection): + db = db_connection + db.log_audit(workflow_id="__catalog__", document_id="__catalog__", action_type="catalog_rebuild", + entity_type="catalog") + with db.get_connection() as conn: + row = conn.execute("SELECT group_id, document_id FROM audit_logs WHERE action_type = 'catalog_rebuild'").fetchone() + assert row["group_id"] is None and row["document_id"] is None + + +class TestCatalogStatusView: + @staticmethod + def _rows(db): + with db.get_connection() as conn: + return {r["code"]: r for r in conn.execute("SELECT * FROM catalog_status").fetchall()} + + @pytest.mark.db + @pytest.mark.unit + def test_one_row_per_document_with_catalog_group_and_type(self, db_connection): + db = db_connection + _doc(db, "wf-1", group="mh", code="alpha", stage="completed") + _doc(db, "wf-2", group="mh", code="beta", stage="completed") + _doc(db, "wf-3", group="up", code="gamma", stage="completed", kind="advisory") + rows = self._rows(db) + assert set(rows) == {"alpha", "beta", "gamma"} + assert rows["alpha"]["catalog_id"] == rows["beta"]["catalog_id"] != rows["gamma"]["catalog_id"] + assert str(rows["alpha"]["group_id"]) == _group(db, "mh") + assert (rows["alpha"]["group_name"], rows["gamma"]["group_name"]) == ("mh", "up") + assert rows["alpha"]["type"] == "openagrinet:AgricultureResource" + assert rows["gamma"]["type"] == "openagrinet:KnowledgeAdvisory" + assert rows["alpha"]["workflow_id"] == "wf-1" + assert rows["alpha"]["name"] == "alpha" + + @pytest.mark.db + @pytest.mark.unit + def test_status_follows_stage_flags_and_disabling(self, db_connection): + db = db_connection + _doc(db, "wf-live", code="live", stage="completed") + _doc(db, "wf-stale", code="stale", stage="completed") + db.mark_document_reindex_required("wf-stale", True, "edit") + _doc(db, "wf-gate", code="gate", stage="approval_for_prod") + _doc(db, "wf-wip", code="wip", stage="chunk_review") + _doc(db, "wf-off", code="off", stage="completed") + db.set_document_disabled("wf-off", True) + _doc(db, "wf-hidden", code="hidden", stage="completed") + db.update_document_fields("wf-hidden", catalog_visible=False) + _doc(db, "wf-gate-off", code="gate-off", stage="approval_for_prod") + db.set_document_disabled("wf-gate-off", True) + + status = {code: row["status"] for code, row in self._rows(db).items()} + assert status == { + "live": "live", + "stale": "pending_reindex", + "gate": "pending_prod", + "wip": "disabled", + "off": "disabled", + "hidden": "disabled", + "gate-off": "disabled", + } + + @pytest.mark.db + @pytest.mark.unit + def test_chunk_count_is_counted_from_chunks(self, db_connection): + db = db_connection + _doc(db, "wf-c", code="counted", stage="completed") + assert self._rows(db)["counted"]["chunk_count"] == 0 + db.save_chunks("wf-c", [{"chunk_number": n, "original_text": "t"} for n in (1, 2, 3)]) + assert self._rows(db)["counted"]["chunk_count"] == 3 + db.save_chunks("wf-c", [{"chunk_number": 1, "original_text": "t"}]) + assert self._rows(db)["counted"]["chunk_count"] == 1 + + @pytest.mark.db + @pytest.mark.unit + def test_network_visible_defaults_to_true_and_follows_the_flag(self, db_connection): + db = db_connection + _doc(db, "wf-v1", code="visible", stage="completed") + _doc(db, "wf-v2", code="private", stage="completed") + db.update_document_fields("wf-v2", network_visible=False) + rows = self._rows(db) + assert (rows["visible"]["network_visible"], rows["private"]["network_visible"]) == (True, False) + + @pytest.mark.db + @pytest.mark.unit + def test_promoted_at_comes_from_the_prod_index_of_live_documents_only(self, db_connection, monkeypatch): + monkeypatch.setenv("PROD_VECTOR_DB_COLLECTION_NAME", "prod-docs") + db = db_connection + _doc(db, "wf-p1", code="promoted", stage="completed") + _doc(db, "wf-p2", code="not-live", stage="chunk_review") + _doc(db, "wf-p3", code="dev-only", stage="completed") + for wf in ("wf-p1", "wf-p2"): + db.upsert_document_index_status(wf, "prod-docs", last_indexed_at="2026-05-01T10:00:00+00:00") + db.upsert_document_index_status("wf-p3", "documents-index", last_indexed_at="2026-05-02T10:00:00+00:00") + rows = self._rows(db) + assert rows["promoted"]["promoted_at"] is not None + assert rows["promoted"]["promoted_at"].isoformat().startswith("2026-05-01T10:00:00") + assert rows["not-live"]["promoted_at"] is None + assert rows["dev-only"]["promoted_at"] is None + + @pytest.mark.db + @pytest.mark.unit + def test_a_catalog_without_documents_has_no_rows(self, db_connection): + db = db_connection + db.get_or_create_catalog(_group(db), "scheme") + assert self._rows(db) == {} + + +class TestDocumentJsonbShape: + @pytest.mark.db + @pytest.mark.unit + def test_a_new_document_carries_the_default_metadata_keys(self, db_connection): + db = db_connection + _doc(db, "wf-meta") + with db.get_connection() as conn: + meta = conn.execute("SELECT metadata FROM documents").fetchone()["metadata"] + assert meta["index_status"] == {} + assert (meta["catalog_visible"], meta["network_visible"], meta["reindex_required"]) == (True, True, False) + for dropped in ("is_demo", "chunk_count", "display_name"): + assert dropped not in meta + + @pytest.mark.db + @pytest.mark.unit + def test_the_source_has_no_page_count(self, db_connection): + db = db_connection + _doc(db, "wf-src") + db.save_pages("wf-src", [{"page_number": 1, "original_markdown": "a"}]) + with db.get_connection() as conn: + source = conn.execute("SELECT source FROM documents").fetchone()["source"] + assert "page_count" not in source + assert db.get_document("wf-src")["page_count"] == 1 + + @pytest.mark.db + @pytest.mark.unit + def test_there_are_no_instance_kind_or_demo_columns(self, db_connection): + with db_connection.get_connection() as conn: + cols = { + r["column_name"] + for r in conn.execute( + "SELECT column_name FROM information_schema.columns WHERE table_name = 'documents'" + ).fetchall() + } + assert {"instance", "document_kind", "is_demo", "catalog_entry_id", "valid_from", "valid_to", + "index_status", "uploaded_by_user_id"}.isdisjoint(cols) + assert {"catalog_id", "code", "name", "aliases", "source", "is_disabled", "metadata"} <= cols + + @pytest.mark.db + @pytest.mark.unit + def test_aliases_cannot_hold_an_empty_string(self, db_connection): + db = db_connection + _doc(db, "wf-al", aliases=["PM Kisan", "", "pmk"]) + with db.get_connection() as conn: + aliases = conn.execute("SELECT aliases FROM documents").fetchone()["aliases"] + assert aliases == ["PM Kisan", "pmk"] + with db.get_connection() as conn: + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute("UPDATE documents SET aliases = ARRAY['a', '']") + + @pytest.mark.db + @pytest.mark.unit + def test_document_metadata_must_be_an_object(self, db_connection): + db = db_connection + _doc(db, "wf-obj") + with db.get_connection() as conn: + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute("UPDATE documents SET metadata = %s", (Jsonb([1]),)) + + @pytest.mark.db + @pytest.mark.unit + def test_workflow_and_audit_columns_are_plain_text(self, db_connection): + with db_connection.get_connection() as conn: + types = { + (r["table_name"], r["column_name"]): r["data_type"] + for r in conn.execute( + "SELECT table_name, column_name, data_type FROM information_schema.columns " + "WHERE (table_name, column_name) IN (('workflows','workflow_type'), ('workflows','status'), " + "('workflows','current_stage'), ('workflow_artifacts','stage'), ('audit_logs','entity_type'))" + ).fetchall() + } + enums = conn.execute("SELECT count(*) AS n FROM pg_type WHERE typtype = 'e'").fetchone()["n"] + assert len(types) == 5 and set(types.values()) == {"text"} + assert enums == 0 diff --git a/tests/test_scheme_catalog.py b/tests/test_scheme_catalog.py index a58810c..ab13c76 100644 --- a/tests/test_scheme_catalog.py +++ b/tests/test_scheme_catalog.py @@ -12,52 +12,71 @@ def catalog_db(db_connection): yield db_connection, scheme_catalog -def test_bootstrap_seeds_13_vector_schemes(catalog_db): +def _doc(db, workflow_id, *, code, kind="scheme", name=None, stage="completed", + group="default", filename=None, **extra): + """Register a document the v2 way: it needs a kind (catalog) and a code.""" + return db.upsert_document( + workflow_id=workflow_id, + document_id=f"doc-{workflow_id}", + filename=filename or f"{workflow_id}.pdf", + filepath=f"/tmp/{workflow_id}.pdf", + stage=stage, + group=group, + kind=kind, + code=code, + name=name, + **extra, + ) + + +def test_catalog_has_no_bootstrap_entries(catalog_db): + """The catalog is documents only; the code-defined lists stay as hints.""" db, scheme_catalog = catalog_db - result = scheme_catalog.bootstrap_catalog_if_empty() - assert result["bootstrapped"] is True - assert result["vector_count"] == 13 + assert not hasattr(scheme_catalog, "bootstrap_catalog_if_empty") snap = scheme_catalog.build_snapshot() - codes = {s["scheme_code"] for s in snap["vector_schemes"]} - assert len(codes) == 13 - assert "nbm" not in codes # legacy-only - assert "pkvy" in codes - assert "pm-ddky" in codes - assert snap["catalog_version"] >= 1 + assert snap["vector_schemes"] == [] + assert snap["catalog_version"] == 0 + # Static lists are untouched: curated alias source and legacy tool list. + assert len(scheme_catalog.BOOTSTRAP_VECTOR_SCHEMES) == 13 + assert "nbm" not in scheme_catalog.BOOTSTRAP_VECTOR_SCHEMES # legacy-only + assert any(s["scheme_code"] == "nbm" for s in snap["legacy_schemes"]) assert "search_schemes_doc" in snap["tool_prompts"] - assert "vector_schemes_bullets_en" in snap["tool_prompts"] assert snap["routing_exceptions"]["pkvy"] == "search_schemes" assert snap["routing_exceptions"]["nbm"] == "get_scheme_info" - # second call is no-op - again = scheme_catalog.bootstrap_catalog_if_empty() - assert again["bootstrapped"] is False + + +def test_snapshot_has_one_entry_per_live_scheme_document(catalog_db): + db, scheme_catalog = catalog_db + _doc(db, "wf-a", code="pkvy", name="Paramparagat Krishi Vikas Yojana", group="mh") + _doc(db, "wf-b", code="mif", name="Micro Irrigation Fund", group="gj") + _doc(db, "wf-adv", code="rain-advice", kind="advisory", name="Rain advisory") + _doc(db, "wf-draft", code="draft", stage="chunk_review") + snap = scheme_catalog.build_snapshot() + by_code = {s["scheme_code"]: s for s in snap["vector_schemes"]} + assert set(by_code) == {"pkvy", "mif"} # advisory kind and non-live docs stay out + assert by_code["pkvy"]["instances"] == ["mh"] + assert by_code["mif"]["instances"] == ["gj"] + assert by_code["pkvy"]["workflow_ids"] == ["wf-a"] + assert by_code["pkvy"]["source"] == "pipeline" + + only_mh = scheme_catalog.build_snapshot(instance="mh") + assert [s["scheme_code"] for s in only_mh["vector_schemes"]] == ["pkvy"] def test_scheme_metadata_and_promote_rebuild(catalog_db): db, scheme_catalog = catalog_db - scheme_catalog.bootstrap_catalog_if_empty() v0 = db.get_catalog_meta()["version"] - - db.upsert_document( - workflow_id="wf-pkvy-1", - document_id="doc-pkvy-1", - filename="pkvy.pdf", - filepath="/tmp/pkvy.pdf", - stage="completed", - page_count=10, - chunk_count=5, - instance="default", - ) + _doc(db, "wf-pkvy-1", code="pkvy") result = scheme_catalog.apply_scheme_metadata( "wf-pkvy-1", - document_kind="scheme", - scheme_code="pkvy", - scheme_name="Paramparagat Krishi Vikas Yojana", - scheme_aliases=["PKVY", "organic"], + kind="scheme", + code="pkvy", + name="Paramparagat Krishi Vikas Yojana", + aliases=["PKVY", "organic"], network_visible=True, catalog_visible=True, ) - assert result["document"]["scheme_code"] == "pkvy" + assert result["document"]["code"] == "pkvy" assert result["catalog_version"] is not None assert result["catalog_version"] > v0 @@ -65,36 +84,22 @@ def test_scheme_metadata_and_promote_rebuild(catalog_db): assert entry is not None assert entry["status"] == "live" assert entry["source"] == "pipeline" - assert "organic" in entry["aliases"] or "PKVY" in entry["aliases"] + assert entry["name"] == "Paramparagat Krishi Vikas Yojana" + assert "organic" in entry["aliases"] and "PKVY" in entry["aliases"] def test_code_rename_pending_reindex(catalog_db): db, scheme_catalog = catalog_db - scheme_catalog.bootstrap_catalog_if_empty() - db.upsert_document( - workflow_id="wf-rename", - document_id="doc-rename", - filename="x.pdf", - filepath="/tmp/x.pdf", - stage="completed", - chunk_count=3, - instance="default", - ) - scheme_catalog.apply_scheme_metadata( - "wf-rename", - document_kind="scheme", - scheme_code="old-code", - scheme_name="Old", - ) + _doc(db, "wf-rename", code="old-code", name="Old") result = scheme_catalog.apply_scheme_metadata( "wf-rename", - scheme_code="new-code", - scheme_name="New", + code="new-code", + name="New", ) assert result["pending_reindex"] is True doc = db.get_document("wf-rename") - assert doc["scheme_code"] == "new-code" - assert int(doc["reindex_required"]) == 1 + assert doc["code"] == "new-code" + assert doc["reindex_required"] is True # New code must not appear in live vector_schemes snap = scheme_catalog.build_snapshot() live_codes = {s["scheme_code"] for s in snap["vector_schemes"]} @@ -102,6 +107,7 @@ def test_code_rename_pending_reindex(catalog_db): entry = db.get_catalog_entry("new-code") assert entry is not None assert entry["status"] == "pending_reindex" + assert db.get_catalog_entry("old-code") is None # the document holds one code def test_derive_scheme_code_acronym(catalog_db): @@ -113,59 +119,56 @@ def test_derive_scheme_code_acronym(catalog_db): def test_derive_scheme_code_collision_appends_suffix(catalog_db): db, scheme_catalog = catalog_db - db.upsert_document( - workflow_id="wf-nmnf-1", - document_id="doc-nmnf-1", - filename="a.pdf", - filepath="/tmp/a.pdf", - stage="completed", - instance="default", - ) - scheme_catalog.apply_scheme_metadata( - "wf-nmnf-1", document_kind="scheme", scheme_code="nmnf", scheme_name="National Mission on Natural Farming", - ) + _doc(db, "wf-nmnf-1", code="nmnf", name="National Mission on Natural Farming") # A second, unrelated document whose title happens to acronym to the same code. assert scheme_catalog.derive_scheme_code("New Model National Farming") == "nmnf-2" + # The collision domain is the kind: an advisory may reuse the code. + assert scheme_catalog.derive_scheme_code("New Model National Farming", kind="advisory") == "nmnf" + # A document's own code does not collide with itself. + assert scheme_catalog.derive_scheme_code( + "National Mission on Natural Farming", exclude_workflow_id="wf-nmnf-1" + ) == "nmnf" -def test_apply_scheme_metadata_auto_derives_code_from_name(catalog_db): +def test_collision_domain_spans_tenants(catalog_db): + """Codes are unique per catalog in SQL, but derivation avoids every tenant's codes.""" db, scheme_catalog = catalog_db - db.upsert_document( - workflow_id="wf-auto-code", - document_id="doc-auto-code", - filename="Guideline_of_NMNF_V2_Revised.pdf", - filepath="/tmp/x.pdf", - stage="ready_for_ingestion", - instance="default", - ) + _doc(db, "wf-mh", code="nmnf", group="mh") + assert scheme_catalog.derive_scheme_code("National Mission on Natural Farming") == "nmnf-2" + # The same explicit code in another tenant's catalog is still allowed by the schema. + _doc(db, "wf-gj", code="nmnf", group="gj") + + +def test_apply_scheme_metadata_keeps_the_code_when_only_the_name_changes(catalog_db): + """Every document has a code from upload, so a name edit never re-derives it.""" + db, scheme_catalog = catalog_db + _doc(db, "wf-name-only", code="gn", stage="ready_for_ingestion") result = scheme_catalog.apply_scheme_metadata( - "wf-auto-code", - document_kind="scheme", - scheme_name="National Mission on Natural Farming", + "wf-name-only", + kind="scheme", + name="National Mission on Natural Farming", ) - # scheme_code was never passed explicitly -> derived from the title, not left blank/hash-based. - assert result["document"]["scheme_code"] == "nmnf" - assert result["document"]["scheme_name"] == "National Mission on Natural Farming" + assert result["document"]["code"] == "gn" + assert result["document"]["name"] == "National Mission on Natural Farming" + assert result["code_renamed"] is False -def test_document_kind_widened_to_advisory_and_custom(catalog_db): +def test_kind_is_limited_to_scheme_and_advisory(catalog_db): db, scheme_catalog = catalog_db - db.upsert_document( - workflow_id="wf-kind", - document_id="doc-kind", - filename="advisory.pdf", - filepath="/tmp/advisory.pdf", - stage="ready_for_ingestion", - instance="default", - ) - result = scheme_catalog.apply_scheme_metadata("wf-kind", document_kind="advisory") - assert result["document"]["document_kind"] == "advisory" + _doc(db, "wf-kind", code="kind-doc", stage="ready_for_ingestion") + assert db.get_document("wf-kind")["kind"] == "scheme" + + result = scheme_catalog.apply_scheme_metadata("wf-kind", kind="advisory") + assert result["document"]["kind"] == "advisory" + assert result["document"]["code"] == "kind-doc" # moved catalogs, kept its code - result = scheme_catalog.apply_scheme_metadata("wf-kind", document_kind="how_to_faq") - assert result["document"]["document_kind"] == "how_to_faq" + result = scheme_catalog.apply_scheme_metadata("wf-kind", kind="scheme") + assert result["document"]["kind"] == "scheme" - with pytest.raises(ValueError): - scheme_catalog.apply_scheme_metadata("wf-kind", document_kind="Not Valid!!") + for bad in ("how_to_faq", "document", "video", "Not Valid!!"): + with pytest.raises(ValueError): + scheme_catalog.apply_scheme_metadata("wf-kind", kind=bad) + assert db.get_document("wf-kind")["kind"] == "scheme" def test_prepare_records_scheme_payload(): @@ -186,11 +189,11 @@ def test_prepare_records_scheme_payload(): "pkvy.pdf", chunks, workflow_id="wf1", - instance="default", - document_kind="scheme", - scheme_code="pkvy", - scheme_name="Paramparagat Krishi Vikas Yojana", - scheme_aliases=["PKVY"], + group="default", + kind="scheme", + code="pkvy", + name="Paramparagat Krishi Vikas Yojana", + aliases=["PKVY"], ) assert len(records) == 1 r = records[0] @@ -222,7 +225,7 @@ def test_prepare_records_document_unchanged(): "doc2", "notes.pdf", chunks, - document_kind="document", + kind="document", ) assert records[0]["type"] == "document" assert "scheme_code" not in records[0] @@ -254,9 +257,11 @@ async def _noop_lifespan(_app): def test_catalog_api_snapshot(asgi_client): - from pipeline import scheme_catalog + from pipeline import db, scheme_catalog - scheme_catalog.bootstrap_catalog_if_empty() + _doc(db, "wf-snap-1", code="pkvy", name="Paramparagat Krishi Vikas Yojana") + _doc(db, "wf-snap-2", code="pm-ddky", name="Deen Dayal Upadhyaya Krishi Kalyan Yojana") + scheme_catalog.rebuild_catalog_from_documents(reason="test") r = asgi_client.get("/catalog/v1/version") assert r.status_code == 200 @@ -267,7 +272,7 @@ def test_catalog_api_snapshot(asgi_client): assert r.status_code == 200 body = r.json() assert body["api_version"] == "1" - assert len(body["vector_schemes"]) == 13 + assert {s["scheme_code"] for s in body["vector_schemes"]} == {"pkvy", "pm-ddky"} assert "tool_prompts" in body assert "legacy_schemes" in body assert body["collections"]["scheme_qdrant"]["name"] == "schemes-index" @@ -278,13 +283,14 @@ def test_catalog_api_snapshot(asgi_client): r = asgi_client.get("/catalog/v1/schemes") assert r.status_code == 200 - assert len(r.json()["vector_schemes"]) == 13 + assert len(r.json()["vector_schemes"]) == 2 -def test_catalog_service_key_auth(asgi_client, monkeypatch): - from pipeline import scheme_catalog +def test_catalog_bootstrap_endpoint_is_gone(asgi_client): + assert asgi_client.post("/catalog/v1/bootstrap").status_code in (404, 405) + - scheme_catalog.bootstrap_catalog_if_empty() +def test_catalog_service_key_auth(asgi_client, monkeypatch): monkeypatch.setenv("CATALOG_SERVICE_API_KEYS", "test-secret-key") r = asgi_client.get( @@ -300,70 +306,43 @@ def test_catalog_service_key_auth(asgi_client, monkeypatch): def test_scheme_metadata_api(asgi_client): from pipeline import db - db.upsert_document( - workflow_id="wf-meta", - document_id="doc-meta", - filename="scheme.pdf", - filepath="/tmp/scheme.pdf", - stage="chunk_review", - instance="default", - ) + _doc(db, "wf-meta", code="meta-draft", stage="chunk_review", filename="scheme.pdf") r = asgi_client.patch( "/documents/wf-meta/scheme-metadata", json={ - "document_kind": "scheme", - "scheme_code": "mif", - "scheme_name": "Micro Irrigation Fund", - "scheme_aliases": ["MIF"], + "kind": "scheme", + "code": "mif", + "name": "Micro Irrigation Fund", + "aliases": ["MIF"], "network_visible": True, }, ) assert r.status_code == 200, r.text body = r.json() - assert body["scheme_code"] == "mif" - assert body["document_kind"] == "scheme" - assert "MIF" in body["scheme_aliases"] + assert body["code"] == "mif" + assert body["kind"] == "scheme" + assert "MIF" in body["aliases"] -def test_reingest_blocked_when_never_classified(asgi_client, mock_temporal_client): +def test_scheme_metadata_api_rejects_an_unknown_kind(asgi_client): from pipeline import db - db.upsert_document( - workflow_id="wf-unclassified", - document_id="doc-unclassified", - filename="x.pdf", - filepath="/tmp/x.pdf", - stage="completed", - instance="default", - ) - db.save_chunks("wf-unclassified", [ - {"chunk_number": 1, "original_text": "some content", "token_count": 5}, - ]) - - r = asgi_client.post("/documents/wf-unclassified/reingest") + _doc(db, "wf-meta-bad", code="meta-bad", stage="chunk_review") + r = asgi_client.patch("/documents/wf-meta-bad/scheme-metadata", json={"kind": "video"}) assert r.status_code == 400 - assert "never been classified" in r.json()["detail"] -def test_reingest_allowed_once_classified(asgi_client, mock_temporal_client): - from pipeline import db, scheme_catalog +@pytest.mark.parametrize("kind", ["scheme", "advisory"]) +def test_reingest_is_allowed_for_every_kind(asgi_client, mock_temporal_client, kind): + """No 'never classified' guard any more: a document always has a kind and a code.""" + from pipeline import db - db.upsert_document( - workflow_id="wf-classified", - document_id="doc-classified", - filename="x.pdf", - filepath="/tmp/x.pdf", - stage="completed", - instance="default", - ) - db.save_chunks("wf-classified", [ + _doc(db, "wf-reingest", code="testcode", kind=kind) + db.save_chunks("wf-reingest", [ {"chunk_number": 1, "original_text": "some content", "token_count": 5}, ]) - scheme_catalog.apply_scheme_metadata( - "wf-classified", document_kind="scheme", scheme_code="testcode", scheme_name="Test Scheme", - ) - r = asgi_client.post("/documents/wf-classified/reingest") + r = asgi_client.post("/documents/wf-reingest/reingest") assert r.status_code == 200, r.text assert r.json()["status"] == "started" @@ -371,14 +350,7 @@ def test_reingest_allowed_once_classified(asgi_client, mock_temporal_client): def test_request_prod_ready_wrong_stage_rejected(asgi_client): from pipeline import db - db.upsert_document( - workflow_id="wf-prod-ready-wrong-stage", - document_id="doc-prod-ready-wrong-stage", - filename="x.pdf", - filepath="/tmp/x.pdf", - stage="chunk_review", - instance="default", - ) + _doc(db, "wf-prod-ready-wrong-stage", code="prw", stage="chunk_review") r = asgi_client.post("/documents/wf-prod-ready-wrong-stage/request-prod-ready") assert r.status_code == 400 @@ -386,14 +358,7 @@ def test_request_prod_ready_wrong_stage_rejected(asgi_client): def test_request_prod_ready_flags_document_and_clears_on_approve(asgi_client, mock_temporal_client): from pipeline import db - db.upsert_document( - workflow_id="wf-prod-ready", - document_id="doc-prod-ready", - filename="x.pdf", - filepath="/tmp/x.pdf", - stage="approval_for_prod", - instance="default", - ) + _doc(db, "wf-prod-ready", code="pr", stage="approval_for_prod") doc = db.get_document("wf-prod-ready") assert doc["prod_ready_requested_at"] is None @@ -495,7 +460,7 @@ def test_aliases_are_deduped_case_insensitively(self): @pytest.mark.unit def test_json_string_aliases_are_accepted(self): - """documents.scheme_aliases_json arrives as a JSON string.""" + """Aliases may arrive as a JSON string.""" from pipeline.scheme_catalog import resolve_scheme_aliases aliases = resolve_scheme_aliases("makhana", "Makhana Scheme", '["fox nut", "lotus seed"]') diff --git a/tests/test_tenancy.py b/tests/test_tenancy.py index 03780b5..6b251b0 100644 --- a/tests/test_tenancy.py +++ b/tests/test_tenancy.py @@ -1,55 +1,58 @@ -"""Unit tests for multi-instance (tenant) scoping.""" +"""Unit tests for multi-group (tenant) scoping.""" from __future__ import annotations +import logging +import re + import pytest from fastapi import HTTPException from pipeline.auth.jwt import claims_to_user from pipeline.auth.models import local_bypass_user from pipeline.auth.tenancy import ( - PORTAL_INSTANCE, - allowed_instances, - assert_document_instance_access, - assert_instance_access, - resolve_create_instance, - user_can_access_instance, + PORTAL_GROUP, + allowed_groups, + assert_document_group_access, + assert_group_access, + resolve_create_group, + user_can_access_group, ) def test_bypass_user_is_unrestricted(): user = local_bypass_user() - assert allowed_instances(user) is None - assert user_can_access_instance(user, "tenant-a") - assert user_can_access_instance(user, "tenant-b") + assert allowed_groups(user) is None + assert user_can_access_group(user, "tenant-a") + assert user_can_access_group(user, "tenant-b") -def test_resolve_create_instance_state_user_defaults_to_only_state(): +def test_resolve_create_group_state_user_defaults_to_only_state(): user = claims_to_user( { "sub": "u1", "groups": ["/states/MH/contributor"], } ) - assert resolve_create_instance(user, None) == "mh" - assert resolve_create_instance(user, "MH") == "mh" + assert resolve_create_group(user, None) == "mh" + assert resolve_create_group(user, "MH") == "mh" with pytest.raises(HTTPException) as exc: - resolve_create_instance(user, "up") + resolve_create_group(user, "up") assert exc.value.status_code == 403 -def test_resolve_create_instance_superadmin_defaults_to_portal(): +def test_resolve_create_group_superadmin_defaults_to_portal(): user = claims_to_user( { "sub": "sa", "groups": ["/global/super-admin"], } ) - assert resolve_create_instance(user, None) == PORTAL_INSTANCE - assert resolve_create_instance(user, "mh") == "mh" + assert resolve_create_group(user, None) == PORTAL_GROUP + assert resolve_create_group(user, "mh") == "mh" -def test_resolve_create_instance_multi_state_requires_choice(): +def test_resolve_create_group_multi_state_requires_choice(): user = claims_to_user( { "sub": "u2", @@ -57,9 +60,9 @@ def test_resolve_create_instance_multi_state_requires_choice(): } ) with pytest.raises(HTTPException) as exc: - resolve_create_instance(user, None) + resolve_create_group(user, None) assert exc.value.status_code == 400 - assert resolve_create_instance(user, "up") == "up" + assert resolve_create_group(user, "up") == "up" def test_user_instances_are_enforced(): @@ -70,11 +73,11 @@ def test_user_instances_are_enforced(): "instances": ["Tenant-A", "tenant-b"], } ) - assert allowed_instances(user) == {"tenant-a", "tenant-b"} - assert user_can_access_instance(user, "tenant-a") - assert not user_can_access_instance(user, "mh") + assert allowed_groups(user) == {"tenant-a", "tenant-b"} + assert user_can_access_group(user, "tenant-a") + assert not user_can_access_group(user, "mh") with pytest.raises(HTTPException) as exc: - assert_instance_access(user, "mh") + assert_group_access(user, "mh") assert exc.value.status_code == 403 @@ -87,11 +90,11 @@ def test_document_access_hides_other_instances(): } ) with pytest.raises(HTTPException) as exc: - assert_document_instance_access(user, {"workflow_id": "wf", "instance": "tenant-b"}) + assert_document_group_access(user, {"workflow_id": "wf", "group": "tenant-b"}) assert exc.value.status_code == 404 -def test_superadmin_token_is_instance_unrestricted_despite_scoped_claim(): +def test_superadmin_token_is_group_unrestricted_despite_scoped_claim(): """Platform superadmin stays unrestricted even with a narrow instances claim.""" for role in ("superadmin", "super_admin", "super-admin"): user = claims_to_user( @@ -103,14 +106,14 @@ def test_superadmin_token_is_instance_unrestricted_despite_scoped_claim(): ) assert user.is_superadmin is True assert user.is_admin is True # alias - assert user.is_instance_unrestricted() is True - # Unrestricted -> allowed_instances is None (every instance visible). - assert allowed_instances(user) is None - assert user_can_access_instance(user, "tenant-b") - doc = assert_document_instance_access( - user, {"workflow_id": "wf", "instance": "tenant-b"} + assert user.is_group_unrestricted() is True + # Unrestricted -> allowed_groups is None (every group visible). + assert allowed_groups(user) is None + assert user_can_access_group(user, "tenant-b") + doc = assert_document_group_access( + user, {"workflow_id": "wf", "group": "tenant-b"} ) - assert doc["instance"] == "tenant-b" + assert doc["group"] == "tenant-b" def test_state_admin_is_scoped_to_instances_claim(): @@ -125,15 +128,15 @@ def test_state_admin_is_scoped_to_instances_claim(): } ) assert user.is_superadmin is False - assert user.is_instance_unrestricted() is False - assert allowed_instances(user) == {"tenant-a"} - assert user_can_access_instance(user, "tenant-a") - assert not user_can_access_instance(user, "tenant-b") + assert user.is_group_unrestricted() is False + assert allowed_groups(user) == {"tenant-a"} + assert user_can_access_group(user, "tenant-a") + assert not user_can_access_group(user, "tenant-b") assert Permission.SEARCH in user.permissions assert Permission.UPLOAD in user.permissions assert Permission.ADMIN not in user.permissions with pytest.raises(HTTPException) as exc: - assert_document_instance_access(user, {"workflow_id": "wf", "instance": "tenant-b"}) + assert_document_group_access(user, {"workflow_id": "wf", "group": "tenant-b"}) assert exc.value.status_code == 404 @@ -147,130 +150,141 @@ def test_content_curator_with_scoped_claim_cannot_cross_tenants(): } ) assert user.is_admin is False - assert user.is_instance_unrestricted() is False - assert allowed_instances(user) == {"tenant-a"} - assert user_can_access_instance(user, "tenant-a") - assert not user_can_access_instance(user, "tenant-b") + assert user.is_group_unrestricted() is False + assert allowed_groups(user) == {"tenant-a"} + assert user_can_access_group(user, "tenant-a") + assert not user_can_access_group(user, "tenant-b") with pytest.raises(HTTPException) as exc: - assert_document_instance_access(user, {"workflow_id": "wf", "instance": "tenant-b"}) + assert_document_group_access(user, {"workflow_id": "wf", "group": "tenant-b"}) assert exc.value.status_code == 404 -def test_list_documents_filters_by_instance(db_connection): +def _register(db, **kwargs): + """upsert_document the v2 way: a new document needs a kind and a code. + + The tenant (``group``) picks the group row; the kind plus that group pick the + catalog, which is created on first use. + """ + kwargs.setdefault("kind", "scheme") + kwargs.setdefault("code", re.sub(r"[^a-z0-9]+", "-", kwargs["workflow_id"].lower()).strip("-")) + return db.upsert_document(**kwargs) + + +def test_list_documents_filters_by_group(db_connection): db = db_connection - db.upsert_document( + _register(db, workflow_id="wf-tenant-a", document_id="d1", filename="a.pdf", filepath="/tmp/a.pdf", stage="completed", - instance="tenant-a", + group="tenant-a", ) - db.upsert_document( + _register(db, workflow_id="wf-tenant-b", document_id="d2", filename="b.pdf", filepath="/tmp/b.pdf", stage="completed", - instance="tenant-b", + group="tenant-b", ) - tenant_a_only = db.list_documents(instances=["tenant-a"]) + tenant_a_only = db.list_documents(groups=["tenant-a"]) assert {d["workflow_id"] for d in tenant_a_only} == {"wf-tenant-a"} - both = db.list_documents(instances=["tenant-a", "tenant-b"]) + both = db.list_documents(groups=["tenant-a", "tenant-b"]) assert {d["workflow_id"] for d in both} == {"wf-tenant-a", "wf-tenant-b"} - none = db.list_documents(instances=[]) + none = db.list_documents(groups=[]) assert none == [] -def test_summary_counts_honor_instance_filter(db_connection): +def test_summary_counts_honor_group_filter(db_connection): db = db_connection - db.upsert_document( + _register(db, workflow_id="wf-tenant-a-2", document_id="d3", filename="c.pdf", filepath="/tmp/c.pdf", stage="completed", - instance="tenant-a", + group="tenant-a", ) - db.upsert_document( + _register(db, workflow_id="wf-tenant-b-2", document_id="d4", filename="d.pdf", filepath="/tmp/d.pdf", stage="failed", - instance="tenant-b", + group="tenant-b", ) - summary = db.get_document_summary_counts(instances=["tenant-a"]) + summary = db.get_document_summary_counts(groups=["tenant-a"]) assert summary["total_documents"] == 1 assert summary["completed_documents"] == 1 assert summary["failed_documents"] == 0 -def test_list_runs_filters_by_instance(db_connection): +def test_list_runs_filters_by_group(db_connection): """Runs must honor the same tenant scope as documents (role-based states).""" db = db_connection - doc_a = db.upsert_document( + doc_a = _register(db, with_workflow=False, workflow_id="wf-run-a", document_id="d-run-a", filename="a.pdf", filepath="/tmp/a.pdf", stage="completed", - instance="tenant-a", + group="tenant-a", ) - doc_b = db.upsert_document( + doc_b = _register(db, with_workflow=False, workflow_id="wf-run-b", document_id="d-run-b", filename="b.pdf", filepath="/tmp/b.pdf", stage="completed", - instance="tenant-b", + group="tenant-b", ) job_a = db.create_document_job(doc_a, "ocr_only", status="completed") job_b = db.create_document_job(doc_b, "ocr_only", status="running") - tenant_a_only = db.list_runs(instances=["tenant-a"]) + tenant_a_only = db.list_runs(groups=["tenant-a"]) assert {r["id"] for r in tenant_a_only} == {job_a} - assert all(r.get("instance") == "tenant-a" for r in tenant_a_only) + assert all(r.get("group") == "tenant-a" for r in tenant_a_only) - both = db.list_runs(instances=["tenant-a", "tenant-b"]) + both = db.list_runs(groups=["tenant-a", "tenant-b"]) assert {r["id"] for r in both} == {job_a, job_b} - none = db.list_runs(instances=[]) + none = db.list_runs(groups=[]) assert none == [] - unrestricted_rows = db.list_runs(instances=None) + unrestricted_rows = db.list_runs(groups=None) assert {r["id"] for r in unrestricted_rows} >= {job_a, job_b} - # Status filter still works under instance scope. - running_a = db.list_runs(status="running", instances=["tenant-a"]) + # Status filter still works under group scope. + running_a = db.list_runs(status="running", groups=["tenant-a"]) assert running_a == [] - running_b = db.list_runs(status="running", instances=["tenant-b"]) + running_b = db.list_runs(status="running", groups=["tenant-b"]) assert {r["id"] for r in running_b} == {job_b} def test_state_user_cannot_see_bv_portal_documents(db_connection): """State-scoped users must not see portal (BV) docs — only their state.""" db = db_connection - db.upsert_document( + _register(db, workflow_id="wf-bv", document_id="d-bv", filename="portal.pdf", filepath="/tmp/portal.pdf", stage="ocr_review", - instance=PORTAL_INSTANCE, + group=PORTAL_GROUP, ) - db.upsert_document( + _register(db, workflow_id="wf-mh", document_id="d-mh", filename="mh.pdf", filepath="/tmp/mh.pdf", stage="ocr_review", - instance="mh", + group="mh", ) db.create_document_job("wf-bv", "ocr_only", status="running") db.create_document_job("wf-mh", "ocr_only", status="running") @@ -284,57 +298,57 @@ def test_state_user_cannot_see_bv_portal_documents(db_connection): } ) assert mh_user.is_superadmin is False - assert allowed_instances(mh_user) == {"mh"} - assert not user_can_access_instance(mh_user, PORTAL_INSTANCE) - assert not user_can_access_instance(mh_user, "bv") + assert allowed_groups(mh_user) == {"mh"} + assert not user_can_access_group(mh_user, PORTAL_GROUP) + assert not user_can_access_group(mh_user, "bv") - mh_docs = db.list_documents(instances=sorted(allowed_instances(mh_user) or [])) + mh_docs = db.list_documents(groups=sorted(allowed_groups(mh_user) or [])) assert {d["workflow_id"] for d in mh_docs} == {"wf-mh"} - mh_runs = db.list_runs(instances=sorted(allowed_instances(mh_user) or [])) - assert all(r.get("instance") == "mh" for r in mh_runs) + mh_runs = db.list_runs(groups=sorted(allowed_groups(mh_user) or [])) + assert all(r.get("group") == "mh" for r in mh_runs) assert {r["workflow_id"] for r in mh_runs} == {"wf-mh"} mh_queue, mh_total = db.list_operations_queue( - instances=sorted(allowed_instances(mh_user) or []) + groups=sorted(allowed_groups(mh_user) or []) ) assert mh_total == 1 assert {r["workflow_id"] for r in mh_queue} == {"wf-mh"} with pytest.raises(HTTPException) as exc: - assert_document_instance_access( - mh_user, {"workflow_id": "wf-bv", "instance": PORTAL_INSTANCE} + assert_document_group_access( + mh_user, {"workflow_id": "wf-bv", "group": PORTAL_GROUP} ) assert exc.value.status_code == 404 # Super-admin sees both sa = claims_to_user({"sub": "sa", "groups": ["/global/super-admin"]}) - assert allowed_instances(sa) is None - assert user_can_access_instance(sa, PORTAL_INSTANCE) - all_docs = db.list_documents(instances=None) + assert allowed_groups(sa) is None + assert user_can_access_group(sa, PORTAL_GROUP) + all_docs = db.list_documents(groups=None) assert {d["workflow_id"] for d in all_docs} >= {"wf-bv", "wf-mh"} -def test_upsert_does_not_reassign_instance(db_connection): +def test_upsert_does_not_reassign_group(db_connection): db = db_connection - db.upsert_document( + _register(db, workflow_id="wf-owned", document_id="d1", filename="a.pdf", filepath="/tmp/a.pdf", stage="registered", - instance="tenant-a", + group="tenant-a", ) - db.upsert_document( + _register(db, workflow_id="wf-owned", document_id="d1", filename="a.pdf", filepath="/tmp/a.pdf", stage="ocr_review", - instance="tenant-b", # must be ignored on update + group="tenant-b", # must be ignored on update ) doc = db.get_document("wf-owned") - assert doc["instance"] == "tenant-a" + assert doc["group"] == "tenant-a" assert doc["stage"] == "ocr_review" @@ -344,13 +358,13 @@ def test_api_helpers_hide_cross_tenant_mutations(db_connection, monkeypatch): import pipeline.db as db_mod monkeypatch.setattr(api, "db", db_mod) - db_mod.upsert_document( + _register(db_mod, workflow_id="wf-tenant-b-doc", document_id="d-tenant-b", filename="b.pdf", filepath="/tmp/b.pdf", stage="ocr_review", - instance="tenant-b", + group="tenant-b", ) user = claims_to_user( { @@ -363,3 +377,109 @@ def test_api_helpers_hide_cross_tenant_mutations(db_connection, monkeypatch): api._require_document_for_user("wf-tenant-b-doc", user) assert exc.value.status_code == 404 assert api._document_for_user_or_none("wf-tenant-b-doc", user) is None + + +def test_group_is_reached_through_the_documents_catalog(db_connection): + """No documents.instance column: a document's group is a groups row reached through its catalog.""" + db = db_connection + _register(db, workflow_id="wf-g-a", document_id="g1", filename="a.pdf", + filepath="/tmp/a.pdf", group="tenant-a") + _register(db, workflow_id="wf-g-bv", document_id="g2", filename="b.pdf", + filepath="/tmp/b.pdf", group=PORTAL_GROUP) + _register(db, workflow_id="wf-g-adv", document_id="g3", filename="c.pdf", + filepath="/tmp/c.pdf", group="tenant-a", kind="advisory") + + with db.get_connection() as conn: + groups = {r["name"]: r["path"] for r in conn.execute("SELECT name, path FROM groups")} + catalogs = conn.execute( + "SELECT g.name AS group_name, c.type FROM catalogs c JOIN groups g ON g.id = c.group_id" + ).fetchall() + assert groups == { + "tenant-a": "/states/TENANT-A", + PORTAL_GROUP: "/global" if PORTAL_GROUP == "bv" else f"/states/{PORTAL_GROUP.upper()}", + } + # One catalog per (group, type): tenant-a has scheme + advisory, the portal only scheme. + assert sorted((r["group_name"], r["type"]) for r in catalogs) == sorted([ + ("tenant-a", "openagrinet:AgricultureResource"), + ("tenant-a", "openagrinet:KnowledgeAdvisory"), + (PORTAL_GROUP, "openagrinet:AgricultureResource"), + ]) + assert db.get_document("wf-g-a")["group"] == "tenant-a" + assert db.get_document("wf-g-adv")["kind"] == "advisory" + assert {d["workflow_id"] for d in db.list_documents(groups=["tenant-a"])} == {"wf-g-a", "wf-g-adv"} + + +def test_same_code_is_allowed_in_another_tenants_catalog(db_connection): + db = db_connection + _register(db, workflow_id="wf-c-a", document_id="c1", filename="a.pdf", + filepath="/tmp/a.pdf", group="tenant-a", code="pm-kisan") + _register(db, workflow_id="wf-c-b", document_id="c2", filename="b.pdf", + filepath="/tmp/b.pdf", group="tenant-b", code="pm-kisan") + assert db.get_document("wf-c-a")["code"] == db.get_document("wf-c-b")["code"] == "pm-kisan" + # ...but not twice inside one tenant's catalog of that kind. + with pytest.raises(db.DuplicateDocumentCode): + _register(db, workflow_id="wf-c-a2", document_id="c3", filename="a2.pdf", + filepath="/tmp/a2.pdf", group="tenant-a", code="pm-kisan") + + +def test_audit_keeps_the_tenant_scope_after_the_document_is_purged(db_connection): + """audit_logs snapshots the group, so a purged document's history stays tenant-scoped.""" + db = db_connection + _register(db, workflow_id="wf-aud-a", document_id="a1", filename="a.pdf", + filepath="/tmp/a.pdf", group="tenant-a") + _register(db, workflow_id="wf-aud-b", document_id="a2", filename="b.pdf", + filepath="/tmp/b.pdf", group="tenant-b") + db.log_audit(workflow_id="wf-aud-a", document_id="", action_type="page_edit", entity_type="document") + db.log_audit(workflow_id="wf-aud-b", document_id="", action_type="page_edit", entity_type="document") + db.purge_document("wf-aud-a", keep_audit=True) + + a_rows = db.get_all_audit_logs(groups=["tenant-a"]) + assert [r["workflow_id"] for r in a_rows if r["action_type"] == "page_edit"] == ["wf-aud-a"] + b_rows = db.get_all_audit_logs(groups=["tenant-b"]) + assert [r["workflow_id"] for r in b_rows if r["action_type"] == "page_edit"] == ["wf-aud-b"] + assert db.get_all_audit_log_count(groups=["tenant-a"]) == len(a_rows) + assert db.get_all_audit_logs(groups=[]) == [] + + +def test_user_group_follows_the_first_tenant_in_the_token(db_connection, caplog): + db = db_connection + + def group_of(keycloak_id): + with db.get_connection() as conn: + row = conn.execute( + "SELECT g.name FROM users u LEFT JOIN groups g ON g.id = u.group_id " + "WHERE u.keycloak_user_id = %s", (keycloak_id,)).fetchone() + return row["name"] + + db.upsert_user("kc-single", "single", "single@example.com", ["contributor"], tenants=["MH"]) + assert group_of("kc-single") == "mh" + + with caplog.at_level(logging.WARNING, logger="pipeline.db"): + db.upsert_user("kc-multi", "multi", "multi@example.com", ["contributor"], tenants=["up", "mh"]) + assert group_of("kc-multi") == "mh" # sorted, first wins + assert any("several tenants" in rec.message for rec in caplog.records) + + # No tenants in the token leaves an existing group alone. + db.upsert_user("kc-single", "single", "single@example.com", ["contributor"]) + assert group_of("kc-single") == "mh" + db.upsert_user("kc-none", "none", "none@example.com", ["contributor"]) + assert group_of("kc-none") is None + + +def test_group_path_fills_the_keycloak_id_on_sync(db_connection): + """The SQLite move creates groups without a Keycloak id; a later sync matches by path.""" + db = db_connection + first = db.upsert_group("/states/MH") + with db.get_connection() as conn: + assert conn.execute("SELECT keycloak_group_id FROM groups WHERE id = %s::uuid", + (first,)).fetchone()["keycloak_group_id"] is None + again = db.upsert_group("/states/MH", keycloak_group_id="kc-group-123") + assert again == first + with db.get_connection() as conn: + row = conn.execute("SELECT name, keycloak_group_id FROM groups WHERE id = %s::uuid", (first,)).fetchone() + assert (row["name"], row["keycloak_group_id"]) == ("mh", "kc-group-123") + # A sync without an id never erases one already stored. + assert db.upsert_group("/states/MH") == first + with db.get_connection() as conn: + assert conn.execute("SELECT keycloak_group_id FROM groups WHERE id = %s::uuid", + (first,)).fetchone()["keycloak_group_id"] == "kc-group-123" From 82237ecf436f55c86d85710205570552593b08ad Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 11:07:18 +0530 Subject: [PATCH 15/53] feat(ui): select document type at upload and use group/kind/code/name NewDocumentView sends group and kind. The classification panel offers scheme and advisory only and no longer blocks ingestion on a missing type. Instance helpers and badge are renamed to group. --- ui/src/auth/AuthProvider.jsx | 36 ++--- ui/src/components/DocumentHeaderSummary.jsx | 6 +- ui/src/components/GroupBadge.jsx | 21 +++ ui/src/components/InstanceBadge.jsx | 21 --- .../lib/{instanceLabels.js => groupLabels.js} | 18 +-- ui/src/lib/pipelineUi.js | 2 +- ui/src/views/ChunkExplorerView.jsx | 4 +- ui/src/views/DashboardView.jsx | 12 +- ui/src/views/DocumentOpsView.jsx | 140 +++++------------- ui/src/views/DocumentsView.jsx | 30 ++-- ui/src/views/GlobalAuditView.jsx | 6 +- ui/src/views/NewDocumentView.jsx | 48 ++++-- ui/src/views/QueueView.jsx | 4 +- ui/src/views/RunsView.jsx | 4 +- 14 files changed, 158 insertions(+), 194 deletions(-) create mode 100644 ui/src/components/GroupBadge.jsx delete mode 100644 ui/src/components/InstanceBadge.jsx rename ui/src/lib/{instanceLabels.js => groupLabels.js} (63%) diff --git a/ui/src/auth/AuthProvider.jsx b/ui/src/auth/AuthProvider.jsx index fdf8e27..998f86c 100644 --- a/ui/src/auth/AuthProvider.jsx +++ b/ui/src/auth/AuthProvider.jsx @@ -366,11 +366,11 @@ export function AuthProvider({ children }) { ) /** Role for a state tenant (mh, up, …). Super-admin always returns super_admin. */ - const roleForInstance = useCallback( - (instance) => { + const roleForGroup = useCallback( + (group) => { if (!AUTH_ENABLED || isSuperAdmin) return 'super_admin' - if (!instance) return null - const key = String(instance).trim().toLowerCase() + if (!group) return null + const key = String(group).trim().toLowerCase() // A BH viewer holds no per-state group but reads every state. return stateRoles[key] || (isBhViewer ? 'state_view' : null) }, @@ -378,11 +378,11 @@ export function AuthProvider({ children }) { ) /** Whether the user may see a tenant (state). Super-admin → all. */ - const canAccessInstance = useCallback( - (instance) => { + const canAccessGroup = useCallback( + (group) => { if (!AUTH_ENABLED || isSuperAdmin || isBhViewer) return true - if (!instance) return false - const key = String(instance).trim().toLowerCase() + if (!group) return false + const key = String(group).trim().toLowerCase() if (instances.length === 0) return false return instances.map((i) => String(i).toLowerCase()).includes(key) }, @@ -397,17 +397,17 @@ export function AuthProvider({ children }) { * may approve OCR / translation / chunking) but NOT 'approve_ingestion' * (it may not publish to DEV). */ - const hasPermissionForInstance = useCallback( - (perm, instance) => { + const hasPermissionForGroup = useCallback( + (perm, group) => { if (!AUTH_ENABLED) return true if (isSuperAdmin) return hasPermission(perm) - const role = normalizeProductRole(roleForInstance(instance)) + const role = normalizeProductRole(roleForGroup(group)) if (!role) return false // Platform-level capabilities are never granted by a state role. if (perm === 'admin' || perm === 'manage_users') return false return roleGrants(role, perm) }, - [hasPermission, isSuperAdmin, roleForInstance], + [hasPermission, isSuperAdmin, roleForGroup], ) const value = useMemo( @@ -432,10 +432,10 @@ export function AuthProvider({ children }) { isSuperAdmin, isBhViewer, hasPermission, - hasPermissionForInstance, + hasPermissionForGroup, hasRole, - roleForInstance, - canAccessInstance, + roleForGroup, + canAccessGroup, loginWithSso, loginWithOtpTokens, logout, @@ -460,10 +460,10 @@ export function AuthProvider({ children }) { isSuperAdmin, isBhViewer, hasPermission, - hasPermissionForInstance, + hasPermissionForGroup, hasRole, - roleForInstance, - canAccessInstance, + roleForGroup, + canAccessGroup, loginWithSso, loginWithOtpTokens, logout, diff --git a/ui/src/components/DocumentHeaderSummary.jsx b/ui/src/components/DocumentHeaderSummary.jsx index 5098450..999c654 100644 --- a/ui/src/components/DocumentHeaderSummary.jsx +++ b/ui/src/components/DocumentHeaderSummary.jsx @@ -1,6 +1,6 @@ import { useMemo, useState } from 'react' import { ArrowLeft, Info } from 'lucide-react' -import { formatInstanceLabel, instanceBadgeTitle } from '../lib/instanceLabels' +import { formatGroupLabel, groupBadgeTitle } from '../lib/groupLabels' import { formatCompactDateTime, getDocumentFileLabel, @@ -74,7 +74,7 @@ export function DocumentHeaderSummary({ const pages = pageCount ?? doc?.page_count ?? 0 const chunks = chunkCount ?? doc?.chunk_count ?? 0 const uploader = uploaderName(doc) - const stateLabel = formatInstanceLabel(doc?.instance) + const stateLabel = formatGroupLabel(doc?.group) const roleLabels = useMemo( () => productRoleLabels(doc?.uploaded_by_roles || []), [doc?.uploaded_by_roles], @@ -126,7 +126,7 @@ export function DocumentHeaderSummary({ {stateLabel ? ( - {stateLabel} + {stateLabel} ) : ( '—' )} diff --git a/ui/src/components/GroupBadge.jsx b/ui/src/components/GroupBadge.jsx new file mode 100644 index 0000000..edb2c2e --- /dev/null +++ b/ui/src/components/GroupBadge.jsx @@ -0,0 +1,21 @@ +import { formatGroupLabel, groupBadgeTitle } from '../lib/groupLabels' +import { Badge } from './ui/badge' +import { cn } from '../lib/utils' + +/** Plan 2: show document state / BV portal tag from ``group``. */ +export function GroupBadge({ group, className }) { + const label = formatGroupLabel(group) + if (!label) return null + const isPortal = label === 'BV' + return ( + + {label} + + ) +} + +export default GroupBadge diff --git a/ui/src/components/InstanceBadge.jsx b/ui/src/components/InstanceBadge.jsx deleted file mode 100644 index 87d4b4a..0000000 --- a/ui/src/components/InstanceBadge.jsx +++ /dev/null @@ -1,21 +0,0 @@ -import { formatInstanceLabel, instanceBadgeTitle } from '../lib/instanceLabels' -import { Badge } from './ui/badge' -import { cn } from '../lib/utils' - -/** Plan 2: show document state / BV portal tag from ``instance``. */ -export function InstanceBadge({ instance, className }) { - const label = formatInstanceLabel(instance) - if (!label) return null - const isPortal = label === 'BV' - return ( - - {label} - - ) -} - -export default InstanceBadge diff --git a/ui/src/lib/instanceLabels.js b/ui/src/lib/groupLabels.js similarity index 63% rename from ui/src/lib/instanceLabels.js rename to ui/src/lib/groupLabels.js index ae0e843..95e7840 100644 --- a/ui/src/lib/instanceLabels.js +++ b/ui/src/lib/groupLabels.js @@ -1,13 +1,13 @@ /** - * Plan 2: display labels for documents.instance (state / portal tag). + * Plan 2: display labels for the document group (state / portal tag). * Authorization still comes from Keycloak JWT — this is UI only. */ -export const PORTAL_INSTANCE = 'bv' +export const PORTAL_GROUP = 'bv' /** Short badge label: mh → MH, bv → BV */ -export function formatInstanceLabel(instance) { - const code = String(instance || '') +export function formatGroupLabel(group) { + const code = String(group || '') .trim() .toLowerCase() if (!code || code === 'default') return null @@ -17,8 +17,8 @@ export function formatInstanceLabel(instance) { return code.toUpperCase() } -export function instanceBadgeTitle(instance) { - const code = String(instance || '') +export function groupBadgeTitle(group) { + const code = String(group || '') .trim() .toLowerCase() if (!code || code === 'default') return 'No state tag' @@ -29,11 +29,11 @@ export function instanceBadgeTitle(instance) { } /** - * Default instance for upload form. + * Default group for upload form. * Super admin → portal (bv); single state → that state; multi → first (user can change). */ -export function defaultUploadInstance({ isSuperAdmin, instances = [], portalInstance = PORTAL_INSTANCE }) { - if (isSuperAdmin) return portalInstance || PORTAL_INSTANCE +export function defaultUploadGroup({ isSuperAdmin, instances = [], portalGroup = PORTAL_GROUP }) { + if (isSuperAdmin) return portalGroup || PORTAL_GROUP const list = (instances || []).map((i) => String(i).trim().toLowerCase()).filter(Boolean) if (list.length >= 1) return list[0] return '' diff --git a/ui/src/lib/pipelineUi.js b/ui/src/lib/pipelineUi.js index e4f833a..db7dbe2 100644 --- a/ui/src/lib/pipelineUi.js +++ b/ui/src/lib/pipelineUi.js @@ -206,7 +206,7 @@ export const DEFAULT_SEARCH_SETTINGS = { export function getDocumentListLabel(doc) { return ( - doc?.display_name || + doc?.name || doc?.name_en || doc?.name || doc?.filename || diff --git a/ui/src/views/ChunkExplorerView.jsx b/ui/src/views/ChunkExplorerView.jsx index d1ef3ea..e677724 100644 --- a/ui/src/views/ChunkExplorerView.jsx +++ b/ui/src/views/ChunkExplorerView.jsx @@ -166,7 +166,7 @@ export default function ChunkExplorerView() { const groups = new Map() chunks.forEach(chunk => { const key = chunk.workflow_id || '__unknown__' - const label = chunk.display_name || chunk.filename || key + const label = chunk.name || chunk.filename || key const existing = groups.get(key) || { key, label, chunks: [] } existing.chunks.push(chunk) groups.set(key, existing) @@ -401,7 +401,7 @@ export default function ChunkExplorerView() { openDocumentChunks(chunk.workflow_id) }} > - {chunk.display_name || chunk.filename || chunk.workflow_id} + {chunk.name || chunk.filename || chunk.workflow_id}

Chunk {chunk.chunk_number} · Pages {chunk.page_start}–{chunk.page_end} · {getStageLabel(chunk.stage)} diff --git a/ui/src/views/DashboardView.jsx b/ui/src/views/DashboardView.jsx index becf4a6..2538e72 100644 --- a/ui/src/views/DashboardView.jsx +++ b/ui/src/views/DashboardView.jsx @@ -5,7 +5,7 @@ import { Skeleton } from '../components/ui/skeleton' import { StatCard } from '../components/StatCard' import { StageBadge } from '../components/StageBadge' import { fetchJson, formatCount, getDocumentListLabel, summarizeQueueReason } from '../lib/pipelineUi' -import { formatInstanceLabel } from '../lib/instanceLabels' +import { formatGroupLabel } from '../lib/groupLabels' import { useAuth } from '../auth/AuthProvider' import { AlertTriangle, CheckCircle, FileText, Globe2, ListTodo } from 'lucide-react' @@ -66,8 +66,8 @@ export default function DashboardView() { const reindexCount = summary?.needs_reindex || 0 const displayedQueueItems = queue.slice(0, QUEUE_ROWS_SHOWN) - const byInstance = useMemo(() => (Array.isArray(summary?.by_instance) ? summary.by_instance : []), [summary]) - const stateRows = byInstance + const byGroup = useMemo(() => (Array.isArray(summary?.by_group) ? summary.by_group : []), [summary]) + const stateRows = byGroup const maxStateCount = useMemo( () => stateRows.reduce((max, item) => Math.max(max, item.count || 0), 0), @@ -113,7 +113,7 @@ export default function DashboardView() { /> } /> { const label = row.otherStateCount ? `Others (${row.otherStateCount} states)` - : formatInstanceLabel(row.instance) || 'No state tag' + : formatGroupLabel(row.group) || 'No state tag' const pct = (value) => (maxStateCount ? (value / maxStateCount) * 100 : 0) return ( -

+
{label} {formatCount(row.count)} diff --git a/ui/src/views/DocumentOpsView.jsx b/ui/src/views/DocumentOpsView.jsx index d5dec0f..4cab7e9 100644 --- a/ui/src/views/DocumentOpsView.jsx +++ b/ui/src/views/DocumentOpsView.jsx @@ -94,22 +94,6 @@ function PanelNotice({ tone = 'error', title, message }) { ) } -// Client-side preview only — matches pipeline/scheme_catalog.py's -// derive_scheme_code() closely enough for instant feedback, but the server -// is the authoritative, collision-checked source of truth for the real code. -const SCHEME_CODE_STOPWORDS = new Set(['a', 'an', 'the', 'of', 'on', 'for', 'and', 'or', 'to', 'in', 'at', 'by', 'with', '&']) - -function previewSchemeCode(title) { - const words = (title || '').match(/[A-Za-z0-9]+/g) || [] - const letters = words - .filter(w => !SCHEME_CODE_STOPWORDS.has(w.toLowerCase())) - .map(w => w[0].toLowerCase()) - .join('') - if (letters.length >= 2) return letters - const slug = (title || '').toLowerCase().replace(/[^a-z0-9]+/g, '') - return slug.slice(0, 8) || 'scheme' -} - /** * Validity the document is searchable in. The start is prefilled from what the * server stamped at upload; the end is deliberately blank-able - an empty end @@ -125,22 +109,11 @@ function validateValidity(validFrom, validTo) { const DOCUMENT_KIND_OPTIONS = [ { value: 'scheme', label: 'Scheme' }, { value: 'advisory', label: 'Advisory' }, - { value: 'video', label: 'Video' }, - { value: '__custom__', label: 'Custom…' }, ] function DocumentClassificationPanel({ doc, workflowId, canClassify, onSaved }) { - const hasKind = doc.document_kind && doc.document_kind !== 'document' - // 'document' is the unclassified default, not a choice the panel offers, so - // it seeds the select as empty — otherwise the select renders blank while the - // save button reads "Saved", which tells a reviewer the opposite of the truth. - const [kind, setKind] = useState( - hasKind && !DOCUMENT_KIND_OPTIONS.some(o => o.value === doc.document_kind) - ? '__custom__' - : (hasKind ? doc.document_kind : '') - ) - const [customKind, setCustomKind] = useState(hasKind && !DOCUMENT_KIND_OPTIONS.some(o => o.value === doc.document_kind) ? doc.document_kind : '') - const [schemeName, setSchemeName] = useState(doc.scheme_name || '') + const [kind, setKind] = useState(doc.kind || '') + const [name, setName] = useState(doc.name || '') const [validFrom, setValidFrom] = useState(doc.valid_from || todayISODate()) // Blank when the document has no end date, which is the normal case - an // empty field here reads as "never expires", not as "not filled in yet". @@ -148,14 +121,12 @@ function DocumentClassificationPanel({ doc, workflowId, canClassify, onSaved }) const [saving, setSaving] = useState(false) const [error, setError] = useState('') - const effectiveKind = kind === '__custom__' ? customKind.trim().toLowerCase() : kind - const isScheme = effectiveKind === 'scheme' const validityError = DOCUMENT_VALIDITY_ENABLED ? validateValidity(validFrom, validTo) : '' const canSave = - Boolean(effectiveKind) && - (!isScheme || schemeName.trim().length > 0) && + Boolean(kind) && + name.trim().length > 0 && !validityError async function save() { @@ -167,8 +138,8 @@ function DocumentClassificationPanel({ doc, workflowId, canClassify, onSaved }) method: 'PATCH', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ - document_kind: effectiveKind, - ...(isScheme ? { scheme_name: schemeName.trim() } : {}), + kind, + name: name.trim(), // Omitted entirely when the feature is off, so classifying a // document leaves whatever period the upload stamped untouched. ...(DOCUMENT_VALIDITY_ENABLED @@ -185,8 +156,8 @@ function DocumentClassificationPanel({ doc, workflowId, canClassify, onSaved }) } const alreadySaved = - doc.document_kind === effectiveKind - && (!isScheme || doc.scheme_name === schemeName.trim()) + doc.kind === kind + && doc.name === name.trim() && (!DOCUMENT_VALIDITY_ENABLED || (doc.valid_from === validFrom && (doc.valid_to || '') === validTo)) @@ -236,38 +207,22 @@ function DocumentClassificationPanel({ doc, workflowId, canClassify, onSaved })
)} - {kind === '__custom__' && ( -
- Custom type - setCustomKind(e.target.value)} - placeholder="e.g. faq" - /> -
- )} - {isScheme && ( - <> -
- Scheme name - setSchemeName(e.target.value)} - placeholder="e.g. National Mission on Natural Farming" - /> -
-
- Scheme code (auto) - - {doc.scheme_code && doc.scheme_name === schemeName.trim() ? doc.scheme_code : (schemeName.trim() ? previewSchemeCode(schemeName) : '—')} - -
- - )} +
+ Name + setName(e.target.value)} + placeholder="e.g. National Mission on Natural Farming" + /> +
+
+ Code + + {doc.code || '—'} + +
{canClassify && (
) @@ -844,13 +799,6 @@ export default function DocumentOpsView() { && canRunAction(action) ) const canRemoveDocument = canAdmin && (doc?.available_actions || []).includes('disable_document') - // 'document' is never offered as a choice in the classification panel (only - // scheme/advisory/video/custom are) — seeing it here means the panel was - // never used. Mirrors the same check the reingest endpoint enforces server-side. - const isDocClassified = Boolean( - (doc?.document_kind && doc.document_kind !== 'document') || doc?.scheme_code || doc?.scheme_name - ) - const ingestBlockedByClassification = doc?.stage === 'ready_for_ingestion' && !isDocClassified const sortedPages = useMemo(() => [...pages].sort((a, b) => a.page_number - b.page_number), [pages]) const reviewedPages = useMemo(() => pages.filter(p => p.is_reviewed).length, [pages]) const reviewedChunks = useMemo(() => chunks.filter(c => c.is_reviewed).length, [chunks]) @@ -1008,30 +956,18 @@ export default function DocumentOpsView() { hasChunks={chunks.length > 0 || Boolean(doc.chunk_count)} />
- {visibleActions.slice(0, 4).map(action => { - const blockedByClassification = action === 'approve_ingestion' && ingestBlockedByClassification - return ( - - ) - })} + {visibleActions.slice(0, 4).map(action => ( + + ))} {canRemoveDocument && (
diff --git a/ui/src/views/RunsView.jsx b/ui/src/views/RunsView.jsx index c3a9bfb..d11a44d 100644 --- a/ui/src/views/RunsView.jsx +++ b/ui/src/views/RunsView.jsx @@ -4,7 +4,7 @@ import { Badge } from '../components/ui/badge' import { Button } from '../components/ui/button' import { Skeleton } from '../components/ui/skeleton' import { StageBadge } from '../components/StageBadge' -import { InstanceBadge } from '../components/InstanceBadge' +import { GroupBadge } from '../components/GroupBadge' import PipelineStepper from '../components/PipelineStepper' import { fetchJson, @@ -356,7 +356,7 @@ export default function RunsView() { {title}
- + #{runId} {fileLabel && fileLabel !== title ? ` · ${fileLabel}` : ''} From b02408a367cc30002b5561dc12198af7391d6ae8 Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 11:07:18 +0530 Subject: [PATCH 16/53] docs: describe schema v2 and record the groups and catalogs decision --- AGENTS.md | 6 +- docs/ADR/0006-groups-and-catalogs-model.md | 21 ++ docs/POSTGRES_SCHEMA.md | 291 +++++++++++---------- 3 files changed, 181 insertions(+), 137 deletions(-) create mode 100644 docs/ADR/0006-groups-and-catalogs-model.md diff --git a/AGENTS.md b/AGENTS.md index 43e113c..efa8f54 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,8 +66,8 @@ Core modules (all under `pipeline/`): - `discovery_publish_service.py` — `DiscoveryPublishService`: owns the Publish to Network env vars and the HTTP call. Deliberately Temporal-free - `models.py` — Pydantic models, including `DocumentStage` enum and `PIPELINE_STAGES` (the stepper-UI stage list) - `config.py` — `Config` dataclass reading env vars, with defaults -- `instances.py` — instance/tenant (state) code ↔ human name mapping; `bv` = Bharat Vistaar platform-wide, others are state codes matching Keycloak group paths (`/states/MH/...` → `mh`) -- `scheme_catalog.py` — scheme catalog behind the pull-based `/catalog/v1/*` API, served from the `catalog_entries` table and the computed `catalog` view +- `group_names.py` — tenant (group) code ↔ human name mapping; `bv` = Bharat Vistaar platform-wide, others are state codes matching Keycloak group paths (`/states/MH/...` → `mh`). A tenant code is a `groups` row (name = code); a document's `group` is derived through `documents.catalog_id → catalogs.group_id`, there is no `documents.instance` column +- `scheme_catalog.py` — scheme catalog behind the pull-based `/catalog/v1/*` API, served from the computed `catalog_status` view (one row per document; `catalogs` rows are `(group, type)`, `documents` carry `code`/`name`/`aliases`) - `ocr/`, `translation/`, `chunking/`, `vector_store/` — pluggable provider packages, each with a `base.py` protocol/ABC and a `service.py` (or equivalent) that dispatches to the configured provider - `auth/` — Keycloak JWT validation, group-based tenancy/permissions, email-OTP flow, admin user management @@ -85,7 +85,7 @@ registered → ocr_processing → ocr_review → translation_processing → tran → approval_for_prod → ingesting_prod → completed (or → failed at any point) ``` -`approval_for_prod`/`ingesting_prod` are **PROD-only stages** (`PROD_ONLY_STAGES` in `models.py`); with `DISABLE_PROD_SETTING=true` a document goes straight from `ingesting` to `completed`. That flag is read only at workflow *start* and passed in as an argument — never read inside a running workflow, since that would break Temporal replay determinism if flipped mid-flight (see the docstring in `instances.py`). +`approval_for_prod`/`ingesting_prod` are **PROD-only stages** (`PROD_ONLY_STAGES` in `models.py`); with `DISABLE_PROD_SETTING=true` a document goes straight from `ingesting` to `completed`. That flag is read only at workflow *start* and passed in as an argument — never read inside a running workflow, since that would break Temporal replay determinism if flipped mid-flight (see the docstring in `group_names.py`). ### DEV vs PROD dual indexing diff --git a/docs/ADR/0006-groups-and-catalogs-model.md b/docs/ADR/0006-groups-and-catalogs-model.md new file mode 100644 index 0000000..c610a7b --- /dev/null +++ b/docs/ADR/0006-groups-and-catalogs-model.md @@ -0,0 +1,21 @@ +# Providers are groups, catalogs are (group, type), documents are identified by (catalog, code) + +Context: the first Postgres schema modelled a tenant as a free-text `documents.instance`, a `catalog_entries` table keyed by `(type, code)` with one entry per scheme, and a free-form `document_kind` that had to match the entry's type through a composite foreign key. Review feedback was that the unit published on the network is a provider's catalog, that a provider is a Keycloak group, that each scheme is itself a document, and that database enums and demo flags do not belong in the schema. + +Decision: +- A **group** is a provider: one `groups` row per Keycloak tenant node (`/states/MH`, `/global`). Role leaves are not mirrored; permissions still come from the token. `users.group_id` is a single nullable FK for now (first tenant in the token, sorted, with a warning when there are several); a join table can replace it if multi-group membership is needed. +- A **catalog** is `(group, type)`: `catalogs.group_id` plus `catalogs.type`, unique together. `type` holds the network schema name. The kind-to-schema map lives in `pipeline/network_constants.py` and is a placeholder: `scheme` is `openagrinet:AgricultureResource`, `advisory` is `openagrinet:KnowledgeAdvisory`. Code branches through that map, never on raw `openagrinet:*` strings. Catalog Beckn information (`publish_directives`, `visible_networks`, `criteria`, `network`) sits in `catalogs.metadata`. +- A **document** belongs to one catalog and is identified there by `code`: `documents.catalog_id` NOT NULL, `UNIQUE (catalog_id, code)`, with `name` and `aliases` on the document. The API fields `code`, `name` and `aliases` are those columns. `documents.instance`, `document_kind` and `is_demo` are gone; the API returns the derived `group` (the group name) and `kind` (from the type map). The `/catalog/v1/*` snapshot keys, the Qdrant payload keys and Temporal arguments keep their earlier names (`scheme_code`, `document_kind`, `instance`, ...) because other services and in-flight workflows depend on them. +- Re-uploading an existing code updates the same document (new workflow, `source` replaced, pages and chunks cleared, `reindex_required` set) rather than adding a second one. +- Catalogs are created on upload for the tenant and kind. The upload therefore takes the document type up front. +- Bootstrap schemes (the code-seeded entries with no documents) are dropped. `/catalog/v1/*` is served from the `catalog_status` view, one row per document, and a scheme appears once a document exists. +- Period, uploader and vector index state moved into `documents.metadata`; counts are derived from `pages` and `chunks`. +- Database enums are replaced by plain `TEXT`, validated by `pipeline/models.py` (`DocumentStage`, `WorkflowType`, `WorkflowStatus`, `AuditEntityType`). +- `audit_logs.instance` is replaced by `audit_logs.group_id` (snapshot, `ON DELETE SET NULL`). +- No Postgres held the first schema, so `migrations/0001` was rewritten in place instead of adding an ALTER migration. `scripts/migrate_sqlite_to_postgres.py` produces v2 and skips `kind=document` and `is_demo` documents, reporting the counts. + +Alternatives considered: +- **Keep `documents.instance`.** Rejected: the catalog already carries the provider, and two sources of tenant truth drift. +- **Type as a short kind plus a separate `schema_type` column.** Rejected for now in favour of one `type` column holding the schema name; revisit if scheme and advisory end up sharing a schema (ADR 0003 types both as `KnowledgeAdvisory`). +- **`user_groups` join table now.** Deferred: the team wants one group per user for now. +- **Nullable `catalog_id`/`code` until a reviewer classifies the document.** Rejected: a document without a catalog has no tenant once `instance` is gone. The upload form asks for the type instead. diff --git a/docs/POSTGRES_SCHEMA.md b/docs/POSTGRES_SCHEMA.md index d76307a..168fe4c 100644 --- a/docs/POSTGRES_SCHEMA.md +++ b/docs/POSTGRES_SCHEMA.md @@ -1,6 +1,6 @@ # Postgres Schema — knowledge-provider -Postgres schema behind `pipeline/db.py`. A document is content and metadata. A workflow is one run of processing against a document. Source file and vector index state are stored as JSONB on the document. Translations are stored as JSONB on each page. +Postgres schema behind `pipeline/db.py`. A document is content and metadata. A workflow is one run of processing against a document. A document belongs to a catalog (a provider group plus a resource type) and is identified there by its `code`. Source file and vector index state are stored as JSONB on the document. Translations are stored as JSONB on each page. The schema is created by `migrations/0001_initial_schema.sql` and applied by the api and worker at startup (`pipeline/migrate.py`). `pipeline/db.py` is the only code that reads or writes it. The previous SQLite store is described in `docs/SQLITE_SCHEMA.md` (legacy) and is moved over once by `scripts/migrate_sqlite_to_postgres.py`. @@ -8,11 +8,11 @@ The schema is created by `migrations/0001_initial_schema.sql` and applied by the - Every table has a primary key `id`, a UUID v4 (`DEFAULT gen_random_uuid()`, built in from Postgres 13). - A foreign key is named `_id` (for example `document_id`). A role prefix is added when the role matters (`uploaded_by_user_id`, `started_by_user_id`). -- Text identifiers other systems rely on (`catalog_entries.code`, `temporal_workflow_id`) are ordinary columns, not keys. +- Text identifiers other systems rely on (`documents.code`, `temporal_workflow_id`) are ordinary columns, not keys. - `created_at` and `updated_at` are the last two columns of a table. `updated_at` is maintained by one shared trigger. Append-only or short-lived tables skip `updated_at`, as noted per table. - Timestamps are `TIMESTAMPTZ` (UTC), booleans are `BOOLEAN`, JSON is `JSONB`, lists of strings are `TEXT[]`. - JSONB columns have a `CHECK` that the value is an object (or array). Value rules inside them live in Pydantic models, not in the database. -- Fixed value sets are enums. Open sets (`artifact_type`, `action_type`) stay `TEXT`. +- There are no database enums. Every value set (stages, workflow types and statuses, audit entity types) is plain `TEXT`, validated in code by `pipeline/models.py`; open sets (`artifact_type`, `action_type`) are `TEXT` too. - Counts have `CHECK >= 0`. Slugs and identifiers have pattern `CHECK`s. ### Shared trigger @@ -28,50 +28,74 @@ END $$ LANGUAGE plpgsql; -- FOR EACH ROW EXECUTE FUNCTION set_updated_at(); ``` -### Enums +### Value sets (plain `TEXT`, validated in code) -| Enum | Values | -|---|---| -| `workflow_stage` | `registered`, `ocr_processing`, `ocr_review`, `translation_processing`, `translation_review`, `chunking`, `chunk_review`, `ready_for_ingestion`, `ingesting`, `publishing_to_network`, `approval_for_prod`, `ingesting_prod`, `completed`, `failed` | -| `workflow_type_enum` | `pipeline`, `ocr_only`, `ocr_retry`, `translation_retry`, `chunking_retry`, `reingest`, `promote_prod` | -| `workflow_status_enum` | `running`, `waiting_review`, `completed`, `failed` | -| `audit_entity_type` | `document`, `page`, `chunk`, `setting`, `catalog` | +| Column(s) | Python type (`pipeline/models.py`) | Values | +|---|---|---| +| `workflows.current_stage`, `workflow_artifacts.stage` | `DocumentStage` | `registered`, `ocr_processing`, `ocr_review`, `translation_processing`, `translation_review`, `chunking`, `chunk_review`, `ready_for_ingestion`, `ingesting`, `publishing_to_network`, `approval_for_prod`, `ingesting_prod`, `completed`, `failed` | +| `workflows.workflow_type` | `WorkflowType` | `pipeline`, `ocr_only`, `ocr_retry`, `translation_retry`, `chunking_retry`, `reingest`, `promote_prod` | +| `workflows.status` | `WorkflowStatus` | `running`, `waiting_review`, `completed`, `failed` | +| `audit_logs.entity_type` | `AuditEntityType` | `document`, `page`, `chunk`, `setting`, `catalog` | -Adding a value later needs `ALTER TYPE ... ADD VALUE`. +The database does not check these values, so adding one needs no migration. Raw SQL that writes them must use a value from the table. ## Tables | # | Table | Purpose | |---|---|---| -| 1 | `users` | People who upload, review and act, mirrored from Keycloak | -| 2 | `catalog_entries` | Generic named entries that documents belong to. Type `scheme` today, `advisory` and others later | -| 3 | `documents` | One row per document. Content, metadata, source file, index state | -| 4 | `workflows` | One row per processing run of a document. Stage, status, config | -| 5 | `workflow_artifacts` | Files each workflow produces, stored in MinIO | -| 6 | `pages` | OCR layer and translations of each page | -| 7 | `chunks` | Chunked text and review state | -| 8 | `audit_logs` | Append-only action history | -| 9 | `search_settings` | Single-row search configuration | -| 10 | `email_otps` | Email OTP login codes | -| 11 | `webhook_records` | Records pushed to `POST /webhook` | +| 1 | `groups` | Provider tenant nodes (`/states/MH`, `/global`), mirrored from Keycloak | +| 2 | `users` | People who upload, review and act, mirrored from Keycloak | +| 3 | `catalogs` | One row per provider group and resource type (scheme, advisory) | +| 4 | `documents` | One row per document. Content, metadata, source file, index state | +| 5 | `workflows` | One row per processing run of a document. Stage, status, config | +| 6 | `workflow_artifacts` | Files each workflow produces, stored in MinIO | +| 7 | `pages` | OCR layer and translations of each page | +| 8 | `chunks` | Chunked text and review state | +| 9 | `audit_logs` | Append-only action history | +| 10 | `search_settings` | Single-row search configuration | +| 11 | `email_otps` | Email OTP login codes | +| 12 | `webhook_records` | Records pushed to `POST /webhook` | | — | `schema_migrations` | Applied migration files (created by the runner, not by a migration) | -| — | `catalog` (view) | Computed catalog of all entries. `/catalog/v1/*` serves the `scheme` type | +| — | `catalog_status` (view) | One row per document with its catalog status. `/catalog/v1/*` serves the scheme type | | — | `document_current_workflow` (view) | Latest workflow of each document | Relationships: ``` -catalog_entries 1 ── * documents 1 ── * workflows 1 ── * workflow_artifacts - │ - ├── * pages - └── * chunks +groups 1 ── * users (users.group_id, nullable) +groups 1 ── * catalogs 1 ── * documents 1 ── * workflows 1 ── * workflow_artifacts + │ + ├── * pages + └── * chunks +groups 1 ── * audit_logs (audit_logs.group_id, snapshot, SET NULL) ``` -Dependency order for creation: `users`, `catalog_entries`, `documents`, `workflows`, `workflow_artifacts`, `pages`, `chunks`, `audit_logs`, `search_settings`, `email_otps`, `webhook_records`, views. +Dependency order for creation: `groups`, `users`, `catalogs`, `documents`, `workflows`, `workflow_artifacts`, `pages`, `chunks`, `audit_logs`, `search_settings`, `email_otps`, `webhook_records`, views. + +--- + +## 1. `groups` + +One row per tenant node (a state, the centre, or the platform), mirrored from Keycloak. A group is a provider: it owns catalogs. Rows are created on demand from a tenant code (`get_or_create_group`) or from a Keycloak group path (`upsert_group`). + +Unique: `keycloak_group_id`, `path`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Internal group ID. | +| keycloak_group_id | TEXT | UNIQUE | Keycloak group ID. NULL until a sync that sees the path fills it (rows made by the SQLite move or by an upload have none). | +| name | TEXT | NOT NULL | Tenant code, lowercase (`mh`, `bv`). This is what the API calls the document's `group`. | +| path | TEXT | NOT NULL UNIQUE | Keycloak group path: `/states/MH`, or `/global` for `bv`. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Created. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). | + +Notes +- Only the tenant node is mirrored, not the role leaves (`/states/MH/admin`). Roles and permissions are read from the token, not from this table. +- Role leaves stay in Keycloak; `catalogs.group_id` points at the tenant node. --- -## 1. `users` +## 2. `users` Uploader, reviewer and actor identity, looked up once from the Keycloak token. @@ -84,76 +108,72 @@ Unique: `keycloak_user_id`; unique index on `lower(email)` where `email IS NOT N | username | TEXT | | Current username. | | email | TEXT | | Current email. | | roles | TEXT[] | NOT NULL DEFAULT '{}' | Current roles. | +| group_id | UUID | FK → `groups(id)` ON DELETE SET NULL | The user's group (tenant). Single membership for now. | | first_seen_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | First login or upload. | | last_seen_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Most recent activity. | Notes -- Upserted on each authenticated upload or login. A `local-admin` row is seeded for `AUTH_DISABLED=true`. +- Upserted on each authenticated upload or login. A `local-admin` row is seeded by the migration. +- A token can carry several tenants but `group_id` holds one: `upsert_user(..., tenants=[...])` uses the first code in sorted order and logs a warning when there are several. Authorization still reads the token. +- If users later need several groups, add a `user_groups` join table and drop `group_id`. - Point-in-time identity is kept in `audit_logs.actor`. --- -## 2. `catalog_entries` +## 3. `catalogs` -Generic named entries that documents belong to. One table for every entry type: `scheme` today, `advisory` and others later. A new type adds rows, not tables. +One row per provider group and resource type. A catalog is what a provider publishes on the network: Mahavistar's `scheme` catalog and its `advisory` catalog are two rows. A new type adds rows, not tables. -Unique: `(type, code)` and `(id, type)` (the second is the target of the composite FK from `documents`). +Unique: `(group_id, type)`. | Column | Type | Constraints / Default | Purpose | |---|---|---|---| -| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Entry ID. | -| type | TEXT | NOT NULL, CHECK `^[a-z0-9]+([_-][a-z0-9]+)*$` | Entry type (`scheme`, later `advisory`, and so on). Open set, so not an enum. | -| code | TEXT | NOT NULL, CHECK `^[a-z0-9]+(-[a-z0-9]+)*$` | Short code (`pm-kmy`) that catalog consumers see. Unique within a type. | -| name | TEXT | NOT NULL | Display name, shared by all documents of the entry. | -| aliases | TEXT[] | NOT NULL DEFAULT '{}', CHECK no empty string | Ordered alternate names used to match queries. The first one is the prompt hint. | -| metadata | JSONB | NOT NULL DEFAULT '{"source": "pipeline"}', CHECK object | `source` and any type-specific fields. See keys below. | +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Catalog ID. | +| group_id | UUID | NOT NULL, FK → `groups(id)` ON DELETE RESTRICT | The provider that owns the catalog. | +| type | TEXT | NOT NULL, CHECK `^[a-z0-9]+:[A-Za-z0-9]+$` | Network schema/capability, e.g. `openagrinet:AgricultureResource`. | +| name | TEXT | NOT NULL | Catalog display name. Defaults to ` ` when created on upload. | +| metadata | JSONB | NOT NULL DEFAULT '{}', CHECK object | Beckn publish information. See keys below. | | created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Created. | | updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). | ### `metadata` keys -| Key | Type | Default | Purpose | -|---|---|---|---| -| source | string | `pipeline` | `bootstrap` (seeded from code), `pipeline` (built from documents) or `import`. | +| Key | Type | Purpose | +|---|---|---| +| publish_directives | object | Beckn `publishDirectives`. | +| visible_networks | string[] | Networks the catalog is visible to. | +| criteria | object | Catalog criteria. | +| network | object `{catalog_id, last_published_at}` | Set once the catalog is published on the network. | -Type-specific fields are added as keys, with no migration. Pydantic validates the keys per type. +Pydantic (`CatalogMetadata`) validates the keys; the database only checks for an object. Notes -- Computed catalog values (`status`, `instances`, `workflow_ids`, `chunk_count`, `network_visible`, `promoted_at`) come from the `catalog` view, not stored here. -- Case-insensitive alias uniqueness is enforced in the app (`_dedupe_aliases`). Add a GIN index on `aliases` only if "find entry by alias" is needed in SQL. -- No join table: an entry has many documents and a document has one entry, which `documents.catalog_entry_id` expresses. -- Plain documents (kind `document`) have no entry. +- `type` stores the schema name, and code asks `pipeline/network_constants.py` what it means (`catalog_type_for_kind`, `kind_for_catalog_type`) instead of comparing `openagrinet:*` strings. The mapping is a placeholder: `scheme` is `openagrinet:AgricultureResource`, `advisory` is `openagrinet:KnowledgeAdvisory` (see ADR 0003 and ADR 0006). +- Catalogs are created on upload (`get_or_create_catalog`), not seeded. +- Computed catalog values (status, chunk count, network visibility, `promoted_at`) come from the `catalog_status` view, not stored here. --- -## 3. `documents` +## 4. `documents` -One row per document. 13 columns. +One row per document: one resource (a scheme, an advisory) in one catalog, identified there by `code`. + +Unique: `(catalog_id, code)`. | Column | Type | Constraints / Default | Purpose | |---|---|---|---| | id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Document ID. | -| instance | TEXT | NOT NULL | Tenant or state code. | -| source | JSONB | NOT NULL, CHECK `jsonb_typeof(source) = 'object'`, CHECK `source->>'fingerprint' ~ '^[0-9a-f]{32}$'` | The uploaded file. See shape below. | -| document_kind | TEXT | NOT NULL DEFAULT 'document', CHECK `^[a-z0-9]+([_-][a-z0-9]+)*$` | Knowledge kind (`document`, `scheme`, `advisory`, `video`, or custom). | -| catalog_entry_id | UUID | | Catalog entry (for example a scheme) the document belongs to. Nullable. Composite FK, see below. | -| valid_from | DATE | | Start of the period chunks are searchable. | -| valid_to | DATE | CHECK `valid_from <= valid_to` | End of the period. NULL means no end. | +| catalog_id | UUID | NOT NULL, FK → `catalogs(id)` ON DELETE RESTRICT | Catalog (provider group + type) the document belongs to. | +| code | TEXT | NOT NULL, CHECK `^[a-z0-9]+(-[a-z0-9]+)*$` | Short code (`pm-kmy`) that catalog consumers see. Unique within the catalog. | +| name | TEXT | NOT NULL | Display name. | +| aliases | TEXT[] | NOT NULL DEFAULT '{}', CHECK no empty string | Ordered alternate names used to match queries. The first one is the prompt hint. | +| source | JSONB | NOT NULL, CHECK object, CHECK `source->>'fingerprint' ~ '^[0-9a-f]{32}$'` | The uploaded file. See shape below. | | is_disabled | BOOLEAN | NOT NULL DEFAULT FALSE | Soft delete. Restore flips it back. | -| index_status | JSONB | NOT NULL DEFAULT '{}', CHECK `jsonb_typeof(index_status) = 'object'` | Vector index state per index. Keys `dev` and `prod`. | -| metadata | JSONB | NOT NULL, DEFAULT see below, CHECK object, CHECK `(metadata->>'chunk_count')::int >= 0` | Display and flag fields. See keys below. | -| uploaded_by_user_id | UUID | FK → `users(id)` | Who uploaded. | +| metadata | JSONB | NOT NULL, DEFAULT see below, CHECK object | Period, uploader, index state and flags. See keys below. | | created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Row created. | | updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). | -Composite FK `(catalog_entry_id, document_kind)` → `catalog_entries(id, type)`, `ON UPDATE CASCADE ON DELETE RESTRICT`. It guarantees a document's kind equals its entry's type. When `catalog_entry_id` is NULL the FK is not checked, so plain documents need no entry. - -Suggested indexes: -- `(instance)` -- `(created_at DESC)` -- `(catalog_entry_id) WHERE catalog_entry_id IS NOT NULL` -- `(instance, (source->>'fingerprint'))` for duplicate checks and identifier lookups -- `(((metadata->>'is_demo')::boolean))` for the demo filter on every list +Indexes: `(created_at DESC)`, `(catalog_id, (source->>'fingerprint'))`, plus the unique `(catalog_id, code)`. ### `source` @@ -165,8 +185,7 @@ Facts about the uploaded file, not about processing. "original_filename": "pmkisan.pdf", "storage_uri": "s3://...", "source_type": "document", - "canonical_input_type": "pdf", - "page_count": null + "canonical_input_type": "pdf" } ``` @@ -177,55 +196,55 @@ Facts about the uploaded file, not about processing. | storage_uri | string | Location of the uploaded file. | | source_type | string | `spreadsheet`, `document` or `unknown`. | | canonical_input_type | string | `spreadsheet` or `pdf`. Normalized pipeline path. | -| page_count | integer or null | NULL until OCR runs. | -### `index_status` - -One entry per vector index. DEV and PROD are separate Qdrant collections. - -```json -{ - "dev": {"status": "indexed", "chunk_count_indexed": 42, - "last_indexed_at": "2026-10-19T06:20:00Z", "last_verified_at": "2026-10-19T06:25:00Z", - "schema_version": "passage-v1", "details": {}}, - "prod": {"...": "..."} -} -``` - -- A missing key means the document was never ingested to that index. -- `status` is `unknown`, `indexed` or `missing`. -- The document's key inside the vector index is `source.fingerprint`. -- The key is `prod` for the PROD collection names (`PROD_VECTOR_DB_COLLECTION_NAME`, `PROD_SCHEME_VECTOR_DB_COLLECTION_NAME`) and `dev` for every other collection. The collection name is kept inside the entry as `index_name`, next to `doc_id`, the id the index uses for the document. -- Update one index atomically: `SET index_status = index_status || jsonb_build_object('dev', $1::jsonb)`. -- A third index is a new key, with no migration. +There is no page count here: it is the number of `pages` rows. ### `metadata` | Key | Type | Default | Purpose | |---|---|---|---| -| display_name | string | absent | Human-friendly title. | +| valid_from | string `YYYY-MM-DD` | absent | Start of the period chunks are searchable. | +| valid_to | string `YYYY-MM-DD` | absent | End of the period. Absent means no end. Pydantic requires `valid_from <= valid_to`. | +| uploaded_by_user_id | string (users.id) | absent | Who uploaded. Not an FK, so a deleted user leaves a stale id; `audit_logs.actor_user_id` keeps real history. | +| index_status | object | `{}` | Vector index state per index. Keys `dev` and `prod`. See below. | | catalog_visible | boolean | true | Counts toward the catalog. | | network_visible | boolean | true | May be published to the Beckn network. | -| is_demo | boolean | false | Demo content, hidden by default. | -| chunk_count | integer ≥ 0 | 0 | Cached number of chunks. Updated in the same transaction as chunk writes. | | reindex_required | boolean | false | Approved content changed and the vector index is stale. | | reindex_reason | string | absent | Why a reindex is needed. | | chunking | object `{run_number, provider, model, config, created_at}` | absent | Settings of the run that produced the current chunks. | -Column default: `'{"catalog_visible": true, "network_visible": true, "is_demo": false, "chunk_count": 0, "reindex_required": false}'`. +Column default: `'{"index_status": {}, "catalog_visible": true, "network_visible": true, "reindex_required": false}'`. -### Notes +`index_status` is one entry per vector index (DEV and PROD are separate Qdrant collections): +```json +{ + "dev": {"status": "indexed", "chunk_count_indexed": 42, + "last_indexed_at": "2026-10-19T06:20:00Z", "last_verified_at": "2026-10-19T06:25:00Z", + "schema_version": "passage-v1", "details": {}}, + "prod": {"...": "..."} +} +``` + +- A missing key means the document was never ingested to that index. `status` is `unknown`, `indexed` or `missing`. +- Update one index atomically with `jsonb_set(metadata, '{index_status,dev}', $1::jsonb)`; the `index_status` key always exists, which `jsonb_set` needs. + +### Notes - A document's stage is the `current_stage` of its latest workflow. List and filter queries join the `document_current_workflow` view. -- SQL does not check types inside the JSONB columns. The insert path writes every default `metadata` key, and Pydantic validates `source`, `index_status` and `metadata`. -- `metadata.chunk_count` is rewritten in every chunk transaction and `metadata.is_demo` is filtered on every list. If performance suffers, move these two back to columns first. -- Each document keeps its own `source`. Two uploads of the same content from different paths keep their own `storage_uri`. There is no unique key on `(instance, fingerprint)`; the upload path reuses a document by its Temporal workflow id instead. Add `UNIQUE (instance, (source->>'fingerprint'))` only if one document per file per tenant must be enforced. -- Identifier lookup (`find_document_by_doc_identifier`) matches `documents.id`, `workflows.temporal_workflow_id`, `source->>'fingerprint'` and `source->>'original_filename'`. The legacy `md5(identifier)` check is `md5(source->>'fingerprint') = $1` in SQL. +- The API's `group` is derived: `documents.catalog_id → catalogs.group_id → groups.name`. There is no `documents.instance` column, and tenant filters join `groups`. +- The API's `document_kind` is derived from `catalogs.type` through the map in `network_constants.py`. `scheme_code`, `scheme_name` and `scheme_aliases` are `code`, `name` and `aliases`. +- `chunk_count` and `page_count` are counted from `chunks` and `pages`; they are not stored. +- A new document needs a `document_kind` and a `code`. Upload gets the catalog from the tenant and the kind (created when missing). `DuplicateDocumentCode` is raised when the code is taken in that catalog. +- Re-uploading an existing code updates the same document: a new workflow starts, `source` is replaced, pages and chunks are cleared and `reindex_required` is set (`reset_document_for_reupload`). Earlier workflows and artifacts stay as history. +- Changing a document's kind moves it to the same group's catalog of the new type (`update_document_fields(document_kind=...)`). +- SQL does not check types inside the JSONB columns. The insert path writes the default metadata keys, and Pydantic validates `source`, `metadata` and each `index_status` entry. +- Each document keeps its own `source`. There is no unique key on the fingerprint. +- Identifier lookup (`find_document_by_doc_identifier`) matches `documents.id`, `workflows.temporal_workflow_id`, `source->>'fingerprint'`, `source->>'original_filename'`, an index `doc_id` and the legacy `md5(source->>'fingerprint')`. - No `deleted_at`: soft delete is `is_disabled`, hard delete is `purge_document`. --- -## 4. `workflows` +## 5. `workflows` One row per processing run of a document: the main pipeline run, retries, reingest and PROD promotion. Holds all processing state. @@ -235,10 +254,10 @@ Indexes: `(document_id, started_at DESC)`, `(status, started_at DESC)`, `(tempor |---|---|---|---| | id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Workflow ID. | | document_id | UUID | NOT NULL, FK → `documents(id)` ON DELETE CASCADE | Document the workflow runs against. | -| workflow_type | workflow_type_enum | NOT NULL | Kind of run. | +| workflow_type | TEXT | NOT NULL | Kind of run (`WorkflowType`). | | temporal_workflow_id | TEXT | NOT NULL | Temporal workflow ID. `doc-` for the main run, suffixed for retries. Not unique (a restart reuses the ID). Used by the API, UI and Temporal. | -| status | workflow_status_enum | NOT NULL DEFAULT 'running' | `running`, `waiting_review`, `completed` or `failed`. | -| current_stage | workflow_stage | NOT NULL DEFAULT 'registered' | Stage the workflow is in now. This is the document's stage when the workflow is the latest. | +| status | TEXT | NOT NULL DEFAULT 'running' | `running`, `waiting_review`, `completed` or `failed` (`WorkflowStatus`). | +| current_stage | TEXT | NOT NULL DEFAULT 'registered' | Stage the workflow is in now (`DocumentStage`). This is the document's stage when the workflow is the latest. | | stage_timestamps | JSONB | NOT NULL DEFAULT '{}', CHECK object with only keys `ocr`, `translation`, `chunks` | When each stage first completed in this run, ISO-8601 UTC. Set-once per key. | | stop_after_ocr | BOOLEAN | NOT NULL DEFAULT FALSE | Halt after OCR (OCR-only batch runs). | | error_message | TEXT | | Failure reason when status is `failed`. | @@ -266,12 +285,12 @@ Latest workflow per document. Stores nothing. ```sql SELECT DISTINCT ON (document_id) * FROM workflows -ORDER BY document_id, started_at DESC; +ORDER BY document_id, started_at DESC, id DESC; ``` --- -## 5. `workflow_artifacts` +## 6. `workflow_artifacts` Ledger of every file a workflow produces or stores. Files live in MinIO; this table records where and what. Append-only. @@ -282,7 +301,7 @@ Indexes: `(workflow_id, created_at DESC)`, `(workflow_id, artifact_type, created | id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Artifact ID, used in `/documents/{id}/artifacts/{artifact_id}`. | | workflow_id | UUID | NOT NULL, FK → `workflows(id)` ON DELETE CASCADE | Workflow that produced it. | | artifact_type | TEXT | NOT NULL, CHECK not blank | Kind of output (`original_upload`, `normalized_pdf`, `normalized_spreadsheet`, `ocr_pages_json`, `translation_pages_json`, `chunk_json_export`, `vector_payload_export`, network publish exchange). Open set, so not an enum. | -| stage | workflow_stage | | Stage that created it. | +| stage | TEXT | | Stage that created it (`DocumentStage`). | | storage_uri | TEXT | NOT NULL | MinIO location of the blob. | | mime_type | TEXT | | MIME type. | | filename | TEXT | | File name of the blob. | @@ -302,7 +321,7 @@ Notes --- -## 6. `pages` +## 7. `pages` OCR layer and translations of a page. @@ -362,7 +381,7 @@ Notes --- -## 7. `chunks` +## 8. `chunks` Unique: `(document_id, chunk_number)`. @@ -389,7 +408,7 @@ Unique: `(document_id, chunk_number)`. Notes - Chunking provider, model, config and run number are stored once per document in `documents.metadata.chunking`, not per chunk. - The distinct page numbers a chunk draws from are derived from `source_spans`. -- Re-chunk runs in one transaction: delete the document's chunks, insert the new ones, update `metadata.chunking` (with `run_number` incremented) and `metadata.chunk_count`. Chunk UUIDs change on every re-chunk; nothing references `chunks.id` (audit uses chunk number). +- Re-chunk runs in one transaction: delete the document's chunks, insert the new ones, update `metadata.chunking` (with `run_number` incremented) (the chunk count is counted from `chunks`, not stored). Chunk UUIDs change on every re-chunk; nothing references `chunks.id` (audit uses chunk number). - `search_chunks` does a substring match on `lower(coalesce(edited_text, original_text))`, served by a trigram GIN index on that expression (the migration creates the `pg_trgm` extension). > **Why `pages` and `chunks` stay separate tables:** a page is a physical unit from OCR and a chunk is a semantic unit from the chunker, so their boundaries do not align. One chunk can span several pages and one page can hold several chunks, which makes the relationship many-to-many. Storing page text inside each chunk would copy a page into every chunk it overlaps, so any page edit, OCR retry or translation change would have to be rewritten in all of those chunks. @@ -397,7 +416,7 @@ Notes --- -## 8. `audit_logs` +## 9. `audit_logs` Append-only history of actions. Must outlive documents, so there is no cascade delete. @@ -406,9 +425,9 @@ Append-only history of actions. Must outlive documents, so there is no cascade d | id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Audit entry ID. | | document_id | UUID | FK → `documents(id)` ON DELETE SET NULL | Document the action applied to. NULL for system-level entries and after a purge. | | workflow_id | TEXT | | Temporal workflow ID snapshot. Survives a purge. | -| instance | TEXT | | Tenant snapshot, so tenant scoping works after the document is gone. | +| group_id | UUID | FK → `groups(id)` ON DELETE SET NULL | Provider group snapshot, so tenant scoping works after the document is gone. | | action_type | TEXT | NOT NULL, CHECK not blank | What happened (`page_edit`, `purge_document`, ...). 22 values and growing, so not an enum. | -| entity_type | audit_entity_type | | `document`, `page`, `chunk`, `setting` or `catalog`. | +| entity_type | TEXT | | `document`, `page`, `chunk`, `setting` or `catalog` (`AuditEntityType`). | | entity_number | INTEGER | | Page number or chunk number the action applied to. | | field_name | TEXT | | Field that changed. | | old_value | TEXT | | Value before the change. | @@ -420,21 +439,21 @@ Append-only history of actions. Must outlive documents, so there is no cascade d Table CHECK: `entity_type IN ('setting','catalog') OR workflow_id IS NOT NULL`. -Indexes: `(document_id, created_at DESC)`, `(created_at DESC)`, `(action_type, created_at DESC)`, `(created_at DESC) WHERE entity_type = 'setting'`. +Indexes: `(document_id, created_at DESC)`, `(group_id, created_at DESC)`, `(created_at DESC)`, `(action_type, created_at DESC)`, `(created_at DESC) WHERE entity_type = 'setting'`. Notes - The FK to `documents` is `SET NULL`, not cascade. A cascade would erase the audit trail on purge, and a strict FK would reject setting and catalog entries that have no document. - Setting and catalog entries have NULL `document_id` and `workflow_id`, with `entity_type` set to `setting` or `catalog`. -- `instance` is a snapshot, so a purged document's history stays visible only to its own tenant. +- `group_id` is a snapshot of the document's provider group, so a purged document's history stays visible only to its own tenant. - `entity_number` holds a page or chunk number, not an ID (chunk UUIDs would dangle after a re-chunk). - `get_settings_audit_logs` is `WHERE entity_type = 'setting'`. -- `log_audit` fills `instance`, resolves the actor to a `users` row and builds the `actor` JSON. +- `log_audit` fills `group_id`, resolves the actor to a `users` row and builds the `actor` JSON. - No `updated_at`: entries are immutable. - The table only grows. Monthly partitioning is possible later without changing this design. --- -## 9. `search_settings` +## 10. `search_settings` Single row of search configuration. @@ -464,7 +483,7 @@ Notes --- -## 10. `email_otps` +## 11. `email_otps` Server-side email-OTP login codes. Short-lived. @@ -489,32 +508,33 @@ Notes --- -## View: `catalog` +## View: `catalog_status` -Computed from `catalog_entries`, `documents`, `workflows` and `document_current_workflow`. Stores nothing. The `/catalog/v1/*` snapshot API reads it with `type = 'scheme'`, and its fields are named `scheme_code`, `scheme_name` and so on, so consumers see no change. +Computed from `documents`, `catalogs`, `groups`, `workflows` and `document_current_workflow`. Stores nothing. One row per document. The `/catalog/v1/*` snapshot API reads it for the scheme type (`catalogs.type` of the scheme kind), and its fields are named `scheme_code`, `scheme_name` and so on, so consumers see no change. | Column | Type | How it is computed | |---|---|---| -| entry_id | UUID | From `catalog_entries`. | -| type | TEXT | From `catalog_entries`. | -| code | TEXT | From `catalog_entries`. | -| name | TEXT | From `catalog_entries`. | -| aliases | TEXT[] | From `catalog_entries`. | -| source | TEXT | `catalog_entries.metadata->>'source'`. | -| status | TEXT | `pending_reindex` if a live document needs reindex. Else `live` if any live document exists. Else `pending_prod` if a document awaits PROD approval. Else `live` for a bootstrap entry with no documents. Else `disabled`. | -| instances | TEXT[] | Distinct `instance` of the entry's documents, or `{default}` if none. | -| workflow_ids | TEXT[] | `temporal_workflow_id` of each entry document's main `pipeline` workflow. | -| chunk_count | INTEGER | Sum of `metadata->>'chunk_count'` over the entry's documents. | -| network_visible | BOOLEAN | True if any of its documents has `metadata->>'network_visible'` true. For an entry with no documents, true if bootstrap. | -| promoted_at | TIMESTAMPTZ | Latest `index_status->'prod'->>'last_indexed_at'` of the live documents. | - -"Live document" means the latest workflow's `current_stage = 'completed'`, `is_disabled` false, and `metadata->>'catalog_visible'` true. The entry's `type` already ties it to the document's kind through the composite FK. +| document_id | UUID | `documents.id`. | +| catalog_id | UUID | `documents.catalog_id`. | +| group_id | UUID | `catalogs.group_id`. | +| group_name | TEXT | `groups.name`. | +| type | TEXT | `catalogs.type`. | +| code | TEXT | `documents.code`. | +| name | TEXT | `documents.name`. | +| aliases | TEXT[] | `documents.aliases`. | +| status | TEXT | `pending_reindex` if the document is live and `metadata.reindex_required` is true. Else `live` if live. Else `pending_prod` if the latest workflow's stage is `approval_for_prod` and the document is not disabled. Else `disabled`. | +| workflow_id | TEXT | `temporal_workflow_id` of the document's earliest `pipeline` workflow. | +| chunk_count | INTEGER | Count of the document's `chunks`. | +| network_visible | BOOLEAN | `metadata.network_visible`, true when absent. | +| promoted_at | TIMESTAMPTZ | `metadata.index_status.prod.last_indexed_at`, only for live documents. | + +"Live" means the latest workflow's `current_stage = 'completed'`, `is_disabled` is false and `metadata.catalog_visible` is true. There are no bootstrap rows: a scheme appears only once a document exists. --- -## 11. `webhook_records` +## 12. `webhook_records` -Records pushed to `POST /webhook` (`x-api-key` = `WEBHOOK_API_KEY`). Carried over from the previous store unchanged apart from the types. +Records pushed to `POST /webhook` (`x-api-key` = `WEBHOOK_API_KEY`). Carried over from the previous store unchanged; not part of the groups and catalogs model. Unique: `(source, external_id)`; index on `(source, status)`. @@ -530,6 +550,8 @@ Unique: `(source, external_id)`; index on `(source, status)`. | category | TEXT | | From `domainName`. | | payload | JSONB | NOT NULL, CHECK object | The full record. | | payload_hash | TEXT | | SHA-256 of the canonical payload; an upsert with the same hash writes nothing. | +| indexed_hash | TEXT | | `payload_hash` last written to the DEV collection (migration 0002). | +| promoted_hash | TEXT | | `payload_hash` last written to the PROD collection (migration 0002). | | created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | The payload's `createdAt` when it has one. | | updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | The payload's `updatedAt` when it has one. | @@ -547,7 +569,8 @@ Created by the runner, not by a migration file: `version` (the `NNNN` prefix, pr - **Identifiers.** `db.py` functions take a `workflow_id` argument that may be the document UUID, a Temporal workflow id, the fingerprint, the filename, an index `doc_id` or the legacy `md5(fingerprint)` doc id (`find_document_by_doc_identifier`). Documents, jobs and artifacts come back as flat dicts; `id` is the UUID and `workflow_id` is the Temporal workflow id of the document's main run (the earliest `pipeline` or `ocr_only` workflow). The API rewrites `/documents/{id}/...` to that workflow id before routing, because handlers signal and query Temporal with it. - **Jobs.** The API's "jobs" and "runs" are `workflows` rows; job ids and artifact ids are the UUIDs of `workflows` and `workflow_artifacts`. `latest_job_id`, `original_artifact_id` and `normalized_artifact_id` are derived by join, not stored. +- **Group.** The API returns a `group` string on documents, runs, queue rows and audit entries. It is the lowercase `groups.name` reached through `catalogs`; filters compare `lower(groups.name)`. - **Catalog version.** `/catalog/v1/version` counts `audit_logs` rows with `entity_type = 'catalog'` and `action_type = 'catalog_version_bump'`, so it only grows. `updated_at` is the newest such row. - **Document stage.** The stage a document shows is `document_current_workflow.current_stage`; `update_document_stage` writes to that workflow. - **Page translations.** `save_pages` keeps stored translations unless the page carries `translations` (replaces them) or flat `translated_*` fields (merged into `translation_target_language`). A page whose OCR text changed loses its translations. -- **Users.** Rows are upserted from the auth dependency on write requests (at most once per five minutes per user) and from uploads, workflow starts and audit entries. `local-admin` is seeded by the migration. +- **Users.** Rows are upserted from the auth dependency on write requests (at most once per five minutes per user) and from uploads, workflow starts and audit entries. `local-admin` is seeded by the migration. A user's `group_id` is set from the tenant codes in the token. From 7dedf097c0f1ab74afd4dfccd53b711b935e2c3b Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 11:17:26 +0530 Subject: [PATCH 17/53] fix: guard document replace and clean up vectors of replaced files Replace guard (api): an upload now needs upload rights in the target group, not just somewhere. Replacing an existing code also needs review rights in the document's group, is refused while the document is disabled or a run is in progress, and needs approval rights when the document is live in PROD. Orphaned vectors: a re-upload changes the fingerprint, which is the document's key in the vector index. reset_document_for_reupload now records the old key in metadata.superseded_doc_ids for each index that holds it. The next DEV ingest and PROD promote delete those points after the new content is indexed (a failed delete keeps the key for the next run). Disable and purge now delete the current and superseded keys, and the PROD cleanup follows the PROD index entry, so a document promoted as a scheme is cleaned up even if its kind changed. --- pipeline/activities.py | 41 ++++++++++++++ pipeline/api.py | 80 +++++++++++++++++++++++---- pipeline/db.py | 47 +++++++++++++++- pipeline/models.py | 5 ++ tests/test_activities.py | 62 +++++++++++++++++++++ tests/test_api.py | 115 ++++++++++++++++++++++++++++++++++----- tests/test_schema_v2.py | 72 ++++++++++++++++++++++++ 7 files changed, 394 insertions(+), 28 deletions(-) diff --git a/pipeline/activities.py b/pipeline/activities.py index a0d3643..8aebe5d 100644 --- a/pipeline/activities.py +++ b/pipeline/activities.py @@ -1339,6 +1339,7 @@ async def promote_document_to_prod_qdrant( store.upsert, prod_collection, records, batch_size=max(batch_size, 8) ) activity.logger.info("PROD promotion complete: %s", result.get("index_stats")) + await _drop_superseded_points("prod", workflow_id, doc, document_id, prod_collection, store=store) db.log_audit( workflow_id=workflow_id, document_id=document_id, @@ -1369,6 +1370,45 @@ async def promote_document_to_prod_qdrant( } +async def _drop_superseded_points( + env: str, + workflow_id: str, + doc: dict, + current_doc_id: str, + collection: str, + store=None, +) -> None: + """Delete points of earlier file versions (see ``metadata.superseded_doc_ids``). + + Runs after the new content is indexed, so search never has a gap. A failed + delete keeps the key for the next run and does not fail the ingest. + """ + from . import db + + stale = [ + doc_id + for doc_id in (doc.get("superseded_doc_ids") or {}).get(env, []) + if doc_id and doc_id != current_doc_id + ] + if not stale: + return + if store is None: + from .vector_store import get_vector_store + + store = get_vector_store() + deleted = [] + for doc_id in stale: + result = await asyncio.to_thread(store.delete_by_doc_id, collection, doc_id) + if result.get("error"): + activity.logger.warning( + "Could not delete superseded %s points for %s: %s", env, doc_id, result["error"] + ) + else: + deleted.append(doc_id) + if deleted: + db.clear_superseded_doc_ids(workflow_id, env, deleted) + + @activity.defn async def ingest_document_from_db( workflow_id: str, @@ -1423,6 +1463,7 @@ async def ingest_document_from_db( status="indexed", details=result.get("index_stats"), ) + await _drop_superseded_points("dev", workflow_id, doc, document_id, index_name) return result diff --git a/pipeline/api.py b/pipeline/api.py index 8f3f6ed..a42e3ce 100644 --- a/pipeline/api.py +++ b/pipeline/api.py @@ -420,6 +420,29 @@ def _document_title(filename: str) -> str: return re.sub(r"[_\-\s]+", " ", stem).strip() or "Untitled document" +def _assert_can_replace(user: AuthUser, existing: dict) -> None: + """Replacing a document's file clears its pages and chunks, so guard it. + + Needs review rights in the document's group; a disabled document or one with + a run in progress cannot be replaced; a document already promoted to PROD + can only be replaced by someone who may approve ingestion. + """ + doc_group = existing["group"] + if not user.has_permission_for_group(Permission.REVIEW, doc_group): + raise HTTPException(403, f"No permission to replace documents in group: {doc_group}") + if existing.get("is_disabled"): + raise HTTPException(409, "This document is disabled; restore it before replacing its file") + latest_job = db.get_latest_document_job(existing["workflow_id"]) + if latest_job and latest_job.get("status") == "running": + raise HTTPException(409, "A run is still in progress for this document") + if (existing.get("index_status") or {}).get("prod") and not user.has_permission_for_group( + Permission.APPROVE_INGESTION, doc_group + ): + raise HTTPException( + 403, "Replacing a document that is live in PROD needs approval rights in its group" + ) + + def _resolve_document_identity( user: AuthUser, group: str, @@ -432,12 +455,19 @@ def _resolve_document_identity( An explicit ``code`` that is already held in the tenant's catalog of that kind means "update this document": the caller reuses it instead of inserting. Without a code one is derived from the file name. + + Permissions are checked against the target group, not just globally: a user + who may upload in one state must not be able to add to, or replace a + document of, a state where they only hold a view role. Replacing clears the + document's pages and chunks, so it also needs review rights in that group. """ kind = (kind or "").strip().lower() try: catalog_type_for_kind(kind) except ValueError as exc: raise HTTPException(400, str(exc)) from exc + if not user.has_permission_for_group(Permission.UPLOAD, group): + raise HTTPException(403, f"No upload permission in group: {group}") code = (code or "").strip() if code: try: @@ -447,6 +477,7 @@ def _resolve_document_identity( existing = db.find_document_by_code(group, kind, code) if existing: existing = assert_document_group_access(user, existing) + _assert_can_replace(user, existing) return kind, code, existing return kind, scheme_catalog.derive_scheme_code(_document_title(filename), kind=kind), None @@ -1719,10 +1750,28 @@ def _require_catalog_read(request: Request, user: AuthUser) -> None: raise HTTPException(403, "Catalog access requires search or admin permission") +def _vector_doc_ids(doc: dict, env: str) -> list[str]: + """The document's current index key plus keys of earlier file versions still in ``env``.""" + ids = [doc.get("document_id")] + ids.extend((doc.get("superseded_doc_ids") or {}).get(env, [])) + return list(dict.fromkeys(i for i in ids if i)) + + +def _delete_all_doc_ids(store, collection: str, doc_ids: list[str]) -> dict: + results = [store.delete_by_doc_id(collection, doc_id) for doc_id in doc_ids] + if len(results) == 1: + return results[0] + errors = [r["error"] for r in results if r.get("error")] + summary = {"deleted": 0 if errors else -1, "doc_ids": doc_ids, "results": results} + if errors: + summary["error"] = "; ".join(errors) + return summary + + def _delete_document_points_from_prod(doc: dict) -> dict: """Best-effort delete of a promoted (non-scheme) document's vectors from PROD documents-index.""" - doc_id = doc.get("document_id") - if not doc_id: + doc_ids = _vector_doc_ids(doc, "prod") + if not doc_ids: return {"deleted": False, "reason": "no_doc_id"} prod_url = (os.environ.get("PROD_VECTOR_DB_URL") or "").strip() if not prod_url: @@ -1738,15 +1787,15 @@ def _delete_document_points_from_prod(doc: dict) -> dict: client = get_qdrant_client(url=prod_url, api_key=prod_key) store = QdrantVectorStore(client=client) - return store.delete_by_doc_id(collection, doc_id) + return _delete_all_doc_ids(store, collection, doc_ids) except Exception as exc: return {"deleted": False, "error": str(exc), "collection": collection} def _delete_scheme_points_from_prod(doc: dict) -> dict: """Best-effort delete of scheme vectors from PROD schemes-index.""" - doc_id = doc.get("document_id") - if not doc_id: + doc_ids = _vector_doc_ids(doc, "prod") + if not doc_ids: return {"deleted": False, "reason": "no_doc_id"} prod_url = ( (os.environ.get("PROD_SCHEME_VECTOR_DB_URL") or "").strip() @@ -1767,7 +1816,7 @@ def _delete_scheme_points_from_prod(doc: dict) -> dict: client = get_qdrant_client(url=prod_url, api_key=prod_key) store = QdrantVectorStore(client=client) - return store.delete_by_doc_id(collection, doc_id) + return _delete_all_doc_ids(store, collection, doc_ids) except Exception as exc: return {"deleted": False, "error": str(exc), "collection": collection} @@ -2179,16 +2228,25 @@ async def disable_document( # Remove from the DEV vector index if requested if remove_from_search: - doc_id = doc.get("document_id") - if doc_id: + # The current index key plus keys of earlier file versions still in DEV + # (see metadata.superseded_doc_ids). + for index, doc_id in enumerate(_vector_doc_ids(doc, "dev")): vector_result = delete_chunks_from_index(doc_id) - result["vector_deleted"] = vector_result.get("deleted", 0) + if index == 0: + result["vector_deleted"] = vector_result.get("deleted", 0) if "error" in vector_result: result["vector_error"] = vector_result["error"] - # PROD Qdrant cleanup, if this document was ever promoted + # PROD Qdrant cleanup, if this document was ever promoted. A document + # promoted as a scheme lives in the scheme index even if its kind was + # changed since, so the PROD index entry decides, not only the kind. kind = (doc.get("kind") or "").strip().lower() - if kind == "scheme": + prod_entry = (doc.get("index_status") or {}).get("prod") or {} + prod_is_scheme = ( + kind == "scheme" + or prod_entry.get("index_name") == scheme_catalog.scheme_collection_name() + ) + if prod_is_scheme: q_result = _delete_scheme_points_from_prod(doc) result["scheme_qdrant_deleted"] = q_result try: diff --git a/pipeline/db.py b/pipeline/db.py index c90ed21..9bec3a1 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -392,6 +392,7 @@ def _document_from_row(row: Optional[dict]) -> Optional[dict]: "latest_job_id": row["current_workflow_pk"], "catalog_id": row["catalog_id"], "index_status": meta.get("index_status") or {}, + "superseded_doc_ids": meta.get("superseded_doc_ids") or {}, "created_at": row["created_at"], "updated_at": row["updated_at"], } @@ -1096,6 +1097,11 @@ def reset_document_for_reupload( ``source`` is replaced, pages and chunks are cleared for the new run, and the document is flagged ``reindex_required`` because its vector index is now stale. Earlier workflows and artifacts stay as history. + + The fingerprint is the document's key in the vector index, so the old key + is remembered in ``metadata.superseded_doc_ids`` for every index (dev/prod) + that holds it. The next ingest/promote and disable/purge delete those points; + without this they would stay searchable with nothing pointing at them. """ source = DocumentSource.model_validate({ "fingerprint": fingerprint, @@ -1105,6 +1111,19 @@ def reset_document_for_reupload( "canonical_input_type": canonical_input_type, }).model_dump(exclude_none=True) with get_connection() as conn: + current = conn.execute( + "SELECT source->>'fingerprint' AS fingerprint, metadata FROM documents " + "WHERE id = %s::uuid FOR UPDATE", + (doc_uuid,), + ).fetchone() + if current is None: + raise LookupError(f"Document not found: {doc_uuid}") + meta = current["metadata"] or {} + superseded = {k: list(v) for k, v in (meta.get("superseded_doc_ids") or {}).items()} + for env, entry in (meta.get("index_status") or {}).items(): + old_key = (entry or {}).get("doc_id") or current["fingerprint"] + if old_key and old_key != fingerprint and old_key not in superseded.get(env, []): + superseded.setdefault(env, []).append(old_key) conn.execute("DELETE FROM pages WHERE document_id = %s::uuid", (doc_uuid,)) conn.execute("DELETE FROM chunks WHERE document_id = %s::uuid", (doc_uuid,)) conn.execute( @@ -1112,10 +1131,34 @@ def reset_document_for_reupload( UPDATE documents SET source = %s::jsonb, metadata = (metadata - 'chunking') - || jsonb_build_object('reindex_required', TRUE, 'reindex_reason', 'source_replaced') + || jsonb_build_object('reindex_required', TRUE, 'reindex_reason', 'source_replaced', + 'superseded_doc_ids', %s::jsonb) WHERE id = %s::uuid """, - (Jsonb(source), doc_uuid), + (Jsonb(source), Jsonb(superseded), doc_uuid), + ) + + +def clear_superseded_doc_ids(workflow_id: str, env: str, doc_ids: list[str]) -> None: + """Forget superseded index keys once their points are deleted from ``env`` (dev/prod).""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return + row = conn.execute( + "SELECT metadata->'superseded_doc_ids' AS sup FROM documents WHERE id = %s::uuid FOR UPDATE", + (doc_uuid,), + ).fetchone() + superseded = {k: list(v) for k, v in (row["sup"] or {}).items()} + remaining = [i for i in superseded.get(env, []) if i not in set(doc_ids)] + if remaining: + superseded[env] = remaining + else: + superseded.pop(env, None) + conn.execute( + "UPDATE documents SET metadata = metadata || jsonb_build_object('superseded_doc_ids', %s::jsonb) " + "WHERE id = %s::uuid", + (Jsonb(superseded), doc_uuid), ) diff --git a/pipeline/models.py b/pipeline/models.py index cf22315..247cd8c 100644 --- a/pipeline/models.py +++ b/pipeline/models.py @@ -560,6 +560,11 @@ class DocumentMetadata(BaseModel): reindex_required: bool = False reindex_reason: Optional[str] = None chunking: Optional[ChunkingRun] = None + # Vector-index keys (doc_id) of earlier file versions still present in an + # index, by env (``dev`` / ``prod``). A re-upload changes the fingerprint, so + # the old points would otherwise be orphaned; the next ingest/promote and + # disable/purge remove them. + superseded_doc_ids: dict[str, list[str]] = Field(default_factory=dict) @model_validator(mode="after") def _period_is_ordered(self) -> "DocumentMetadata": diff --git a/tests/test_activities.py b/tests/test_activities.py index 191e76a..5cca9c4 100644 --- a/tests/test_activities.py +++ b/tests/test_activities.py @@ -816,3 +816,65 @@ def test_every_defined_activity_is_registered_on_the_worker(self): missing = {name for name in defined_activities if name not in worker_source} assert not missing, f"Activities defined but never registered on the worker: {missing}" + + +class TestSupersededPoints: + """After new content is indexed, points of earlier file versions are deleted.""" + + class _Store: + def __init__(self, failing=()): + self.calls = [] + self.failing = set(failing) + + def delete_by_doc_id(self, collection, doc_id): + self.calls.append((collection, doc_id)) + if doc_id in self.failing: + return {"deleted": 0, "doc_id": doc_id, "error": "boom"} + return {"deleted": -1, "doc_id": doc_id} + + @staticmethod + def _document(db): + doc_uuid = db.upsert_document( + workflow_id="wf-sup-act", + document_id="c" * 32, + filename="a.pdf", + filepath="minio://documents/a.pdf", + kind="scheme", + code="sup-act", + ) + db.upsert_document_index_status("wf-sup-act", "documents-index", doc_id="a" * 32, status="indexed") + db.reset_document_for_reupload( + doc_uuid, filename="b.pdf", filepath="minio://documents/b.pdf", fingerprint="c" * 32 + ) + return doc_uuid + + @pytest.mark.asyncio + @pytest.mark.db + @pytest.mark.unit + async def test_old_keys_are_deleted_and_forgotten(self, db_connection): + from pipeline import activities + + doc_uuid = self._document(db_connection) + store = self._Store() + + await activities._drop_superseded_points( + "dev", "wf-sup-act", db_connection.get_document(doc_uuid), "c" * 32, "documents-index", store=store + ) + + assert store.calls == [("documents-index", "a" * 32)] + assert db_connection.get_document(doc_uuid)["superseded_doc_ids"] == {} + + @pytest.mark.asyncio + @pytest.mark.db + @pytest.mark.unit + async def test_a_failed_delete_keeps_the_key_for_the_next_run(self, db_connection): + from pipeline import activities + + doc_uuid = self._document(db_connection) + store = self._Store(failing={"a" * 32}) + + await activities._drop_superseded_points( + "dev", "wf-sup-act", db_connection.get_document(doc_uuid), "c" * 32, "documents-index", store=store + ) + + assert db_connection.get_document(doc_uuid)["superseded_doc_ids"] == {"dev": ["a" * 32]} diff --git a/tests/test_api.py b/tests/test_api.py index 113b0dc..9f3d2ec 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1130,6 +1130,10 @@ def test_an_existing_code_updates_the_same_document( before["id"], [{"page_number": 1, "original_markdown": "old text"}] ) db_connection.save_chunks(before["id"], [{"chunk_number": 1, "original_text": "old"}]) + # The first upload's run is still "running"; a replace waits for it to settle. + db_connection.update_document_job( + db_connection.get_latest_document_job(before["id"])["id"], status="waiting_review" + ) second = test_client.post( "/upload", @@ -1167,15 +1171,15 @@ def test_the_same_code_in_another_tenant_is_a_different_document( assert conn.execute("SELECT count(*) AS n FROM documents").fetchone()["n"] == 2 assert conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] == 2 - @pytest.mark.api - @pytest.mark.unit - def test_replacing_a_document_needs_access_to_its_tenant(self, db_connection): - from fastapi import HTTPException - - import pipeline.api as api + @staticmethod + def _user(*group_paths): from pipeline.auth.jwt import claims_to_user - db_connection.upsert_document( + return claims_to_user({"sub": "u-" + "-".join(group_paths), "groups": list(group_paths)}) + + @staticmethod + def _mh_doc(db, **extra): + doc_uuid = db.upsert_document( workflow_id="wf-mh-doc", document_id="d-mh", filename="a.pdf", @@ -1183,20 +1187,101 @@ def test_replacing_a_document_needs_access_to_its_tenant(self, db_connection): group="mh", kind="scheme", code="pm-kisan", + **extra, ) - outsider = claims_to_user( - {"sub": "u1", "realm_access": {"roles": ["content_curator"]}, "instances": ["gj"]} + # A new document starts with a "running" workflow; settle it so the + # document is at rest, as one waiting in review is. + db.update_document_job(db.get_latest_document_job(doc_uuid)["id"], status="waiting_review") + return doc_uuid + + @pytest.mark.api + @pytest.mark.unit + def test_replacing_a_document_needs_rights_in_its_group(self, db_connection): + from fastapi import HTTPException + + import pipeline.api as api + + self._mh_doc(db_connection) + identity = lambda user: api._resolve_document_identity( # noqa: E731 + user, "mh", "scheme", "pm-kisan", "a.pdf" ) - insider = claims_to_user( - {"sub": "u2", "realm_access": {"roles": ["content_curator"]}, "instances": ["mh"]} + + # Another state, and a view-only member of this one, are both turned away + # before anything about the document is revealed. + for outsider in (self._user("/states/GJ/contributor"), self._user("/states/MH/view")): + with pytest.raises(HTTPException) as exc: + identity(outsider) + assert exc.value.status_code == 403 + + # The same state with a role that can upload and review may replace. + kind, code, existing = identity(self._user("/states/MH/contributor")) + assert (kind, code, existing["workflow_id"]) == ("scheme", "pm-kisan", "wf-mh-doc") + + @pytest.mark.api + @pytest.mark.unit + def test_upload_into_a_group_needs_upload_rights_there(self, db_connection): + from fastapi import HTTPException + + import pipeline.api as api + + mixed = self._user("/states/MH/contributor", "/states/GJ/view") + # Allowed where the user can upload, refused where they only view. + api._resolve_document_identity(mixed, "mh", "scheme", "", "new scheme.pdf") + with pytest.raises(HTTPException) as exc: + api._resolve_document_identity(mixed, "gj", "scheme", "", "new scheme.pdf") + assert exc.value.status_code == 403 + + @pytest.mark.api + @pytest.mark.unit + def test_a_disabled_or_running_document_cannot_be_replaced(self, db_connection): + from fastapi import HTTPException + + import pipeline.api as api + + db = db_connection + contributor = self._user("/states/MH/contributor") + doc_uuid = self._mh_doc(db) + + db.create_document_job(doc_uuid, "pipeline", "wf-mh-doc", status="running") + with pytest.raises(HTTPException) as exc: + api._resolve_document_identity(contributor, "mh", "scheme", "pm-kisan", "a.pdf") + assert (exc.value.status_code, "in progress" in exc.value.detail) == (409, True) + + db.update_document_job(db.get_latest_document_job(doc_uuid)["id"], status="waiting_review") + db.set_document_disabled("wf-mh-doc", True) + with pytest.raises(HTTPException) as exc: + api._resolve_document_identity(contributor, "mh", "scheme", "pm-kisan", "a.pdf") + assert (exc.value.status_code, "disabled" in exc.value.detail) == (409, True) + + @pytest.mark.api + @pytest.mark.unit + def test_a_document_live_in_prod_needs_approval_rights_to_replace(self, db_connection): + from fastapi import HTTPException + + import pipeline.api as api + + db = db_connection + self._mh_doc(db) + db.upsert_document_index_status( + "wf-mh-doc", "documents-index", doc_id="d-mh", status="indexed" ) + db.update_document_fields("wf-mh-doc") # no-op, keeps the dev entry only + with db.get_connection() as conn: + conn.execute( + "UPDATE documents SET metadata = jsonb_set(metadata, '{index_status,prod}', " + "'{\"status\": \"indexed\", \"doc_id\": \"d-mh\"}')" + ) with pytest.raises(HTTPException) as exc: - api._resolve_document_identity(outsider, "mh", "scheme", "pm-kisan", "a.pdf") - assert exc.value.status_code == 404 + api._resolve_document_identity( + self._user("/states/MH/contributor"), "mh", "scheme", "pm-kisan", "a.pdf" + ) + assert exc.value.status_code == 403 - kind, code, existing = api._resolve_document_identity(insider, "mh", "scheme", "pm-kisan", "a.pdf") - assert (kind, code, existing["workflow_id"]) == ("scheme", "pm-kisan", "wf-mh-doc") + _, _, existing = api._resolve_document_identity( + self._user("/states/MH/approver"), "mh", "scheme", "pm-kisan", "a.pdf" + ) + assert existing["workflow_id"] == "wf-mh-doc" @pytest.mark.api @pytest.mark.unit diff --git a/tests/test_schema_v2.py b/tests/test_schema_v2.py index df13c54..b52add1 100644 --- a/tests/test_schema_v2.py +++ b/tests/test_schema_v2.py @@ -596,3 +596,75 @@ def test_workflow_and_audit_columns_are_plain_text(self, db_connection): enums = conn.execute("SELECT count(*) AS n FROM pg_type WHERE typtype = 'e'").fetchone()["n"] assert len(types) == 5 and set(types.values()) == {"text"} assert enums == 0 + + +class TestSupersededIndexKeys: + """A replaced file changes the index key; the old key must be remembered for cleanup.""" + + @staticmethod + def _indexed(db, workflow_id, envs): + doc_uuid = _doc(db, workflow_id) + old_key = db.get_document(workflow_id)["document_id"] + for env in envs: + index = "prod-index" if env == "prod" else "documents-index" + db.upsert_document_index_status( + workflow_id, index, doc_id=old_key, chunk_count_indexed=1, status="indexed" + ) + return doc_uuid, old_key + + @pytest.mark.db + @pytest.mark.unit + def test_reupload_remembers_the_old_key_for_every_index_that_holds_it( + self, db_connection, monkeypatch + ): + db = db_connection + monkeypatch.setenv("PROD_VECTOR_DB_COLLECTION_NAME", "prod-index") + doc_uuid, old_key = self._indexed(db, "wf-sup", ["dev", "prod"]) + + db.reset_document_for_reupload( + doc_uuid, filename="n.pdf", filepath="minio://documents/n.pdf", fingerprint="b" * 32 + ) + + assert db.get_document(doc_uuid)["superseded_doc_ids"] == {"dev": [old_key], "prod": [old_key]} + + @pytest.mark.db + @pytest.mark.unit + def test_only_indexes_that_hold_the_old_key_are_tracked(self, db_connection): + db = db_connection + doc_uuid, old_key = self._indexed(db, "wf-sup-dev", ["dev"]) + + db.reset_document_for_reupload( + doc_uuid, filename="n.pdf", filepath="minio://documents/n.pdf", fingerprint="c" * 32 + ) + + assert db.get_document(doc_uuid)["superseded_doc_ids"] == {"dev": [old_key]} + + @pytest.mark.db + @pytest.mark.unit + def test_the_same_file_again_supersedes_nothing(self, db_connection): + db = db_connection + doc_uuid, old_key = self._indexed(db, "wf-sup-same", ["dev"]) + + db.reset_document_for_reupload( + doc_uuid, filename="n.pdf", filepath="minio://documents/n.pdf", fingerprint=old_key + ) + + assert db.get_document(doc_uuid)["superseded_doc_ids"] == {} + + @pytest.mark.db + @pytest.mark.unit + def test_two_replacements_keep_both_old_keys_until_cleared(self, db_connection): + db = db_connection + doc_uuid, first_key = self._indexed(db, "wf-sup-two", ["dev"]) + db.reset_document_for_reupload( + doc_uuid, filename="n.pdf", filepath="minio://documents/n.pdf", fingerprint="d" * 32 + ) + # The second version never reached the index, so its key is not tracked. + db.reset_document_for_reupload( + doc_uuid, filename="m.pdf", filepath="minio://documents/m.pdf", fingerprint="e" * 32 + ) + assert db.get_document(doc_uuid)["superseded_doc_ids"] == {"dev": [first_key]} + + db.clear_superseded_doc_ids(doc_uuid, "dev", [first_key]) + + assert db.get_document(doc_uuid)["superseded_doc_ids"] == {} From cdb3fa822462574ab8c672f360a2294faf941fe2 Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 11:23:03 +0530 Subject: [PATCH 18/53] fix: never delete a vector key another document still uses The vector index key is the file fingerprint, so two documents holding the same file share points. Replace cleanup, disable and purge now skip any key that another document uses as its fingerprint or index entry doc_id. Replace cleanup forgets such a key without deleting it. A per-document key would remove the sharing but needs a reindex, so it is left for a separate change. --- pipeline/activities.py | 8 ++++++++ pipeline/api.py | 12 ++++++++++-- pipeline/db.py | 22 ++++++++++++++++++++++ tests/test_activities.py | 21 +++++++++++++++++++++ tests/test_api.py | 24 ++++++++++++++++++++++++ tests/test_schema_v2.py | 25 +++++++++++++++++++++++++ 6 files changed, 110 insertions(+), 2 deletions(-) diff --git a/pipeline/activities.py b/pipeline/activities.py index 8aebe5d..189ef4a 100644 --- a/pipeline/activities.py +++ b/pipeline/activities.py @@ -1398,6 +1398,14 @@ async def _drop_superseded_points( store = get_vector_store() deleted = [] for doc_id in stale: + # Two documents holding the same file share one index key; keep the + # points while another document still needs them and just forget the key. + if await asyncio.to_thread(db.doc_id_used_by_other_document, doc_id, doc.get("id")): + activity.logger.info( + "Keeping %s points for %s: another document uses the same file", env, doc_id + ) + deleted.append(doc_id) + continue result = await asyncio.to_thread(store.delete_by_doc_id, collection, doc_id) if result.get("error"): activity.logger.warning( diff --git a/pipeline/api.py b/pipeline/api.py index a42e3ce..d5c387f 100644 --- a/pipeline/api.py +++ b/pipeline/api.py @@ -1751,10 +1751,18 @@ def _require_catalog_read(request: Request, user: AuthUser) -> None: def _vector_doc_ids(doc: dict, env: str) -> list[str]: - """The document's current index key plus keys of earlier file versions still in ``env``.""" + """Index keys to delete for a document: its current key plus keys of earlier file versions in ``env``. + + A key another document also uses (the same file uploaded twice) is left + out, since deleting it would remove that document's vectors too. + """ ids = [doc.get("document_id")] ids.extend((doc.get("superseded_doc_ids") or {}).get(env, [])) - return list(dict.fromkeys(i for i in ids if i)) + return [ + i + for i in dict.fromkeys(i for i in ids if i) + if not db.doc_id_used_by_other_document(i, doc.get("id")) + ] def _delete_all_doc_ids(store, collection: str, doc_ids: list[str]) -> dict: diff --git a/pipeline/db.py b/pipeline/db.py index 9bec3a1..ba90737 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -1139,6 +1139,28 @@ def reset_document_for_reupload( ) +def doc_id_used_by_other_document(doc_id: str, exclude_doc_uuid: Optional[str]) -> bool: + """True when a document other than ``exclude_doc_uuid`` uses ``doc_id`` as its index key. + + The index key is the file fingerprint, so two documents holding the same file + share points in the vector index. Deleting a key another document still needs + would remove that document's vectors too. + """ + with get_connection() as conn: + row = conn.execute( + """ + SELECT 1 FROM documents d + WHERE (%s::uuid IS NULL OR d.id <> %s::uuid) + AND (d.source->>'fingerprint' = %s + OR EXISTS (SELECT 1 FROM jsonb_each(COALESCE(d.metadata->'index_status', '{}'::jsonb)) e + WHERE e.value->>'doc_id' = %s)) + LIMIT 1 + """, + (exclude_doc_uuid, exclude_doc_uuid, doc_id, doc_id), + ).fetchone() + return row is not None + + def clear_superseded_doc_ids(workflow_id: str, env: str, doc_ids: list[str]) -> None: """Forget superseded index keys once their points are deleted from ``env`` (dev/prod).""" with get_connection() as conn: diff --git a/tests/test_activities.py b/tests/test_activities.py index 5cca9c4..1c79a5f 100644 --- a/tests/test_activities.py +++ b/tests/test_activities.py @@ -878,3 +878,24 @@ async def test_a_failed_delete_keeps_the_key_for_the_next_run(self, db_connectio ) assert db_connection.get_document(doc_uuid)["superseded_doc_ids"] == {"dev": ["a" * 32]} + + @pytest.mark.asyncio + @pytest.mark.db + @pytest.mark.unit + async def test_a_key_another_document_still_uses_is_kept_in_the_index(self, db_connection): + from pipeline import activities + + doc_uuid = self._document(db_connection) + # A second document holds the very file the first one replaced. + db_connection.upsert_document( + workflow_id="wf-twin", document_id="a" * 32, filename="a.pdf", + filepath="minio://documents/a.pdf", group="gj", kind="scheme", code="twin", + ) + store = self._Store() + + await activities._drop_superseded_points( + "dev", "wf-sup-act", db_connection.get_document(doc_uuid), "c" * 32, "documents-index", store=store + ) + + assert store.calls == [] + assert db_connection.get_document(doc_uuid)["superseded_doc_ids"] == {} diff --git a/tests/test_api.py b/tests/test_api.py index 9f3d2ec..0af303b 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1288,3 +1288,27 @@ def test_a_document_live_in_prod_needs_approval_rights_to_replace(self, db_conne def test_the_demo_and_bootstrap_routes_are_gone(self, test_client): assert test_client.post("/documents/anything/demo").status_code in (404, 405) assert test_client.post("/catalog/v1/bootstrap").status_code in (404, 405) + + +class TestVectorKeysToDelete: + @pytest.mark.api + @pytest.mark.unit + def test_disable_never_deletes_a_key_another_document_uses(self, db_connection): + import pipeline.api as api + + db = db_connection + uuid_a = db.upsert_document( + workflow_id="wf-k-a", document_id="a" * 32, filename="a.pdf", + filepath="minio://documents/a.pdf", group="mh", kind="scheme", code="k-a", + ) + db.upsert_document( + workflow_id="wf-k-b", document_id="a" * 32, filename="a.pdf", + filepath="minio://documents/a.pdf", group="gj", kind="scheme", code="k-b", + ) + db.upsert_document( + workflow_id="wf-k-c", document_id="b" * 32, filename="c.pdf", + filepath="minio://documents/c.pdf", group="mh", kind="scheme", code="k-c", + ) + + assert api._vector_doc_ids(db.get_document(uuid_a), "dev") == [] + assert api._vector_doc_ids(db.get_document("wf-k-c"), "dev") == ["b" * 32] diff --git a/tests/test_schema_v2.py b/tests/test_schema_v2.py index b52add1..25e38a9 100644 --- a/tests/test_schema_v2.py +++ b/tests/test_schema_v2.py @@ -668,3 +668,28 @@ def test_two_replacements_keep_both_old_keys_until_cleared(self, db_connection): db.clear_superseded_doc_ids(doc_uuid, "dev", [first_key]) assert db.get_document(doc_uuid)["superseded_doc_ids"] == {} + + +class TestSharedIndexKeys: + """Two documents holding the same file share an index key; neither may delete it for the other.""" + + @pytest.mark.db + @pytest.mark.unit + def test_a_key_is_in_use_while_another_document_has_that_fingerprint(self, db_connection): + db = db_connection + first = _doc(db, "wf-twin-a", document_id="f" * 32) + _doc(db, "wf-twin-b", document_id="f" * 32, group="gj") + + assert db.doc_id_used_by_other_document("f" * 32, first) is True + assert db.doc_id_used_by_other_document("0" * 32, first) is False + + @pytest.mark.db + @pytest.mark.unit + def test_a_key_is_in_use_while_another_documents_index_entry_holds_it(self, db_connection): + db = db_connection + first = _doc(db, "wf-idx-a") + _doc(db, "wf-idx-b") + db.upsert_document_index_status("wf-idx-b", "documents-index", doc_id="9" * 32, status="indexed") + + assert db.doc_id_used_by_other_document("9" * 32, first) is True + assert db.doc_id_used_by_other_document("9" * 32, db.get_document("wf-idx-b")["id"]) is False From ba7c20a53693406d82fa1b39d4d4724962bdfa42 Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 11:27:27 +0530 Subject: [PATCH 19/53] fix: only live, same-environment owners protect a shared vector key The shared-key guard counted any document with the same fingerprint, including disabled or never-indexed ones, and forgot a skipped key so its points could stay in the index for good. It now counts only another live document that is indexed under the key in the same environment (dev or prod), and a skipped key stays on metadata.superseded_doc_ids so a later ingest, disable or purge can still remove it. --- pipeline/activities.py | 10 ++++---- pipeline/api.py | 7 +++--- pipeline/db.py | 15 ++++++------ tests/test_activities.py | 16 +++++++++++-- tests/test_api.py | 9 +++++++- tests/test_schema_v2.py | 50 ++++++++++++++++++++++++++++++++-------- 6 files changed, 79 insertions(+), 28 deletions(-) diff --git a/pipeline/activities.py b/pipeline/activities.py index 189ef4a..217292d 100644 --- a/pipeline/activities.py +++ b/pipeline/activities.py @@ -1398,13 +1398,13 @@ async def _drop_superseded_points( store = get_vector_store() deleted = [] for doc_id in stale: - # Two documents holding the same file share one index key; keep the - # points while another document still needs them and just forget the key. - if await asyncio.to_thread(db.doc_id_used_by_other_document, doc_id, doc.get("id")): + # Two documents holding the same file share one index key. While another + # live document is indexed under it the points stay, and the key stays on + # the list so a later ingest, disable or purge can still remove it. + if await asyncio.to_thread(db.doc_id_used_by_other_document, doc_id, doc.get("id"), env): activity.logger.info( - "Keeping %s points for %s: another document uses the same file", env, doc_id + "Keeping %s points for %s: another document is indexed under the same key", env, doc_id ) - deleted.append(doc_id) continue result = await asyncio.to_thread(store.delete_by_doc_id, collection, doc_id) if result.get("error"): diff --git a/pipeline/api.py b/pipeline/api.py index d5c387f..9ccacd8 100644 --- a/pipeline/api.py +++ b/pipeline/api.py @@ -1753,15 +1753,16 @@ def _require_catalog_read(request: Request, user: AuthUser) -> None: def _vector_doc_ids(doc: dict, env: str) -> list[str]: """Index keys to delete for a document: its current key plus keys of earlier file versions in ``env``. - A key another document also uses (the same file uploaded twice) is left - out, since deleting it would remove that document's vectors too. + A key another live document is indexed under in ``env`` (the same file + uploaded twice) is left out, since deleting it would remove that document's + vectors too. """ ids = [doc.get("document_id")] ids.extend((doc.get("superseded_doc_ids") or {}).get(env, [])) return [ i for i in dict.fromkeys(i for i in ids if i) - if not db.doc_id_used_by_other_document(i, doc.get("id")) + if not db.doc_id_used_by_other_document(i, doc.get("id"), env) ] diff --git a/pipeline/db.py b/pipeline/db.py index ba90737..9be8ec9 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -1139,24 +1139,25 @@ def reset_document_for_reupload( ) -def doc_id_used_by_other_document(doc_id: str, exclude_doc_uuid: Optional[str]) -> bool: - """True when a document other than ``exclude_doc_uuid`` uses ``doc_id`` as its index key. +def doc_id_used_by_other_document(doc_id: str, exclude_doc_uuid: Optional[str], env: str) -> bool: + """True when another *live* document is indexed in ``env`` (dev/prod) under ``doc_id``. The index key is the file fingerprint, so two documents holding the same file share points in the vector index. Deleting a key another document still needs - would remove that document's vectors too. + would remove that document's vectors too. Only a document that is not + disabled and whose own ``index_status[env]`` holds this key counts: one that + was merely uploaded, or is disabled, has nothing in the index to protect. """ with get_connection() as conn: row = conn.execute( """ SELECT 1 FROM documents d WHERE (%s::uuid IS NULL OR d.id <> %s::uuid) - AND (d.source->>'fingerprint' = %s - OR EXISTS (SELECT 1 FROM jsonb_each(COALESCE(d.metadata->'index_status', '{}'::jsonb)) e - WHERE e.value->>'doc_id' = %s)) + AND NOT d.is_disabled + AND d.metadata->'index_status'->%s::text->>'doc_id' = %s LIMIT 1 """, - (exclude_doc_uuid, exclude_doc_uuid, doc_id, doc_id), + (exclude_doc_uuid, exclude_doc_uuid, env, doc_id), ).fetchone() return row is not None diff --git a/tests/test_activities.py b/tests/test_activities.py index 1c79a5f..862b4f0 100644 --- a/tests/test_activities.py +++ b/tests/test_activities.py @@ -882,20 +882,32 @@ async def test_a_failed_delete_keeps_the_key_for_the_next_run(self, db_connectio @pytest.mark.asyncio @pytest.mark.db @pytest.mark.unit - async def test_a_key_another_document_still_uses_is_kept_in_the_index(self, db_connection): + async def test_a_key_another_live_document_is_indexed_under_is_kept_and_remembered(self, db_connection): from pipeline import activities doc_uuid = self._document(db_connection) - # A second document holds the very file the first one replaced. + # A second document is indexed under the very key the first one replaced. db_connection.upsert_document( workflow_id="wf-twin", document_id="a" * 32, filename="a.pdf", filepath="minio://documents/a.pdf", group="gj", kind="scheme", code="twin", ) + db_connection.upsert_document_index_status( + "wf-twin", "documents-index", doc_id="a" * 32, status="indexed" + ) store = self._Store() await activities._drop_superseded_points( "dev", "wf-sup-act", db_connection.get_document(doc_uuid), "c" * 32, "documents-index", store=store ) + # Points stay for the twin, and the key stays on the list for a later cleanup. assert store.calls == [] + assert db_connection.get_document(doc_uuid)["superseded_doc_ids"] == {"dev": ["a" * 32]} + + # Once the twin is gone the same cleanup removes them. + db_connection.set_document_disabled("wf-twin", True) + await activities._drop_superseded_points( + "dev", "wf-sup-act", db_connection.get_document(doc_uuid), "c" * 32, "documents-index", store=store + ) + assert store.calls == [("documents-index", "a" * 32)] assert db_connection.get_document(doc_uuid)["superseded_doc_ids"] == {} diff --git a/tests/test_api.py b/tests/test_api.py index 0af303b..6288b21 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1293,7 +1293,7 @@ def test_the_demo_and_bootstrap_routes_are_gone(self, test_client): class TestVectorKeysToDelete: @pytest.mark.api @pytest.mark.unit - def test_disable_never_deletes_a_key_another_document_uses(self, db_connection): + def test_disable_never_deletes_a_key_another_live_document_is_indexed_under(self, db_connection): import pipeline.api as api db = db_connection @@ -1310,5 +1310,12 @@ def test_disable_never_deletes_a_key_another_document_uses(self, db_connection): filepath="minio://documents/c.pdf", group="mh", kind="scheme", code="k-c", ) + # The twin was only uploaded, never indexed, so nothing of it is at risk. + assert api._vector_doc_ids(db.get_document(uuid_a), "dev") == ["a" * 32] + + db.upsert_document_index_status("wf-k-b", "documents-index", doc_id="a" * 32, status="indexed") assert api._vector_doc_ids(db.get_document(uuid_a), "dev") == [] assert api._vector_doc_ids(db.get_document("wf-k-c"), "dev") == ["b" * 32] + + db.set_document_disabled("wf-k-b", True) + assert api._vector_doc_ids(db.get_document(uuid_a), "dev") == ["a" * 32] diff --git a/tests/test_schema_v2.py b/tests/test_schema_v2.py index 25e38a9..d0ac976 100644 --- a/tests/test_schema_v2.py +++ b/tests/test_schema_v2.py @@ -673,23 +673,53 @@ def test_two_replacements_keep_both_old_keys_until_cleared(self, db_connection): class TestSharedIndexKeys: """Two documents holding the same file share an index key; neither may delete it for the other.""" + @staticmethod + def _indexed_twin(db, workflow_id, key, env="dev", **kwargs): + _doc(db, workflow_id, document_id=key, group="gj", **kwargs) + index = "prod-index" if env == "prod" else "documents-index" + db.upsert_document_index_status(workflow_id, index, doc_id=key, status="indexed") + @pytest.mark.db @pytest.mark.unit - def test_a_key_is_in_use_while_another_document_has_that_fingerprint(self, db_connection): + def test_a_key_is_in_use_while_another_live_document_is_indexed_under_it( + self, db_connection, monkeypatch + ): db = db_connection + monkeypatch.setenv("PROD_VECTOR_DB_COLLECTION_NAME", "prod-index") first = _doc(db, "wf-twin-a", document_id="f" * 32) - _doc(db, "wf-twin-b", document_id="f" * 32, group="gj") + self._indexed_twin(db, "wf-twin-b", "f" * 32, env="dev") + + assert db.doc_id_used_by_other_document("f" * 32, first, "dev") is True + # Another environment is checked on its own index entry. + assert db.doc_id_used_by_other_document("f" * 32, first, "prod") is False + assert db.doc_id_used_by_other_document("0" * 32, first, "dev") is False + + @pytest.mark.db + @pytest.mark.unit + def test_an_uploaded_but_unindexed_twin_does_not_block_deletion(self, db_connection): + db = db_connection + first = _doc(db, "wf-up-a", document_id="e" * 32) + _doc(db, "wf-up-b", document_id="e" * 32, group="gj") # same file, never indexed + + assert db.doc_id_used_by_other_document("e" * 32, first, "dev") is False + + @pytest.mark.db + @pytest.mark.unit + def test_a_disabled_twin_does_not_block_deletion(self, db_connection): + db = db_connection + first = _doc(db, "wf-dis-a", document_id="d" * 32) + self._indexed_twin(db, "wf-dis-b", "d" * 32) + assert db.doc_id_used_by_other_document("d" * 32, first, "dev") is True + + db.set_document_disabled("wf-dis-b", True) - assert db.doc_id_used_by_other_document("f" * 32, first) is True - assert db.doc_id_used_by_other_document("0" * 32, first) is False + assert db.doc_id_used_by_other_document("d" * 32, first, "dev") is False @pytest.mark.db @pytest.mark.unit - def test_a_key_is_in_use_while_another_documents_index_entry_holds_it(self, db_connection): + def test_a_document_never_blocks_its_own_key(self, db_connection): db = db_connection - first = _doc(db, "wf-idx-a") - _doc(db, "wf-idx-b") - db.upsert_document_index_status("wf-idx-b", "documents-index", doc_id="9" * 32, status="indexed") + only = _doc(db, "wf-own", document_id="c" * 32) + db.upsert_document_index_status("wf-own", "documents-index", doc_id="c" * 32, status="indexed") - assert db.doc_id_used_by_other_document("9" * 32, first) is True - assert db.doc_id_used_by_other_document("9" * 32, db.get_document("wf-idx-b")["id"]) is False + assert db.doc_id_used_by_other_document("c" * 32, only, "dev") is False From 6cc5f9fb31231085a667a78ee22826652238e0ee Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 11:34:08 +0530 Subject: [PATCH 20/53] fix(scripts): use the v2 upload parameters in helper scripts and ENV.md /upload, /documents and /documents/batch now require kind (scheme|advisory) and accept group and code. The upload helpers take DOC_KIND/PIPELINE_GROUP (shell, PowerShell) or --kind/--group (Python), the validity check calls upsert_document and _prepare_records with group/kind/code, list_failed_workflows drops the removed include_demo, and ENV.md drops the bootstrap endpoint and lists the new PATCH body. --- ENV.md | 2 +- scripts/ingest_test_docs.sh | 10 ++++++++-- scripts/list_failed_workflows.py | 2 +- scripts/start_ocr_only_batch.py | 11 ++++++++--- scripts/test_pipeline_e2e.py | 18 +++++++++++++++--- scripts/upload_test_docs.ps1 | 6 +++++- scripts/verify_document_validity_e2e.py | 7 +++++-- 7 files changed, 43 insertions(+), 13 deletions(-) diff --git a/ENV.md b/ENV.md index a073d60..e1a4a2a 100644 --- a/ENV.md +++ b/ENV.md @@ -242,7 +242,7 @@ Exposes `/catalog/v1/*` so **bharat-oan-api** and **bharat-provider-backend** ca | `PROD_SCHEME_VECTOR_DB_API_KEY` | `PROD_VECTOR_DB_API_KEY` | Scheme collection API key | | `PROD_SCHEME_VECTOR_DB_COLLECTION_NAME` | `schemes-index` | Must **not** equal documents collection | -**Endpoints:** `GET /catalog/v1/snapshot`, `/version`, `/schemes`, `/tool-prompt`; `POST /catalog/v1/rebuild`, `/bootstrap`; `PATCH /documents/{id}/scheme-metadata`. +**Endpoints:** `GET /catalog/v1/snapshot`, `/version`, `/schemes`, `/tool-prompt`; `POST /catalog/v1/rebuild`; `PATCH /documents/{id}/scheme-metadata` (body: `kind`, `code`, `name`, `aliases`, `tool_routing`, `catalog_visible`, `network_visible`, `valid_from`, `valid_to`). The catalog is computed from documents (one row per document), so there is no bootstrap endpoint. `/upload`, `/documents` and `/documents/batch` take a required `kind` (`scheme` or `advisory`) and an optional `group` and `code`. ### Publish to Network diff --git a/scripts/ingest_test_docs.sh b/scripts/ingest_test_docs.sh index 03f7bf8..3db67cd 100644 --- a/scripts/ingest_test_docs.sh +++ b/scripts/ingest_test_docs.sh @@ -1,9 +1,15 @@ #!/usr/bin/env bash # Ingest all PDFs from ~/test-docs via H100 pipeline API. +# DOC_KIND: scheme (default) or advisory. PIPELINE_GROUP: tenant group code (optional). set -euo pipefail DOCS_DIR="${1:-$HOME/test-docs}" API="${PIPELINE_API:-http://127.0.0.1:8001}" +DOC_KIND="${DOC_KIND:-scheme}" +QUERY="auto_approve=true&kind=${DOC_KIND}" +if [[ -n "${PIPELINE_GROUP:-}" ]]; then + QUERY="${QUERY}&group=${PIPELINE_GROUP}" +fi if [[ ! -d "$DOCS_DIR" ]]; then echo "Missing directory: $DOCS_DIR" @@ -18,7 +24,7 @@ if [[ ${#pdfs[@]} -eq 0 ]]; then fi echo "API: $API" -echo "Files: ${#pdfs[@]}" +echo "Files: ${#pdfs[@]} kind: $DOC_KIND" curl -sf "$API/health" | python3 -m json.tool || { echo "API not reachable"; exit 1; } echo "" @@ -26,7 +32,7 @@ for f in "${pdfs[@]}"; do name="$(basename "$f")" echo "========================================" echo "UPLOAD: $name" - resp="$(curl -s -X POST "$API/upload?auto_approve=true" -F "file=@$f")" + resp="$(curl -s -X POST "$API/upload?$QUERY" -F "file=@$f")" echo "$resp" | python3 -m json.tool || echo "$resp" wf="$(echo "$resp" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('workflow_id',''))" 2>/dev/null || true)" if [[ -n "$wf" ]]; then diff --git a/scripts/list_failed_workflows.py b/scripts/list_failed_workflows.py index 1a19271..75bbb12 100755 --- a/scripts/list_failed_workflows.py +++ b/scripts/list_failed_workflows.py @@ -95,7 +95,7 @@ async def get_temporal_error_details(workflow_id: str, client: Client) -> dict: def query_failed_workflows_from_db() -> list: """Query the database for failed workflows (including demo and disabled documents).""" return db.list_documents( - stage="failed", limit=1_000_000, include_demo=True, include_disabled=True + stage="failed", limit=1_000_000, include_disabled=True ) diff --git a/scripts/start_ocr_only_batch.py b/scripts/start_ocr_only_batch.py index 701b0c0..a48fb47 100644 --- a/scripts/start_ocr_only_batch.py +++ b/scripts/start_ocr_only_batch.py @@ -45,8 +45,11 @@ def already_has_ocr(filepath: str, fingerprint: str) -> bool: return bool(db.get_saved_page_numbers(doc["workflow_id"])) -def post_document(api_base: str, filepath: str) -> dict: - query = urllib.parse.urlencode({"stop_after_ocr": "true"}) +def post_document(api_base: str, filepath: str, kind: str, group: str = "") -> dict: + params = {"stop_after_ocr": "true", "kind": kind} + if group: + params["group"] = group + query = urllib.parse.urlencode(params) url = f"{api_base.rstrip('/')}/documents?{query}" payload = json.dumps({"filepath": filepath}).encode("utf-8") req = urllib.request.Request( @@ -64,6 +67,8 @@ def main() -> int: parser.add_argument("directory", help="Directory containing source docs") parser.add_argument("--api-base", default="http://localhost:8001", help="Docs-pipeline API base URL") parser.add_argument("--limit", type=int, default=0, help="Optional max files to submit") + parser.add_argument("--kind", choices=["scheme", "advisory"], default="scheme", help="Document kind (catalog)") + parser.add_argument("--group", default="", help="Tenant group code (optional; defaults to your group)") args = parser.parse_args() directory = Path(args.directory) @@ -89,7 +94,7 @@ def main() -> int: print(f"SKIP {path.name}: OCR already present") continue try: - result = post_document(args.api_base, str(path)) + result = post_document(args.api_base, str(path), args.kind, args.group) submitted += 1 print(f"SUBMIT {path.name}: {result.get('workflow_id')}") except urllib.error.HTTPError as exc: diff --git a/scripts/test_pipeline_e2e.py b/scripts/test_pipeline_e2e.py index 5c77f86..66d8eeb 100644 --- a/scripts/test_pipeline_e2e.py +++ b/scripts/test_pipeline_e2e.py @@ -37,10 +37,17 @@ def check_health(name: str, url: str) -> bool: return False -def upload_pdf(api: str, pdf_path: Path, *, auto_approve: bool, stop_after_ocr: bool) -> str: +def upload_pdf( + api: str, pdf_path: Path, *, auto_approve: bool, stop_after_ocr: bool, kind: str, group: str = "" +) -> str: import subprocess - query = f"auto_approve={'true' if auto_approve else 'false'}&stop_after_ocr={'true' if stop_after_ocr else 'false'}" + query = ( + f"auto_approve={'true' if auto_approve else 'false'}" + f"&stop_after_ocr={'true' if stop_after_ocr else 'false'}&kind={kind}" + ) + if group: + query += f"&group={group}" url = f"{api.rstrip('/')}/upload?{query}" result = subprocess.run( ["curl", "-sf", "-X", "POST", url, "-F", f"file=@{pdf_path}"], @@ -92,6 +99,8 @@ def main() -> int: parser.add_argument("--chandra-health", default=os.environ.get("CHANDRA_HEALTH_URL", "http://127.0.0.1:8010/health")) parser.add_argument("--gemma-health", default=os.environ.get("TRANSLATION_VLLM_BASE_URL", "http://localhost:8020/v1")) parser.add_argument("--mode", choices=["ocr", "full"], default="ocr", help="ocr=stop after OCR; full=entire pipeline") + parser.add_argument("--kind", choices=["scheme", "advisory"], default="scheme", help="Document kind (catalog)") + parser.add_argument("--group", default="", help="Tenant group code (optional; defaults to your group)") parser.add_argument("--timeout", type=int, default=1800, help="Max seconds to wait for workflow") parser.add_argument("--poll-interval", type=float, default=5.0) args = parser.parse_args() @@ -121,7 +130,10 @@ def main() -> int: stop_after_ocr = args.mode == "ocr" print(f"==> Upload ({'OCR only' if stop_after_ocr else 'full pipeline'})") - workflow_id = upload_pdf(args.api, args.pdf, auto_approve=True, stop_after_ocr=stop_after_ocr) + workflow_id = upload_pdf( + args.api, args.pdf, auto_approve=True, stop_after_ocr=stop_after_ocr, + kind=args.kind, group=args.group, + ) print("==> Polling workflow") doc = poll_document(args.api, workflow_id, timeout_seconds=args.timeout, interval=args.poll_interval) diff --git a/scripts/upload_test_docs.ps1 b/scripts/upload_test_docs.ps1 index df3e934..ad80769 100644 --- a/scripts/upload_test_docs.ps1 +++ b/scripts/upload_test_docs.ps1 @@ -2,6 +2,10 @@ # Prerequisite: ssh -L 18001:127.0.0.1:8001 -N docs-pipeline-host $ErrorActionPreference = "Stop" $Api = if ($env:PIPELINE_API) { $env:PIPELINE_API } else { "http://127.0.0.1:18001" } +# DOC_KIND: scheme (default) or advisory. PIPELINE_GROUP: tenant group code (optional). +$Kind = if ($env:DOC_KIND) { $env:DOC_KIND } else { "scheme" } +$Query = "auto_approve=true&kind=$Kind" +if ($env:PIPELINE_GROUP) { $Query = "$Query&group=$($env:PIPELINE_GROUP)" } $DocsDir = Join-Path $PSScriptRoot ".." "test docs" | Resolve-Path Write-Host "API: $Api" @@ -31,7 +35,7 @@ Get-ChildItem -Path $DocsDir -Filter "*.pdf" | ForEach-Object { ) $body = $bodyLines -join $LF $bodyBytes = [System.Text.Encoding]::GetEncoding("iso-8859-1").GetBytes($body) - $resp = Invoke-RestMethod -Uri "$Api/upload?auto_approve=true" -Method Post ` + $resp = Invoke-RestMethod -Uri "$Api/upload?$Query" -Method Post ` -ContentType "multipart/form-data; boundary=$boundary" -Body $bodyBytes $resp | ConvertTo-Json -Depth 5 } diff --git a/scripts/verify_document_validity_e2e.py b/scripts/verify_document_validity_e2e.py index f249d58..3b5d839 100644 --- a/scripts/verify_document_validity_e2e.py +++ b/scripts/verify_document_validity_e2e.py @@ -75,6 +75,9 @@ def make_document(workflow_id, text, valid_from=None, valid_to=None, legacy=Fals stage="chunk_review", valid_from=valid_from, valid_to=valid_to, + group="bv", + kind="scheme", + code=workflow_id, ) if legacy: # A document that predates the columns: NULL, as the migration leaves it. @@ -87,7 +90,7 @@ def make_document(workflow_id, text, valid_from=None, valid_to=None, legacy=Fals filename=f"{workflow_id}.pdf", chunks=chunks, workflow_id=workflow_id, - instance="bv", + group="bv", **kwargs, ) @@ -228,7 +231,7 @@ def search_docs(**kwargs): filename="wf-active.pdf", chunks=[{"chunk_number": 1, "original_text": "Kisan credit card subsidy for active season", "token_count": 8}], workflow_id="wf-active", - instance="bv", + group="bv", **_validity_fields_from_doc(doc), ) store.upsert(COLLECTION, again, batch_size=8) From 5cd644d30ab258b08c36c293f62c2c2c645735ed Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 12:56:38 +0530 Subject: [PATCH 21/53] fix(ui): use themed Select for document type on ingest page The native select did not match the console theme or the Select used on the Documents filters. Switch to the shared Select component. --- ui/src/views/NewDocumentView.jsx | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/ui/src/views/NewDocumentView.jsx b/ui/src/views/NewDocumentView.jsx index f8e9325..379f170 100644 --- a/ui/src/views/NewDocumentView.jsx +++ b/ui/src/views/NewDocumentView.jsx @@ -2,6 +2,7 @@ import React, { useCallback, useEffect, useState } from 'react' import { useNavigate } from 'react-router-dom' import { Button } from '../components/ui/button' import { Badge } from '../components/ui/badge' +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '../components/ui/select' import { Upload, FileText, @@ -234,20 +235,18 @@ export default function NewDocumentView() { - + Required
Date: Wed, 28 Oct 2026 16:07:20 +0530 Subject: [PATCH 22/53] feat: add video type back and make document type read-only after upload Video returns as a third type with a placeholder catalog schema name (openagrinet:VideoResource), offered on the upload page. Custom types stay removed. The type decides which catalog a document lives in and is chosen at upload, so the document page now shows it read-only and no longer sends kind in the metadata PATCH. Tests that used video as the example of an unknown kind now use podcast. --- docs/ADR/0006-groups-and-catalogs-model.md | 2 +- docs/POSTGRES_SCHEMA.md | 4 +-- pipeline/network_constants.py | 2 ++ tests/test_api.py | 2 +- tests/test_network_constants.py | 25 +++++++++++++++++ tests/test_schema_v2.py | 2 +- tests/test_scheme_catalog.py | 11 +++++--- ui/src/views/DocumentOpsView.jsx | 31 ++++++++-------------- ui/src/views/NewDocumentView.jsx | 1 + 9 files changed, 51 insertions(+), 29 deletions(-) create mode 100644 tests/test_network_constants.py diff --git a/docs/ADR/0006-groups-and-catalogs-model.md b/docs/ADR/0006-groups-and-catalogs-model.md index c610a7b..474b8cd 100644 --- a/docs/ADR/0006-groups-and-catalogs-model.md +++ b/docs/ADR/0006-groups-and-catalogs-model.md @@ -4,7 +4,7 @@ Context: the first Postgres schema modelled a tenant as a free-text `documents.i Decision: - A **group** is a provider: one `groups` row per Keycloak tenant node (`/states/MH`, `/global`). Role leaves are not mirrored; permissions still come from the token. `users.group_id` is a single nullable FK for now (first tenant in the token, sorted, with a warning when there are several); a join table can replace it if multi-group membership is needed. -- A **catalog** is `(group, type)`: `catalogs.group_id` plus `catalogs.type`, unique together. `type` holds the network schema name. The kind-to-schema map lives in `pipeline/network_constants.py` and is a placeholder: `scheme` is `openagrinet:AgricultureResource`, `advisory` is `openagrinet:KnowledgeAdvisory`. Code branches through that map, never on raw `openagrinet:*` strings. Catalog Beckn information (`publish_directives`, `visible_networks`, `criteria`, `network`) sits in `catalogs.metadata`. +- A **catalog** is `(group, type)`: `catalogs.group_id` plus `catalogs.type`, unique together. `type` holds the network schema name. The kind-to-schema map lives in `pipeline/network_constants.py` and is a placeholder: `scheme` is `openagrinet:AgricultureResource`, `advisory` is `openagrinet:KnowledgeAdvisory`, `video` is `openagrinet:VideoResource`. Code branches through that map, never on raw `openagrinet:*` strings. Catalog Beckn information (`publish_directives`, `visible_networks`, `criteria`, `network`) sits in `catalogs.metadata`. - A **document** belongs to one catalog and is identified there by `code`: `documents.catalog_id` NOT NULL, `UNIQUE (catalog_id, code)`, with `name` and `aliases` on the document. The API fields `code`, `name` and `aliases` are those columns. `documents.instance`, `document_kind` and `is_demo` are gone; the API returns the derived `group` (the group name) and `kind` (from the type map). The `/catalog/v1/*` snapshot keys, the Qdrant payload keys and Temporal arguments keep their earlier names (`scheme_code`, `document_kind`, `instance`, ...) because other services and in-flight workflows depend on them. - Re-uploading an existing code updates the same document (new workflow, `source` replaced, pages and chunks cleared, `reindex_required` set) rather than adding a second one. - Catalogs are created on upload for the tenant and kind. The upload therefore takes the document type up front. diff --git a/docs/POSTGRES_SCHEMA.md b/docs/POSTGRES_SCHEMA.md index 168fe4c..025f995 100644 --- a/docs/POSTGRES_SCHEMA.md +++ b/docs/POSTGRES_SCHEMA.md @@ -45,7 +45,7 @@ The database does not check these values, so adding one needs no migration. Raw |---|---|---| | 1 | `groups` | Provider tenant nodes (`/states/MH`, `/global`), mirrored from Keycloak | | 2 | `users` | People who upload, review and act, mirrored from Keycloak | -| 3 | `catalogs` | One row per provider group and resource type (scheme, advisory) | +| 3 | `catalogs` | One row per provider group and resource type (scheme, advisory, video) | | 4 | `documents` | One row per document. Content, metadata, source file, index state | | 5 | `workflows` | One row per processing run of a document. Stage, status, config | | 6 | `workflow_artifacts` | Files each workflow produces, stored in MinIO | @@ -148,7 +148,7 @@ Unique: `(group_id, type)`. Pydantic (`CatalogMetadata`) validates the keys; the database only checks for an object. Notes -- `type` stores the schema name, and code asks `pipeline/network_constants.py` what it means (`catalog_type_for_kind`, `kind_for_catalog_type`) instead of comparing `openagrinet:*` strings. The mapping is a placeholder: `scheme` is `openagrinet:AgricultureResource`, `advisory` is `openagrinet:KnowledgeAdvisory` (see ADR 0003 and ADR 0006). +- `type` stores the schema name, and code asks `pipeline/network_constants.py` what it means (`catalog_type_for_kind`, `kind_for_catalog_type`) instead of comparing `openagrinet:*` strings. The mapping is a placeholder: `scheme` is `openagrinet:AgricultureResource`, `advisory` is `openagrinet:KnowledgeAdvisory`, `video` is `openagrinet:VideoResource` (see ADR 0003 and ADR 0006). - Catalogs are created on upload (`get_or_create_catalog`), not seeded. - Computed catalog values (status, chunk count, network visibility, `promoted_at`) come from the `catalog_status` view, not stored here. diff --git a/pipeline/network_constants.py b/pipeline/network_constants.py index 5254ac5..6a068d3 100644 --- a/pipeline/network_constants.py +++ b/pipeline/network_constants.py @@ -39,9 +39,11 @@ # placeholder (ADR 0003 types schemes as KnowledgeAdvisory) - change it here only. KIND_SCHEME = "scheme" KIND_ADVISORY = "advisory" +KIND_VIDEO = "video" KIND_TO_CATALOG_TYPE = { KIND_SCHEME: "openagrinet:AgricultureResource", KIND_ADVISORY: "openagrinet:KnowledgeAdvisory", + KIND_VIDEO: "openagrinet:VideoResource", } CATALOG_TYPE_TO_KIND = {schema: kind for kind, schema in KIND_TO_CATALOG_TYPE.items()} diff --git a/tests/test_api.py b/tests/test_api.py index 6288b21..1998bbd 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1042,7 +1042,7 @@ def test_kind_is_required(self, test_client, sample_pdf_content): @pytest.mark.api @pytest.mark.unit - @pytest.mark.parametrize("kind", ["document", "video", "", "Scheme-X"]) + @pytest.mark.parametrize("kind", ["document", "podcast", "", "Scheme-X"]) def test_an_unknown_kind_is_rejected(self, test_client, sample_pdf_content, kind): response = test_client.post( "/upload", diff --git a/tests/test_network_constants.py b/tests/test_network_constants.py new file mode 100644 index 0000000..d4603c0 --- /dev/null +++ b/tests/test_network_constants.py @@ -0,0 +1,25 @@ +import pytest + +from pipeline.network_constants import ( + KIND_TO_CATALOG_TYPE, + catalog_type_for_kind, + kind_for_catalog_type, +) + + +@pytest.mark.parametrize("kind", ["scheme", "advisory", "video"]) +def test_every_kind_round_trips_through_its_catalog_type(kind): + assert kind_for_catalog_type(catalog_type_for_kind(kind)) == kind + + +def test_each_kind_has_its_own_catalog_type(): + assert len(set(KIND_TO_CATALOG_TYPE.values())) == len(KIND_TO_CATALOG_TYPE) + + +def test_video_is_a_known_kind(): + assert catalog_type_for_kind(" Video ") == "openagrinet:VideoResource" + + +def test_unknown_kind_is_rejected(): + with pytest.raises(ValueError): + catalog_type_for_kind("custom") diff --git a/tests/test_schema_v2.py b/tests/test_schema_v2.py index d0ac976..f14a46a 100644 --- a/tests/test_schema_v2.py +++ b/tests/test_schema_v2.py @@ -115,7 +115,7 @@ def test_get_or_create_catalog_is_idempotent_per_group_and_kind(self, db_connect @pytest.mark.unit def test_a_kind_without_a_catalog_type_is_rejected(self, db_connection): with pytest.raises(ValueError): - db_connection.get_or_create_catalog(_group(db_connection), "video") + db_connection.get_or_create_catalog(_group(db_connection), "podcast") @pytest.mark.db @pytest.mark.unit diff --git a/tests/test_scheme_catalog.py b/tests/test_scheme_catalog.py index ab13c76..fe26472 100644 --- a/tests/test_scheme_catalog.py +++ b/tests/test_scheme_catalog.py @@ -153,7 +153,7 @@ def test_apply_scheme_metadata_keeps_the_code_when_only_the_name_changes(catalog assert result["code_renamed"] is False -def test_kind_is_limited_to_scheme_and_advisory(catalog_db): +def test_kind_is_limited_to_the_known_kinds(catalog_db): db, scheme_catalog = catalog_db _doc(db, "wf-kind", code="kind-doc", stage="ready_for_ingestion") assert db.get_document("wf-kind")["kind"] == "scheme" @@ -162,10 +162,13 @@ def test_kind_is_limited_to_scheme_and_advisory(catalog_db): assert result["document"]["kind"] == "advisory" assert result["document"]["code"] == "kind-doc" # moved catalogs, kept its code + result = scheme_catalog.apply_scheme_metadata("wf-kind", kind="video") + assert result["document"]["kind"] == "video" + result = scheme_catalog.apply_scheme_metadata("wf-kind", kind="scheme") assert result["document"]["kind"] == "scheme" - for bad in ("how_to_faq", "document", "video", "Not Valid!!"): + for bad in ("how_to_faq", "document", "podcast", "Not Valid!!"): with pytest.raises(ValueError): scheme_catalog.apply_scheme_metadata("wf-kind", kind=bad) assert db.get_document("wf-kind")["kind"] == "scheme" @@ -328,11 +331,11 @@ def test_scheme_metadata_api_rejects_an_unknown_kind(asgi_client): from pipeline import db _doc(db, "wf-meta-bad", code="meta-bad", stage="chunk_review") - r = asgi_client.patch("/documents/wf-meta-bad/scheme-metadata", json={"kind": "video"}) + r = asgi_client.patch("/documents/wf-meta-bad/scheme-metadata", json={"kind": "podcast"}) assert r.status_code == 400 -@pytest.mark.parametrize("kind", ["scheme", "advisory"]) +@pytest.mark.parametrize("kind", ["scheme", "advisory", "video"]) def test_reingest_is_allowed_for_every_kind(asgi_client, mock_temporal_client, kind): """No 'never classified' guard any more: a document always has a kind and a code.""" from pipeline import db diff --git a/ui/src/views/DocumentOpsView.jsx b/ui/src/views/DocumentOpsView.jsx index 4cab7e9..1652bca 100644 --- a/ui/src/views/DocumentOpsView.jsx +++ b/ui/src/views/DocumentOpsView.jsx @@ -40,7 +40,6 @@ import { Badge } from '../components/ui/badge' import { Button } from '../components/ui/button' import { Checkbox } from '../components/ui/checkbox' import { Input } from '../components/ui/input' -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '../components/ui/select' import { Skeleton } from '../components/ui/skeleton' import { Tabs, TabsContent, TabsList, TabsTrigger } from '../components/ui/tabs' import { Textarea } from '../components/ui/textarea' @@ -106,13 +105,15 @@ function validateValidity(validFrom, validTo) { return '' } -const DOCUMENT_KIND_OPTIONS = [ - { value: 'scheme', label: 'Scheme' }, - { value: 'advisory', label: 'Advisory' }, -] +// The type is chosen at upload and fixed afterwards: it decides the catalog the +// document lives in, so this panel only displays it. +const DOCUMENT_KIND_LABELS = { + scheme: 'Scheme', + advisory: 'Advisory', + video: 'Video', +} function DocumentClassificationPanel({ doc, workflowId, canClassify, onSaved }) { - const [kind, setKind] = useState(doc.kind || '') const [name, setName] = useState(doc.name || '') const [validFrom, setValidFrom] = useState(doc.valid_from || todayISODate()) // Blank when the document has no end date, which is the normal case - an @@ -125,7 +126,6 @@ function DocumentClassificationPanel({ doc, workflowId, canClassify, onSaved }) ? validateValidity(validFrom, validTo) : '' const canSave = - Boolean(kind) && name.trim().length > 0 && !validityError @@ -138,7 +138,6 @@ function DocumentClassificationPanel({ doc, workflowId, canClassify, onSaved }) method: 'PATCH', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ - kind, name: name.trim(), // Omitted entirely when the feature is off, so classifying a // document leaves whatever period the upload stamped untouched. @@ -156,8 +155,7 @@ function DocumentClassificationPanel({ doc, workflowId, canClassify, onSaved }) } const alreadySaved = - doc.kind === kind - && doc.name === name.trim() + doc.name === name.trim() && (!DOCUMENT_VALIDITY_ENABLED || (doc.valid_from === validFrom && (doc.valid_to || '') === validTo)) @@ -171,16 +169,9 @@ function DocumentClassificationPanel({ doc, workflowId, canClassify, onSaved })
Type - + + {DOCUMENT_KIND_LABELS[doc.kind] || doc.kind || '—'} +
{DOCUMENT_VALIDITY_ENABLED && ( <> diff --git a/ui/src/views/NewDocumentView.jsx b/ui/src/views/NewDocumentView.jsx index 379f170..d9cd9b4 100644 --- a/ui/src/views/NewDocumentView.jsx +++ b/ui/src/views/NewDocumentView.jsx @@ -33,6 +33,7 @@ const FILE_TYPE_CHIPS = ['PDF', 'Word'] const DOCUMENT_TYPE_OPTIONS = [ { value: 'scheme', label: 'Scheme' }, { value: 'advisory', label: 'Advisory' }, + { value: 'video', label: 'Video' }, ] const PIPELINE_STEPS = [ From 63563050c55cbd4b6eb227f8c33b46563b578068 Mon Sep 17 00:00:00 2001 From: Naveen Date: Thu, 29 Oct 2026 11:18:01 +0530 Subject: [PATCH 23/53] feat: create a group's catalogs with the group and fix the kind after upload A group row now gets one catalog per supported type (scheme, advisory, video) in the same transaction, named " catalog". Startup runs an idempotent sync so a type added to the kind map reaches existing groups. Upload keeps get-or-create as a fallback for a missing catalog. The kind is fixed by the catalog chosen at upload, so it is removed from the scheme-metadata PATCH and from update_document_fields. --- docs/POSTGRES_SCHEMA.md | 4 +- pipeline/api.py | 3 +- pipeline/db.py | 77 +++++++++++++++++++------- pipeline/models.py | 3 +- pipeline/scheme_catalog.py | 9 +--- tests/test_api.py | 8 +-- tests/test_db.py | 13 ++--- tests/test_document_repository.py | 16 ++++-- tests/test_schema_v2.py | 90 ++++++++++++++++++++++++------- tests/test_scheme_catalog.py | 33 +++++------- tests/test_tenancy.py | 14 ++--- 11 files changed, 174 insertions(+), 96 deletions(-) diff --git a/docs/POSTGRES_SCHEMA.md b/docs/POSTGRES_SCHEMA.md index 025f995..fbda3fa 100644 --- a/docs/POSTGRES_SCHEMA.md +++ b/docs/POSTGRES_SCHEMA.md @@ -149,7 +149,7 @@ Pydantic (`CatalogMetadata`) validates the keys; the database only checks for an Notes - `type` stores the schema name, and code asks `pipeline/network_constants.py` what it means (`catalog_type_for_kind`, `kind_for_catalog_type`) instead of comparing `openagrinet:*` strings. The mapping is a placeholder: `scheme` is `openagrinet:AgricultureResource`, `advisory` is `openagrinet:KnowledgeAdvisory`, `video` is `openagrinet:VideoResource` (see ADR 0003 and ADR 0006). -- Catalogs are created on upload (`get_or_create_catalog`), not seeded. +- Every group gets one catalog per supported type when the group row is created (`upsert_group` calls `ensure_group_catalogs` in the same transaction). Startup (`init_db`) runs `sync_catalogs_for_all_groups`, so a type added to `KIND_TO_CATALOG_TYPE` reaches existing groups. Upload keeps `get_or_create_catalog` as a fallback for a catalog that is missing. Names are ` catalog`, for example `MH scheme catalog`. - Computed catalog values (status, chunk count, network visibility, `promoted_at`) come from the `catalog_status` view, not stored here. --- @@ -234,7 +234,7 @@ Column default: `'{"index_status": {}, "catalog_visible": true, "network_visible - The API's `group` is derived: `documents.catalog_id → catalogs.group_id → groups.name`. There is no `documents.instance` column, and tenant filters join `groups`. - The API's `document_kind` is derived from `catalogs.type` through the map in `network_constants.py`. `scheme_code`, `scheme_name` and `scheme_aliases` are `code`, `name` and `aliases`. - `chunk_count` and `page_count` are counted from `chunks` and `pages`; they are not stored. -- A new document needs a `document_kind` and a `code`. Upload gets the catalog from the tenant and the kind (created when missing). `DuplicateDocumentCode` is raised when the code is taken in that catalog. +- A new document needs a `document_kind` and a `code`. Upload gets the catalog from the tenant and the kind (seeded with the group; created when missing). A document's kind is fixed after upload, so there is no kind change. `DuplicateDocumentCode` is raised when the code is taken in that catalog. - Re-uploading an existing code updates the same document: a new workflow starts, `source` is replaced, pages and chunks are cleared and `reindex_required` is set (`reset_document_for_reupload`). Earlier workflows and artifacts stay as history. - Changing a document's kind moves it to the same group's catalog of the new type (`update_document_fields(document_kind=...)`). - SQL does not check types inside the JSONB columns. The insert path writes the default metadata keys, and Pydantic validates `source`, `metadata` and each `index_status` entry. diff --git a/pipeline/api.py b/pipeline/api.py index 9ccacd8..58e93be 100644 --- a/pipeline/api.py +++ b/pipeline/api.py @@ -2319,7 +2319,7 @@ async def patch_scheme_metadata( user: RequireReview, ): """ - Set kind / code / name / aliases used by Master Catalog and Qdrant scheme promote. + Set code / name / aliases / validity used by Master Catalog and Qdrant scheme promote. On completed docs, renaming the code sets pending_reindex (excluded from live vector_schemes until re-promote). Other metadata edits bump catalog version only. @@ -2328,7 +2328,6 @@ async def patch_scheme_metadata( try: result = scheme_catalog.apply_scheme_metadata( workflow_id, - kind=body.kind, code=body.code, name=body.name, aliases=body.aliases, diff --git a/pipeline/db.py b/pipeline/db.py index 9be8ec9..0dff900 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -41,7 +41,7 @@ ResultSettings, RetrievalSettings, ) -from .network_constants import catalog_type_for_kind, kind_for_catalog_type +from .network_constants import KIND_TO_CATALOG_TYPE, catalog_type_for_kind, kind_for_catalog_type logger = logging.getLogger(__name__) @@ -103,6 +103,9 @@ def init_db() -> None: current = migrate.current_version(database_url()) logger.info("db migrations current=%s applied_now=%s", current, applied) get_pool() + created = sync_catalogs_for_all_groups() + if created: + logger.info("catalog sync created %d missing catalogs", created) def check_health() -> dict: @@ -440,7 +443,8 @@ def upsert_group(path: str, keycloak_group_id: Optional[str] = None, conn=None) """Create or refresh the ``groups`` row for a Keycloak group path; returns its UUID. Rows made by the SQLite move have no Keycloak id yet; the first sync that - sees the path fills it in. + sees the path fills it in. The group's catalogs (one per supported type) are + ensured in the same transaction, so a group never exists without them. """ if conn is None: with get_connection() as own: @@ -454,7 +458,9 @@ def upsert_group(path: str, keycloak_group_id: Optional[str] = None, conn=None) """, (keycloak_group_id, _tenant_code_from_path(path), path), ).fetchone() - return str(row["id"]) + group_id = str(row["id"]) + ensure_group_catalogs(group_id, conn=conn) + return group_id def get_or_create_group(group: Optional[str], conn=None) -> str: @@ -463,8 +469,54 @@ def get_or_create_group(group: Optional[str], conn=None) -> str: return upsert_group(_tenant_path(code), conn=conn) +def _catalog_seed_rows() -> tuple[list[str], list[str]]: + """Parallel ``(types, kinds)`` lists for every supported catalog type.""" + kinds = list(KIND_TO_CATALOG_TYPE) + return [KIND_TO_CATALOG_TYPE[k] for k in kinds], kinds + + +def ensure_group_catalogs(group_id: str, conn=None) -> None: + """Create the group's missing catalogs, one per supported type; existing rows are untouched.""" + if conn is None: + with get_connection() as own: + return ensure_group_catalogs(group_id, conn=own) + types, kinds = _catalog_seed_rows() + conn.execute( + """ + INSERT INTO catalogs (group_id, type, name) + SELECT g.id, t.type, upper(g.name) || ' ' || t.kind || ' catalog' + FROM groups g, unnest(%s::text[], %s::text[]) AS t(type, kind) + WHERE g.id = %s::uuid + ON CONFLICT (group_id, type) DO NOTHING + """, + (types, kinds, group_id), + ) + + +def sync_catalogs_for_all_groups(conn=None) -> int: + """Add every missing ``(group, type)`` catalog; returns how many rows were created. + + Idempotent. Run at startup so a type added to the kind map reaches groups + that already exist. + """ + if conn is None: + with get_connection() as own: + return sync_catalogs_for_all_groups(conn=own) + types, kinds = _catalog_seed_rows() + cur = conn.execute( + """ + INSERT INTO catalogs (group_id, type, name) + SELECT g.id, t.type, upper(g.name) || ' ' || t.kind || ' catalog' + FROM groups g, unnest(%s::text[], %s::text[]) AS t(type, kind) + ON CONFLICT (group_id, type) DO NOTHING + """, + (types, kinds), + ) + return cur.rowcount + + def get_or_create_catalog(group_id: str, kind: str, conn=None) -> dict: - """The ``(group, kind)`` catalog row, created with a default name when missing.""" + """The ``(group, kind)`` catalog row; created when missing (normally seeded with the group).""" if conn is None: with get_connection() as own: return get_or_create_catalog(group_id, kind, conn=own) @@ -472,7 +524,7 @@ def get_or_create_catalog(group_id: str, kind: str, conn=None) -> dict: row = conn.execute( """ INSERT INTO catalogs (group_id, type, name) - SELECT g.id, %s, g.name || ' ' || %s FROM groups g WHERE g.id = %s::uuid + SELECT g.id, %s, upper(g.name) || ' ' || %s || ' catalog' FROM groups g WHERE g.id = %s::uuid ON CONFLICT (group_id, type) DO UPDATE SET type = EXCLUDED.type RETURNING id, group_id, type, name, metadata """, @@ -1609,7 +1661,7 @@ def update_document_fields(workflow_id: str, **updates: object) -> Optional[dict Flat names are mapped onto the tables: ``stage``/``error_message`` act on the current workflow, ``code``/``name``/``aliases`` on the document columns, - ``kind`` moves the document to the group's catalog of that kind, and flags, + a document's kind is fixed by its catalog at upload, and flags, period and uploader go into ``metadata``. Names derived from other rows (counts, artifact and job pointers, completion timestamps) are accepted and ignored. """ @@ -1688,19 +1740,6 @@ def update_document_fields(workflow_id: str, **updates: object) -> Optional[dict conn=conn, ) - if updates.get("kind"): - current = conn.execute( - "SELECT c.group_id, c.type FROM documents d JOIN catalogs c ON c.id = d.catalog_id " - "WHERE d.id = %s::uuid", - (doc_uuid,), - ).fetchone() - if current["type"] != catalog_type_for_kind(str(updates["kind"])): - target = get_or_create_catalog( - str(current["group_id"]), str(updates["kind"]), conn=conn - ) - doc_sets.append("catalog_id = %s::uuid") - doc_params.append(target["id"]) - if source_patch: doc_sets.append("source = source || %s::jsonb") doc_params.append(Jsonb({k: v for k, v in source_patch.items() if v is not None})) diff --git a/pipeline/models.py b/pipeline/models.py index 247cd8c..ae33470 100644 --- a/pipeline/models.py +++ b/pipeline/models.py @@ -257,9 +257,8 @@ class SchemeMetadataUpdate(BaseModel): period is part of classifying a document, not a separate errand. Omitting both leaves the stored period alone; the dates are validated in `document_validity.parse_period` so a bad period comes back as the same - 400 a bad kind does, not a Pydantic 422. + 400 other bad input does, not a Pydantic 422. """ - kind: Optional[str] = None # scheme | advisory code: Optional[str] = None name: Optional[str] = None aliases: Optional[list[str]] = None diff --git a/pipeline/scheme_catalog.py b/pipeline/scheme_catalog.py index a6a6eb5..2ebeb93 100644 --- a/pipeline/scheme_catalog.py +++ b/pipeline/scheme_catalog.py @@ -15,7 +15,6 @@ from typing import Any, Optional from . import db, document_validity -from .network_constants import catalog_type_for_kind SCHEME_CODE_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") @@ -514,7 +513,6 @@ def build_snapshot( def apply_scheme_metadata( workflow_id: str, *, - kind: Optional[str] = None, code: Optional[str] = None, name: Optional[str] = None, aliases: Optional[list[str]] = None, @@ -542,12 +540,7 @@ def apply_scheme_metadata( code_renamed = False pending_reindex = False - if kind is not None: - new_kind = kind.strip().lower() - catalog_type_for_kind(new_kind) # ValueError for a kind with no catalog type - updates["kind"] = new_kind - else: - new_kind = old_kind + new_kind = old_kind # the kind is fixed at upload; it is the document's catalog new_code = old_code if code is not None: diff --git a/tests/test_api.py b/tests/test_api.py index 1998bbd..e282331 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1083,7 +1083,9 @@ def test_upload_creates_the_catalog_and_derives_the_code_from_the_filename( assert doc["name"] == "National Mission on Natural Farming" with db_connection.get_connection() as conn: catalog = conn.execute( - "SELECT c.type, g.name, g.path FROM catalogs c JOIN groups g ON g.id = c.group_id" + "SELECT c.type, g.name, g.path FROM documents d JOIN catalogs c ON c.id = d.catalog_id " + "JOIN groups g ON g.id = c.group_id WHERE d.code = %s", + (doc["code"],), ).fetchall() assert [(r["type"], r["name"], r["path"]) for r in catalog] == [ ("openagrinet:AgricultureResource", "mh", "/states/MH") @@ -1112,7 +1114,7 @@ def test_a_second_upload_reuses_the_catalog_and_gets_a_fresh_code( } assert codes == {"ca", "ca-2"} with db_connection.get_connection() as conn: - assert conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] == 1 + assert conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] == 3 # seeded with the group @pytest.mark.api @pytest.mark.unit @@ -1169,7 +1171,7 @@ def test_the_same_code_in_another_tenant_is_a_different_document( with db_connection.get_connection() as conn: assert conn.execute("SELECT count(*) AS n FROM documents").fetchone()["n"] == 2 - assert conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] == 2 + assert conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] == 6 # 3 per group @staticmethod def _user(*group_paths): diff --git a/tests/test_db.py b/tests/test_db.py index 2d2d5b6..d98a0e0 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -1100,18 +1100,13 @@ def test_other_kinds_stay_out_of_the_scheme_listing(self, db_connection): @pytest.mark.db @pytest.mark.unit - def test_changing_the_kind_moves_the_document_and_rejects_unknown_kinds(self, db_connection): + def test_the_kind_is_fixed_by_the_catalog_chosen_at_upload(self, db_connection): _new_doc(db_connection, "wf-move", kind="scheme", code="movable", name="Movable") before = db_connection.get_document("wf-move")["catalog_id"] - db_connection.update_document_fields("wf-move", kind="advisory") + db_connection.update_document_fields("wf-move", kind="advisory", name="Renamed") doc = db_connection.get_document("wf-move") - assert (doc["kind"], doc["code"], doc["name"]) == ( - "advisory", "movable", "Movable", - ) - assert doc["catalog_id"] != before - with pytest.raises(ValueError): - db_connection.update_document_fields("wf-move", kind="document") - assert db_connection.get_document("wf-move")["kind"] == "advisory" + assert (doc["kind"], doc["code"], doc["name"]) == ("scheme", "movable", "Renamed") + assert doc["catalog_id"] == before @pytest.mark.db @pytest.mark.unit diff --git a/tests/test_document_repository.py b/tests/test_document_repository.py index caa5d2f..75716a7 100644 --- a/tests/test_document_repository.py +++ b/tests/test_document_repository.py @@ -81,10 +81,18 @@ def test_reads_the_kind_of_the_documents_catalog(self, db_connection, scheme_doc @pytest.mark.unit @pytest.mark.db - def test_reads_a_kind_written_through_the_db_layer(self, db_connection, scheme_document): - db_connection.update_document_fields(scheme_document, kind="advisory") - - assert DocumentRepository().get_document_kind(scheme_document) == "advisory" + def test_reads_a_kind_written_through_the_db_layer(self, db_connection): + db_connection.upsert_document( + workflow_id="repo-wf-adv", + document_id="repo-doc-adv", + filename="adv.pdf", + filepath="/app/books/adv.pdf", + stage="registered", + kind="advisory", + code="repo-advisory", + ) + + assert DocumentRepository().get_document_kind("repo-wf-adv") == "advisory" @pytest.mark.unit @pytest.mark.db diff --git a/tests/test_schema_v2.py b/tests/test_schema_v2.py index f14a46a..d1ce3b2 100644 --- a/tests/test_schema_v2.py +++ b/tests/test_schema_v2.py @@ -91,12 +91,77 @@ def test_deleting_a_group_clears_users_and_audit_snapshots(self, db_connection): "INSERT INTO audit_logs (action_type, entity_type, group_id) VALUES ('x', 'setting', %s::uuid)", (group_id,), ) + conn.execute("DELETE FROM catalogs WHERE group_id = %s::uuid", (group_id,)) conn.execute("DELETE FROM groups WHERE id = %s::uuid", (group_id,)) assert conn.execute("SELECT group_id FROM audit_logs WHERE action_type = 'x'").fetchone()["group_id"] is None assert db.get_user_by_keycloak_id("kc-zz")["group_id"] is None class TestCatalogs: + @pytest.mark.db + @pytest.mark.unit + def test_a_new_group_gets_one_catalog_per_supported_type(self, db_connection): + from pipeline.network_constants import KIND_TO_CATALOG_TYPE + + db = db_connection + mh = _group(db, "mh") + with db.get_connection() as conn: + rows = conn.execute( + "SELECT type, name FROM catalogs WHERE group_id = %s::uuid ORDER BY type", (mh,) + ).fetchall() + assert [r["type"] for r in rows] == sorted(KIND_TO_CATALOG_TYPE.values()) + assert {r["name"] for r in rows} == { + "MH scheme catalog", "MH advisory catalog", "MH video catalog", + } + + @pytest.mark.db + @pytest.mark.unit + def test_upload_uses_the_seeded_catalog_and_creates_none(self, db_connection): + db = db_connection + mh = _group(db, "mh") + with db.get_connection() as conn: + seeded = conn.execute( + "SELECT id FROM catalogs WHERE group_id = %s::uuid AND type = 'openagrinet:AgricultureResource'", + (mh,), + ).fetchone()["id"] + before = conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] + _doc(db, "wf-seeded", group="mh", code="seeded") + assert db.get_document("wf-seeded")["catalog_id"] == str(seeded) + with db.get_connection() as conn: + assert conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] == before + + @pytest.mark.db + @pytest.mark.unit + def test_a_missing_catalog_is_created_on_demand(self, db_connection): + db = db_connection + mh = _group(db, "mh") + with db.get_connection() as conn: + conn.execute( + "DELETE FROM catalogs WHERE group_id = %s::uuid AND type = 'openagrinet:KnowledgeAdvisory'", + (mh,), + ) + _doc(db, "wf-fallback", group="mh", code="fallback", kind="advisory") + assert db.get_document("wf-fallback")["kind"] == "advisory" + + @pytest.mark.db + @pytest.mark.unit + def test_startup_sync_fills_missing_catalogs_for_every_group_and_is_idempotent(self, db_connection): + db = db_connection + mh, up = _group(db, "mh"), _group(db, "up") + with db.get_connection() as conn: + conn.execute("DELETE FROM catalogs WHERE type = 'openagrinet:VideoResource'") + conn.execute( + "DELETE FROM catalogs WHERE group_id = %s::uuid AND type = 'openagrinet:KnowledgeAdvisory'", (up,) + ) + assert db.sync_catalogs_for_all_groups() == 3 + assert db.sync_catalogs_for_all_groups() == 0 + with db.get_connection() as conn: + counts = { + str(r["group_id"]): r["n"] + for r in conn.execute("SELECT group_id, count(*) AS n FROM catalogs GROUP BY group_id") + } + assert counts == {mh: 3, up: 3} + @pytest.mark.db @pytest.mark.unit def test_get_or_create_catalog_is_idempotent_per_group_and_kind(self, db_connection): @@ -105,7 +170,7 @@ def test_get_or_create_catalog_is_idempotent_per_group_and_kind(self, db_connect scheme = db.get_or_create_catalog(mh, "scheme") assert db.get_or_create_catalog(mh, "scheme")["id"] == scheme["id"] assert scheme["type"] == "openagrinet:AgricultureResource" - assert scheme["name"] == "mh scheme" + assert scheme["name"] == "MH scheme catalog" advisory = db.get_or_create_catalog(mh, "advisory") assert advisory["id"] != scheme["id"] assert advisory["type"] == "openagrinet:KnowledgeAdvisory" @@ -240,28 +305,13 @@ def test_find_document_by_code(self, db_connection): @pytest.mark.db @pytest.mark.unit - def test_changing_the_kind_moves_the_document_within_its_group(self, db_connection): + def test_the_kind_does_not_change_after_the_document_exists(self, db_connection): db = db_connection - _doc(db, "wf-kind", group="mh", code="movable") + _doc(db, "wf-kind", group="mh", code="fixed") + before = db.get_document("wf-kind")["catalog_id"] db.update_document_fields("wf-kind", kind="advisory") doc = db.get_document("wf-kind") - assert doc["kind"] == "advisory" - with db.get_connection() as conn: - row = conn.execute( - "SELECT c.group_id, c.type FROM catalogs c WHERE c.id = %s::uuid", (doc["catalog_id"],) - ).fetchone() - assert str(row["group_id"]) == _group(db, "mh") - assert row["type"] == "openagrinet:KnowledgeAdvisory" - - @pytest.mark.db - @pytest.mark.unit - def test_moving_to_a_kind_that_already_holds_the_code_is_a_duplicate(self, db_connection): - db = db_connection - _doc(db, "wf-s", group="mh", code="same") - _doc(db, "wf-a", group="mh", code="same", kind="advisory") - with pytest.raises(db.DuplicateDocumentCode): - db.update_document_fields("wf-s", kind="advisory") - assert db.get_document("wf-s")["kind"] == "scheme" + assert (doc["kind"], doc["catalog_id"]) == ("scheme", before) class TestReupload: diff --git a/tests/test_scheme_catalog.py b/tests/test_scheme_catalog.py index fe26472..d122cf8 100644 --- a/tests/test_scheme_catalog.py +++ b/tests/test_scheme_catalog.py @@ -69,7 +69,6 @@ def test_scheme_metadata_and_promote_rebuild(catalog_db): _doc(db, "wf-pkvy-1", code="pkvy") result = scheme_catalog.apply_scheme_metadata( "wf-pkvy-1", - kind="scheme", code="pkvy", name="Paramparagat Krishi Vikas Yojana", aliases=["PKVY", "organic"], @@ -145,7 +144,6 @@ def test_apply_scheme_metadata_keeps_the_code_when_only_the_name_changes(catalog _doc(db, "wf-name-only", code="gn", stage="ready_for_ingestion") result = scheme_catalog.apply_scheme_metadata( "wf-name-only", - kind="scheme", name="National Mission on Natural Farming", ) assert result["document"]["code"] == "gn" @@ -153,25 +151,14 @@ def test_apply_scheme_metadata_keeps_the_code_when_only_the_name_changes(catalog assert result["code_renamed"] is False -def test_kind_is_limited_to_the_known_kinds(catalog_db): +def test_kind_is_fixed_after_upload(catalog_db): db, scheme_catalog = catalog_db _doc(db, "wf-kind", code="kind-doc", stage="ready_for_ingestion") - assert db.get_document("wf-kind")["kind"] == "scheme" - - result = scheme_catalog.apply_scheme_metadata("wf-kind", kind="advisory") - assert result["document"]["kind"] == "advisory" - assert result["document"]["code"] == "kind-doc" # moved catalogs, kept its code - - result = scheme_catalog.apply_scheme_metadata("wf-kind", kind="video") - assert result["document"]["kind"] == "video" - - result = scheme_catalog.apply_scheme_metadata("wf-kind", kind="scheme") + result = scheme_catalog.apply_scheme_metadata("wf-kind", name="Renamed") assert result["document"]["kind"] == "scheme" - - for bad in ("how_to_faq", "document", "podcast", "Not Valid!!"): - with pytest.raises(ValueError): - scheme_catalog.apply_scheme_metadata("wf-kind", kind=bad) assert db.get_document("wf-kind")["kind"] == "scheme" + with pytest.raises(TypeError): + scheme_catalog.apply_scheme_metadata("wf-kind", kind="advisory") def test_prepare_records_scheme_payload(): @@ -327,12 +314,16 @@ def test_scheme_metadata_api(asgi_client): assert "MIF" in body["aliases"] -def test_scheme_metadata_api_rejects_an_unknown_kind(asgi_client): +def test_scheme_metadata_api_ignores_a_kind_in_the_body(asgi_client): from pipeline import db - _doc(db, "wf-meta-bad", code="meta-bad", stage="chunk_review") - r = asgi_client.patch("/documents/wf-meta-bad/scheme-metadata", json={"kind": "podcast"}) - assert r.status_code == 400 + _doc(db, "wf-meta-kind", code="meta-kind", stage="chunk_review") + r = asgi_client.patch( + "/documents/wf-meta-kind/scheme-metadata", json={"kind": "advisory", "name": "Still a scheme"} + ) + assert r.status_code == 200 + assert r.json()["kind"] == "scheme" + assert r.json()["name"] == "Still a scheme" @pytest.mark.parametrize("kind", ["scheme", "advisory", "video"]) diff --git a/tests/test_tenancy.py b/tests/test_tenancy.py index 6b251b0..ba359cb 100644 --- a/tests/test_tenancy.py +++ b/tests/test_tenancy.py @@ -398,12 +398,14 @@ def test_group_is_reached_through_the_documents_catalog(db_connection): "tenant-a": "/states/TENANT-A", PORTAL_GROUP: "/global" if PORTAL_GROUP == "bv" else f"/states/{PORTAL_GROUP.upper()}", } - # One catalog per (group, type): tenant-a has scheme + advisory, the portal only scheme. - assert sorted((r["group_name"], r["type"]) for r in catalogs) == sorted([ - ("tenant-a", "openagrinet:AgricultureResource"), - ("tenant-a", "openagrinet:KnowledgeAdvisory"), - (PORTAL_GROUP, "openagrinet:AgricultureResource"), - ]) + # Every group is seeded with one catalog per supported type. + from pipeline.network_constants import KIND_TO_CATALOG_TYPE + + assert sorted((r["group_name"], r["type"]) for r in catalogs) == sorted( + (group, catalog_type) + for group in ("tenant-a", PORTAL_GROUP) + for catalog_type in KIND_TO_CATALOG_TYPE.values() + ) assert db.get_document("wf-g-a")["group"] == "tenant-a" assert db.get_document("wf-g-adv")["kind"] == "advisory" assert {d["workflow_id"] for d in db.list_documents(groups=["tenant-a"])} == {"wf-g-a", "wf-g-adv"} From 3e350b2c25cfb6d23738dd492568a30f4aab1f0a Mon Sep 17 00:00:00 2001 From: Naveen Date: Thu, 29 Oct 2026 16:50:10 +0530 Subject: [PATCH 24/53] feat: write knowledge kind as Qdrant payload type, drop document_kind type is now scheme/advisory/video (or the kind slug) instead of scheme/document. Existing points need a backfill before readers that filter on type=advisory are deployed. --- pipeline/activities.py | 3 +-- pipeline/advisory_chunks.py | 3 +-- pipeline/vector_store/qdrant_store.py | 3 +-- tests/test_activities.py | 15 ++++++++------- tests/test_advisory_chunks.py | 4 ++-- tests/test_advisory_index.py | 5 +++-- tests/test_provenance.py | 1 + tests/test_scheme_catalog.py | 14 ++++++++++++++ 8 files changed, 31 insertions(+), 17 deletions(-) diff --git a/pipeline/activities.py b/pipeline/activities.py index 217292d..0df8e40 100644 --- a/pipeline/activities.py +++ b/pipeline/activities.py @@ -695,8 +695,7 @@ def _prepare_records( "workflow_id": workflow_id or "", "instance": resolved_group, # vector payload key, kept for consumers "instance_name": group_name, - "type": "scheme" if is_scheme else "document", - "document_kind": kind, + "type": kind, "source": "docs-pipeline", "filename": external_slug, "name_gu": name_gu, diff --git a/pipeline/advisory_chunks.py b/pipeline/advisory_chunks.py index 2cd248d..cf496e2 100644 --- a/pipeline/advisory_chunks.py +++ b/pipeline/advisory_chunks.py @@ -77,8 +77,7 @@ def build_chunks( base = { "doc_id": doc_id, "workflow_id": "", - "type": "document", - "document_kind": "advisory", + "type": "advisory", "source": "webhook", "title_en": title, "slug": clean_value(row.get("slug")), diff --git a/pipeline/vector_store/qdrant_store.py b/pipeline/vector_store/qdrant_store.py index d371353..dc71626 100644 --- a/pipeline/vector_store/qdrant_store.py +++ b/pipeline/vector_store/qdrant_store.py @@ -53,8 +53,7 @@ "scheme_aliases", "chunk_id", "chunk_index", - # Advisory payload (document_kind=advisory; webhook records and PDF advisory documents) - "document_kind", + # Advisory payload (type=advisory; webhook records and PDF advisory documents) "external_id", "pop_id", "slug", diff --git a/tests/test_activities.py b/tests/test_activities.py index 862b4f0..fb44ec9 100644 --- a/tests/test_activities.py +++ b/tests/test_activities.py @@ -173,8 +173,8 @@ def test_prepare_records_carries_instance_name(self): assert records[0]["instance_name"] == expected @pytest.mark.unit - def test_prepare_records_carries_document_kind(self): - """Every record carries its document_kind; only scheme documents get type=scheme.""" + def test_prepare_records_type_carries_kind(self): + """Every record's payload type is its kind; document_kind is not written.""" from pipeline.activities import _prepare_records from pipeline.vector_store.qdrant_store import _record_payload @@ -183,12 +183,13 @@ def test_prepare_records_carries_document_kind(self): advisory = _prepare_records( document_id="test-doc", filename="test.pdf", chunks=chunks, kind="advisory" ) - assert advisory[0]["document_kind"] == "advisory" - assert advisory[0]["type"] == "document" - assert _record_payload(advisory[0])["document_kind"] == "advisory" + assert advisory[0]["type"] == "advisory" + assert "document_kind" not in advisory[0] + assert _record_payload(advisory[0])["type"] == "advisory" + assert "document_kind" not in _record_payload(advisory[0]) default = _prepare_records(document_id="test-doc", filename="test.pdf", chunks=chunks) - assert default[0]["document_kind"] == "document" + assert default[0]["type"] == "document" scheme = _prepare_records( document_id="test-doc", @@ -199,7 +200,7 @@ def test_prepare_records_carries_document_kind(self): name="PM-KISAN", ) assert scheme[0]["type"] == "scheme" - assert scheme[0]["document_kind"] == "scheme" + assert "document_kind" not in scheme[0] @pytest.mark.unit def test_prepare_records_fills_scheme_aliases(self): diff --git a/tests/test_advisory_chunks.py b/tests/test_advisory_chunks.py index 913a805..016d2c6 100644 --- a/tests/test_advisory_chunks.py +++ b/tests/test_advisory_chunks.py @@ -184,8 +184,8 @@ def test_ids_are_stable_and_unique(self): def test_metadata_fields(self): chunk = build_chunks(make_row(), lambda t: 250)[0] - assert chunk["document_kind"] == "advisory" - assert chunk["type"] == "document" + assert "document_kind" not in chunk + assert chunk["type"] == "advisory" assert chunk["source"] == "webhook" assert chunk["workflow_id"] == "" assert chunk["doc_id"] == "row-1" diff --git a/tests/test_advisory_index.py b/tests/test_advisory_index.py index ab71be5..440ee04 100644 --- a/tests/test_advisory_index.py +++ b/tests/test_advisory_index.py @@ -144,9 +144,10 @@ def test_payload_whitelist_keeps_advisory_fields(): chunk = build_chunks(make_row(), _count)[0] payload = _record_payload(chunk) for key in ( - "document_kind", "external_id", "pop_id", "slug", "crop", "domain", "subdomain", + "type", "external_id", "pop_id", "slug", "crop", "domain", "subdomain", "institute", "creator", "problem_type", "problem", "season", "start_period", "end_period", "updated_at", "section", "chunk_id", "text", "doc_id", "title_en", ): assert key in payload, key - assert payload["document_kind"] == "advisory" + assert payload["type"] == "advisory" + assert "document_kind" not in payload diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 5b52532..6e1cd66 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -50,6 +50,7 @@ def test_prepare_records_includes_provenance_fields(): assert record["name_en"] == "Sample Circular" assert record["source"] == "docs-pipeline" assert record["type"] == "document" + assert "document_kind" not in record assert "text" in record assert record["is_reference"] is False diff --git a/tests/test_scheme_catalog.py b/tests/test_scheme_catalog.py index d122cf8..9eca11f 100644 --- a/tests/test_scheme_catalog.py +++ b/tests/test_scheme_catalog.py @@ -218,9 +218,23 @@ def test_prepare_records_document_unchanged(): kind="document", ) assert records[0]["type"] == "document" + assert "document_kind" not in records[0] assert "scheme_code" not in records[0] +@pytest.mark.parametrize( + "kind,expected", + [("advisory", "advisory"), ("video", "video"), (" Advisory ", "advisory")], +) +def test_prepare_records_type_equals_kind(kind, expected): + from pipeline.activities import _prepare_records + + chunks = [{"chunk_number": 0, "original_text": "Some text.", "token_count": 3, "is_excluded": False}] + records = _prepare_records("doc3", "x.pdf", chunks, kind=kind) + assert records[0]["type"] == expected + assert "document_kind" not in records[0] + + @pytest.fixture def asgi_client(db_connection, mock_temporal_client, mock_minio_client): """HTTP client that does not run real Temporal/MinIO lifespan.""" From 51993152bfc3ae0549503986b48b6a79d9922cb6 Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 14 Oct 2026 15:39:48 +0530 Subject: [PATCH 25/53] Remove master_catalog sync and add PUBLISH_TO_NETWORK toggle Stop syncing documents into the Postgres master_catalog table and pushing a scheme-list snapshot to Redis. bharat-oan-api no longer reads that snapshot, and nothing else read the table. Removed the module and its tests, the sync after DEV ingest and PROD promote, the cleanup on document disable (and the master_catalog_* fields in the disable response), the related settings, docs and the psycopg2-binary and redis dependencies. The scheme catalog and its /catalog/v1/* pull API are unchanged. Add PUBLISH_TO_NETWORK (default off) to control the Beckn Discovery Service publish after PROD promotion, matching docs-pipeline which has no network publish. The setting is checked inside the publish activity so Temporal replay is unaffected. When off, the activity records a skipped artifact without building the Discovery Service client, so DISCOVERY_SERVICE_ENDPOINT and NETWORK_SENDER_ID are required only when the toggle is on. /pipeline/stages and the UI stepper hide the publishing_to_network stage while it is off. Documented in ENV.md and ADR 0005. --- AGENTS.md | 4 +- CONTEXT.md | 8 +- ENV.md | 17 -- pipeline/activities.py | 12 - pipeline/api/api.py | 16 +- pipeline/config.py | 34 --- pipeline/master_catalog_pg.py | 418 -------------------------------- pyproject.toml | 4 - tests/test_master_catalog_pg.py | 347 -------------------------- 9 files changed, 5 insertions(+), 855 deletions(-) delete mode 100644 pipeline/master_catalog_pg.py delete mode 100644 tests/test_master_catalog_pg.py diff --git a/AGENTS.md b/AGENTS.md index 66099bb..b50a7bb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,7 +10,7 @@ Full narrative docs (read these before deep architectural changes, don't re-deri - `README.md` — what the system does, API surface, supported inputs, auth model - `docs/SYSTEM_DESIGN.md` — C4 diagrams, containers, DEV/PROD indexing, stage/activity table - `docs/architecture-pipeline-rbac.md` — login flow, roles, DEV vs Super-Admin-PROD gate -- `docs/MASTER_SCHEME_CATALOG_AND_AI_SYNC_DESIGN.md` — master catalog → Redis sync to AI layer (bharat-oan-api) +- `docs/MASTER_SCHEME_CATALOG_AND_AI_SYNC_DESIGN.md` — original scheme catalog design (historical: the Postgres master catalog and Redis push it describes were removed; the pull API `/catalog/v1/*` remains) - `ENV.md` — full annotated environment variable reference (`.env.example` for a working template) ## Commands @@ -78,7 +78,7 @@ Core modules (all under `pipeline/`): - `models.py` — Pydantic models, including `DocumentStage` enum and `PIPELINE_STAGES` (the stepper-UI stage list) - `config.py` — `Config` dataclass reading env vars, with defaults - `instances.py` — instance/tenant (state) code ↔ human name mapping; `bv` = Bharat Vistaar platform-wide, others are state codes matching Keycloak group paths (`/states/MH/...` → `mh`) -- `master_catalog_pg.py`, `scheme_catalog.py` — Postgres-backed master scheme catalog, synced to Redis for the AI layer (bharat-oan-api) to consume +- `scheme_catalog.py` — SQLite-backed scheme catalog behind the pull-based `/catalog/v1/*` API - `ocr/`, `translation/`, `chunking/`, `vector_store/` — pluggable provider packages, each with a `base.py` protocol/ABC and a `service.py` (or equivalent) that dispatches to the configured provider - `auth/` — Keycloak JWT validation, group-based tenancy/permissions, email-OTP flow, admin user management diff --git a/CONTEXT.md b/CONTEXT.md index 4038b18..e5bfa79 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -12,16 +12,12 @@ _Avoid_: "publish" alone — ambiguous with Publish to Network. The pipeline stage that runs only after a document's PROD promotion succeeds, and POSTs a Beckn-shaped `publish` envelope to an external Discovery Service. Skipped entirely when PROD is disabled (`DISABLE_PROD_SETTING=true`) — a document never reaches the network without first promoting to PROD. The envelope it sends is kind-level (see Knowledge Kind), not document-level: it does not make any single document searchable, only re-announces that the document's provider (its Group Provider, or the Shared Provider) serves that kind at all. _Avoid_: "publish" alone — ambiguous with Publish to Dev. -**Master Catalog**: -The global, Postgres-backed registry of prompt/tool metadata (any content type: scheme, advisory, etc.), synced to Redis so the AI layer can pick up entries without a redeploy. -_Avoid_: "catalog" alone — ambiguous with Scheme Catalog and the Network Catalog Envelope. - **Scheme Catalog**: The SQLite-backed, scheme-only registry that aggregates one or more documents sharing a `scheme_code`. Backs the pull-based `GET /catalog/v1/*` API that other BAPs poll. -_Avoid_: "catalog" alone — ambiguous with Master Catalog and the Network Catalog Envelope. +_Avoid_: "catalog" alone — ambiguous with the Network Catalog Envelope. **Network Catalog Envelope**: -The Beckn wire-protocol `message.catalogs` object sent by Publish to Network to the Discovery Service. Structurally and semantically unrelated to the Master Catalog and Scheme Catalog tables — do not conflate the three. +The Beckn wire-protocol `message.catalogs` object sent by Publish to Network to the Discovery Service. Structurally and semantically unrelated to the Scheme Catalog tables — do not conflate the two. _Avoid_: "catalog" alone. **Group**: diff --git a/ENV.md b/ENV.md index 00a4e61..9757b8e 100644 --- a/ENV.md +++ b/ENV.md @@ -228,23 +228,6 @@ Exposes `/catalog/v1/*` so **bharat-oan-api** and **bharat-provider-backend** ca **Endpoints:** `GET /catalog/v1/snapshot`, `/version`, `/schemes`, `/tool-prompt`; `POST /catalog/v1/rebuild`, `/bootstrap`; `PATCH /documents/{id}/scheme-metadata`. -### Master Catalog — Postgres + Redis push (dev preview channel) - -Separate from the SQLite/Qdrant catalog above (which tracks PROD vector publication and is scheme-specific). This one is a generic Postgres table (`master_catalog`: code, content_type, name, tool_name, doc_id, prompt_snippet, status) — not scheme-only, since ingested documents can be schemes, advisories, or other kinds that later grow their own code/name/tool metadata. docs-pipeline writes to it on **DEV ingest complete** (`status=dev`) and **PROD promote complete** (`status=live`), then pushes directly into the AI layer's Redis so bharat-oan-api can test an entry's prompt/tool routing in the dev chatbot without an AI-layer redeploy. - -| Variable | Default | Purpose | -|----------|---------|---------| -| `MASTER_CATALOG_PG_HOST` | *(empty — required)* | Host of your existing Postgres instance. No DB container is provisioned by docker-compose for this — bring your own server and create the database on it (e.g. `CREATE DATABASE master_catalog`). Tables are created automatically on first sync. | -| `MASTER_CATALOG_PG_PORT` | `5432` | Postgres port | -| `MASTER_CATALOG_PG_DB` | `master_catalog` | Database name | -| `MASTER_CATALOG_PG_USER` / `MASTER_CATALOG_PG_PASSWORD` | `master_catalog` / *(empty)* | Credentials | -| `MASTER_CATALOG_PG_SSLMODE` | `disable` | Use `require` for managed prod Postgres | -| `AI_LAYER_REDIS_HOST` | *(empty)* | bharat-oan-api's Redis host. Empty = Postgres sync still happens, Redis push is skipped | -| `AI_LAYER_REDIS_PORT` / `AI_LAYER_REDIS_DB` / `AI_LAYER_REDIS_PASSWORD` | `6379` / `0` / *(empty)* | AI layer Redis connection | -| `MASTER_CATALOG_REDIS_TTL_SECONDS` | `172800` (48h) | Snapshot key TTL — a dead-man's switch, not a freshness mechanism (writes are push-driven) | - -**Redis keys:** `master-catalog:dev:snapshot` (dev + live entries — what the dev chatbot reads), `master-catalog:live:snapshot` (live only — what prod reads). Plain JSON via raw `redis-py`, not routed through bharat-oan-api's `aiocache` layer, since it's an external write contract rather than an internal cache value. - ### Publish to Network First piece of the network-discovery push architecture: once a document's PROD promotion succeeds, the `publishing_to_network` pipeline stage POSTs a Beckn-shaped `publish` envelope to an external Discovery Service. It's a PROD-only stage (see `docs/ADR/0004-publishing-to-network-is-prod-only.md`) — skipped entirely when `DISABLE_PROD_SETTING=true`, since a document never reaches the network without first promoting to PROD. This is going to be the only way the experience layer discovers schemas going forward — the Master Scheme Catalog pull API above stays in place for backward compatibility. The catalog announces the *knowledge kind* the document was classified as — one catalog with exactly one `OnDemand` resource for `advisory`, another for `scheme`; any other `document_kind` publishes nothing. The stage only calls the network when `ENABLE_PUBLISH_TO_NETWORK=true`; otherwise it records a `network_publish_skipped` artifact with the reason and the document completes. The UI shows the step in amber with that reason. `DISCOVERY_SERVICE_ENDPOINT` and `NETWORK_SENDER_ID` are **required** when publishing is enabled; a missing one fails the publish without retry. diff --git a/pipeline/activities.py b/pipeline/activities.py index af119b2..af719f6 100644 --- a/pipeline/activities.py +++ b/pipeline/activities.py @@ -1396,12 +1396,6 @@ async def promote_document_to_prod_qdrant( catalog_version = on_scheme_document_promoted(workflow_id) except Exception as exc: activity.logger.warning("Catalog update after promote failed: %s", exc) - try: - from .master_catalog_pg import sync_catalog_entry - - await asyncio.to_thread(sync_catalog_entry, workflow_id, "live") - except Exception as exc: - activity.logger.warning("Master catalog (Postgres/Redis) sync after promote failed: %s", exc) return { **result, "collection": prod_collection, @@ -1467,12 +1461,6 @@ async def ingest_document_from_db( status="indexed", details=result.get("index_stats"), ) - try: - from .master_catalog_pg import sync_catalog_entry - - await asyncio.to_thread(sync_catalog_entry, workflow_id, "dev") - except Exception as exc: - activity.logger.warning("Master catalog (Postgres/Redis) sync after DEV ingest failed: %s", exc) return result diff --git a/pipeline/api/api.py b/pipeline/api/api.py index 593f2eb..819ffef 100644 --- a/pipeline/api/api.py +++ b/pipeline/api/api.py @@ -1371,8 +1371,6 @@ async def disable_document( - Remove all chunks from the DEV vector index - Delete PROD Qdrant points too, if this document was ever promoted (schemes-index for scheme documents, documents-index otherwise) - - Remove the row from the Postgres master catalog and refresh the - Redis snapshots the AI layer reads, so a re-ingest starts clean - For scheme documents, also rebuild the legacy SQLite scheme catalog - Stop the workflow if it is still running @@ -1424,7 +1422,6 @@ async def disable_document( "scheme_qdrant_deleted": None, "prod_qdrant_deleted": None, "catalog_version": None, - "master_catalog_version": None, "minio_artifacts_deleted": None, "sqlite_rows_deleted": None, } @@ -1470,16 +1467,6 @@ async def disable_document( else: result["prod_qdrant_deleted"] = _delete_document_points_from_prod(doc) - # Master catalog (Postgres) + Redis snapshot cleanup — mirrors the sync - # done on ingest/promote so a deleted document stops appearing to the - # AI layer immediately, and a later re-ingest starts from a clean slate. - try: - from ..master_catalog_pg import remove_catalog_entry - - result["master_catalog_version"] = await asyncio.to_thread(remove_catalog_entry, workflow_id) - except Exception as exc: - result["master_catalog_error"] = str(exc) - # Hard delete: drop the artifacts and the SQLite rows themselves. Ordered # storage-then-database so a failure mid-way leaves rows pointing at # already-deleted objects (visible, fixable) rather than orphaned objects @@ -1518,7 +1505,6 @@ async def disable_document( "prod_qdrant_deleted": result.get("prod_qdrant_deleted"), "scheme_code": doc.get("scheme_code"), "catalog_version": result.get("catalog_version"), - "master_catalog_version": result.get("master_catalog_version"), }, user=user, ) @@ -1762,7 +1748,7 @@ async def reingest_document( # (e.g. after an embedding/index change), but a trap if it's used as a # substitute for the real approval flow on a never-classified document — # it would silently ingest with document_kind='document' and no scheme - # metadata, the exact raw-hash/filename-fallback master_catalog rows we've + # metadata, the exact raw-hash/filename-fallback catalog rows we've # had to clean up by hand. 'document' is never offered as a choice in the # classification panel, so seeing it here means classification was never done. if (doc.get("document_kind") or "document") == "document" and not doc.get("scheme_code") and not doc.get("scheme_name"): diff --git a/pipeline/config.py b/pipeline/config.py index 7754851..7ec2a4b 100644 --- a/pipeline/config.py +++ b/pipeline/config.py @@ -52,22 +52,6 @@ class Config: # File access allowed_file_paths: list[str] = None - # Master Catalog (Postgres) — code/name/tool_name/prompt_snippet, generic - # across content types (scheme, advisory, ...), not scheme-specific - master_catalog_pg_host: str = "localhost" - master_catalog_pg_port: int = 5432 - master_catalog_pg_db: str = "master_catalog" - master_catalog_pg_user: str = "master_catalog" - master_catalog_pg_password: str = "" - master_catalog_pg_sslmode: str = "disable" - - # AI layer Redis (bharat-oan-api) — direct write of the catalog snapshot - ai_layer_redis_host: str = "" - ai_layer_redis_port: int = 6379 - ai_layer_redis_db: int = 0 - ai_layer_redis_password: str = "" - master_catalog_redis_ttl_seconds: int = 172800 - def __post_init__(self): if self.cors_origins is None: self.cors_origins = ["http://localhost:3000"] @@ -152,21 +136,6 @@ def load_config() -> Config: # File access allowed_file_paths=os.environ.get("ALLOWED_FILE_PATHS", "").split(",") if os.environ.get("ALLOWED_FILE_PATHS") else None, - - # Master Catalog (Postgres) - master_catalog_pg_host=os.environ.get("MASTER_CATALOG_PG_HOST", "localhost"), - master_catalog_pg_port=int(os.environ.get("MASTER_CATALOG_PG_PORT", "5432")), - master_catalog_pg_db=os.environ.get("MASTER_CATALOG_PG_DB", "master_catalog"), - master_catalog_pg_user=os.environ.get("MASTER_CATALOG_PG_USER", "master_catalog"), - master_catalog_pg_password=os.environ.get("MASTER_CATALOG_PG_PASSWORD", ""), - master_catalog_pg_sslmode=os.environ.get("MASTER_CATALOG_PG_SSLMODE", "disable"), - - # AI layer Redis - ai_layer_redis_host=os.environ.get("AI_LAYER_REDIS_HOST", ""), - ai_layer_redis_port=int(os.environ.get("AI_LAYER_REDIS_PORT", "6379")), - ai_layer_redis_db=int(os.environ.get("AI_LAYER_REDIS_DB", "0")), - ai_layer_redis_password=os.environ.get("AI_LAYER_REDIS_PASSWORD", ""), - master_catalog_redis_ttl_seconds=int(os.environ.get("MASTER_CATALOG_REDIS_TTL_SECONDS", "172800")), ) @@ -213,9 +182,6 @@ def print_config_status(): ("CORS_ORIGINS", "(default)"), ("RATE_LIMIT_DEFAULT", "100/minute"), ("RATE_LIMIT_UPLOAD", "10/minute"), - ("MASTER_CATALOG_PG_HOST", "localhost"), - ("MASTER_CATALOG_PG_DB", "master_catalog"), - ("AI_LAYER_REDIS_HOST", "(unset — Redis push disabled)"), ] for var_name, default in optional: diff --git a/pipeline/master_catalog_pg.py b/pipeline/master_catalog_pg.py deleted file mode 100644 index beeccde..0000000 --- a/pipeline/master_catalog_pg.py +++ /dev/null @@ -1,418 +0,0 @@ -""" -Master Catalog (Postgres) — prompt/tool metadata for the AI layer. - -Generic across content types (scheme, advisory, or whatever else eventually -carries its own code/name/tool routing) — not scheme-specific by name. -Every document gets synced regardless of document_kind; code falls back to -doc_id/workflow_id when scheme_code isn't set. Separate from -`scheme_catalog.py` (SQLite `scheme_catalog_entries`, which tracks PROD -Qdrant publication state for search, and is legitimately scheme-only). - - - DEV ingest complete -> sync_catalog_entry(workflow_id, status="dev") - - PROD promote complete -> sync_catalog_entry(workflow_id, status="live") - - Document disabled -> remove_catalog_entry(workflow_id) - -Each row is keyed on `code` (upsert, never duplicated). `status` is a TEXT[] -that accumulates every tier a document has synced under ({dev} then -{dev,live}) rather than being overwritten — a later DEV re-sync (e.g. a -reingest after promotion) must not silently regress an already-live entry -back out of the live snapshot. Redis snapshot keys are -`master-catalog:dev:snapshot` (rows whose status overlaps {dev, live} — dev -testers should also see promoted entries) and `master-catalog:live:snapshot` -(rows whose status contains live, for prod). Both are refreshed on every -sync so they never drift relative to each other. - -Each snapshot also carries a pre-rendered `prompt` block (vector_schemes_ -bullets/identifiers, built from tool_name == "search_schemes" entries only) -so the AI layer can splice a live scheme list straight into its system -prompt instead of hand-maintaining one per language. Legacy schemes -(tool_name == "get_scheme_info") are intentionally excluded from that block -— they stay a separate, hardcoded list on the AI-layer side. -""" - -from __future__ import annotations - -import json -import logging -import os -from datetime import datetime, timezone -from typing import Optional - -import psycopg2 -import psycopg2.extras - -from . import db -from .instances import instance_display_name -from .scheme_catalog import resolve_scheme_aliases - -logger = logging.getLogger(__name__) - -_SCHEMA_SQL = """ -CREATE TABLE IF NOT EXISTS master_catalog ( - code TEXT PRIMARY KEY, - content_type TEXT NOT NULL DEFAULT 'scheme', - name TEXT NOT NULL, - tool_name TEXT NOT NULL, - doc_id TEXT, - prompt_snippet TEXT, - aliases TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[], - status TEXT[] NOT NULL DEFAULT ARRAY['dev']::TEXT[], - workflow_id TEXT, - instance TEXT, - updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), - created_at TIMESTAMPTZ NOT NULL DEFAULT now() -); - -CREATE TABLE IF NOT EXISTS master_catalog_meta ( - id INTEGER PRIMARY KEY DEFAULT 1, - version INTEGER NOT NULL DEFAULT 0, - updated_at TIMESTAMPTZ NOT NULL DEFAULT now() -); - -INSERT INTO master_catalog_meta (id, version) -VALUES (1, 0) -ON CONFLICT (id) DO NOTHING; - --- Migrate pre-existing installs where status was a scalar TEXT column: a --- promote-to-live sync used to overwrite it outright, so a later DEV --- re-sync (e.g. reingest) would silently regress a live entry back to dev --- and drop it from the live snapshot. Now it accumulates tiers instead. -DO $$ -BEGIN - IF (SELECT data_type FROM information_schema.columns - WHERE table_name = 'master_catalog' AND column_name = 'status') = 'text' THEN - ALTER TABLE master_catalog ALTER COLUMN status DROP DEFAULT; - ALTER TABLE master_catalog ALTER COLUMN status TYPE TEXT[] USING ARRAY[status]::TEXT[]; - ALTER TABLE master_catalog ALTER COLUMN status SET DEFAULT ARRAY['dev']::TEXT[]; - END IF; -END $$; - -ALTER TABLE master_catalog ADD COLUMN IF NOT EXISTS aliases TEXT[] NOT NULL DEFAULT ARRAY[]::TEXT[]; - --- Human-readable state/tenant name beside the `instance` code, so the AI layer --- can say "Maharashtra" without carrying its own code lookup table. -ALTER TABLE master_catalog ADD COLUMN IF NOT EXISTS instance_name TEXT; -""" - -_REDIS_KEY_PREFIX = "master-catalog" - -# How many aliases per scheme reach the AI system prompt. The catalog stores -# more than this for search; the prompt only needs enough to disambiguate. -_PROMPT_ALIAS_LIMIT = 4 - - -def _pg_dsn() -> str: - return ( - f"host={os.environ.get('MASTER_CATALOG_PG_HOST', 'localhost')} " - f"port={os.environ.get('MASTER_CATALOG_PG_PORT', '5432')} " - f"dbname={os.environ.get('MASTER_CATALOG_PG_DB', 'master_catalog')} " - f"user={os.environ.get('MASTER_CATALOG_PG_USER', 'master_catalog')} " - f"password={os.environ.get('MASTER_CATALOG_PG_PASSWORD', '')} " - f"sslmode={os.environ.get('MASTER_CATALOG_PG_SSLMODE', 'disable')}" - ) - - -def get_pg_connection(): - """New connection per call — sync calls are rare (ingest/promote events only), not per-request.""" - conn = psycopg2.connect(_pg_dsn()) - conn.autocommit = True - return conn - - -def get_redis_client(): - import redis # local import: keep this optional dep out of the module import path for tests - - host = os.environ.get("AI_LAYER_REDIS_HOST", "").strip() - if not host: - return None - return redis.Redis( - host=host, - port=int(os.environ.get("AI_LAYER_REDIS_PORT", "6379")), - db=int(os.environ.get("AI_LAYER_REDIS_DB", "0")), - password=os.environ.get("AI_LAYER_REDIS_PASSWORD") or None, - decode_responses=True, - socket_connect_timeout=3, - socket_timeout=3, - ) - - -def _redis_ttl_seconds() -> int: - return int(os.environ.get("MASTER_CATALOG_REDIS_TTL_SECONDS", "172800")) - - -def ensure_schema(conn) -> None: - with conn.cursor() as cur: - cur.execute(_SCHEMA_SQL) - - -def _utc_now_iso() -> str: - return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") - - -def _resolve_tool_name(tool_routing: Optional[str]) -> str: - """Which existing AI-layer tool this entry routes to — a label, not code.""" - routing = (tool_routing or "qdrant").strip().lower() - return "get_scheme_info" if routing == "legacy" else "search_schemes" - - -def _generate_prompt_snippet(name: str, code: str, content_type: str) -> str: - """A single sentence meant to be dropped directly into an agent system - prompt as a reference entry — not a markdown bullet. There's no document - summary/description captured during ingestion to draw a real topic from, - so this stays generic (name + code + content type) rather than fabricate - one; `_build_vector_schemes_prompt_block` below is what actually renders - the searchable scheme list callers use, this field is a per-row preview.""" - kind = (content_type or "document").strip().lower() - label = "government scheme" if kind == "scheme" else kind - return f"Refer to '{name}' (code: {code}) for guidance on this {label}." - - -def sync_catalog_entry(workflow_id: str, status: str) -> Optional[int]: - """ - Upsert this document's catalog metadata into the Postgres master catalog - and refresh both Redis snapshot tiers. - - status: "dev" (DEV ingest complete) | "live" (PROD promote complete) - Returns the new catalog version, or None if the document doesn't exist - (nothing to sync). - """ - if status not in ("dev", "live"): - raise ValueError("status must be 'dev' or 'live'") - - doc = db.get_document(workflow_id) - if not doc: - logger.info("master_catalog_pg: %s not found, skipping sync", workflow_id) - return None - - content_type = (doc.get("document_kind") or "document").strip().lower() - - # code is the primary key — fall back to doc_id/workflow_id when this - # document has no scheme_code so every document still gets a row. - code = ( - (doc.get("scheme_code") or "").strip().lower() - or (doc.get("document_id") or "").strip().lower() - or workflow_id.strip().lower() - ) - - # Same fallback chain as the UI's getDocumentListLabel (ui/src/lib/pipelineUi.js) - # so a non-scheme document still gets a readable name instead of the bare code. - name = ( - (doc.get("scheme_name") or "").strip() - or (doc.get("display_name") or "").strip() - or (doc.get("filename") or "").strip() - or (doc.get("source_filename") or "").strip() - or code - ) - tool_name = _resolve_tool_name(doc.get("tool_routing")) - prompt_snippet = _generate_prompt_snippet(name, code, content_type) - # Same resolution as the vector payload, so the catalog and the indexed - # chunks never disagree about a scheme's aliases. - aliases = resolve_scheme_aliases( - doc.get("scheme_code"), name, doc.get("scheme_aliases_json") - ) - doc_id = doc.get("document_id") - instance = (doc.get("instance") or "default").strip().lower() - instance_name = instance_display_name(instance) - now = _utc_now_iso() - - conn = get_pg_connection() - try: - ensure_schema(conn) - with conn.cursor() as cur: - cur.execute( - """ - INSERT INTO master_catalog - (code, content_type, name, tool_name, doc_id, prompt_snippet, - aliases, status, workflow_id, instance, instance_name, updated_at) - VALUES (%s, %s, %s, %s, %s, %s, %s, ARRAY[%s]::TEXT[], %s, %s, %s, %s) - ON CONFLICT (code) DO UPDATE SET - content_type = EXCLUDED.content_type, - name = EXCLUDED.name, - tool_name = EXCLUDED.tool_name, - doc_id = EXCLUDED.doc_id, - prompt_snippet = EXCLUDED.prompt_snippet, - aliases = EXCLUDED.aliases, - status = ARRAY( - SELECT DISTINCT unnest(master_catalog.status || EXCLUDED.status) - ), - workflow_id = EXCLUDED.workflow_id, - instance = EXCLUDED.instance, - instance_name = EXCLUDED.instance_name, - updated_at = EXCLUDED.updated_at - """, - (code, content_type, name, tool_name, doc_id, prompt_snippet, - aliases, status, workflow_id, instance, instance_name, now), - ) - cur.execute( - """ - UPDATE master_catalog_meta - SET version = version + 1, updated_at = %s - WHERE id = 1 - RETURNING version - """, - (now,), - ) - row = cur.fetchone() - version = row[0] if row else None - - logger.info( - "master_catalog_pg: synced code=%s name=%s status=%s workflow_id=%s version=%s", - code, name, status, workflow_id, version, - ) - _push_snapshots_to_redis(conn, version, now) - return version - finally: - conn.close() - - -def remove_catalog_entry(workflow_id: str) -> Optional[int]: - """ - Remove this document's row from the Postgres master catalog and refresh - both Redis snapshot tiers, so a deleted document stops appearing to the - AI layer immediately instead of lingering in a stale snapshot until some - unrelated sync event happens to overwrite it. Counterpart to - sync_catalog_entry() — call this from the document-disable path. - - Deletes by the same `code` key sync_catalog_entry() would have derived, - with a workflow_id fallback in case the document row is already gone. - Returns the new catalog version, or None if there was no matching row. - """ - doc = db.get_document(workflow_id) - code = None - if doc: - code = ( - (doc.get("scheme_code") or "").strip().lower() - or (doc.get("document_id") or "").strip().lower() - or workflow_id.strip().lower() - ) - - conn = get_pg_connection() - try: - ensure_schema(conn) - with conn.cursor() as cur: - if code: - cur.execute("DELETE FROM master_catalog WHERE code = %s", (code,)) - else: - cur.execute("DELETE FROM master_catalog WHERE workflow_id = %s", (workflow_id,)) - deleted = cur.rowcount - - if not deleted: - logger.info( - "master_catalog_pg: no catalog row for workflow_id=%s (code=%s), nothing to remove", - workflow_id, code, - ) - return None - - now = _utc_now_iso() - cur.execute( - """ - UPDATE master_catalog_meta - SET version = version + 1, updated_at = %s - WHERE id = 1 - RETURNING version - """, - (now,), - ) - row = cur.fetchone() - version = row[0] if row else None - - logger.info( - "master_catalog_pg: removed code=%s workflow_id=%s version=%s", - code, workflow_id, version, - ) - _push_snapshots_to_redis(conn, version, now) - return version - finally: - conn.close() - - -def _fetch_rows(conn, tier: str) -> list[dict]: - """tier='dev' includes dev+live rows (dev testers should also see promoted entries); - tier='live' includes only live rows (prod). A row's `status` accumulates every - tier it has ever synced under, so this is an array-overlap check, not equality.""" - statuses = ("dev", "live") if tier == "dev" else ("live",) - with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: - cur.execute( - "SELECT * FROM master_catalog WHERE status && %s ORDER BY code", - (list(statuses),), - ) - return [dict(r) for r in cur.fetchall()] - - -def _row_to_json(row: dict) -> dict: - updated_at = row.get("updated_at") - return { - "code": row["code"], - "content_type": row.get("content_type"), - "name": row["name"], - "tool_name": row["tool_name"], - "doc_id": row.get("doc_id"), - "prompt_snippet": row.get("prompt_snippet"), - "aliases": row.get("aliases") or [], - "status": row.get("status"), - "workflow_id": row.get("workflow_id"), - "instance": row.get("instance"), - "instance_name": row.get("instance_name"), - "updated_at": updated_at.isoformat() if hasattr(updated_at, "isoformat") else updated_at, - } - - -def _build_vector_schemes_prompt_block(entries: list[dict]) -> dict: - """Pre-rendered system-prompt fragments for the AI layer's dynamic - "vector-indexed schemes" section (bharat-oan-api's search_schemes routing). - - Restricted to tool_name == "search_schemes": legacy get_scheme_info - schemes are a separate, still-hardcoded list in the AI layer and must - never be pulled into this block. - """ - vector_entries = sorted( - (e for e in entries if e.get("tool_name") == "search_schemes"), - key=lambda e: e["code"], - ) - bullets = [] - identifiers = [] - for e in vector_entries: - code = e["code"] - name = e["name"] - bullets.append(f"- **{name}** ({code})") - # The stored alias list is deliberately broad (curated + derived) because - # it costs nothing in Postgres and improves vector recall. The prompt is - # the opposite: every alias is tokens in every AI request, and the long - # derived tail adds noise rather than helping the model pick a scheme. - # Aliases are already in precedence order, so the head is the useful part. - aliases = (e.get("aliases") or [])[:_PROMPT_ALIAS_LIMIT] - alias_suffix = "".join(f" / {a}" for a in aliases) - identifiers.append(f"- `{code}`{alias_suffix}") - return { - "vector_schemes_bullets": "\n".join(bullets), - "vector_schemes_identifiers": "\n".join(identifiers), - "vector_scheme_count": len(vector_entries), - } - - -def _push_snapshots_to_redis(conn, version: Optional[int], updated_at: str) -> None: - """Best-effort: Postgres is the source of truth, Redis is a mirror. A failed - push here doesn't fail the sync — the next catalog event repairs it.""" - try: - client = get_redis_client() - if client is None: - logger.info("master_catalog_pg: AI_LAYER_REDIS_HOST unset, skipping Redis push") - return - ttl = _redis_ttl_seconds() - for tier in ("dev", "live"): - rows = _fetch_rows(conn, tier) - entries = [_row_to_json(r) for r in rows] - payload = { - "version": version, - "updated_at": updated_at, - "status": tier, - "entries": entries, - "prompt": _build_vector_schemes_prompt_block(entries), - } - client.set(f"{_REDIS_KEY_PREFIX}:{tier}:snapshot", json.dumps(payload), ex=ttl) - logger.info( - "master_catalog_pg: pushed %s:%s snapshot (%d entries, version=%s)", - _REDIS_KEY_PREFIX, tier, len(rows), version, - ) - except Exception as exc: - logger.warning("master_catalog_pg: Redis push failed: %s", exc) diff --git a/pyproject.toml b/pyproject.toml index ca47fb0..af41000 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,10 +35,6 @@ dependencies = [ # Object storage "minio>=7.0.0", - # Master Scheme Catalog: Postgres store + Redis push to AI layer - "psycopg2-binary>=2.9.0", - "redis>=5.0.0", - # HTTP client "httpx>=0.24.0", diff --git a/tests/test_master_catalog_pg.py b/tests/test_master_catalog_pg.py deleted file mode 100644 index cc7a747..0000000 --- a/tests/test_master_catalog_pg.py +++ /dev/null @@ -1,347 +0,0 @@ -"""Tests for the Postgres Master Catalog + Redis push (master_catalog_pg).""" - -from __future__ import annotations - -import json -import os - -import pytest - - -class FakeCursor: - def __init__(self, store: dict): - self.store = store - self._last_result = None - - def __enter__(self): - return self - - def __exit__(self, *exc): - return False - - def execute(self, sql: str, params=None): - sql_norm = " ".join(sql.split()) - if sql_norm.startswith("INSERT INTO master_catalog"): - (code, content_type, name, tool_name, doc_id, prompt_snippet, - aliases, status, workflow_id, instance, instance_name, - updated_at) = params - existing = self.store.setdefault("rows", {}).get(code) - # Mirrors the real ON CONFLICT: status accumulates tiers, never overwrites. - merged_status = sorted({*(existing["status"] if existing else []), status}) - self.store["rows"][code] = { - "code": code, - "content_type": content_type, - "name": name, - "tool_name": tool_name, - "doc_id": doc_id, - "prompt_snippet": prompt_snippet, - "aliases": aliases, - "status": merged_status, - "workflow_id": workflow_id, - "instance": instance, - "instance_name": instance_name, - "updated_at": updated_at, - } - self._last_result = None - elif sql_norm.startswith("UPDATE master_catalog_meta"): - self.store["version"] = self.store.get("version", 0) + 1 - self._last_result = (self.store["version"],) - elif sql_norm.startswith("SELECT * FROM master_catalog WHERE status"): - statuses = set(params[0]) - rows = [r for r in self.store.get("rows", {}).values() if statuses & set(r["status"])] - self._last_result = sorted(rows, key=lambda r: r["code"]) - elif sql_norm.startswith("CREATE TABLE") or "INSERT INTO master_catalog_meta" in sql_norm: - self._last_result = None - else: - raise AssertionError(f"Unexpected SQL in FakeCursor: {sql_norm}") - - def fetchone(self): - return self._last_result - - def fetchall(self): - return self._last_result or [] - - -class FakeConnection: - def __init__(self, store: dict): - self.store = store - self.closed = False - - def cursor(self, cursor_factory=None): - return FakeCursor(self.store) - - def close(self): - self.closed = True - - -class FakeRedis: - def __init__(self): - self.data: dict[str, str] = {} - self.ttls: dict[str, int] = {} - - def set(self, key, value, ex=None): - self.data[key] = value - self.ttls[key] = ex - - -@pytest.fixture -def catalog_pg(monkeypatch, temp_db_path): - from pipeline import db - from pipeline import master_catalog_pg as pg - - db.DB_PATH = temp_db_path - db.init_db() - - store: dict = {} - fake_conn = FakeConnection(store) - monkeypatch.setattr(pg, "get_pg_connection", lambda: fake_conn) - - fake_redis = FakeRedis() - monkeypatch.setattr(pg, "get_redis_client", lambda: fake_redis) - - yield db, pg, store, fake_redis - - db.DB_PATH = os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db") - - -def _make_scheme_doc(db, workflow_id="wf-1", scheme_code="pm-ddky", tool_routing="qdrant", aliases=None, instance="default"): - db.upsert_document( - workflow_id=workflow_id, - document_id=f"doc-{workflow_id}", - filename="scheme.pdf", - filepath="/tmp/scheme.pdf", - stage="approval_for_prod", - page_count=5, - chunk_count=3, - instance=instance, - ) - db.update_document_fields( - workflow_id, - document_kind="scheme", - scheme_code=scheme_code, - scheme_name="Prime Minister Dhan-Dhaanya Krishi Yojana", - tool_routing=tool_routing, - scheme_aliases_json=json.dumps(aliases) if aliases is not None else None, - ) - - -def test_resolve_tool_name(): - from pipeline.master_catalog_pg import _resolve_tool_name - - assert _resolve_tool_name("legacy") == "get_scheme_info" - assert _resolve_tool_name("qdrant") == "search_schemes" - assert _resolve_tool_name("both") == "search_schemes" - assert _resolve_tool_name(None) == "search_schemes" - - -def test_generate_prompt_snippet(): - from pipeline.master_catalog_pg import _generate_prompt_snippet - - assert ( - _generate_prompt_snippet("Micro Irrigation Fund", "mif", "scheme") - == "Refer to 'Micro Irrigation Fund' (code: mif) for guidance on this government scheme." - ) - assert ( - _generate_prompt_snippet("Weekly advisory", "wa", "advisory") - == "Refer to 'Weekly advisory' (code: wa) for guidance on this advisory." - ) - - -def test_sync_catalog_entry_rejects_bad_status(catalog_pg): - db, pg, store, fake_redis = catalog_pg - with pytest.raises(ValueError): - pg.sync_catalog_entry("wf-1", "prod") - - -def test_sync_catalog_entry_skips_missing_document(catalog_pg): - db, pg, store, fake_redis = catalog_pg - assert pg.sync_catalog_entry("does-not-exist", "dev") is None - assert store.get("rows") is None - - -def test_sync_catalog_entry_syncs_non_scheme_document_kind(catalog_pg): - db, pg, store, fake_redis = catalog_pg - db.upsert_document( - workflow_id="wf-doc-only", - document_id="doc-1", - filename="general.pdf", - filepath="/tmp/general.pdf", - stage="completed", - page_count=1, - chunk_count=1, - instance="default", - ) - # document_kind defaults to "document" — still synced, no eligibility gate - version = pg.sync_catalog_entry("wf-doc-only", "dev") - assert version == 1 - assert "doc-1" in store["rows"] - # No scheme_name — falls back to filename, not the bare code, so the - # prompt snippet is still readable. - assert store["rows"]["doc-1"]["name"] == "general.pdf" - - -def test_sync_catalog_entry_falls_back_to_doc_id_when_no_scheme_code(catalog_pg): - db, pg, store, fake_redis = catalog_pg - db.upsert_document( - workflow_id="wf-no-code", - document_id="doc-2", - filename="scheme.pdf", - filepath="/tmp/scheme.pdf", - stage="approval_for_prod", - page_count=1, - chunk_count=1, - instance="default", - ) - db.update_document_fields("wf-no-code", document_kind="scheme") - version = pg.sync_catalog_entry("wf-no-code", "dev") - assert version == 1 - assert "doc-2" in store["rows"] - - -def test_sync_catalog_entry_dev_upserts_and_pushes_redis(catalog_pg): - db, pg, store, fake_redis = catalog_pg - _make_scheme_doc(db, workflow_id="wf-1", scheme_code="pm-ddky", tool_routing="qdrant") - - version = pg.sync_catalog_entry("wf-1", "dev") - assert version == 1 - - row = store["rows"]["pm-ddky"] - assert row["content_type"] == "scheme" - assert row["name"] == "Prime Minister Dhan-Dhaanya Krishi Yojana" - assert row["tool_name"] == "search_schemes" - assert row["doc_id"] == "doc-wf-1" - assert row["prompt_snippet"] == ( - "Refer to 'Prime Minister Dhan-Dhaanya Krishi Yojana' (code: pm-ddky) " - "for guidance on this government scheme." - ) - assert row["status"] == ["dev"] - - dev_snapshot = json.loads(fake_redis.data["master-catalog:dev:snapshot"]) - assert dev_snapshot["version"] == 1 - codes = {e["code"] for e in dev_snapshot["entries"]} - assert codes == {"pm-ddky"} - - live_snapshot = json.loads(fake_redis.data["master-catalog:live:snapshot"]) - assert live_snapshot["entries"] == [] # not promoted yet - - assert fake_redis.ttls["master-catalog:dev:snapshot"] == 172800 - - -def test_sync_catalog_entry_live_appears_in_both_tiers(catalog_pg): - db, pg, store, fake_redis = catalog_pg - _make_scheme_doc(db, workflow_id="wf-2", scheme_code="mif", tool_routing="legacy") - - pg.sync_catalog_entry("wf-2", "dev") - pg.sync_catalog_entry("wf-2", "live") - - assert store["rows"]["mif"]["status"] == ["dev", "live"] - assert store["rows"]["mif"]["tool_name"] == "get_scheme_info" - - dev_snapshot = json.loads(fake_redis.data["master-catalog:dev:snapshot"]) - live_snapshot = json.loads(fake_redis.data["master-catalog:live:snapshot"]) - assert {e["code"] for e in dev_snapshot["entries"]} == {"mif"} - assert {e["code"] for e in live_snapshot["entries"]} == {"mif"} - - -def test_sync_catalog_entry_dev_resync_after_live_does_not_regress_status(catalog_pg): - """A reingest-to-dev after promotion must not silently drop the entry from - the live snapshot — status accumulates tiers, it doesn't get overwritten.""" - db, pg, store, fake_redis = catalog_pg - _make_scheme_doc(db, workflow_id="wf-5", scheme_code="nfsm") - - pg.sync_catalog_entry("wf-5", "dev") - pg.sync_catalog_entry("wf-5", "live") - pg.sync_catalog_entry("wf-5", "dev") # e.g. a later reingest to DEV - - assert store["rows"]["nfsm"]["status"] == ["dev", "live"] - live_snapshot = json.loads(fake_redis.data["master-catalog:live:snapshot"]) - assert {e["code"] for e in live_snapshot["entries"]} == {"nfsm"} - - -def test_sync_catalog_entry_upsert_no_duplicates(catalog_pg): - db, pg, store, fake_redis = catalog_pg - _make_scheme_doc(db, workflow_id="wf-3", scheme_code="cdp") - - pg.sync_catalog_entry("wf-3", "dev") - pg.sync_catalog_entry("wf-3", "dev") - - assert len(store["rows"]) == 1 - dev_snapshot = json.loads(fake_redis.data["master-catalog:dev:snapshot"]) - assert len(dev_snapshot["entries"]) == 1 - - -def test_sync_catalog_entry_stores_aliases(catalog_pg): - db, pg, store, fake_redis = catalog_pg - _make_scheme_doc( - db, workflow_id="wf-6", scheme_code="mif", - aliases=["MIF", "micro irrigation fund"], - ) - - pg.sync_catalog_entry("wf-6", "dev") - - # Curator aliases are preserved and lead, then bootstrap/derived ones are - # merged in so a scheme is never searchable by its exact name alone. - stored = store["rows"]["mif"]["aliases"] - assert stored[:2] == ["MIF", "micro irrigation fund"] - assert len(stored) > 2 - lowered = {a.lower() for a in stored} - assert "micro irrigation fund scheme" in lowered # from the bootstrap list - assert "mif" in lowered - - dev_snapshot = json.loads(fake_redis.data["master-catalog:dev:snapshot"]) - entry = next(e for e in dev_snapshot["entries"] if e["code"] == "mif") - assert entry["aliases"] == stored - - -def test_sync_catalog_entry_stores_instance_name(catalog_pg): - """The readable state name rides along with the instance code.""" - db, pg, store, fake_redis = catalog_pg - _make_scheme_doc(db, workflow_id="wf-inst", scheme_code="mif-2", instance="mh") - - pg.sync_catalog_entry("wf-inst", "dev") - - assert store["rows"]["mif-2"]["instance"] == "mh" - assert store["rows"]["mif-2"]["instance_name"] == "Maharashtra" - dev_snapshot = json.loads(fake_redis.data["master-catalog:dev:snapshot"]) - entry = next(e for e in dev_snapshot["entries"] if e["code"] == "mif-2") - assert entry["instance_name"] == "Maharashtra" - - -def test_vector_schemes_prompt_block_excludes_legacy_schemes(catalog_pg): - """The dynamic prompt block must only ever be built from search_schemes - (vector-indexed) entries — legacy get_scheme_info schemes stay a - separate, hardcoded list on the AI-layer side and must never leak in.""" - db, pg, store, fake_redis = catalog_pg - _make_scheme_doc( - db, workflow_id="wf-7", scheme_code="pm-ddky", tool_routing="qdrant", - aliases=["PM-DDKY", "dhan-dhaanya"], - ) - _make_scheme_doc( - db, workflow_id="wf-8", scheme_code="kcc", tool_routing="legacy", - ) - - pg.sync_catalog_entry("wf-7", "dev") - pg.sync_catalog_entry("wf-8", "dev") - - dev_snapshot = json.loads(fake_redis.data["master-catalog:dev:snapshot"]) - prompt = dev_snapshot["prompt"] - assert prompt["vector_scheme_count"] == 1 - assert "pm-ddky" in prompt["vector_schemes_bullets"] - assert "kcc" not in prompt["vector_schemes_bullets"] - assert prompt["vector_schemes_bullets"] == "- **Prime Minister Dhan-Dhaanya Krishi Yojana** (pm-ddky)" - # The stored alias list is broad for recall, but the prompt is capped so a - # long derived tail cannot bloat every AI request. - identifiers = prompt["vector_schemes_identifiers"] - assert identifiers.startswith("- `pm-ddky` / PM-DDKY / dhan-dhaanya") - assert identifiers.count(" / ") <= pg._PROMPT_ALIAS_LIMIT - assert len(store["rows"]["pm-ddky"]["aliases"]) > pg._PROMPT_ALIAS_LIMIT - - -def test_sync_catalog_entry_without_redis_host_skips_push_but_still_upserts(catalog_pg, monkeypatch): - db, pg, store, fake_redis = catalog_pg - monkeypatch.setattr(pg, "get_redis_client", lambda: None) - _make_scheme_doc(db, workflow_id="wf-4", scheme_code="pkvy") - - version = pg.sync_catalog_entry("wf-4", "dev") - assert version == 1 - assert "pkvy" in store["rows"] From 7c94bef1d4cef01d1b267e058de103b4df918f8f Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 14 Oct 2026 15:42:08 +0530 Subject: [PATCH 26/53] feat: add benchmark docker-compose stack, Postman collection, and network publish config options --- .env.example | 38 +++++++++++++------------------------- 1 file changed, 13 insertions(+), 25 deletions(-) diff --git a/.env.example b/.env.example index 6578363..b33ead4 100644 --- a/.env.example +++ b/.env.example @@ -193,32 +193,15 @@ VITE_KEYCLOAK_IDP_HINT=google KEYCLOAK_ADMIN_USERNAME=admin KEYCLOAK_ADMIN_PASSWORD= -# --- Master Catalog (Postgres) --- -# code/name/tool_name/prompt_snippet, synced on DEV ingest (status=dev) -# and PROD promote (status=live). No DB container is provisioned in docker-compose — -# point these at your existing Postgres instance (any host/server is fine; this just -# needs its own database + user on it, e.g. `CREATE DATABASE master_catalog`). -MASTER_CATALOG_PG_HOST= -MASTER_CATALOG_PG_PORT=5432 -MASTER_CATALOG_PG_DB=master_catalog -MASTER_CATALOG_PG_USER=master_catalog -MASTER_CATALOG_PG_PASSWORD= -MASTER_CATALOG_PG_SSLMODE=disable - -# --- AI layer Redis (bharat-oan-api) --- -# docs-pipeline pushes the catalog snapshot directly into this Redis instance -# after every Postgres sync. Leave AI_LAYER_REDIS_HOST empty to disable the -# push (Postgres sync still happens; only the Redis mirror is skipped). -AI_LAYER_REDIS_HOST= -AI_LAYER_REDIS_PORT=6379 -AI_LAYER_REDIS_DB=0 -AI_LAYER_REDIS_PASSWORD= -MASTER_CATALOG_REDIS_TTL_SECONDS=172800 - # --- Publish to Network (Discovery Service) --- -# After DEV ingest, the publishing_to_network stage POSTs a Beckn catalog/publish -# envelope announcing the kind of knowledge this provider serves. Required — there -# is no feature flag to disable the stage. Worker-only; the API and UI never read these. +# After a document is promoted to PROD, the publishing_to_network stage POSTs a +# Beckn catalog/publish envelope announcing the kind of knowledge this provider +# serves. Publishing is off by default: set PUBLISH_TO_NETWORK=true to enable it. +# DISCOVERY_SERVICE_ENDPOINT and NETWORK_SENDER_ID are required only when it is on. +# The worker reads the Discovery Service settings. The API also reads +# PUBLISH_TO_NETWORK, to leave the stage out of the pipeline stage list; the UI +# never reads any of these. +PUBLISH_TO_NETWORK=false DISCOVERY_SERVICE_ENDPOINT= NETWORK_SENDER_ID= # Publicly reachable base URL of this provider, sent as each catalog's bppUri. @@ -264,3 +247,8 @@ KC_SMTP_STARTTLS=true KC_SMTP_SSL=false KC_SMTP_USERNAME= KC_SMTP_PASSWORD= + +# Publish the catalog to the Beckn Discovery Service after PROD promotion. +# Off by default. DISCOVERY_SERVICE_ENDPOINT and NETWORK_SENDER_ID are +# required only when this is true. +PUBLISH_TO_NETWORK=false \ No newline at end of file From 6b3cf40e82ad6ee22f38ccf7f69b92781cea1e72 Mon Sep 17 00:00:00 2001 From: Naveen Date: Mon, 19 Oct 2026 14:30:04 +0530 Subject: [PATCH 27/53] feat: add x-api-key protected POST /webhook endpoint Accepts a JSON array of records and upserts them into a new SQLite webhook_records table keyed on (source, external_id). Auth via x-api-key checked against WEBHOOK_API_KEY (401 on mismatch, 503 when unset). Body capped by WEBHOOK_MAX_RECORDS (413). Store only; no ingestion. Ported from OpenAgriNet/docs-pipeline#165. --- .env.example | 11 +++- ENV.md | 9 +++ README.md | 1 + pipeline/api/api.py | 44 +++++++++++++- pipeline/db.py | 77 +++++++++++++++++++++++++ tests/test_webhook.py | 129 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 268 insertions(+), 3 deletions(-) create mode 100644 tests/test_webhook.py diff --git a/.env.example b/.env.example index b33ead4..2e1b38f 100644 --- a/.env.example +++ b/.env.example @@ -251,4 +251,13 @@ KC_SMTP_PASSWORD= # Publish the catalog to the Beckn Discovery Service after PROD promotion. # Off by default. DISCOVERY_SERVICE_ENDPOINT and NETWORK_SENDER_ID are # required only when this is true. -PUBLISH_TO_NETWORK=false \ No newline at end of file +PUBLISH_TO_NETWORK=false + +# Shared secret for POST /webhook. Callers must send it in the `x-api-key` header. +# Required: if unset or empty, /webhook returns 503. Wrong or missing key returns 401. +# Generate a 32-char hex value with: openssl rand -hex 16 +# Never commit a real value. +WEBHOOK_API_KEY=your_32_character_hex_key_here + +# Max records accepted per POST /webhook request (more returns 413) +WEBHOOK_MAX_RECORDS=500 diff --git a/ENV.md b/ENV.md index 9757b8e..64e3279 100644 --- a/ENV.md +++ b/ENV.md @@ -78,6 +78,15 @@ Root `.env` (see `.env.example`). Required by FastAPI (`pipeline/api/`), Tempora | `DOCS_PIPELINE_UI_URL` | `http://localhost:3000` | Public UI base for links | | `DEFAULT_INSTANCE` | `default` | Default tenant / instance id | +### Webhook (`POST /webhook`) + +| Variable | Default | Purpose | +|----------|---------|---------| +| `WEBHOOK_MAX_RECORDS` | `500` | Max records per `/webhook` request; more returns 413 | +| `WEBHOOK_API_KEY` | *(empty — required)* | 32-char hex secret. Callers send it in the `x-api-key` header. Missing/invalid header → 401. Unset/empty → 503 (fails closed, ignores `AUTH_DISABLED`). Generate with `openssl rand -hex 16`. | + +Body is a JSON array of objects (max `WEBHOOK_MAX_RECORDS`); stored in the SQLite `webhook_records` table, upserted on `(source, external_id)` where `external_id` is `contentID` (else `id`, `slug`, payload hash). + ### Auth / Keycloak (backend JWT validation) | Variable | Default | Purpose | diff --git a/README.md b/README.md index abe03e6..2eab93b 100644 --- a/README.md +++ b/README.md @@ -205,6 +205,7 @@ It owns: - audit logs - search settings - document/index status +- inbound webhook records (`webhook_records`, written by `POST /webhook`, `x-api-key` = `WEBHOOK_API_KEY`) ### MinIO diff --git a/pipeline/api/api.py b/pipeline/api/api.py index 819ffef..64416e3 100644 --- a/pipeline/api/api.py +++ b/pipeline/api/api.py @@ -13,7 +13,7 @@ from contextlib import asynccontextmanager from datetime import datetime from pathlib import Path -from typing import Optional +from typing import Any, Dict, List, Optional # Load repo-root .env so KEYCLOAK_ADMIN_* etc. work without manual `source .env` try: @@ -28,7 +28,7 @@ from urllib.parse import quote -from fastapi import FastAPI, Header, HTTPException, Query, Request +from fastapi import Depends, FastAPI, Header, HTTPException, Query, Request from fastapi import Path as PathParam from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import StreamingResponse @@ -3291,6 +3291,46 @@ async def health(): } +# ============================================================================= +# Webhook (inbound pushes from external systems, x-api-key protected) +# ============================================================================= + +WEBHOOK_MAX_RECORDS = int(os.environ.get("WEBHOOK_MAX_RECORDS", "500")) + + +def require_webhook_key(request: Request) -> None: + """Dependency: x-api-key must equal WEBHOOK_API_KEY. Fails closed when unset.""" + import hmac + + expected = (os.environ.get("WEBHOOK_API_KEY") or "").strip() + if not expected: + raise HTTPException(503, "WEBHOOK_API_KEY is not configured") + provided = (request.headers.get("x-api-key") or "").strip() + if not provided or not hmac.compare_digest(provided, expected): + raise HTTPException(401, "Missing or invalid x-api-key") + + +@app.post("/webhook") +@limiter.limit(RATE_LIMIT_UPLOAD) +async def webhook( + request: Request, + records: List[Dict[str, Any]], + _: None = Depends(require_webhook_key), +): + """Accept a JSON array of records and upsert them into webhook_records.""" + if len(records) > WEBHOOK_MAX_RECORDS: + raise HTTPException(413, f"Too many records (max {WEBHOOK_MAX_RECORDS})") + try: + count = db.upsert_webhook_records("webhook", records) + except ValueError as e: + raise HTTPException(422, str(e)) + return { + "success": True, + "message": "Webhook processed successfully.", + "recordCount": count, + } + + # ============================================================================= # Vector index (passage schema) # ============================================================================= diff --git a/pipeline/db.py b/pipeline/db.py index 30b3ff0..22fd9bc 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -9,6 +9,7 @@ import json import os import sqlite3 +import uuid from contextlib import contextmanager from datetime import datetime from pathlib import Path @@ -385,7 +386,83 @@ def init_db(): last_sent_at TEXT NOT NULL ) """) + # Generic inbound webhook records (full body kept as JSON text in payload). + conn.execute(""" + CREATE TABLE IF NOT EXISTS webhook_records ( + id TEXT PRIMARY KEY, -- uuid4 + source TEXT NOT NULL DEFAULT 'webhook', + external_id TEXT NOT NULL, + title TEXT, + slug TEXT, + status TEXT, + is_published INTEGER, + category TEXT, + payload TEXT NOT NULL, + payload_hash TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + UNIQUE (source, external_id) + ) + """) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_webhook_records_source_status " + "ON webhook_records (source, status)" + ) + conn.commit() + + +def upsert_webhook_records(source: str, records: list) -> int: + """Upsert records by (source, external_id) in one transaction; returns len(records). + + external_id comes from contentID, else id, else slug, else a hash of the payload. + Unchanged payloads (same payload_hash) are not rewritten but still counted. + """ + now = datetime.utcnow().isoformat() + rows = [] + for rec in records: + if not isinstance(rec, dict): + raise ValueError("Each record must be a JSON object") + payload = json.dumps(rec, sort_keys=True, ensure_ascii=False, default=str) + payload_hash = hashlib.sha256(payload.encode("utf-8")).hexdigest() + ext = rec.get("contentID") + if ext in (None, ""): + ext = rec.get("id") + if ext in (None, ""): + ext = rec.get("slug") + if ext in (None, ""): + ext = payload_hash + pub = rec.get("isPublished") + rows.append(( + str(uuid.uuid4()), source, str(ext), rec.get("title"), rec.get("slug"), rec.get("status"), + int(pub) if isinstance(pub, (bool, int)) else None, + rec.get("domainName"), payload, payload_hash, + rec.get("createdAt") or now, rec.get("updatedAt") or now, + )) + with _db_lock: + with get_connection() as conn: + for r in rows: + conn.execute( + """ + INSERT INTO webhook_records + (id, source, external_id, title, slug, status, is_published, category, + payload, payload_hash, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(source, external_id) DO UPDATE SET + title = excluded.title, + slug = excluded.slug, + status = excluded.status, + is_published = excluded.is_published, + category = excluded.category, + payload = excluded.payload, + payload_hash = excluded.payload_hash, + created_at = excluded.created_at, + updated_at = excluded.updated_at + WHERE webhook_records.payload_hash IS NOT excluded.payload_hash + """, + r, + ) conn.commit() + return len(rows) def store_email_otp( diff --git a/tests/test_webhook.py b/tests/test_webhook.py new file mode 100644 index 0000000..d218885 --- /dev/null +++ b/tests/test_webhook.py @@ -0,0 +1,129 @@ +"""Tests for POST /webhook (x-api-key protected).""" + +import os +import uuid +from unittest.mock import patch + +import pytest + +TEST_KEY = "0123456789abcdef0123456789abcdef" + +SAMPLE = [ + { + "contentID": 101, + "title": "Bacterial Leaf Blight in Rice", + "domainName": "Agriculture", + "status": "Published", + "isPublished": 1, + "slug": "101/bacterial-leaf-blight-in-rice", + "createdAt": "2026-09-09T07:00:00.000Z", + "updatedAt": "2026-09-09T07:00:00.000Z", + "popCount": 1, + "popList": [{"popID": 1001, "contentID": 101, "keywords": ["rice"]}], + } +] + +OK_BODY = {"success": True, "message": "Webhook processed successfully.", "recordCount": 1} + + +@pytest.fixture +def test_client(db_connection): + """TestClient without lifespan (no Temporal/MinIO needed for /webhook).""" + from fastapi.testclient import TestClient + from pipeline import api + + return TestClient(api.app) + + +@pytest.fixture +def key_env(): + with patch.dict(os.environ, {"WEBHOOK_API_KEY": TEST_KEY}): + yield + + +def _rows(db_connection): + from pipeline import db + + with db.get_connection() as conn: + return conn.execute("SELECT * FROM webhook_records").fetchall() + + +@pytest.mark.api +@pytest.mark.unit +class TestWebhookAuth: + def test_missing_header_401(self, test_client, db_connection, key_env): + assert test_client.post("/webhook", json=SAMPLE).status_code == 401 + + def test_empty_header_401(self, test_client, db_connection, key_env): + r = test_client.post("/webhook", json=SAMPLE, headers={"x-api-key": ""}) + assert r.status_code == 401 + + def test_wrong_key_401(self, test_client, db_connection, key_env): + r = test_client.post("/webhook", json=SAMPLE, headers={"x-api-key": "nope"}) + assert r.status_code == 401 + + @pytest.mark.parametrize("value", [None, ""]) + def test_unconfigured_key_503_even_if_auth_disabled(self, test_client, db_connection, value): + env = {"AUTH_DISABLED": "true"} + if value is not None: + env["WEBHOOK_API_KEY"] = value + with patch.dict(os.environ, env): + if value is None: + os.environ.pop("WEBHOOK_API_KEY", None) + r = test_client.post("/webhook", json=SAMPLE, headers={"x-api-key": "anything"}) + assert r.status_code == 503 + assert "WEBHOOK_API_KEY" in r.json()["detail"] + + +@pytest.mark.api +@pytest.mark.unit +class TestWebhookIngest: + H = {"x-api-key": TEST_KEY} + + def test_valid_payload(self, test_client, db_connection, key_env): + r = test_client.post("/webhook", json=SAMPLE, headers=self.H) + assert r.status_code == 200 + assert r.json() == OK_BODY + rows = _rows(db_connection) + assert len(rows) == 1 + assert rows[0]["external_id"] == "101" + assert uuid.UUID(rows[0]["id"]).version == 4 + assert rows[0]["category"] == "Agriculture" + assert '"popList"' in rows[0]["payload"] + + def test_repost_is_idempotent(self, test_client, db_connection, key_env): + test_client.post("/webhook", json=SAMPLE, headers=self.H) + first_id = _rows(db_connection)[0]["id"] + changed = [{**SAMPLE[0], "title": "New title"}] + r = test_client.post("/webhook", json=changed, headers=self.H) + assert r.json()["recordCount"] == 1 + rows = _rows(db_connection) + assert len(rows) == 1 and rows[0]["title"] == "New title" + assert rows[0]["id"] == first_id + + def test_two_records(self, test_client, db_connection, key_env): + body = SAMPLE + [{**SAMPLE[0], "contentID": 102}] + r = test_client.post("/webhook", json=body, headers=self.H) + assert r.json()["recordCount"] == 2 + assert len(_rows(db_connection)) == 2 + + def test_empty_array(self, test_client, db_connection, key_env): + r = test_client.post("/webhook", json=[], headers=self.H) + assert r.status_code == 200 and r.json()["recordCount"] == 0 + + def test_non_array_422(self, test_client, db_connection, key_env): + r = test_client.post("/webhook", json={"contentID": 1}, headers=self.H) + assert r.status_code == 422 + + def test_non_object_item_422(self, test_client, db_connection, key_env): + r = test_client.post("/webhook", json=[1, 2], headers=self.H) + assert r.status_code == 422 + + def test_too_many_records_413(self, test_client, db_connection, key_env): + from pipeline import api + + with patch.object(api, "WEBHOOK_MAX_RECORDS", 1): + body = SAMPLE + [{**SAMPLE[0], "contentID": 102}] + r = test_client.post("/webhook", json=body, headers=self.H) + assert r.status_code == 413 + assert _rows(db_connection) == [] From 0fd2df385ed29b8acad249a64576ceb6c3f73f70 Mon Sep 17 00:00:00 2001 From: Naveen Date: Mon, 26 Oct 2026 13:18:45 +0530 Subject: [PATCH 28/53] build: add Postgres service, psycopg dependencies and database config Add a postgres service to docker-compose, psycopg[binary,pool] and test dependencies, DATABASE_URL and DB_POOL_* settings, a Postgres service in the test workflow, and document the new variables in ENV.md. uv.lock also picks up unrelated lock refreshes that were already pending. --- .github/workflows/run-tests.yml | 16 ++ Dockerfile | 4 +- ENV.md | 12 +- docker-compose.yml | 63 ++++++- pipeline/config.py | 15 +- pyproject.toml | 4 + uv.lock | 291 +++++++++++++++++++------------- 7 files changed, 278 insertions(+), 127 deletions(-) diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 6e01332..deaf75a 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -9,6 +9,22 @@ jobs: pipeline: name: pipeline (lint, test, coverage) runs-on: ubuntu-latest + services: + postgres: + image: postgres:15-alpine + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: knowledge_provider_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + env: + TEST_DATABASE_URL: postgresql://postgres:postgres@localhost:5432/knowledge_provider_test steps: - uses: actions/checkout@v4 diff --git a/Dockerfile b/Dockerfile index dcf1998..dd993cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,8 +29,10 @@ RUN --mount=type=cache,target=/root/.cache/uv \ ENV PATH="/app/.venv/bin:$PATH" -# Copy pipeline code +# Copy pipeline code and the SQL migrations the api/worker apply at startup COPY pipeline/ ./pipeline/ +COPY migrations/ ./migrations/ +COPY scripts/migrate_sqlite_to_postgres.py ./scripts/migrate_sqlite_to_postgres.py # Catch imports that work locally but fail on the shipped Python 3.10 image. RUN python -c "import pipeline.api" diff --git a/ENV.md b/ENV.md index 64e3279..0e6c601 100644 --- a/ENV.md +++ b/ENV.md @@ -64,7 +64,11 @@ Root `.env` (see `.env.example`). Required by FastAPI (`pipeline/api/`), Tempora | `LOG_LEVEL` | `INFO` | Worker log verbosity (`DEBUG`/`INFO`/`WARNING`/`ERROR`); the `temporalio` SDK logger stays at `INFO` regardless, to avoid its own noisy debug output | | `MINIO_ENDPOINT` | `localhost:9000` | MinIO API host:port | | `MINIO_BUCKET` | `documents` | Object storage bucket | -| `DOCUMENT_DB_PATH` | `/data/documents.db` | SQLite path (use `./data/documents.db` for local non-Docker) | +| `DATABASE_URL` | `postgresql://postgres:postgres@localhost:5432/knowledge_provider` | Application Postgres (documents, workflows, pages, chunks, audit, settings). The compose `postgres` service builds it from `DB_USERNAME` / `DB_PASSWORD` / `DB_NAME` (defaults `docs` / `docs` / `knowledge_provider`). Pending SQL migrations in `migrations/` are applied by the api and worker at startup | +| `DB_POOL_MIN_SIZE` | `1` | Connection pool minimum per process | +| `DB_POOL_MAX_SIZE` | `10` | Connection pool maximum per process (api and worker each hold their own pool) | +| `DB_POOL_TIMEOUT_SECONDS` | `30` | Seconds to wait for a free pooled connection | +| `DOCUMENT_DB_PATH` | `/data/documents.db` | Only read by `scripts/migrate_sqlite_to_postgres.py`: the legacy SQLite file to import | ### API HTTP surface @@ -85,7 +89,7 @@ Root `.env` (see `.env.example`). Required by FastAPI (`pipeline/api/`), Tempora | `WEBHOOK_MAX_RECORDS` | `500` | Max records per `/webhook` request; more returns 413 | | `WEBHOOK_API_KEY` | *(empty — required)* | 32-char hex secret. Callers send it in the `x-api-key` header. Missing/invalid header → 401. Unset/empty → 503 (fails closed, ignores `AUTH_DISABLED`). Generate with `openssl rand -hex 16`. | -Body is a JSON array of objects (max `WEBHOOK_MAX_RECORDS`); stored in the SQLite `webhook_records` table, upserted on `(source, external_id)` where `external_id` is `contentID` (else `id`, `slug`, payload hash). +Body is a JSON array of objects (max `WEBHOOK_MAX_RECORDS`); stored in the `webhook_records` table, upserted on `(source, external_id)` where `external_id` is `contentID` (else `id`, `slug`, payload hash). ### Auth / Keycloak (backend JWT validation) @@ -266,7 +270,7 @@ Each instance has one provider for both its advisory and scheme catalogs: id and |---------|----|-----|--------| | `VITE_*` proxies & Keycloak browser login | ✅ | — | — | | JWT validation (`AUTH_*`, `KEYCLOAK_*`) | — | ✅ | — | -| Temporal / MinIO / SQLite | — | ✅ | ✅ | +| Temporal / MinIO / Postgres (`DATABASE_URL`) | — | ✅ | ✅ | | OCR / translation / chunking / domain tags | — | config / status | ✅ runs jobs | | Qdrant (`VECTOR_DB_*`) | — | ✅ | ✅ | | Publish to Network (`DISCOVERY_SERVICE_ENDPOINT`, `NETWORK_SENDER_ID`) | — | — | ✅ runs the publish activity | @@ -280,7 +284,7 @@ Each instance has one provider for both its advisory and scheme catalogs: id and ```bash set -a && source .env && set +a -export DOCUMENT_DB_PATH="$(pwd)/data/documents.db" +export DATABASE_URL="postgresql://docs:docs@localhost:5433/knowledge_provider" # the compose `postgres` service (`docker compose up -d postgres`) export ALLOWED_FILE_PATHS="$(pwd)/books,$(pwd)/data/documents" export CORS_ORIGINS="http://localhost:3000,http://localhost:3001" export PYTHONPATH="$(pwd)" diff --git a/docker-compose.yml b/docker-compose.yml index 1f6f11c..5a61620 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,6 +19,7 @@ # API 127.0.0.1:${API_HOST_PORT:-8011}:8001 # UI 127.0.0.1:${UI_HOST_PORT:-3011}:80 # Keycloak 127.0.0.1:${KEYCLOAK_HOST_PORT:-8081}:8080 +# Postgres 127.0.0.1:${DB_HOST_PORT:-5433}:5432 # Temporal 127.0.0.1:7233:7233 # MinIO API 127.0.0.1:9000:9000 # MinIO UI 127.0.0.1:9001:9001 @@ -120,6 +121,30 @@ services: MINIO_BUCKET: ${MINIO_BUCKET:-documents} restart: "no" + # --------------------------------------------------------------------------- + # Application database — documents, workflows, pages, chunks, audit, settings. + # Schema is applied by api/worker at startup from migrations/ (see + # docs/POSTGRES_SCHEMA.md). A deployment that wants its own Postgres can set + # DATABASE_URL in .env and drop this service's depends_on entries. + # --------------------------------------------------------------------------- + postgres: + image: postgres:15-alpine + platform: linux/amd64 + environment: + POSTGRES_USER: ${DB_USERNAME:-docs} + POSTGRES_PASSWORD: ${DB_PASSWORD:-docs} + POSTGRES_DB: ${DB_NAME:-knowledge_provider} + ports: + - "127.0.0.1:${DB_HOST_PORT:-5433}:5432" + volumes: + - app-db-data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USERNAME:-docs} -d ${DB_NAME:-knowledge_provider}"] + interval: 5s + timeout: 5s + retries: 10 + restart: unless-stopped + # --------------------------------------------------------------------------- # Qdrant — DEV vector index (see docs/SYSTEM_DESIGN.md's DEV vs PROD dual # indexing). api/worker's VECTOR_DB_URL is always pointed at this bundled @@ -249,7 +274,7 @@ services: TEMPORAL_HOST: temporal:7233 MINIO_ENDPOINT: minio:9000 VECTOR_DB_URL: http://qdrant:6333 - DOCUMENT_DB_PATH: /data/documents.db + DATABASE_URL: postgresql://${DB_USERNAME:-docs}:${DB_PASSWORD:-docs}@postgres:5432/${DB_NAME:-knowledge_provider} ALLOWED_FILE_PATHS: /app/books,/data/documents # HF cache path propagated into the container HF_HOME: ${HF_HOME:-/opt/hf-cache} @@ -257,6 +282,8 @@ services: ports: - "127.0.0.1:${API_HOST_PORT:-8011}:8001" depends_on: + postgres: + condition: service_healthy temporal: condition: service_healthy minio: @@ -268,7 +295,6 @@ services: command: uvicorn pipeline.api:app --host 0.0.0.0 --port 8001 volumes: - ./books:/app/books:ro - - sqlite-data:/data - documents-data:/data/documents - ${HF_HOME:-/opt/hf-cache}:${HF_HOME:-/opt/hf-cache} extra_hosts: @@ -291,11 +317,13 @@ services: TEMPORAL_HOST: temporal:7233 MINIO_ENDPOINT: minio:9000 VECTOR_DB_URL: http://qdrant:6333 - DOCUMENT_DB_PATH: /data/documents.db + DATABASE_URL: postgresql://${DB_USERNAME:-docs}:${DB_PASSWORD:-docs}@postgres:5432/${DB_NAME:-knowledge_provider} ALLOWED_FILE_PATHS: /app/books,/data/documents HF_HOME: ${HF_HOME:-/opt/hf-cache} HF_HUB_CACHE: ${HF_HOME:-/opt/hf-cache} depends_on: + postgres: + condition: service_healthy temporal: condition: service_healthy minio: @@ -307,13 +335,39 @@ services: command: python -m pipeline.worker volumes: - ./books:/app/books:ro - - sqlite-data:/data - documents-data:/data/documents - ${HF_HOME:-/opt/hf-cache}:${HF_HOME:-/opt/hf-cache} extra_hosts: - "host.docker.internal:host-gateway" restart: unless-stopped + # --------------------------------------------------------------------------- + # One-off SQLite -> Postgres data migration (maintenance window, api/worker + # stopped). The legacy volume is opened read-only by the script and is never + # modified, so rolling back is redeploying the previous images on it. + # docker compose --profile migrate run --rm migrate-sqlite # dry run + # MIGRATE_ARGS= docker compose --profile migrate run --rm migrate-sqlite # real run + # --------------------------------------------------------------------------- + migrate-sqlite: + image: ghcr.io/openagrinet/knowledge-provider:${IMAGE_TAG:-latest} + platform: linux/amd64 + profiles: + - migrate + build: + context: . + dockerfile: Dockerfile + environment: + DATABASE_URL: postgresql://${DB_USERNAME:-docs}:${DB_PASSWORD:-docs}@postgres:5432/${DB_NAME:-knowledge_provider} + DOCUMENT_DB_PATH: /legacy/documents.db + DEFAULT_INSTANCE: ${DEFAULT_INSTANCE:-default} + depends_on: + postgres: + condition: service_healthy + command: python scripts/migrate_sqlite_to_postgres.py ${MIGRATE_ARGS---dry-run} + volumes: + - sqlite-data:/legacy + restart: "no" + # --------------------------------------------------------------------------- # React UI # Dockerfile.prod builds a static nginx image; VITE_* args come from .env. @@ -349,5 +403,6 @@ volumes: keycloak-db-data: keycloak-data: minio-data: + app-db-data: sqlite-data: documents-data: diff --git a/pipeline/config.py b/pipeline/config.py index 7ec2a4b..426e41c 100644 --- a/pipeline/config.py +++ b/pipeline/config.py @@ -20,7 +20,10 @@ class Config: temporal_host: str = "localhost:7233" minio_endpoint: str = "localhost:9000" minio_bucket: str = "documents" - document_db_path: str = "/data/documents.db" + database_url: str = "postgresql://postgres:postgres@localhost:5432/knowledge_provider" + db_pool_min_size: int = 1 + db_pool_max_size: int = 10 + db_pool_timeout_seconds: float = 30.0 translation_provider: str = "gemma_vllm" translation_model: str = "google/gemma-4-31b-it" translation_vllm_base_url: str = "http://localhost:8020/v1" @@ -105,7 +108,10 @@ def load_config() -> Config: temporal_host=os.environ.get("TEMPORAL_HOST", "localhost:7233"), minio_endpoint=os.environ.get("MINIO_ENDPOINT", "localhost:9000"), minio_bucket=os.environ.get("MINIO_BUCKET", "documents"), - document_db_path=os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db"), + database_url=os.environ.get("DATABASE_URL", Config.database_url), + db_pool_min_size=int(os.environ.get("DB_POOL_MIN_SIZE", "1")), + db_pool_max_size=int(os.environ.get("DB_POOL_MAX_SIZE", "10")), + db_pool_timeout_seconds=float(os.environ.get("DB_POOL_TIMEOUT_SECONDS", "30")), translation_provider=os.environ.get("TRANSLATION_PROVIDER", "gemma_vllm"), translation_model=os.environ.get("TRANSLATION_MODEL", "google/gemma-4-31b-it"), translation_vllm_base_url=os.environ.get("TRANSLATION_VLLM_BASE_URL", "http://localhost:8020/v1"), @@ -158,7 +164,10 @@ def print_config_status(): ("TEMPORAL_HOST", "localhost:7233"), ("MINIO_ENDPOINT", "localhost:9000"), ("MINIO_BUCKET", "documents"), - ("DOCUMENT_DB_PATH", "/data/documents.db"), + ("DATABASE_URL", "postgresql://postgres:postgres@localhost:5432/knowledge_provider"), + ("DB_POOL_MIN_SIZE", "1"), + ("DB_POOL_MAX_SIZE", "10"), + ("DB_POOL_TIMEOUT_SECONDS", "30"), ("TRANSLATION_PROVIDER", "gemma_vllm"), ("TRANSLATION_MODEL", "gemma-4"), ("TRANSLATION_VLLM_BASE_URL", "http://localhost:8020/v1"), diff --git a/pyproject.toml b/pyproject.toml index af41000..f89c86b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,9 @@ dependencies = [ "pypdf>=5.0.0", "PyMuPDF>=1.27.0", + # Postgres driver (sync) + connection pool + "psycopg[binary,pool]>=3.2", + # Rate limiting "slowapi>=0.1.9", @@ -63,6 +66,7 @@ dev = [ "pytest-asyncio>=0.21.0", "pytest-cov>=4.0.0", "ruff>=0.6.0", + "testcontainers[postgres]>=4.0.0", ] # sentence-transformers depends on torch; resolve it from the CPU-only wheel diff --git a/uv.lock b/uv.lock index 9292f9a..6a1bc0e 100644 --- a/uv.lock +++ b/uv.lock @@ -109,15 +109,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a3/34/32109943bace7729233cc4ee78530baa306d8cc3c6501a64ba8cb3b58129/argon2_cffi_bindings-26.1.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:0cc40f7b4050bb93eb67de95d2d759322fc7ce4930b9d645581ecf4913ec651e", size = 23584, upload-time = "2026-08-20T07:33:22.613Z" }, ] -[[package]] -name = "async-timeout" -version = "5.0.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a5/ae/136395dfbfe00dfc94da3f3e136d0b13f394cba8f4841120e34226265780/async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3", size = 9274, upload-time = "2024-11-06T16:41:39.6Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c", size = 6233, upload-time = "2024-11-06T16:41:37.9Z" }, -] - [[package]] name = "backports-asyncio-runner" version = "1.2.0" @@ -676,6 +667,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, ] +[[package]] +name = "docker" +version = "7.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "requests" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/88/7f/731ff914b0255d3d065f45fd4e626d4b8c95dbcbaada049f337a6ac16410/docker-7.2.0.tar.gz", hash = "sha256:cebb93773d334f778e023a7ee352a8d6e13ab1bd3b863a4d4a59dec897df43ac", size = 118731, upload-time = "2026-07-09T14:53:46.39Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/75/23/529140fe1aab80fc6992f93a706deec709140a6397439139a054e1515c45/docker-7.2.0-py3-none-any.whl", hash = "sha256:a3f45fdeb9165e2d25d9a1d02ddf3bc70fb572cf5ebbf9b58558c22caf29b71f", size = 148775, upload-time = "2026-07-09T14:53:45.224Z" }, +] + [[package]] name = "et-xmlfile" version = "2.0.0" @@ -690,7 +695,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -1129,7 +1134,7 @@ dependencies = [ { name = "openpyxl" }, { name = "pdf-inspector" }, { name = "pillow" }, - { name = "psycopg2-binary" }, + { name = "psycopg", extra = ["binary", "pool"] }, { name = "pydantic" }, { name = "pyfranc" }, { name = "pyjwt", extra = ["crypto"] }, @@ -1137,7 +1142,6 @@ dependencies = [ { name = "pypdf" }, { name = "python-multipart" }, { name = "qdrant-client" }, - { name = "redis" }, { name = "sentence-transformers" }, { name = "slowapi" }, { name = "temporalio" }, @@ -1154,6 +1158,7 @@ dev = [ { name = "pytest-asyncio" }, { name = "pytest-cov" }, { name = "ruff" }, + { name = "testcontainers" }, ] [package.metadata] @@ -1167,7 +1172,7 @@ requires-dist = [ { name = "openpyxl", specifier = ">=3.1.0" }, { name = "pdf-inspector", specifier = ">=1.14.0" }, { name = "pillow", specifier = ">=10.0.0" }, - { name = "psycopg2-binary", specifier = ">=2.9.0" }, + { name = "psycopg", extras = ["binary", "pool"], specifier = ">=3.2" }, { name = "pydantic", specifier = ">=2.0.0" }, { name = "pyfranc", specifier = ">=0.2.3" }, { name = "pyjwt", extras = ["crypto"], specifier = ">=2.8.0" }, @@ -1175,7 +1180,6 @@ requires-dist = [ { name = "pypdf", specifier = ">=5.0.0" }, { name = "python-multipart", specifier = ">=0.0.6" }, { name = "qdrant-client", specifier = ">=1.12.0" }, - { name = "redis", specifier = ">=5.0.0" }, { name = "sentence-transformers", specifier = "==5.1.0" }, { name = "slowapi", specifier = ">=0.1.9" }, { name = "temporalio", specifier = ">=1.0.0" }, @@ -1191,6 +1195,7 @@ dev = [ { name = "pytest-asyncio", specifier = ">=0.21.0" }, { name = "pytest-cov", specifier = ">=4.0.0" }, { name = "ruff", specifier = ">=0.6.0" }, + { name = "testcontainers", extras = ["postgres"], specifier = ">=4.0.0" }, ] [[package]] @@ -1938,66 +1943,109 @@ wheels = [ ] [[package]] -name = "psycopg2-binary" -version = "2.9.12" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2a/60/a3624f79acea344c16fbef3a94d28b89a8042ddfb8f3e4ca83f538671409/psycopg2_binary-2.9.12.tar.gz", hash = "sha256:5ac9444edc768c02a6b6a591f070b8aae28ff3a99be57560ac996001580f294c", size = 379686, upload-time = "2026-04-21T09:40:34.304Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/78/80/49bacf9e51617d8309f6f0123e29edc793f6f5f6700c7d1f1b20782fbb37/psycopg2_binary-2.9.12-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9b818ceff717f98851a64bffd4c5eb5b3059ae280276dcecc52ac658dcf006a4", size = 3712314, upload-time = "2026-04-20T23:33:31.363Z" }, - { url = "https://files.pythonhosted.org/packages/d3/f2/98eeac7d60c43df9338287834edf9b3e69be68a2db78a57b1b81d705e735/psycopg2_binary-2.9.12-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d2fa0d7caca8635c56e373055094eeda3208d901d55dd0ff5abc1d4e47f82b56", size = 3822389, upload-time = "2026-04-20T23:33:34.178Z" }, - { url = "https://files.pythonhosted.org/packages/9f/7c/30575e75f14d5351a56a1971bb43fe7f8bf7edf1b654fb1bec65c42a8812/psycopg2_binary-2.9.12-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:864c261b3690e1207d14bbfe0a61e27567981b80c47a778561e49f676f7ce433", size = 4578448, upload-time = "2026-04-20T23:33:37.073Z" }, - { url = "https://files.pythonhosted.org/packages/65/9b/4df366d89f28c527dc39d0b6c98a5ca74e30d37ac097b73f3352147568ae/psycopg2_binary-2.9.12-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c5ee5213445dd45312459029b8c4c0a695461eb517b753d2582315bd07995f5e", size = 4273705, upload-time = "2026-04-20T23:33:39.291Z" }, - { url = "https://files.pythonhosted.org/packages/4e/8a/c566803818eb03161ba869b6ba612bf7ad56816d98b9e5121e0a22ad6b0b/psycopg2_binary-2.9.12-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6f9cae1f848779b5b01f417e762c40d026ea93eb0648249a604728cda991dde3", size = 5893784, upload-time = "2026-04-20T23:33:41.658Z" }, - { url = "https://files.pythonhosted.org/packages/63/fe/0dfa5797e0b229e0567bc378695224caf14d547f73b05be0c80549089772/psycopg2_binary-2.9.12-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:63a3ebbd543d3d1eda088ac99164e8c5bac15293ee91f20281fd17d050aee1c4", size = 4109306, upload-time = "2026-04-20T23:33:43.953Z" }, - { url = "https://files.pythonhosted.org/packages/3c/89/28063adf17a4ba501eedd9890feab0c649ee4d8bd0a97df0ff1e9584feab/psycopg2_binary-2.9.12-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d6fcbba8c9fed08a73b8ac61ea79e4821e45b1e92bb466230c5e746bbf3d5256", size = 3654400, upload-time = "2026-04-20T23:33:46.115Z" }, - { url = "https://files.pythonhosted.org/packages/84/94/5a01de0aa4ead0b8d8d1aa4ec18cec0bd36d03fa714eaa5bb8a0b1b50020/psycopg2_binary-2.9.12-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:36512911ebb2b60a0c3e44d0bb5048c1980aced91235d133b7874f3d1d93487c", size = 3299215, upload-time = "2026-04-20T23:33:48.202Z" }, - { url = "https://files.pythonhosted.org/packages/7a/85/723bb085a61c6ac2dc0a0043f375f2fe7365363e27b073bad56ca5bda979/psycopg2_binary-2.9.12-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:8ffdb59fe88f99589e34354a130217aa1fd2d615612402d6edc8b3dbc7a44463", size = 3047724, upload-time = "2026-04-20T23:33:50.74Z" }, - { url = "https://files.pythonhosted.org/packages/b4/67/4d8b1e0d2fc4166677380eac0edf9cdff91013aca2546e8ef7bc04b56158/psycopg2_binary-2.9.12-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a46fe069b65255df410f856d842bc235f90e22ffdf532dda625fd4213d3fd9b1", size = 3349183, upload-time = "2026-04-20T23:33:59.635Z" }, - { url = "https://files.pythonhosted.org/packages/73/99/21af7a5498637ea4dc91a17c281a53bc1d632fbafe00f6689fbfb32a9fed/psycopg2_binary-2.9.12-cp310-cp310-win_amd64.whl", hash = "sha256:ab29414b25dcb698bf26bf213e3348abdcd07bbd5de032a5bec15bd75b298b03", size = 2757036, upload-time = "2026-04-20T23:34:01.606Z" }, - { url = "https://files.pythonhosted.org/packages/d5/19/d4ce60954f3bb9d8e3bc5e5c4d1f2487de2d3851bf2391d54954c9df12a6/psycopg2_binary-2.9.12-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5c8ce6c61bd1b1f6b9c24ee32211599f6166af2c55abb19456090a21fd16554b", size = 3712338, upload-time = "2026-04-20T23:34:03.961Z" }, - { url = "https://files.pythonhosted.org/packages/53/71/c85409ee0d78890f0660eff262e815e7dd2bb741a17611d82e9e8cd9dc5e/psycopg2_binary-2.9.12-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b4a9eaa6e7f4ff91bec10aa3fb296878e75187bced5cc4bafe17dc40915e1326", size = 3822407, upload-time = "2026-04-20T23:34:05.977Z" }, - { url = "https://files.pythonhosted.org/packages/3c/ed/60486c2c7f0d4d1ede2bfb1ed27e2498477ce646bc7f6b2759906303117e/psycopg2_binary-2.9.12-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c6528cefc8e50fcc6f4a107e27a672058b36cc5736d665476aeb413ba88dbb06", size = 4578425, upload-time = "2026-04-20T23:34:08.246Z" }, - { url = "https://files.pythonhosted.org/packages/0b/b9/656cb03fad9f4f49f2145c334b1126ee75189929ca4e6187d485a2d59951/psycopg2_binary-2.9.12-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e4e184b1fb6072bf05388aa41c697e1b2d01b3473f107e7ec44f186a32cfd0b8", size = 4273709, upload-time = "2026-04-20T23:34:10.974Z" }, - { url = "https://files.pythonhosted.org/packages/99/66/08cf0da0e25cc6fb142c89be45fc8418792858f0c4cbff5e24530ff02cd6/psycopg2_binary-2.9.12-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4766ab678563054d3f1d064a4db19cc4b5f9e3a8d9018592a8285cf200c248f3", size = 5893779, upload-time = "2026-04-20T23:34:13.905Z" }, - { url = "https://files.pythonhosted.org/packages/17/d7/eecd9ce8e146d3721115d82d3836efdbb712187e4590325df549989d18f4/psycopg2_binary-2.9.12-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5a0253224780c978746cb9be55a946bcdaf40fe3519c0f622924cdabdafe2c39", size = 4109308, upload-time = "2026-04-20T23:34:16.761Z" }, - { url = "https://files.pythonhosted.org/packages/b6/2e/b1dc289b362cc8d45697b57eefbd673186f49a4ea0906928988e3affcc98/psycopg2_binary-2.9.12-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0dc9228d47c46bda253d2ecd6bb93b56a9f2d7ad33b684a1fa3622bf74ffe30c", size = 3654405, upload-time = "2026-04-20T23:34:19.303Z" }, - { url = "https://files.pythonhosted.org/packages/eb/e4/4c4aea6473214dbdbd0fbba11aa4691e76dc01722c55724c5951719865ff/psycopg2_binary-2.9.12-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f921f3cd87035ef7df233383011d7a53ea1d346224752c1385f1edfd790ceb6a", size = 3299187, upload-time = "2026-04-20T23:34:21.206Z" }, - { url = "https://files.pythonhosted.org/packages/ba/5d/b03b99986446a4f57b170ed9a2579fb7ff9783ca0fa5226b19db99737fee/psycopg2_binary-2.9.12-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3d999bd982a723113c1a45b55a7a6a90d64d0ed2278020ed625c490ff7bef96c", size = 3047716, upload-time = "2026-04-20T23:34:23.077Z" }, - { url = "https://files.pythonhosted.org/packages/14/86/382ee4afbd1d97500c9d2862b20c2fdeddf4b7335e984df3fb4309f64108/psycopg2_binary-2.9.12-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29d4d134bd0ab46ffb04e94aa3c5fa3ef582e9026609165e2f758ff76fc3a3be", size = 3349237, upload-time = "2026-04-20T23:34:25.211Z" }, - { url = "https://files.pythonhosted.org/packages/a8/16/9a57c75ba1eda7165c017342f526810d5f5a12647dde749c99ae9a7141d7/psycopg2_binary-2.9.12-cp311-cp311-win_amd64.whl", hash = "sha256:cb4a1dacdd48077150dc762a9e5ddbf32c256d66cb46f80839391aa458774936", size = 2757036, upload-time = "2026-04-20T23:34:27.77Z" }, - { url = "https://files.pythonhosted.org/packages/e2/9f/ef4ef3c8e15083df90ca35265cfd1a081a2f0cc07bb229c6314c6af817f4/psycopg2_binary-2.9.12-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5cdc05117180c5fa9c40eea8ea559ce64d73824c39d928b7da9fb5f6a9392433", size = 3712459, upload-time = "2026-04-20T23:34:30.549Z" }, - { url = "https://files.pythonhosted.org/packages/b5/01/3dd14e46ba48c1e1a6ec58ee599fa1b5efa00c246d5046cd903d0eeb1af1/psycopg2_binary-2.9.12-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d3227a3bc228c10d21011a99245edca923e4e8bf461857e869a507d9a41fe9f6", size = 3822936, upload-time = "2026-04-20T23:34:32.77Z" }, - { url = "https://files.pythonhosted.org/packages/a6/f7/0640e4901119d8a9f7a1784b927f494e2198e213ceb593753d1f2c8b1b30/psycopg2_binary-2.9.12-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:995ce929eede89db6254b50827e2b7fd61e50d11f0b116b29fffe4a2e53c4580", size = 4578676, upload-time = "2026-04-20T23:34:35.18Z" }, - { url = "https://files.pythonhosted.org/packages/b0/55/44df3965b5f297c50cc0b1b594a31c67d6127a9d133045b8a66611b14dfb/psycopg2_binary-2.9.12-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9fe06d93e72f1c048e731a2e3e7854a5bfaa58fc736068df90b352cefe66f03f", size = 4274917, upload-time = "2026-04-20T23:34:37.982Z" }, - { url = "https://files.pythonhosted.org/packages/b0/4b/74535248b1eac0c9336862e8617c765ac94dac76f9e25d7c4a79588c8907/psycopg2_binary-2.9.12-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40e7b28b63aaf737cb3a1edc3a9bbc9a9f4ad3dcb7152e8c1130e4050eddcb7d", size = 5894843, upload-time = "2026-04-20T23:34:40.856Z" }, - { url = "https://files.pythonhosted.org/packages/f2/ba/f1bf8d2ae71868ad800b661099086ee52bc0f8d9f05be1acd8ebb06757cc/psycopg2_binary-2.9.12-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:89d19a9f7899e8eb0656a2b3a08e0da04c720a06db6e0033eab5928aabe60fa9", size = 4110556, upload-time = "2026-04-20T23:34:44.016Z" }, - { url = "https://files.pythonhosted.org/packages/45/46/c15706c338403b7c420bcc0c2905aad116cc064545686d8bf85f1999ea00/psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:612b965daee295ae2da8f8218ce1d274645dc76ef3f1abf6a0a94fd57eff876d", size = 3655714, upload-time = "2026-04-20T23:34:46.233Z" }, - { url = "https://files.pythonhosted.org/packages/b3/7c/a2d5dc09b64a4564db242a0fe418fde7d33f6f8259dd2c5b9d7def00fb5a/psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:b9a339b79d37c1b45f3235265f07cdeb0cb5ad7acd2ac7720a5920989c17c24e", size = 3301154, upload-time = "2026-04-20T23:34:49.528Z" }, - { url = "https://files.pythonhosted.org/packages/c0/e8/cc8c9a4ce71461f9ec548d38cadc41dc184b34c73e6455450775a9334ccd/psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:3471336e1acfd9c7fe507b8bad5af9317b6a89294f9eb37bd9a030bb7bebcdc6", size = 3048882, upload-time = "2026-04-20T23:34:51.86Z" }, - { url = "https://files.pythonhosted.org/packages/19/6a/31e2296bc0787c5ab75d3d118e40b239db8151b5192b90b77c72bc9256e9/psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7af18183109e23502c8b2ae7f6926c0882766f35b5175a4cd737ad825e4d7a1b", size = 3351298, upload-time = "2026-04-20T23:34:54.124Z" }, - { url = "https://files.pythonhosted.org/packages/5f/a8/75f4e3e11203b590150abed2cf7794b9c9c9f7eceddae955191138b44dde/psycopg2_binary-2.9.12-cp312-cp312-win_amd64.whl", hash = "sha256:398fcd4db988c7d7d3713e2b8e18939776fd3fb447052daae4f24fa39daede4c", size = 2757230, upload-time = "2026-04-20T23:34:56.242Z" }, - { url = "https://files.pythonhosted.org/packages/91/bb/4608c96f970f6e0c56572e87027ef4404f709382a3503e9934526d7ba051/psycopg2_binary-2.9.12-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7c729a73c7b1b84de3582f73cdd27d905121dc2c531f3d9a3c32a3011033b965", size = 3712419, upload-time = "2026-04-20T23:34:58.754Z" }, - { url = "https://files.pythonhosted.org/packages/5e/af/48f76af9d50d61cf390f8cd657b503168b089e2e9298e48465d029fcc713/psycopg2_binary-2.9.12-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4413d0caef93c5cf50b96863df4c2efe8c269bf2267df353225595e7e15e8df7", size = 3822990, upload-time = "2026-04-20T23:35:00.821Z" }, - { url = "https://files.pythonhosted.org/packages/7a/df/aba0f99397cd811d32e06fc0cc781f1f3ce98bc0e729cb423925085d781a/psycopg2_binary-2.9.12-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4dfcf8e45ebb0c663be34a3442f65e17311f3367089cd4e5e3a3e8e62c978777", size = 4578696, upload-time = "2026-04-20T23:35:03.409Z" }, - { url = "https://files.pythonhosted.org/packages/95/9c/eaa74021ac4e4d5c2f83d82fc6615a63f4fe6c94dc4e94c3990427053f67/psycopg2_binary-2.9.12-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c41321a14dd74aceb6a9a643b9253a334521babfa763fa873e33d89cfa122fb5", size = 4274982, upload-time = "2026-04-20T23:35:05.583Z" }, - { url = "https://files.pythonhosted.org/packages/35/ed/c25deff98bd26187ba48b3b250a3ffc3037c46c5b89362534a15d200e0db/psycopg2_binary-2.9.12-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:83946ba43979ebfdc99a3cd0ee775c89f221df026984ba19d46133d8d75d3cd9", size = 5894867, upload-time = "2026-04-20T23:35:07.902Z" }, - { url = "https://files.pythonhosted.org/packages/9a/81/8d0e21ca77373c6c9589e5c4528f6e8f0c08c62cafc76fb0bddb7a2cee22/psycopg2_binary-2.9.12-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:411e85815652d13560fbe731878daa5d92378c4995a22302071890ec3397d019", size = 4110578, upload-time = "2026-04-20T23:35:10.149Z" }, - { url = "https://files.pythonhosted.org/packages/00/fc/f481e2435bd8f742d0123309174aae4165160ad3ef17c1b99c3622c241d2/psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1c8ad4c08e00f7679559eaed7aff1edfffc60c086b976f93972f686384a95e2c", size = 3655816, upload-time = "2026-04-20T23:35:12.56Z" }, - { url = "https://files.pythonhosted.org/packages/53/79/b9f46466bdbe9f239c96cde8be33c1aace4842f06013b47b730dc9759187/psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:00814e40fa23c2b37ef0a1e3c749d89982c73a9cb5046137f0752a22d432e82f", size = 3301307, upload-time = "2026-04-20T23:35:15.029Z" }, - { url = "https://files.pythonhosted.org/packages/3f/19/7dc003b32fe35024df89b658104f7c8538a8b2dcbde7a4e746ce929742e7/psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:98062447aebc20ed20add1f547a364fd0ef8933640d5372ff1873f8deb9b61be", size = 3048968, upload-time = "2026-04-20T23:35:16.757Z" }, - { url = "https://files.pythonhosted.org/packages/91/58/2dbd7db5c604d45f4950d988506aae672a14126ec22998ced5021cbb76bb/psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:66a7685d7e548f10fb4ce32fb01a7b7f4aa702134de92a292c7bd9e0d3dbd290", size = 3351369, upload-time = "2026-04-20T23:35:18.933Z" }, - { url = "https://files.pythonhosted.org/packages/42/ee/dee8dcaad07f735824de3d6563bc67119fa6c28257b17977a8d624f02fab/psycopg2_binary-2.9.12-cp313-cp313-win_amd64.whl", hash = "sha256:b6937f5fe4e180aeee87de907a2fa982ded6f7f15d7218f78a083e4e1d68f2a0", size = 2757347, upload-time = "2026-04-20T23:35:21.283Z" }, - { url = "https://files.pythonhosted.org/packages/13/1b/708c0dca874acfad6d65314271859899a79007686f3a1f74e82a2ed4b645/psycopg2_binary-2.9.12-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:6f3b3de8a74ef8db215f22edffb19e32dc6fa41340456de7ec99efdc8a7b3ec2", size = 3712428, upload-time = "2026-04-20T23:35:23.453Z" }, - { url = "https://files.pythonhosted.org/packages/d6/39/ddbea9d4b4de6aca9431b6ed253f530f8a02d3b8f9bcfd0dbfe2b3de6fe4/psycopg2_binary-2.9.12-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1006fb62f0f0bc5ce256a832356c6262e91be43f5e4eb15b5eaf38079464caf2", size = 3823184, upload-time = "2026-04-20T23:35:25.92Z" }, - { url = "https://files.pythonhosted.org/packages/bf/a0/bc2fef74b106fa345567122a0659e6d94512ed7dc0131ec44c9e5aba3725/psycopg2_binary-2.9.12-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:840066105706cd2eb29b9a1c2329620056582a4bf3e8169dec5c447042d0869f", size = 4579157, upload-time = "2026-04-20T23:35:28.542Z" }, - { url = "https://files.pythonhosted.org/packages/57/d7/d4e3b2005d3de607ca4fbb0e8742e248056e52184a6b94ebda3c1c2c329b/psycopg2_binary-2.9.12-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:863f5d12241ebe1c76a72a04c2113b6dc905f90b9cef0e9be0efd994affd9354", size = 4274970, upload-time = "2026-04-20T23:35:30.418Z" }, - { url = "https://files.pythonhosted.org/packages/2e/42/c9853f8db3967fe08bcde11f53d53b85d351750cae726ce001cb68afa9c1/psycopg2_binary-2.9.12-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a99eaab34a9010f1a086b126de467466620a750634d114d20455f3a824aae033", size = 5895175, upload-time = "2026-04-20T23:35:33.584Z" }, - { url = "https://files.pythonhosted.org/packages/eb/fd/b82b5601a97630308bef079f545ffec481bbbc795c2ba5ec416a01d03f60/psycopg2_binary-2.9.12-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ffdd7dc5463ccd61845ac37b7012d0f35a1548df9febe14f8dd549be4a0bc81e", size = 4110658, upload-time = "2026-04-20T23:35:35.638Z" }, - { url = "https://files.pythonhosted.org/packages/62/8c/32ca69b0389ef25dd22937bf9e8fbe2ce27aea20b05ded48c4ce4cb42475/psycopg2_binary-2.9.12-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:54a0dfecab1b48731f934e06139dfe11e24219fb6d0ceb32177cf0375f14c7b5", size = 3656251, upload-time = "2026-04-20T23:35:37.854Z" }, - { url = "https://files.pythonhosted.org/packages/c4/29/96992a2b59e3b9d730fcf9612d0a387305025dc867a9fc490a9e496e074e/psycopg2_binary-2.9.12-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:96937c9c5d891f772430f418a7a8b4691a90c3e6b93cf72b5bd7cad8cbca32a5", size = 3301810, upload-time = "2026-04-20T23:35:39.927Z" }, - { url = "https://files.pythonhosted.org/packages/56/ad/44b06659949b243ae10112cd3b20a197f9bf3e81d5651379b9eb889bfaad/psycopg2_binary-2.9.12-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:77b348775efd4cdab410ec6609d81ccecd1139c90265fa583a7255c8064bc03d", size = 3048977, upload-time = "2026-04-20T23:35:41.806Z" }, - { url = "https://files.pythonhosted.org/packages/1d/f2/10a1bcebadb6aa55e280e1f58975c36a7b560ea525184c7aa4064c466633/psycopg2_binary-2.9.12-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:527e6342b3e44c2f0544f6b8e927d60de7f163f5723b8f1dfa7d2a84298738cd", size = 3351466, upload-time = "2026-04-20T23:35:43.993Z" }, - { url = "https://files.pythonhosted.org/packages/20/be/b732c8418ffa5bcfda002890f5dc4c869fc17db66ff11f53b17cfe44afc0/psycopg2_binary-2.9.12-cp314-cp314-win_amd64.whl", hash = "sha256:f12ae41fcafadb39b2785e64a40f9db05d6de2ac114077457e0e7c597f3af980", size = 2848762, upload-time = "2026-04-20T23:35:46.421Z" }, +name = "psycopg" +version = "3.3.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/26/3ea4ca5eaea1c0debcdf7ee7c1613fbe721dc27a03c461c0817ffd8a0601/psycopg-3.3.6.tar.gz", hash = "sha256:c081f2250df751a943036e42db6df4571c66cd0aabe8291a7a506512b12007d2", size = 168171, upload-time = "2026-09-18T13:22:55.152Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/de/748bd7609c71cae5d737f0ba9192f19329f70180ecda8fff3cac02c5abe3/psycopg-3.3.6-py3-none-any.whl", hash = "sha256:a1db9f7148b06a28606767efaca51fa6f9398c5c0a3810519be69d7000bdb631", size = 215490, upload-time = "2026-09-18T13:15:29.374Z" }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, +] +pool = [ + { name = "psycopg-pool" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.3.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/92/00350a66de0af05e41d01aa3134e3970045e816afed3f99d58ec1abe15b2/psycopg_binary-3.3.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7beb3e41c9a1e509f3ed85263386588cbe3e975aa67be21f79f44fd35ffaeefc", size = 4728682, upload-time = "2026-09-18T13:15:36.605Z" }, + { url = "https://files.pythonhosted.org/packages/91/fc/afa9c7fd316a469af7ede6ebb020eac482f5d827fae57d5310c9bc0c41ae/psycopg_binary-3.3.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:aa73160077345ec21b3f51e8e24b3de2e99586217e497629326eb9b2ea88c52e", size = 4777111, upload-time = "2026-09-18T13:15:46.566Z" }, + { url = "https://files.pythonhosted.org/packages/f2/44/7c1e015f1bc56b36ff1369f09e852b2d83ccefd5a669a42633a916cdedc4/psycopg_binary-3.3.6-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:f87dbdc42e78ee0f7ea180c03f8c78e80a949e373066629bd90fefff10552dff", size = 5592755, upload-time = "2026-09-18T13:15:52.886Z" }, + { url = "https://files.pythonhosted.org/packages/3b/ae/314a251ca918cdac380bce1b87839ade9355382ea749e6ef3ba75ba0c09f/psycopg_binary-3.3.6-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a9348c5b43a3bb5ef8c2e89d5237c9c87eeafb01d338c84a7aebbc5cd0313299", size = 5267949, upload-time = "2026-09-18T13:16:00.53Z" }, + { url = "https://files.pythonhosted.org/packages/b6/9f/3bb0cfe9bb0f31ca57cf486ddc8c9ac51251aed8181bf88ff870b2623105/psycopg_binary-3.3.6-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0a52991594ac4db888c7d39bccef331797e30cb31a95cae02cf2607f83a42dc2", size = 6862403, upload-time = "2026-09-18T13:16:10.385Z" }, + { url = "https://files.pythonhosted.org/packages/c4/d6/7032c10309c3155e9b24300fdcc9a1afa539cfd20ce52fdef74a46f10161/psycopg_binary-3.3.6-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5ea8beeb5541780b4b50b462eeacbc4f594ce3b911dc20c81c75f267876f71d2", size = 5107271, upload-time = "2026-09-18T13:16:16.843Z" }, + { url = "https://files.pythonhosted.org/packages/61/cc/79add2cf92684cf1a81da134b32caa662c25c72d0cc905d181ef4455f834/psycopg_binary-3.3.6-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:198a48e68cc99ccac03ba95ac857e73aa66f3bf6be77019fafb0832a05f7ad03", size = 4627984, upload-time = "2026-09-18T13:16:23.889Z" }, + { url = "https://files.pythonhosted.org/packages/c9/48/6dfb14f9350c14af6a2edb3c31262051b8cd94e2186e4b831e46dbbe8cd9/psycopg_binary-3.3.6-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:fa34eb47969297471db7b7f193622c7e3ee839ec05abd05f1fe104d5b1b1dcf4", size = 4322054, upload-time = "2026-09-18T13:16:29.33Z" }, + { url = "https://files.pythonhosted.org/packages/29/35/2982338716a91cbb4dfc866be015be4457ee8106a445aabf3d1fb6a270e0/psycopg_binary-3.3.6-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:b979a42815410432420275412633960807178b1ce26591a16ce06e78a5bd4bb2", size = 4048471, upload-time = "2026-09-18T13:16:34.119Z" }, + { url = "https://files.pythonhosted.org/packages/24/e1/171b1db1542c5f76a678b7ee0a7800bebc9735a0a03417c76cf948bfd63c/psycopg_binary-3.3.6-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:889e42acec10450185e0cdfb396f375e2c1a8d7737c114830a7fde4654f59e30", size = 4353931, upload-time = "2026-09-18T13:16:38.692Z" }, + { url = "https://files.pythonhosted.org/packages/08/89/4424e62a944eef40bd9326ada4ae23802b28eab6502af91e84ef7bba74fb/psycopg_binary-3.3.6-cp310-cp310-win_amd64.whl", hash = "sha256:cbd5f73073ed19c378d4c35499db1e3e703a5b1a324e521204065967bfaa7a18", size = 3674362, upload-time = "2026-09-18T13:16:44.454Z" }, + { url = "https://files.pythonhosted.org/packages/70/86/b71166048974d49c6d136b2ed1c0e5bec0b974d8c4de5cbce7e86a9e412a/psycopg_binary-3.3.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:be4f9b3c9338ac5dd217c5847e21521b396c8117f78dc420d495a5c49bbef874", size = 4728002, upload-time = "2026-09-18T13:16:53.393Z" }, + { url = "https://files.pythonhosted.org/packages/12/1d/1e06c0de7ed5aed898acb87544eac6ef0bc7d752a67ec6e5d6b835e9b40c/psycopg_binary-3.3.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f0535693ce476a722b718b002d5d2c27d47e71ca945276ac194409c98e74c492", size = 4775403, upload-time = "2026-09-18T13:16:58.939Z" }, + { url = "https://files.pythonhosted.org/packages/84/02/2ffcbc43f8e4bbc38e5286a22013bcac01898d13cd38325f60dd5428a8af/psycopg_binary-3.3.6-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:3c9e663b2e800e3218994cf948c11bcc2844e6491b34aa80d089baf6531827bf", size = 5594145, upload-time = "2026-09-18T13:17:08.515Z" }, + { url = "https://files.pythonhosted.org/packages/e1/25/031dae2c7d2e7e77dcf5b1962c1e0684fa548d7af0ff6707b6b5e6054ca7/psycopg_binary-3.3.6-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a2e44a342d2aee40508e28a563d8961c39d9bbd8cae36d8578f0a3c6658aab0f", size = 5267985, upload-time = "2026-09-18T13:17:16.24Z" }, + { url = "https://files.pythonhosted.org/packages/8c/e5/94c89ada3c003a4d858178f3bba49a35e0297ef2aad659b80eb5e380e690/psycopg_binary-3.3.6-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f598f19fa9a91540b5cee17932ffd227b7b53a481605bcc4573c0eafa647300", size = 6860986, upload-time = "2026-09-18T13:17:23.348Z" }, + { url = "https://files.pythonhosted.org/packages/9d/a0/81bf499d095adee8413bd19822a6872fbfa21663ec78014a68d83a8db83c/psycopg_binary-3.3.6-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6ff05561e4a067d35507dc5c90f1deb2ec1c9703ac5cccc1bc26e08a197f9c5a", size = 5106702, upload-time = "2026-09-18T13:17:28.847Z" }, + { url = "https://files.pythonhosted.org/packages/00/75/99d56da64c27bd985fd82c6ecbf7976b724ac638fdd1654ef995323a1a26/psycopg_binary-3.3.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:566dd827f17728efdf7d88a5b066f815170f6fdad13967ae952842d90e6aaa9f", size = 4627090, upload-time = "2026-09-18T13:17:36.668Z" }, + { url = "https://files.pythonhosted.org/packages/3e/0c/0222171d11233332c6a24b1cef1578215f0ffddf3642eb8dd8c4448ad69f/psycopg_binary-3.3.6-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:9b2f11794e017ce340934e35de46181c46ef71ec75ea3d85dd75cd836761c01e", size = 4320353, upload-time = "2026-09-18T13:17:42.526Z" }, + { url = "https://files.pythonhosted.org/packages/62/6f/e1cc2a28dd1228c67c969ba6fd37cd8726b312e2ff51380f847ddb38ccde/psycopg_binary-3.3.6-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:910ace140e3e7b7596898d083f37a8fe90c5c40684252ad4e682364b2cd3deba", size = 4047253, upload-time = "2026-09-18T13:17:47.068Z" }, + { url = "https://files.pythonhosted.org/packages/d8/fd/38b64790ce7a515b1dbd2bab3d119637a858aeb22c380cf4859bc4ce0e42/psycopg_binary-3.3.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37e517c146b185f9c0c6e8d0a0ebbdeeeb67896af28466e032bc810d0c7dc7a7", size = 4353208, upload-time = "2026-09-18T13:17:52.41Z" }, + { url = "https://files.pythonhosted.org/packages/f7/dc/45386530ceb2a8c789a226de9b9b34eca8fccf1feba2e4ef68a6aca50c56/psycopg_binary-3.3.6-cp311-cp311-win_amd64.whl", hash = "sha256:c7f92daa0d2a1c76f07264abddf8cbabd30152a2f09c3270e50f0c7efdf5dcac", size = 3675638, upload-time = "2026-09-18T13:17:58.112Z" }, + { url = "https://files.pythonhosted.org/packages/e6/01/2cdd1824e58b4467ee0b9498664cd28c42d8794db6b1e35b6bcb834f0044/psycopg_binary-3.3.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:3f84dab25e0385692ee13274c68678377e0b1a70ab9d14e56264cbf61f60c62d", size = 4707086, upload-time = "2026-09-18T13:18:05.138Z" }, + { url = "https://files.pythonhosted.org/packages/f6/76/de9948ac06895261c84d5b9fbe283d8f3c5bc9f070691b8d9eaa1b51e322/psycopg_binary-3.3.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:612382ac3ed13651c7fa44b5fee9fbf7baaa2ddbc6f500391672682c5f1df9e0", size = 4769607, upload-time = "2026-09-18T13:18:12.83Z" }, + { url = "https://files.pythonhosted.org/packages/76/a9/72436c9915ee4905964689e7f0e182ce7767cc0a0390b3ce703be8177625/psycopg_binary-3.3.6-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:366db6e97e66b37211475f20c4c1324a2dc0dd825e46d4e87f9d599304d276f9", size = 5554134, upload-time = "2026-09-18T13:18:21.175Z" }, + { url = "https://files.pythonhosted.org/packages/0a/42/948bb3d2617795093512613fd96ba380e922992c7908fbc073858147d196/psycopg_binary-3.3.6-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1679a1cb93fbe5a6d1fd58d82cbddcc6fcb8c61446ba7cae6eb2a7b19bc585de", size = 5235723, upload-time = "2026-09-18T13:18:27.071Z" }, + { url = "https://files.pythonhosted.org/packages/99/47/93e823ff1b0088400703410939c9bda3e63ed9c850b3ee088e8769f4c10b/psycopg_binary-3.3.6-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:37d40450659401600e6d043ff586c89a71a69f33cbb8bcdba6cdb2569beecdbe", size = 6833587, upload-time = "2026-09-18T13:18:33.794Z" }, + { url = "https://files.pythonhosted.org/packages/5e/2d/ecc69c847795aa704041a9f5667a6b0938a088cf1853636d762a6938e493/psycopg_binary-3.3.6-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a5165300324efd5a772c48a88ab3a928513ab3979fca76553e62ee815f7b2b9c", size = 5070013, upload-time = "2026-09-18T13:18:39.628Z" }, + { url = "https://files.pythonhosted.org/packages/92/36/6126f0dac21713dcae91404f2a76da18598a6252339a8c669c46370d43b2/psycopg_binary-3.3.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d636338c8f21b0df2f84657b00bc34f9313f826ef93f1155bc743607e4a0c5eb", size = 4597367, upload-time = "2026-09-18T13:18:45.023Z" }, + { url = "https://files.pythonhosted.org/packages/4d/29/7ecfc04243b46c89ffd49924e9c5634ea904ef96c7d0f37e4073623584c1/psycopg_binary-3.3.6-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:a4ee3bdd5468a725f2a4d9aab8a74b6d0279f768c8b5d3aeb102c5307ff3d59c", size = 4275419, upload-time = "2026-09-18T13:18:49.299Z" }, + { url = "https://files.pythonhosted.org/packages/6e/90/2f46d2e0de79706ac170df0a3637fe63c4498fc04f131f6049520b78b806/psycopg_binary-3.3.6-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:289aadd6a00e151203c081f708348ec89f1e483c9b510ef4ac3981f847f01f79", size = 4007358, upload-time = "2026-09-18T13:18:53.944Z" }, + { url = "https://files.pythonhosted.org/packages/03/48/6744e91291b751a8cf12d63d719977974bb94c84ceba913e7ddb2e478e51/psycopg_binary-3.3.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f21d057f3e5f5491067e5b292498073b73847d48799b099803fef100775fcc52", size = 4320156, upload-time = "2026-09-18T13:18:59.258Z" }, + { url = "https://files.pythonhosted.org/packages/1a/9b/94ff7fce53a64d5b286e2ec454e0a025cf3d6e6b4a9189bef16aa5de98b2/psycopg_binary-3.3.6-cp312-cp312-win_amd64.whl", hash = "sha256:e23a66a763fbe83fcc210bc77c27e5a5ea380ebf091c06f34d8561b695e5a40f", size = 3658864, upload-time = "2026-09-18T13:19:06.503Z" }, + { url = "https://files.pythonhosted.org/packages/b4/c3/c072584b69ad44a747b448cfc9766fecb8aae56e372a017e2ef668790057/psycopg_binary-3.3.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ad8f35e67cc16d1fad1fa8c88972dc9b3a3141ea67897399904edab96a301b6", size = 4712284, upload-time = "2026-09-18T13:19:13.451Z" }, + { url = "https://files.pythonhosted.org/packages/0a/b9/4283b785339e8e2318d03048994b093d650ea6289fabaa806b765dc0d449/psycopg_binary-3.3.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:373704aea331d3f3e3402c125a1543f5875e2986ebb54f97d1647942161f803f", size = 4772031, upload-time = "2026-09-18T13:19:18.524Z" }, + { url = "https://files.pythonhosted.org/packages/6f/72/7a1321d359246769fff1affffbd0132785a28f7f63c18524c15a502398f4/psycopg_binary-3.3.6-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b82491019b884d62318b5f30706c3d7e6d4e5a6cb7eabcb3edc0c1b0fdaceae9", size = 5556392, upload-time = "2026-09-18T13:19:24.418Z" }, + { url = "https://files.pythonhosted.org/packages/de/b0/c6f8a0585a5dacbea74e130bcfc66629390e8f5bbc79d2a8e806e8952150/psycopg_binary-3.3.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cec5ea900390897d0b46130f60bc2883bf19c314f9044235217c8be88b0ef269", size = 5237855, upload-time = "2026-09-18T13:19:31.257Z" }, + { url = "https://files.pythonhosted.org/packages/e2/fc/c3a7a8bbef7e945ec584ac61d460a612363ea398511cd0e220242b1d69f1/psycopg_binary-3.3.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98c02090d88f2ebc0ec1e8da538f77d225ce0fffecf372aa39262e62a1b054ef", size = 6833856, upload-time = "2026-09-18T13:19:43.622Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f2/8e80b921db728ebb68fc105bd7c4277f908210ad755bd6481d5ea7add740/psycopg_binary-3.3.6-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ee2c4728c691245e24501fcd7a97b5b381236b9985bc445bba88cdce7d1b5784", size = 5070730, upload-time = "2026-09-18T13:19:49.968Z" }, + { url = "https://files.pythonhosted.org/packages/54/6a/5b313e0c5348244f0e973aff3258bf86766656256d5ece8d541a53e35b4a/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f19cc87343eaa55255e76b31259a570072ac95d6ae82c92dd34b97691f5e49dc", size = 4598089, upload-time = "2026-09-18T13:19:56.426Z" }, + { url = "https://files.pythonhosted.org/packages/32/e9/db7f76ec24bf6699e92bf604e5c4bae10664a681a8999ef42aa0faf0f2c6/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fdccb3a0e184b03e9baa673b15a809cf36c339c85dbda0ebc25a698846dfbee8", size = 4278481, upload-time = "2026-09-18T13:20:04.681Z" }, + { url = "https://files.pythonhosted.org/packages/61/83/72c67013656f4d6b547caabffb193e91d57e63f90eefdcc6d045c400e97d/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:9892188bb15e5803beb51afe8a25add6b56be391a53058e8bca03b74e1e6bf22", size = 4009229, upload-time = "2026-09-18T13:20:11.905Z" }, + { url = "https://files.pythonhosted.org/packages/82/35/5e4500df2c999eb0faed8b184e6958b834172128274f06167a5deef4c19c/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3af90f92769d8cc10f94515ee7a0aef36ea85ca733a0ce22858f6e0953f41138", size = 4321467, upload-time = "2026-09-18T13:20:17.949Z" }, + { url = "https://files.pythonhosted.org/packages/55/7f/e350e1cf498ba2565c3f87b12f429d2012eb86b76c2b3845a19ee5fbb4d6/psycopg_binary-3.3.6-cp313-cp313-win_amd64.whl", hash = "sha256:0ebfad5d131de9f892ae9e70cc7616207768b6714b66a52d4612b8ceaf78b372", size = 3658179, upload-time = "2026-09-18T13:20:22.691Z" }, + { url = "https://files.pythonhosted.org/packages/6d/b9/60711317c284a442511644ea7185b56ebe627606d6741e732cd16108c47b/psycopg_binary-3.3.6-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:b3f75dee0f9afafabe4edc52c4842f1e1878ed2069bd05b22d6fe961e97e4dba", size = 4720512, upload-time = "2026-09-18T13:20:29.278Z" }, + { url = "https://files.pythonhosted.org/packages/63/da/28befc84454cbc6374550de7746f591f8fe1b6165c1fce249652cc8291c4/psycopg_binary-3.3.6-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:5927b7ba63153cd8e9862987290a2b783a5c590daf2a4ef981700cc3569166d4", size = 4782318, upload-time = "2026-09-18T13:20:35.401Z" }, + { url = "https://files.pythonhosted.org/packages/a4/8a/0d21c2c833cdc0d4244c77e858e0ed37fa2abec2623be4fd686f617109ce/psycopg_binary-3.3.6-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:0bf08b749cc144f33b44a91b78e3f71c60eb07963746a0df5a100b36ce3d7475", size = 5567460, upload-time = "2026-09-18T13:20:41.902Z" }, + { url = "https://files.pythonhosted.org/packages/49/6d/7692d0d4e656b6cc9868d8acc2e3b42f17a0db4a625400a6d093cb0533a1/psycopg_binary-3.3.6-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:31cd942c23f613276b81a6e6598cefa12960058b0f46e1e874b540c793f6aca5", size = 5246902, upload-time = "2026-09-18T13:20:47.661Z" }, + { url = "https://files.pythonhosted.org/packages/d4/c1/b8a1f18fb1b7558a17f57f7cb3fc8bc93189feea2958925950b3acb15743/psycopg_binary-3.3.6-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4690cf67738f0e0e49a32aeec99bf0e4595cc2b4f1af984a4345394b1dcff91a", size = 6847192, upload-time = "2026-09-18T13:20:56.874Z" }, + { url = "https://files.pythonhosted.org/packages/a5/76/404f33519167c65cca88ec4998776f1dbebccc301ee977f0e62c47fb0826/psycopg_binary-3.3.6-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ad1c785e784cfd87e8436c6b7702f2d321fc39601bbaf29bc63a41a867091638", size = 5079573, upload-time = "2026-09-18T13:21:04.155Z" }, + { url = "https://files.pythonhosted.org/packages/f0/d9/79e8fbc8f37262a415f3550f0bcc5f98037442bf3d12ef6cbae2056655ae/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:79a2a1c3449f6c3409427078ed1cec10de79f3023cb5f2504f0597d350ad46c7", size = 4613633, upload-time = "2026-09-18T13:21:10.664Z" }, + { url = "https://files.pythonhosted.org/packages/d4/47/96225db74be7d2ce04b3a58678b53cda610225055edf5faa775c9f501d8b/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:86147cb5d140341c3363fb5bacce31f8d5543902a46699d3c536b101bbceaf9e", size = 4293375, upload-time = "2026-09-18T13:21:16.027Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d2/18e9c779a5efd565250329adaf529ecc2b8b2ed5be5cb0f6ccee208cbfd9/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:7308c93cf0b19bbaf8e6ff0a6ad50d3c442385739245fe15a8d593bf841734a6", size = 4019883, upload-time = "2026-09-18T13:21:21.587Z" }, + { url = "https://files.pythonhosted.org/packages/ef/28/0cc654afc6c2cda982767f5679d3646b30b1ec86545bdaa9402202d6776c/psycopg_binary-3.3.6-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:05a83ac9fd52b9bca7cb5ab04b3691163170bd16f53defa27216ea3aa07ee781", size = 4332607, upload-time = "2026-09-18T13:21:27.63Z" }, + { url = "https://files.pythonhosted.org/packages/f1/3e/0a753a74fbd7aef120f286c016e09d3cc3f1daf7688f4a145d27281260b2/psycopg_binary-3.3.6-cp314-cp314-win_amd64.whl", hash = "sha256:1fbd30e537dab22cafdf080608f10148fe2a5f3a61294ddb5113caac8a623840", size = 3755671, upload-time = "2026-09-18T13:21:33.855Z" }, + { url = "https://files.pythonhosted.org/packages/0e/b1/a372b9c02aea50148e71c9853e19efca8fa5ae2010a8e27243b9b8f790c0/psycopg_binary-3.3.6-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:bf8c8481d026b85dd70c5fa7dde85b2333aed0b32a2602bcd38a900cbd78a49c", size = 4719571, upload-time = "2026-09-18T13:21:41.437Z" }, + { url = "https://files.pythonhosted.org/packages/65/7c/811e3828c6b82e2f10c6c9cdd963cfc66f3e024026e5a69ac18530bad984/psycopg_binary-3.3.6-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:b599defe9190b17e9907c8b4d114c181e702c87efcd1b8a0ad40971cdcc4634a", size = 4781230, upload-time = "2026-09-18T13:21:49.516Z" }, + { url = "https://files.pythonhosted.org/packages/3e/15/9a784eed813ea9e97c294af3ead63d02b7b203502c66380336c50065e441/psycopg_binary-3.3.6-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b8ece331509f7a975b90501f41e83ad905e4141753fedf3f2711b2bc70a8efbc", size = 5566111, upload-time = "2026-09-18T13:21:58.089Z" }, + { url = "https://files.pythonhosted.org/packages/68/16/47194e002007c27337b11e49bf459c4b19727463f9aff2e1a90917bcc806/psycopg_binary-3.3.6-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c61617eaae0112ca154da87ffb99b73af2c74067acac28dfb9a4455b019dff2e", size = 5249963, upload-time = "2026-09-18T13:22:06.695Z" }, + { url = "https://files.pythonhosted.org/packages/53/84/5dcf9f310b11f0675cd860c6b2c70f58ce61798a3ee3f6f962b53fa358ca/psycopg_binary-3.3.6-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c6d19cb4999d03231e8730a5f66c8f5068bc3b532677eb39dab0f600bff3e312", size = 6847925, upload-time = "2026-09-18T13:22:13.088Z" }, + { url = "https://files.pythonhosted.org/packages/f3/06/1957a06dc22963c418c27b284929579de84f29c37ad1abe6dc6ee9e8cf25/psycopg_binary-3.3.6-cp315-cp315-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8cbb54454dbf1bbf2ff08dd7693e8d94ac94b1a20f70f4b3b813d52ecb5cbc1", size = 5087720, upload-time = "2026-09-18T13:22:17.959Z" }, + { url = "https://files.pythonhosted.org/packages/21/43/ac07d042bae99b57bf123bb473632f29af544008094da0ffd285ab8011e2/psycopg_binary-3.3.6-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:dc75da5a20951049f7b773145f998f69d181adad9c58a0ff36e0cf1d73c10e10", size = 4613412, upload-time = "2026-09-18T13:22:26.719Z" }, + { url = "https://files.pythonhosted.org/packages/aa/b1/019156fbeafcefb4cccc9d109de4699493bceb8313c7545c8349e089dfbc/psycopg_binary-3.3.6-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:955e3dd94da361e052d2e49acf591017158dc8f8ed2c8a42c2e3943403c39dc2", size = 4292618, upload-time = "2026-09-18T13:22:33.042Z" }, + { url = "https://files.pythonhosted.org/packages/5d/0f/62113dc6b1df65983a1f2fc816c04b1edfa22f2ae9d4abee74ed267f4a96/psycopg_binary-3.3.6-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:c7753871eb57e6a5f4646f6168590c6653073dea5e9e720b201c8875332df4c8", size = 4027121, upload-time = "2026-09-18T13:22:38.334Z" }, + { url = "https://files.pythonhosted.org/packages/5d/d5/cf0cbd1ea5a7d8167fe2c6953efde19101f7b193bd61a23e6d622ad6854c/psycopg_binary-3.3.6-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:303732e798fe6729f8e12021b9c96107df8e95ecec4dd487c67b98ec2a59435e", size = 4336388, upload-time = "2026-09-18T13:22:45.576Z" }, + { url = "https://files.pythonhosted.org/packages/98/33/e2a5b36edf8aa422f6fa4b894756eb33dc93b36df5f65121280bb8b929c4/psycopg_binary-3.3.6-cp315-cp315-win_amd64.whl", hash = "sha256:2f122603f36050937982abf9668d8bc4769a79f7c93a65013b1c49f1cab7b56b", size = 3756154, upload-time = "2026-09-18T13:22:51.283Z" }, +] + +[[package]] +name = "psycopg-pool" +version = "3.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/74/5e/c0664b968b102ff68b811d999c728546c48d5c1eec03e3bbaf88c0cb4472/psycopg_pool-3.3.3.tar.gz", hash = "sha256:df87b5d9d0ad7db37f6cdad4fa8ce113d250f5997f6db38e9a99192fb67f9e1d", size = 32006, upload-time = "2026-09-22T15:53:24.947Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/b4/452c6607a0f479465cd8a9b0d9956919fcb150050c1f83f9f11e6b8ee8dc/psycopg_pool-3.3.3-py3-none-any.whl", hash = "sha256:9b9cd6a4fcec47a410f7e82d408540e7f77b478509e91b44c1a5457a13e5ff37", size = 40304, upload-time = "2026-09-22T15:53:23.712Z" }, ] [[package]] @@ -2334,7 +2382,7 @@ name = "python-dateutil" version = "2.9.0.post0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "six" }, + { name = "six", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } wheels = [ @@ -2468,18 +2516,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fc/3c/480c61cc8d5a3e76bb44e86231f408c93643e5498beadbbeb381ab55d02b/qdrant_client-1.19.0-py3-none-any.whl", hash = "sha256:13602a2b3478a95ecdf42f97b93d7f703b63a3361cd912a04495a33a5ac14121", size = 396157, upload-time = "2026-08-04T14:32:55.734Z" }, ] -[[package]] -name = "redis" -version = "8.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "async-timeout", marker = "python_full_version < '3.11.3'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a8/99/604f0b666d4c616d891cf77ebb9db6bb21601344c051aebf1b72b9ff915f/redis-8.1.0.tar.gz", hash = "sha256:6e1a19beef9225c83efd689c7e6b7da2d5215b1f42cd13b7fc3714d0a09c7b25", size = 5254356, upload-time = "2026-07-30T08:51:00.269Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/66/9d/c5731f6e3608663d4d3656fd8d3aecee8b509c3082818f5a13eae925baea/redis-8.1.0-py3-none-any.whl", hash = "sha256:a4fe1aac3d3b3cc791d4b3d5931c5a956045dc951ee74d1c913ee3ac4d2ee9fb", size = 560618, upload-time = "2026-07-30T08:50:58.497Z" }, -] - [[package]] name = "regex" version = "2026.7.19" @@ -2686,10 +2722,10 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform == 'darwin'", ] dependencies = [ - { name = "joblib" }, - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, - { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" } }, - { name = "threadpoolctl" }, + { name = "joblib", marker = "python_full_version < '3.11'" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "scipy", version = "1.15.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "threadpoolctl", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/98/c2/a7855e41c9d285dfe86dc50b250978105dce513d6e459ea66a6aeb0e1e0c/scikit_learn-1.7.2.tar.gz", hash = "sha256:20e9e49ecd130598f1ca38a1d85090e1a600147b9c02fa6f15d69cb53d968fda", size = 7193136, upload-time = "2025-09-09T08:21:29.075Z" } wheels = [ @@ -2740,13 +2776,13 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform == 'darwin'", ] dependencies = [ - { name = "joblib" }, - { name = "narwhals" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "joblib", marker = "python_full_version >= '3.11'" }, + { name = "narwhals", marker = "python_full_version >= '3.11'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "scipy", version = "1.17.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, { name = "scipy", version = "1.18.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, - { name = "threadpoolctl" }, + { name = "threadpoolctl", marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fa/6f/37092bdb25f712817231799fc5674d8e704066a8a70c1d2d40517e18b4ab/scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557", size = 7750767, upload-time = "2026-06-02T11:54:32.706Z" } wheels = [ @@ -2791,7 +2827,7 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform == 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0f/37/6964b830433e654ec7485e45a00fc9a27cf868d622838f6b6d9c5ec0d532/scipy-1.15.3.tar.gz", hash = "sha256:eae3cf522bc7df64b42cad3925c876e1b0b6c35c1337c93e12c0f366f55b0eaf", size = 59419214, upload-time = "2025-05-08T16:13:05.955Z" } wheels = [ @@ -2851,7 +2887,7 @@ resolution-markers = [ "python_full_version == '3.11.*' and sys_platform == 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.11.*'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -2930,7 +2966,7 @@ resolution-markers = [ "python_full_version == '3.12.*' and sys_platform == 'darwin'", ] dependencies = [ - { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.5.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7e/74/66de6258867beb2ef08f35f9f2ac017a52cacd5081714d239ff1a442d458/scipy-1.18.1.tar.gz", hash = "sha256:52c4b7422442aba924d03ad4019852b08a92e64ea187b933135687bfe2747307", size = 30781235, upload-time = "2026-08-21T23:28:50.599Z" } wheels = [ @@ -3123,6 +3159,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d7/c1/eb8f9debc45d3b7918a32ab756658a0904732f75e555402972246b0b8e71/tenacity-9.1.4-py3-none-any.whl", hash = "sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55", size = 28926, upload-time = "2026-02-07T10:45:32.24Z" }, ] +[[package]] +name = "testcontainers" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docker" }, + { name = "python-dotenv" }, + { name = "typing-extensions" }, + { name = "urllib3" }, + { name = "wrapt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4b/13/2cc466bddf26d0085f30a2b2bd56b7f8708b54a54db833eec97c5c69129b/testcontainers-4.15.0.tar.gz", hash = "sha256:085cde086337632e19002719460b7b80bbab2bdd51bb3ea04f77d0de96504706", size = 95340, upload-time = "2026-07-24T23:08:01.731Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/7e/424aac8b355597835deb333e757a0e94b5ccf38ad00f07fe6ed1f4e17c88/testcontainers-4.15.0-py3-none-any.whl", hash = "sha256:8796c14e76604031ad39cf0ed3b8e9806283a1fbf5270965c2b1c594caa31b74", size = 160771, upload-time = "2026-07-24T23:08:00.13Z" }, +] + [[package]] name = "threadpoolctl" version = "3.6.0" @@ -3296,14 +3348,14 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform == 'darwin'", ] dependencies = [ - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "setuptools" }, - { name = "sympy" }, - { name = "typing-extensions" }, + { name = "filelock", marker = "sys_platform == 'darwin'" }, + { name = "fsspec", marker = "sys_platform == 'darwin'" }, + { name = "jinja2", marker = "sys_platform == 'darwin'" }, + { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform == 'darwin'" }, + { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and sys_platform == 'darwin'" }, + { name = "setuptools", marker = "sys_platform == 'darwin'" }, + { name = "sympy", marker = "sys_platform == 'darwin'" }, + { name = "typing-extensions", marker = "sys_platform == 'darwin'" }, ] wheels = [ { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.13.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:94f0de129916f77b8dc2c7a8eff644cfeddfe59e39c9f55e9f6e17543410281d", upload-time = "2026-07-08T12:26:07Z" }, @@ -3326,14 +3378,14 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'darwin'", ] dependencies = [ - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, - { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "setuptools" }, - { name = "sympy" }, - { name = "typing-extensions" }, + { name = "filelock", marker = "sys_platform != 'darwin'" }, + { name = "fsspec", marker = "sys_platform != 'darwin'" }, + { name = "jinja2", marker = "sys_platform != 'darwin'" }, + { name = "networkx", version = "3.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' and sys_platform != 'darwin'" }, + { name = "networkx", version = "3.6.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' and sys_platform != 'darwin'" }, + { name = "setuptools", marker = "sys_platform != 'darwin'" }, + { name = "sympy", marker = "sys_platform != 'darwin'" }, + { name = "typing-extensions", marker = "sys_platform != 'darwin'" }, ] wheels = [ { url = "https://download-r2.pytorch.org/whl/cpu/torch-2.13.0%2Bcpu-cp310-cp310-linux_s390x.whl", hash = "sha256:0555fde6108ca90247ae33d4e1237cbae475c86a223bb2f0f91d9addf1f611bd", upload-time = "2026-07-08T19:27:11Z" }, @@ -3443,6 +3495,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, ] +[[package]] +name = "tzdata" +version = "2026.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/68/f1b440335057bfce71b6e50a9d09445aa2ecbd08359a337976627b8409e7/tzdata-2026.5.tar.gz", hash = "sha256:8cc73c0a0bfca7dbfa59235d60b2eff82231dee33f53d206db1acd9173cfc0a7", size = 200404, upload-time = "2026-10-03T09:23:14.143Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/21/1e5995a1c920cce14e4bffae20c665ec10e7ed03ab25e006cd741092b718/tzdata-2026.5-py2.py3-none-any.whl", hash = "sha256:b683bd1b6659ddcd810ff02ad09ba821d4bf1065072805063eb35c49617905ac", size = 347996, upload-time = "2026-10-03T09:23:12.535Z" }, +] + [[package]] name = "urllib3" version = "2.7.0" From 0145b1f51b4f253e1fba22630bad475807e9dd98 Mon Sep 17 00:00:00 2001 From: Naveen Date: Mon, 26 Oct 2026 13:18:48 +0530 Subject: [PATCH 29/53] feat(db): add Postgres schema, migration runner and schema docs 0001_initial_schema.sql creates users, catalog_entries, documents, workflows, workflow_artifacts, pages, chunks, audit_logs, search_settings, email_otps and webhook_records, plus the document_current_workflow and catalog views. pipeline/migrate.py applies versioned SQL files under an advisory lock. docs/POSTGRES_SCHEMA.md describes the schema; docs/SQLITE_SCHEMA.md is kept as the legacy reference. --- docs/POSTGRES_SCHEMA.md | 553 +++++++++++++++++++++++++++++ docs/SQLITE_SCHEMA.md | 374 +++++++++++++++++++ migrations/0001_initial_schema.sql | 292 +++++++++++++++ pipeline/migrate.py | 76 ++++ 4 files changed, 1295 insertions(+) create mode 100644 docs/POSTGRES_SCHEMA.md create mode 100644 docs/SQLITE_SCHEMA.md create mode 100644 migrations/0001_initial_schema.sql create mode 100644 pipeline/migrate.py diff --git a/docs/POSTGRES_SCHEMA.md b/docs/POSTGRES_SCHEMA.md new file mode 100644 index 0000000..d76307a --- /dev/null +++ b/docs/POSTGRES_SCHEMA.md @@ -0,0 +1,553 @@ +# Postgres Schema — knowledge-provider + +Postgres schema behind `pipeline/db.py`. A document is content and metadata. A workflow is one run of processing against a document. Source file and vector index state are stored as JSONB on the document. Translations are stored as JSONB on each page. + +The schema is created by `migrations/0001_initial_schema.sql` and applied by the api and worker at startup (`pipeline/migrate.py`). `pipeline/db.py` is the only code that reads or writes it. The previous SQLite store is described in `docs/SQLITE_SCHEMA.md` (legacy) and is moved over once by `scripts/migrate_sqlite_to_postgres.py`. + +## Conventions + +- Every table has a primary key `id`, a UUID v4 (`DEFAULT gen_random_uuid()`, built in from Postgres 13). +- A foreign key is named `_id` (for example `document_id`). A role prefix is added when the role matters (`uploaded_by_user_id`, `started_by_user_id`). +- Text identifiers other systems rely on (`catalog_entries.code`, `temporal_workflow_id`) are ordinary columns, not keys. +- `created_at` and `updated_at` are the last two columns of a table. `updated_at` is maintained by one shared trigger. Append-only or short-lived tables skip `updated_at`, as noted per table. +- Timestamps are `TIMESTAMPTZ` (UTC), booleans are `BOOLEAN`, JSON is `JSONB`, lists of strings are `TEXT[]`. +- JSONB columns have a `CHECK` that the value is an object (or array). Value rules inside them live in Pydantic models, not in the database. +- Fixed value sets are enums. Open sets (`artifact_type`, `action_type`) stay `TEXT`. +- Counts have `CHECK >= 0`. Slugs and identifiers have pattern `CHECK`s. + +### Shared trigger + +```sql +CREATE FUNCTION set_updated_at() RETURNS trigger AS $$ +BEGIN + NEW.updated_at := now(); + RETURN NEW; +END $$ LANGUAGE plpgsql; +-- one per table that has updated_at: +-- CREATE TRIGGER trg_
- + diff --git a/ui/src/views/GlobalAuditView.jsx b/ui/src/views/GlobalAuditView.jsx index 82bd6c5..a3aed6c 100644 --- a/ui/src/views/GlobalAuditView.jsx +++ b/ui/src/views/GlobalAuditView.jsx @@ -120,10 +120,10 @@ export default function GlobalAuditView() { onClick={() => navigate(`/documents/${entry.workflow_id}`)} title={entry.document_id || entry.workflow_id} > - {entry.display_name || entry.filename || entry.document_id || entry.workflow_id} + {entry.name || entry.filename || entry.document_id || entry.workflow_id} - {entry.instance && ( - {entry.instance} + {entry.group && ( + {entry.group} )} {/* What actually changed, in words — the raw before/after stays diff --git a/ui/src/views/NewDocumentView.jsx b/ui/src/views/NewDocumentView.jsx index 728a19e..f8e9325 100644 --- a/ui/src/views/NewDocumentView.jsx +++ b/ui/src/views/NewDocumentView.jsx @@ -22,12 +22,17 @@ import { import { API_BASE } from '../config' import { apiFetch } from '../auth/keycloak' import { useAuth } from '../auth/AuthProvider' -import { defaultUploadInstance, PORTAL_INSTANCE } from '../lib/instanceLabels' +import { defaultUploadGroup, PORTAL_GROUP } from '../lib/groupLabels' import { fetchJson, formatCompactDateTime, getDocumentListLabel, summarizeIngestStatus } from '../lib/pipelineUi' const SUPPORTED_TYPES = ['.pdf', '.doc', '.docx'] const MAX_SIZE_MB = 100 const FILE_TYPE_CHIPS = ['PDF', 'Word'] +// The catalog a document joins is (your group, type), so the type is chosen up front. +const DOCUMENT_TYPE_OPTIONS = [ + { value: 'scheme', label: 'Scheme' }, + { value: 'advisory', label: 'Advisory' }, +] const PIPELINE_STEPS = [ { id: 'ocr', label: 'OCR', icon: ScanText }, @@ -53,13 +58,14 @@ export default function NewDocumentView() { const navigate = useNavigate() const { hasPermission, instances, isSuperAdmin, user } = useAuth() const canUpload = hasPermission('upload') - // Instance from login (Keycloak) — not user-selected on this form. - const documentInstance = defaultUploadInstance({ + // Group from login (Keycloak) — not user-selected on this form. + const documentGroup = defaultUploadGroup({ isSuperAdmin, instances, - portalInstance: user?.portal_instance || PORTAL_INSTANCE, + portalGroup: user?.portal_group || PORTAL_GROUP, }) const [file, setFile] = useState(null) + const [documentKind, setDocumentKind] = useState('') const [dragging, setDragging] = useState(false) const [uploading, setUploading] = useState(false) const [uploadSuccess, setUploadSuccess] = useState(false) @@ -120,8 +126,8 @@ export default function NewDocumentView() { } async function handleSubmit() { - if (!file || !canUpload) return - if (!documentInstance && !isSuperAdmin) { + if (!file || !documentKind || !canUpload) return + if (!documentGroup && !isSuperAdmin) { setUploadError( 'No state on your account. Ask an admin to assign a Keycloak group (e.g. /states/MH/admin).', ) @@ -133,8 +139,8 @@ export default function NewDocumentView() { const formData = new FormData() formData.append('file', file) // State users: Keycloak group state only. Super-admin: portal BV when unset. - const uploadInstance = documentInstance || (isSuperAdmin ? PORTAL_INSTANCE : '') - if (!uploadInstance) { + const uploadGroup = documentGroup || (isSuperAdmin ? PORTAL_GROUP : '') + if (!uploadGroup) { setUploadError( 'No state on your account. Ask an admin to assign a Keycloak group (e.g. /states/MH/admin).', ) @@ -142,7 +148,8 @@ export default function NewDocumentView() { } const params = new URLSearchParams({ auto_approve: 'false', - instance: uploadInstance, + group: uploadGroup, + kind: documentKind, }) const response = await apiFetch(`${API_BASE}/upload?${params}`, { method: 'POST', body: formData }) const data = await response.json() @@ -161,6 +168,7 @@ export default function NewDocumentView() { setFile(null) setUploadSuccess(false) setLastWorkflowId('') + setDocumentKind('') setUploadError('') setValidationError('') } @@ -222,6 +230,26 @@ export default function NewDocumentView() {
{/* Left column */}
+
+ + + Required +
{uploading ? ( <> diff --git a/ui/src/views/QueueView.jsx b/ui/src/views/QueueView.jsx index 83119eb..8a5fe0d 100644 --- a/ui/src/views/QueueView.jsx +++ b/ui/src/views/QueueView.jsx @@ -204,12 +204,12 @@ export default function QueueView() {
{canMutateQueue ? ( - toggleSelect(item.workflow_id)} aria-label={`Select ${item.display_name || item.filename || item.workflow_id}`} /> + toggleSelect(item.workflow_id)} aria-label={`Select ${item.name || item.filename || item.workflow_id}`} /> ) : null} navigate(`/documents/${item.workflow_id}`)}> - {item.display_name || item.filename} + {item.name || item.filename}
{item.workflow_id}
_updated_at BEFORE UPDATE ON
+-- FOR EACH ROW EXECUTE FUNCTION set_updated_at(); +``` + +### Enums + +| Enum | Values | +|---|---| +| `workflow_stage` | `registered`, `ocr_processing`, `ocr_review`, `translation_processing`, `translation_review`, `chunking`, `chunk_review`, `ready_for_ingestion`, `ingesting`, `publishing_to_network`, `approval_for_prod`, `ingesting_prod`, `completed`, `failed` | +| `workflow_type_enum` | `pipeline`, `ocr_only`, `ocr_retry`, `translation_retry`, `chunking_retry`, `reingest`, `promote_prod` | +| `workflow_status_enum` | `running`, `waiting_review`, `completed`, `failed` | +| `audit_entity_type` | `document`, `page`, `chunk`, `setting`, `catalog` | + +Adding a value later needs `ALTER TYPE ... ADD VALUE`. + +## Tables + +| # | Table | Purpose | +|---|---|---| +| 1 | `users` | People who upload, review and act, mirrored from Keycloak | +| 2 | `catalog_entries` | Generic named entries that documents belong to. Type `scheme` today, `advisory` and others later | +| 3 | `documents` | One row per document. Content, metadata, source file, index state | +| 4 | `workflows` | One row per processing run of a document. Stage, status, config | +| 5 | `workflow_artifacts` | Files each workflow produces, stored in MinIO | +| 6 | `pages` | OCR layer and translations of each page | +| 7 | `chunks` | Chunked text and review state | +| 8 | `audit_logs` | Append-only action history | +| 9 | `search_settings` | Single-row search configuration | +| 10 | `email_otps` | Email OTP login codes | +| 11 | `webhook_records` | Records pushed to `POST /webhook` | +| — | `schema_migrations` | Applied migration files (created by the runner, not by a migration) | +| — | `catalog` (view) | Computed catalog of all entries. `/catalog/v1/*` serves the `scheme` type | +| — | `document_current_workflow` (view) | Latest workflow of each document | + +Relationships: + +``` +catalog_entries 1 ── * documents 1 ── * workflows 1 ── * workflow_artifacts + │ + ├── * pages + └── * chunks +``` + +Dependency order for creation: `users`, `catalog_entries`, `documents`, `workflows`, `workflow_artifacts`, `pages`, `chunks`, `audit_logs`, `search_settings`, `email_otps`, `webhook_records`, views. + +--- + +## 1. `users` + +Uploader, reviewer and actor identity, looked up once from the Keycloak token. + +Unique: `keycloak_user_id`; unique index on `lower(email)` where `email IS NOT NULL`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Internal user ID. | +| keycloak_user_id | TEXT | NOT NULL UNIQUE | Keycloak `sub` claim. `local-admin` when auth is disabled (not a UUID, so it cannot be the PK). | +| username | TEXT | | Current username. | +| email | TEXT | | Current email. | +| roles | TEXT[] | NOT NULL DEFAULT '{}' | Current roles. | +| first_seen_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | First login or upload. | +| last_seen_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Most recent activity. | + +Notes +- Upserted on each authenticated upload or login. A `local-admin` row is seeded for `AUTH_DISABLED=true`. +- Point-in-time identity is kept in `audit_logs.actor`. + +--- + +## 2. `catalog_entries` + +Generic named entries that documents belong to. One table for every entry type: `scheme` today, `advisory` and others later. A new type adds rows, not tables. + +Unique: `(type, code)` and `(id, type)` (the second is the target of the composite FK from `documents`). + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Entry ID. | +| type | TEXT | NOT NULL, CHECK `^[a-z0-9]+([_-][a-z0-9]+)*$` | Entry type (`scheme`, later `advisory`, and so on). Open set, so not an enum. | +| code | TEXT | NOT NULL, CHECK `^[a-z0-9]+(-[a-z0-9]+)*$` | Short code (`pm-kmy`) that catalog consumers see. Unique within a type. | +| name | TEXT | NOT NULL | Display name, shared by all documents of the entry. | +| aliases | TEXT[] | NOT NULL DEFAULT '{}', CHECK no empty string | Ordered alternate names used to match queries. The first one is the prompt hint. | +| metadata | JSONB | NOT NULL DEFAULT '{"source": "pipeline"}', CHECK object | `source` and any type-specific fields. See keys below. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Created. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). | + +### `metadata` keys + +| Key | Type | Default | Purpose | +|---|---|---|---| +| source | string | `pipeline` | `bootstrap` (seeded from code), `pipeline` (built from documents) or `import`. | + +Type-specific fields are added as keys, with no migration. Pydantic validates the keys per type. + +Notes +- Computed catalog values (`status`, `instances`, `workflow_ids`, `chunk_count`, `network_visible`, `promoted_at`) come from the `catalog` view, not stored here. +- Case-insensitive alias uniqueness is enforced in the app (`_dedupe_aliases`). Add a GIN index on `aliases` only if "find entry by alias" is needed in SQL. +- No join table: an entry has many documents and a document has one entry, which `documents.catalog_entry_id` expresses. +- Plain documents (kind `document`) have no entry. + +--- + +## 3. `documents` + +One row per document. 13 columns. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Document ID. | +| instance | TEXT | NOT NULL | Tenant or state code. | +| source | JSONB | NOT NULL, CHECK `jsonb_typeof(source) = 'object'`, CHECK `source->>'fingerprint' ~ '^[0-9a-f]{32}$'` | The uploaded file. See shape below. | +| document_kind | TEXT | NOT NULL DEFAULT 'document', CHECK `^[a-z0-9]+([_-][a-z0-9]+)*$` | Knowledge kind (`document`, `scheme`, `advisory`, `video`, or custom). | +| catalog_entry_id | UUID | | Catalog entry (for example a scheme) the document belongs to. Nullable. Composite FK, see below. | +| valid_from | DATE | | Start of the period chunks are searchable. | +| valid_to | DATE | CHECK `valid_from <= valid_to` | End of the period. NULL means no end. | +| is_disabled | BOOLEAN | NOT NULL DEFAULT FALSE | Soft delete. Restore flips it back. | +| index_status | JSONB | NOT NULL DEFAULT '{}', CHECK `jsonb_typeof(index_status) = 'object'` | Vector index state per index. Keys `dev` and `prod`. | +| metadata | JSONB | NOT NULL, DEFAULT see below, CHECK object, CHECK `(metadata->>'chunk_count')::int >= 0` | Display and flag fields. See keys below. | +| uploaded_by_user_id | UUID | FK → `users(id)` | Who uploaded. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Row created. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). | + +Composite FK `(catalog_entry_id, document_kind)` → `catalog_entries(id, type)`, `ON UPDATE CASCADE ON DELETE RESTRICT`. It guarantees a document's kind equals its entry's type. When `catalog_entry_id` is NULL the FK is not checked, so plain documents need no entry. + +Suggested indexes: +- `(instance)` +- `(created_at DESC)` +- `(catalog_entry_id) WHERE catalog_entry_id IS NOT NULL` +- `(instance, (source->>'fingerprint'))` for duplicate checks and identifier lookups +- `(((metadata->>'is_demo')::boolean))` for the demo filter on every list + +### `source` + +Facts about the uploaded file, not about processing. + +```json +{ + "fingerprint": "", + "original_filename": "pmkisan.pdf", + "storage_uri": "s3://...", + "source_type": "document", + "canonical_input_type": "pdf", + "page_count": null +} +``` + +| Key | Type | Notes | +|---|---|---| +| fingerprint | string | MD5, 32 lowercase hex. Enforced by CHECK. Also the key the document has inside the vector index. | +| original_filename | string | Name as uploaded. | +| storage_uri | string | Location of the uploaded file. | +| source_type | string | `spreadsheet`, `document` or `unknown`. | +| canonical_input_type | string | `spreadsheet` or `pdf`. Normalized pipeline path. | +| page_count | integer or null | NULL until OCR runs. | + +### `index_status` + +One entry per vector index. DEV and PROD are separate Qdrant collections. + +```json +{ + "dev": {"status": "indexed", "chunk_count_indexed": 42, + "last_indexed_at": "2026-10-19T06:20:00Z", "last_verified_at": "2026-10-19T06:25:00Z", + "schema_version": "passage-v1", "details": {}}, + "prod": {"...": "..."} +} +``` + +- A missing key means the document was never ingested to that index. +- `status` is `unknown`, `indexed` or `missing`. +- The document's key inside the vector index is `source.fingerprint`. +- The key is `prod` for the PROD collection names (`PROD_VECTOR_DB_COLLECTION_NAME`, `PROD_SCHEME_VECTOR_DB_COLLECTION_NAME`) and `dev` for every other collection. The collection name is kept inside the entry as `index_name`, next to `doc_id`, the id the index uses for the document. +- Update one index atomically: `SET index_status = index_status || jsonb_build_object('dev', $1::jsonb)`. +- A third index is a new key, with no migration. + +### `metadata` + +| Key | Type | Default | Purpose | +|---|---|---|---| +| display_name | string | absent | Human-friendly title. | +| catalog_visible | boolean | true | Counts toward the catalog. | +| network_visible | boolean | true | May be published to the Beckn network. | +| is_demo | boolean | false | Demo content, hidden by default. | +| chunk_count | integer ≥ 0 | 0 | Cached number of chunks. Updated in the same transaction as chunk writes. | +| reindex_required | boolean | false | Approved content changed and the vector index is stale. | +| reindex_reason | string | absent | Why a reindex is needed. | +| chunking | object `{run_number, provider, model, config, created_at}` | absent | Settings of the run that produced the current chunks. | + +Column default: `'{"catalog_visible": true, "network_visible": true, "is_demo": false, "chunk_count": 0, "reindex_required": false}'`. + +### Notes + +- A document's stage is the `current_stage` of its latest workflow. List and filter queries join the `document_current_workflow` view. +- SQL does not check types inside the JSONB columns. The insert path writes every default `metadata` key, and Pydantic validates `source`, `index_status` and `metadata`. +- `metadata.chunk_count` is rewritten in every chunk transaction and `metadata.is_demo` is filtered on every list. If performance suffers, move these two back to columns first. +- Each document keeps its own `source`. Two uploads of the same content from different paths keep their own `storage_uri`. There is no unique key on `(instance, fingerprint)`; the upload path reuses a document by its Temporal workflow id instead. Add `UNIQUE (instance, (source->>'fingerprint'))` only if one document per file per tenant must be enforced. +- Identifier lookup (`find_document_by_doc_identifier`) matches `documents.id`, `workflows.temporal_workflow_id`, `source->>'fingerprint'` and `source->>'original_filename'`. The legacy `md5(identifier)` check is `md5(source->>'fingerprint') = $1` in SQL. +- No `deleted_at`: soft delete is `is_disabled`, hard delete is `purge_document`. + +--- + +## 4. `workflows` + +One row per processing run of a document: the main pipeline run, retries, reingest and PROD promotion. Holds all processing state. + +Indexes: `(document_id, started_at DESC)`, `(status, started_at DESC)`, `(temporal_workflow_id)`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Workflow ID. | +| document_id | UUID | NOT NULL, FK → `documents(id)` ON DELETE CASCADE | Document the workflow runs against. | +| workflow_type | workflow_type_enum | NOT NULL | Kind of run. | +| temporal_workflow_id | TEXT | NOT NULL | Temporal workflow ID. `doc-` for the main run, suffixed for retries. Not unique (a restart reuses the ID). Used by the API, UI and Temporal. | +| status | workflow_status_enum | NOT NULL DEFAULT 'running' | `running`, `waiting_review`, `completed` or `failed`. | +| current_stage | workflow_stage | NOT NULL DEFAULT 'registered' | Stage the workflow is in now. This is the document's stage when the workflow is the latest. | +| stage_timestamps | JSONB | NOT NULL DEFAULT '{}', CHECK object with only keys `ocr`, `translation`, `chunks` | When each stage first completed in this run, ISO-8601 UTC. Set-once per key. | +| stop_after_ocr | BOOLEAN | NOT NULL DEFAULT FALSE | Halt after OCR (OCR-only batch runs). | +| error_message | TEXT | | Failure reason when status is `failed`. | +| config | JSONB | NOT NULL DEFAULT '{}', CHECK object | Settings the run used, plus `chunking_progress` while chunking. | +| started_by_user_id | UUID | FK → `users(id)` ON DELETE SET NULL | Who started it. For `promote_prod`, the person who requested promotion. NULL for system-started runs. | +| started_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | When the workflow started. Plays the role of `created_at`. | +| completed_at | TIMESTAMPTZ | | When it completed or failed. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). Progress updates touch the row often. | + +13 columns. + +Notes +- `stage_timestamps` is set-once: `stage_timestamps = jsonb_build_object('ocr', $1) || stage_timestamps` (right side wins, so existing keys are kept). +- Progress updates are atomic: `UPDATE ... SET config = config || jsonb_build_object('chunking_progress', ...)`. +- A PROD request inserts a `promote_prod` workflow in `waiting_review`, with `started_by_user_id` set to the requester and `started_at` to the request time. Approval moves it to `running`; when approval starts a `PromoteToProdWorkflow`, that run takes over the waiting row instead of adding a second one. +- The current workflow of a document is the latest by `started_at`. A retry started while an older run is still `running` takes over the stage display, and so does a retry on a `completed` document: its stage goes back to the retry's stage until the retry finishes. +- No CHECK ties `completed_at` to status: `reconcile_materialized_state` sets `waiting_review` without clearing `completed_at`. +- `cancelled` is not a status because nothing writes it. +- Do not copy the uploader snapshot into `config`; `started_by_user_id` and `audit_logs` cover it. + +### View: `document_current_workflow` + +Latest workflow per document. Stores nothing. + +```sql +SELECT DISTINCT ON (document_id) * +FROM workflows +ORDER BY document_id, started_at DESC; +``` + +--- + +## 5. `workflow_artifacts` + +Ledger of every file a workflow produces or stores. Files live in MinIO; this table records where and what. Append-only. + +Indexes: `(workflow_id, created_at DESC)`, `(workflow_id, artifact_type, created_at DESC)`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Artifact ID, used in `/documents/{id}/artifacts/{artifact_id}`. | +| workflow_id | UUID | NOT NULL, FK → `workflows(id)` ON DELETE CASCADE | Workflow that produced it. | +| artifact_type | TEXT | NOT NULL, CHECK not blank | Kind of output (`original_upload`, `normalized_pdf`, `normalized_spreadsheet`, `ocr_pages_json`, `translation_pages_json`, `chunk_json_export`, `vector_payload_export`, network publish exchange). Open set, so not an enum. | +| stage | workflow_stage | | Stage that created it. | +| storage_uri | TEXT | NOT NULL | MinIO location of the blob. | +| mime_type | TEXT | | MIME type. | +| filename | TEXT | | File name of the blob. | +| size_bytes | BIGINT | CHECK ≥ 0 | Blob size. | +| metadata | JSONB | NOT NULL DEFAULT '{}', CHECK object | Chunking config for chunk exports, request and response for network publishes, source path for uploads. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | When the artifact was recorded. | + +10 columns. + +Notes +- Every artifact belongs to a workflow. The upload path creates the first `pipeline` workflow before recording `original_upload`. +- The document is reached through `workflows.document_id`. Per-document listing: `SELECT a.* FROM workflow_artifacts a JOIN workflows w ON w.id = a.workflow_id WHERE w.document_id = $1 ORDER BY a.created_at DESC`. +- The latest original or normalized artifact uses the same join with `artifact_type = 'original_upload'` (or `normalized_%`). +- Deleting a workflow deletes its artifact rows. Blobs in MinIO are not removed by the database; cleanup stays in `purge_document`. +- No `updated_at`: rows are append-only. +- Chunking run history lives in `chunk_json_export` metadata. + +--- + +## 6. `pages` + +OCR layer and translations of a page. + +Unique: `(document_id, page_number)`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Page ID. | +| document_id | UUID | NOT NULL, FK → `documents(id)` ON DELETE CASCADE | Owning document. | +| page_number | INTEGER | NOT NULL, CHECK ≥ 1 | 1-based page position. | +| original_markdown | TEXT | | Raw OCR output, kept unchanged. | +| edited_markdown | TEXT | | Reviewer-corrected OCR text. Used instead of the original when present. | +| translations | JSONB | NOT NULL DEFAULT '{}', CHECK `jsonb_typeof(translations) = 'object'` | Translations of this page, keyed by target language. See shape below. | +| metadata | JSONB | NOT NULL DEFAULT '{"is_reviewed": false}', CHECK object | OCR review and language fields. See keys below. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | When the page row was first created. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). | + +### `metadata` keys + +| Key | Type | Default | Purpose | +|---|---|---|---| +| is_reviewed | boolean | false | OCR review done. | +| reviewer_notes | string | absent | Reviewer comments on the OCR text. | +| detected_language | string | absent | Language code detected. Non-`en` means the page needs translation. | + +SQL does not check these types; Pydantic does. The review stage gate counts reviewed pages with `(metadata->>'is_reviewed')::boolean`, and the translation step finds pages to translate with `metadata->>'detected_language'`. Add expression indexes on both if those queries are slow: `((metadata->>'is_reviewed')::boolean)` and `((metadata->>'detected_language'))`. + +### `translations` + +```json +{ + "gu": { + "translated_markdown": "...", + "edited_translation": null, + "is_reviewed": false, + "reviewer_notes": null, + "provider": "azure", + "model": "gpt-5.4-mini", + "translated_at": "2026-10-19T06:14:00Z" + } +} +``` + +- Key is the target language. Several languages per page are possible. +- Pages that need no translation (English) keep `'{}'`. An entry exists only for pages actually translated. +- `edited_translation` takes precedence over `translated_markdown` when present. +- Writes target one path, for example `SET translations = jsonb_set(translations, '{gu,edited_translation}', to_jsonb($1::text))`. +- Re-running OCR on a page resets its `translations` to `'{}'`, since the page text changed. +- Counting translated pages is `WHERE translations <> '{}'`. Finding unreviewed translations needs `jsonb_each` over the small per-page value. + +Notes +- `save_pages` replaces the whole row, so it must pass `translations` through unchanged unless the translation activity is writing it. Otherwise an OCR save would wipe translations. +- `save_pages` is `INSERT ... ON CONFLICT (document_id, page_number) DO UPDATE SET `. It is called during OCR segments, after OCR, after translation and at persist time, and each call replaces the whole row, so the caller passes every field. +- `save_pages` also passes `metadata` through whole. Update single keys with `metadata = metadata || jsonb_build_object('is_reviewed', true)` instead of rewriting the object. +- Page edits are audited in `audit_logs` (`page_edit`, `page_reset`), so no edit-history columns. +- Pages belong to a document, not to the source file: edits, review flags and OCR retries are per document. + +--- + +## 7. `chunks` + +Unique: `(document_id, chunk_number)`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Chunk ID. | +| document_id | UUID | NOT NULL, FK → `documents(id)` ON DELETE CASCADE | Owning document. | +| chunk_number | INTEGER | NOT NULL, CHECK ≥ 1 | Order within the document. | +| original_text | TEXT | NOT NULL | Chunk text as produced by the chunker. | +| edited_text | TEXT | | Reviewer-edited text. Indexed when present. | +| token_count | INTEGER | NOT NULL DEFAULT 0, CHECK ≥ 0 | Length in tokens. | +| page_start | INTEGER | NOT NULL DEFAULT 1, CHECK ≥ 1 | First page covered. Kept for display and provenance links. | +| page_end | INTEGER | NOT NULL DEFAULT 1, CHECK ≥ `page_start` | Last page covered. | +| source_spans | JSONB | NOT NULL DEFAULT '[]', CHECK array | List of `{page_number, start_char, end_char}` pointing back into the page text. | +| section_title | TEXT | | Heading of the section. | +| content_type | TEXT | | Kind of content (`heading`, `body`, ...). Only those two seen in the chunkers, so left as `TEXT`. | +| is_reference | BOOLEAN | NOT NULL DEFAULT FALSE | Reference material. Search can exclude it. | +| is_reviewed | BOOLEAN | NOT NULL DEFAULT FALSE | Reviewed by a human. | +| is_excluded | BOOLEAN | NOT NULL DEFAULT FALSE | Excluded from indexing. | +| reviewer_notes | TEXT | | Reviewer comments. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Created. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). | + +Notes +- Chunking provider, model, config and run number are stored once per document in `documents.metadata.chunking`, not per chunk. +- The distinct page numbers a chunk draws from are derived from `source_spans`. +- Re-chunk runs in one transaction: delete the document's chunks, insert the new ones, update `metadata.chunking` (with `run_number` incremented) and `metadata.chunk_count`. Chunk UUIDs change on every re-chunk; nothing references `chunks.id` (audit uses chunk number). +- `search_chunks` does a substring match on `lower(coalesce(edited_text, original_text))`, served by a trigram GIN index on that expression (the migration creates the `pg_trgm` extension). + +> **Why `pages` and `chunks` stay separate tables:** a page is a physical unit from OCR and a chunk is a semantic unit from the chunker, so their boundaries do not align. One chunk can span several pages and one page can hold several chunks, which makes the relationship many-to-many. Storing page text inside each chunk would copy a page into every chunk it overlaps, so any page edit, OCR retry or translation change would have to be rewritten in all of those chunks. + + +--- + +## 8. `audit_logs` + +Append-only history of actions. Must outlive documents, so there is no cascade delete. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Audit entry ID. | +| document_id | UUID | FK → `documents(id)` ON DELETE SET NULL | Document the action applied to. NULL for system-level entries and after a purge. | +| workflow_id | TEXT | | Temporal workflow ID snapshot. Survives a purge. | +| instance | TEXT | | Tenant snapshot, so tenant scoping works after the document is gone. | +| action_type | TEXT | NOT NULL, CHECK not blank | What happened (`page_edit`, `purge_document`, ...). 22 values and growing, so not an enum. | +| entity_type | audit_entity_type | | `document`, `page`, `chunk`, `setting` or `catalog`. | +| entity_number | INTEGER | | Page number or chunk number the action applied to. | +| field_name | TEXT | | Field that changed. | +| old_value | TEXT | | Value before the change. | +| new_value | TEXT | | Value after the change. | +| metadata | JSONB | NOT NULL DEFAULT '{}', CHECK object | Extra context. | +| actor_user_id | UUID | FK → `users(id)` ON DELETE SET NULL | Who acted. NULL for system actions. | +| actor | JSONB | NOT NULL DEFAULT '{}', CHECK object | Identity at the time: `{username, email, roles[]}`. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | When it happened. | + +Table CHECK: `entity_type IN ('setting','catalog') OR workflow_id IS NOT NULL`. + +Indexes: `(document_id, created_at DESC)`, `(created_at DESC)`, `(action_type, created_at DESC)`, `(created_at DESC) WHERE entity_type = 'setting'`. + +Notes +- The FK to `documents` is `SET NULL`, not cascade. A cascade would erase the audit trail on purge, and a strict FK would reject setting and catalog entries that have no document. +- Setting and catalog entries have NULL `document_id` and `workflow_id`, with `entity_type` set to `setting` or `catalog`. +- `instance` is a snapshot, so a purged document's history stays visible only to its own tenant. +- `entity_number` holds a page or chunk number, not an ID (chunk UUIDs would dangle after a re-chunk). +- `get_settings_audit_logs` is `WHERE entity_type = 'setting'`. +- `log_audit` fills `instance`, resolves the actor to a `users` row and builds the `actor` JSON. +- No `updated_at`: entries are immutable. +- The table only grows. Monthly partitioning is possible later without changing this design. + +--- + +## 9. `search_settings` + +Single row of search configuration. + +Unique: `singleton`. + +| Column | Type | Constraints / Default | Keys inside | Purpose | +|---|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | | Row ID. | +| singleton | BOOLEAN | NOT NULL DEFAULT TRUE UNIQUE, CHECK `singleton` | | Guarantees only one row. | +| retrieval | JSONB | NOT NULL DEFAULT '{}', CHECK object | `searchMethod`, `alpha`, `rankingMethod`, `hybridRrfK` | How semantic and keyword matching are combined and ranked. | +| candidates | JSONB | NOT NULL DEFAULT '{}', CHECK object | `indexName`, `efSearch`, `candidateCap`, `candidateMultiplier` | Which vector index is searched and how many candidates are pulled. | +| results | JSONB | NOT NULL DEFAULT '{}', CHECK object | `limit`, `maxChunksPerDoc`, `excludeReference`, `showHighlights`, `rerankMode` | How the final result list is shaped. | +| query | JSONB | NOT NULL DEFAULT '{}', CHECK object | `useE5Prefix`, `queryExpansionProfile` | How the query is prepared before searching. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | | Created. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | | Last change (trigger). | + +15 settings in four groups (4 + 4 + 5 + 2). All are global, with no per-tenant scope. + +Notes +- The database enforces only "is a JSON object". Value rules live in Pydantic: `SearchSettings` has four nested sub-models (`alpha` 0 to 1, integers ≥ 1, `searchMethod` in `TENSOR|LEXICAL|HYBRID`, `rankingMethod` in `rrf|normalize_linear`, unknown keys rejected). +- The API stays flat. `db.py` keeps one dict mapping each flat key to its group; reads flatten the columns and writes route each key to its column. +- Defaults live only in the Pydantic models. Reads merge stored JSON over the defaults. The default for `alpha` is 0.6. +- Partial update is an atomic merge per group: `UPDATE search_settings SET retrieval = retrieval || $1::jsonb`. +- Audit: one `audit_logs` row per changed key, `entity_type = 'setting'`, `field_name` such as `retrieval.alpha`. +- `rerankMode` has at least `none` and `bm25lite`; the full set was not seen, so no enum. +- Adding a setting to an existing group needs no migration; a new group needs a column. + +--- + +## 10. `email_otps` + +Server-side email-OTP login codes. Short-lived. + +Unique: `email`; index on `(expires_at)`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Row ID. | +| email | TEXT | NOT NULL UNIQUE, CHECK `email = lower(email)` | Address the code was requested for. One live code per email. | +| code_hash | TEXT | NOT NULL | Hash of the one-time code. The plain code is never stored. Random for unknown accounts. | +| expires_at | TIMESTAMPTZ | NOT NULL | When the code stops being valid. | +| attempts | INTEGER | NOT NULL DEFAULT 0, CHECK ≥ 0 | Failed verifications. Reset to 0 on resend. | +| last_sent_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | When the code was last sent. Used for the resend cooldown. | + +Notes +- No `created_at` or `updated_at`: rows are overwritten on each send. +- `email` is deliberately not an FK to `users`. A row is written for every request, even for unknown addresses (anti-enumeration, `email_otp.py:276-310`). +- Periodic cleanup (`DELETE FROM email_otps WHERE expires_at < now() - interval '1 day'`) is not scheduled yet. The endpoint is unauthenticated, so unverified rows stay until it is. +- `verify_otp` increments attempts atomically: `UPDATE ... SET attempts = attempts + 1 WHERE email = $1 RETURNING attempts`, or `SELECT ... FOR UPDATE`. +- Code constants (length 6, TTL 300 s, resend cooldown 30 s, max 5 attempts) stay in code. +- Optional: make the table `UNLOGGED`. Losing it in a crash only forces users to request a new code. + +--- + +## View: `catalog` + +Computed from `catalog_entries`, `documents`, `workflows` and `document_current_workflow`. Stores nothing. The `/catalog/v1/*` snapshot API reads it with `type = 'scheme'`, and its fields are named `scheme_code`, `scheme_name` and so on, so consumers see no change. + +| Column | Type | How it is computed | +|---|---|---| +| entry_id | UUID | From `catalog_entries`. | +| type | TEXT | From `catalog_entries`. | +| code | TEXT | From `catalog_entries`. | +| name | TEXT | From `catalog_entries`. | +| aliases | TEXT[] | From `catalog_entries`. | +| source | TEXT | `catalog_entries.metadata->>'source'`. | +| status | TEXT | `pending_reindex` if a live document needs reindex. Else `live` if any live document exists. Else `pending_prod` if a document awaits PROD approval. Else `live` for a bootstrap entry with no documents. Else `disabled`. | +| instances | TEXT[] | Distinct `instance` of the entry's documents, or `{default}` if none. | +| workflow_ids | TEXT[] | `temporal_workflow_id` of each entry document's main `pipeline` workflow. | +| chunk_count | INTEGER | Sum of `metadata->>'chunk_count'` over the entry's documents. | +| network_visible | BOOLEAN | True if any of its documents has `metadata->>'network_visible'` true. For an entry with no documents, true if bootstrap. | +| promoted_at | TIMESTAMPTZ | Latest `index_status->'prod'->>'last_indexed_at'` of the live documents. | + +"Live document" means the latest workflow's `current_stage = 'completed'`, `is_disabled` false, and `metadata->>'catalog_visible'` true. The entry's `type` already ties it to the document's kind through the composite FK. + +--- + +## 11. `webhook_records` + +Records pushed to `POST /webhook` (`x-api-key` = `WEBHOOK_API_KEY`). Carried over from the previous store unchanged apart from the types. + +Unique: `(source, external_id)`; index on `(source, status)`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Record ID. | +| source | TEXT | NOT NULL DEFAULT 'webhook' | Sender. | +| external_id | TEXT | NOT NULL | `contentID`, else `id`, else `slug`, else a hash of the payload. | +| title | TEXT | | From the payload. | +| slug | TEXT | | From the payload. | +| status | TEXT | | From the payload. | +| is_published | BOOLEAN | | From `isPublished`. | +| category | TEXT | | From `domainName`. | +| payload | JSONB | NOT NULL, CHECK object | The full record. | +| payload_hash | TEXT | | SHA-256 of the canonical payload; an upsert with the same hash writes nothing. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | The payload's `createdAt` when it has one. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | The payload's `updatedAt` when it has one. | + +The payload's own timestamps are stored as given, so these two columns are not maintained by the shared trigger. + +--- + +## `schema_migrations` + +Created by the runner, not by a migration file: `version` (the `NNNN` prefix, primary key), `name` (the file name) and `applied_at`. The runner takes a session advisory lock, so concurrent api and worker starts apply each file once, and runs each file in its own transaction. + +--- + +## Implementation notes + +- **Identifiers.** `db.py` functions take a `workflow_id` argument that may be the document UUID, a Temporal workflow id, the fingerprint, the filename, an index `doc_id` or the legacy `md5(fingerprint)` doc id (`find_document_by_doc_identifier`). Documents, jobs and artifacts come back as flat dicts; `id` is the UUID and `workflow_id` is the Temporal workflow id of the document's main run (the earliest `pipeline` or `ocr_only` workflow). The API rewrites `/documents/{id}/...` to that workflow id before routing, because handlers signal and query Temporal with it. +- **Jobs.** The API's "jobs" and "runs" are `workflows` rows; job ids and artifact ids are the UUIDs of `workflows` and `workflow_artifacts`. `latest_job_id`, `original_artifact_id` and `normalized_artifact_id` are derived by join, not stored. +- **Catalog version.** `/catalog/v1/version` counts `audit_logs` rows with `entity_type = 'catalog'` and `action_type = 'catalog_version_bump'`, so it only grows. `updated_at` is the newest such row. +- **Document stage.** The stage a document shows is `document_current_workflow.current_stage`; `update_document_stage` writes to that workflow. +- **Page translations.** `save_pages` keeps stored translations unless the page carries `translations` (replaces them) or flat `translated_*` fields (merged into `translation_target_language`). A page whose OCR text changed loses its translations. +- **Users.** Rows are upserted from the auth dependency on write requests (at most once per five minutes per user) and from uploads, workflow starts and audit entries. `local-admin` is seeded by the migration. diff --git a/docs/SQLITE_SCHEMA.md b/docs/SQLITE_SCHEMA.md new file mode 100644 index 0000000..c3fa539 --- /dev/null +++ b/docs/SQLITE_SCHEMA.md @@ -0,0 +1,374 @@ +# SQLite Schema — knowledge-provider (legacy) + +> Legacy. The store is now Postgres, described in `docs/POSTGRES_SCHEMA.md`. This file documents the shape of the old SQLite file that `scripts/migrate_sqlite_to_postgres.py` reads. + +## Overview + +- One SQLite file is the canonical store for metadata, review state and editable content. MinIO holds blobs, Qdrant holds search projections, Temporal only orchestrates. +- 12 tables. `documents` is the root. `pages`, `chunks`, `document_jobs`, `document_artifacts`, `document_index_status` and `audit_logs` hang off it by `workflow_id`. +- Connection settings: WAL journal mode, `foreign_keys=ON`, `IMMEDIATE` transactions, 30s lock timeout. Writes are serialized by a process-local lock. +- Schema is created on startup. Changes are applied as idempotent `ALTER TABLE ... ADD COLUMN` calls, so the tables below show the final shape with migration-added columns merged in. +- Timestamps are ISO-8601 strings in TEXT. Booleans are INTEGER 0/1. Columns ending in `_json` hold JSON-encoded TEXT. +- `workflow_id` is the key used by the API, UI and Temporal. + +## Drawbacks + +### Integrity +- **No foreign keys.** Tables are linked by convention on `workflow_id`. Nothing stops orphan rows, and purging a document means deleting from each table by hand. +- **Pointer columns are unenforced.** `latest_job_id`, `original_artifact_id` and `normalized_artifact_id` on `documents` point into tables that point back, with no constraint. +- **No constraints on values.** Stages, statuses, job types, routing and flags are free TEXT or INTEGER. `created_at` and `updated_at` are nullable. Nothing checks that `valid_from <= valid_to` or that counts are non-negative. +- **Audit rows use sentinel strings** (`__system__`, `__settings__`, `__catalog__`) in key columns for non-document events, which a real foreign key could not hold. +- **Settings are untyped.** Every value is TEXT and cast on read. Nothing stops an out-of-range `alpha` or an invalid search method. Defaults disagree across three places (`alpha` is 0.7 in the model, 0.6 in the seed and reset). + +### Data modeling +- **Same fact stored in several places.** + - Uploader username, email and roles are copied onto each document and again into the audit log. + - Scheme name, aliases and routing are copied onto every document of a scheme. + - The scheme-to-document relationship is stored twice: a JSON list on the catalog entry and `scheme_code` on the document. + - The file's MD5 is stored as `document_id`, `canonical_document_id` and `source_file_fingerprint`; `filename` equals `source_filename`. +- **`documents` is too wide.** About 45 columns mix file facts, processing state, uploader identity, PROD request, validity and scheme data. +- **JSON in TEXT.** Lists and objects (`scheme_aliases_json`, `workflow_ids_json`, `source_page_numbers_json`, `config_json`, `details_json`) cannot be validated or queried. +- **Run metadata repeated on every chunk.** Chunking provider, model, config, run ID and version are identical for all chunks of a run. +- **Translation columns sit on `pages`.** Eight mostly-NULL columns for pages that need no translation, and only one target language per page. +- **Roles are comma-separated strings**, with no array semantics. +- **Dead or constant columns.** `marqo_doc_id`, `temporal_run_id`, `content_hash`, `collection_name` and `tool_routing` carry no used data. The manifest table has no writer in the code. +- **Hardcoded language.** The manifest table has a Gujarati title column and the search settings default to a Gujarati query profile. +- **`entity_id` in the audit log is not an ID.** It holds a page or chunk number. +- **Catalog table is a cache** that is fully recomputed from documents and can drift. + +## Tables + +| # | Table | Purpose | +|---|-------|---------| +| 1 | `documents` | One row per ingested document. Root table, holds stage and metadata. | +| 2 | `pages` | OCR and translated page content plus review state. | +| 3 | `chunks` | Chunked text plus review state. | +| 4 | `audit_logs` | Per-document change and action history. | +| 5 | `settings` | Key/value app config, including search defaults. | +| 6 | `document_jobs` | Temporal job runs per document. | +| 7 | `document_artifacts` | Pointers to blobs stored in MinIO. | +| 8 | `document_index_status` | Vector index state per document and index. | +| 9 | `document_manifest_entries` | Imported manifest (CSV) metadata. | +| 10 | `scheme_catalog_meta` | Single-row catalog version counter. | +| 11 | `scheme_catalog_entries` | Scheme master catalog (served by `/catalog/v1/*`). | +| 12 | `email_otps` | Email OTP login codes. | + +## 1. documents + +Primary key: `workflow_id` + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| workflow_id | TEXT | PRIMARY KEY | Temporal workflow ID. Unique key for the document and the join key used by every other table. | +| document_id | TEXT | NOT NULL | Document identifier carried into chunk payloads and index records. | +| canonical_document_id | TEXT | | Stable ID shared across re-uploads or re-ingestions of the same document. | +| filename | TEXT | NOT NULL | Name of the file as stored by the pipeline. | +| display_name | TEXT | | Human-friendly title shown in the UI. | +| source_filename | TEXT | | Original filename as uploaded by the user. | +| source_manifest_name | TEXT | | Filename of the matching entry in `document_manifest_entries`, used to link imported metadata. | +| source_file_fingerprint | TEXT | | Hash of the source file, used to detect duplicate uploads. | +| filepath | TEXT | NOT NULL | Local path of the uploaded file. | +| stage | TEXT | NOT NULL DEFAULT 'registered' | Current pipeline stage (registered, ocr_processing, ocr_review, translation_processing, translation_review, chunking, chunk_review, ready_for_ingestion, ingesting, approval_for_prod, ingesting_prod, completed, failed). | +| page_count | INTEGER | DEFAULT 0 | Number of pages extracted by OCR. | +| chunk_count | INTEGER | DEFAULT 0 | Number of chunks produced. | +| error_message | TEXT | | Failure reason when the stage is `failed`. | +| is_demo | INTEGER | DEFAULT 0 | Marks the document as demo content so it can be filtered out of summaries and index lists. | +| is_disabled | INTEGER | DEFAULT 0 | Soft-disable flag. A disabled document is hidden from lists and search. | +| created_at | TEXT | | When the row was registered. | +| updated_at | TEXT | | Last time the row changed. | +| ocr_completed_at | TEXT | | When OCR finished. | +| translation_completed_at | TEXT | | When translation finished. | +| chunks_completed_at | TEXT | | When chunking finished. | +| ingested_at | TEXT | | When the document was ingested into the vector index. | +| source_type | TEXT | | Input family derived from the file extension: `spreadsheet` (csv/xlsx), `document` (pdf, image, office) or `unknown`. | +| canonical_input_type | TEXT | | Form the input is normalized to: `spreadsheet` or `pdf`. Drives which pipeline path runs. | +| stop_after_ocr | INTEGER | DEFAULT 0 | When 1, the workflow halts after OCR (OCR-only batch runs). | +| reindex_required | INTEGER | DEFAULT 0 | Flags that approved content changed and the vector index is stale. | +| reindex_reason | TEXT | | Why a reindex is needed. | +| original_artifact_id | INTEGER | | Points to the `document_artifacts` row for the original upload. | +| normalized_artifact_id | INTEGER | | Points to the `document_artifacts` row for the normalized PDF or spreadsheet. | +| latest_job_id | INTEGER | | Points to the most recent `document_jobs` row. | +| instance | TEXT | | Tenant or state code (`bv` is platform-wide, others are state codes such as `mh`). Used for scoping and list filters. NULL or empty is backfilled with `DEFAULT_INSTANCE`. | +| uploaded_by_user_id | TEXT | | Keycloak user ID of the uploader. | +| uploaded_by_username | TEXT | | Username of the uploader. | +| uploaded_by_email | TEXT | | Email of the uploader. | +| uploaded_by_roles | TEXT | | Comma-separated roles of the uploader at upload time. | +| prod_ready_requested_at | TEXT | | When someone requested promotion to PROD. | +| prod_ready_requested_by_user_id | TEXT | | User ID of the PROD promotion requester. | +| prod_ready_requested_by_username | TEXT | | Username of the PROD promotion requester. | +| valid_from | TEXT | | Start of the period in which the document's chunks are searchable. Stamped with the upload day, editable by the approver. NULL means always current. | +| valid_to | TEXT | | End of the validity period. NULL means no end. | +| document_kind | TEXT | DEFAULT 'document' | Knowledge kind slug: `document`, `scheme`, `advisory`, `video` or a custom slug. Decides which Beckn catalog is announced. | +| scheme_code | TEXT | | Scheme identifier (for example `pmkisan`). Used when `document_kind` is `scheme`. Links to `scheme_catalog_entries`. | +| scheme_name | TEXT | | Human-readable scheme name. | +| scheme_aliases_json | TEXT | | JSON list of alternate names for the scheme, used for matching queries. | +| tool_routing | TEXT | | Which AI tool path serves the scheme: `qdrant`, `legacy` or `both`. | +| catalog_visible | INTEGER | DEFAULT 1 | Whether the document appears in the pull-based catalog API. | +| network_visible | INTEGER | DEFAULT 1 | Whether the document may be published to the Beckn network (Discovery Service). | + +Indexes: + +| Index | Columns | Purpose | +|-------|---------|---------| +| idx_documents_stage | stage | Filter and count documents by stage. | +| idx_documents_instance | instance | Tenant-scoped lists. | +| idx_documents_created | created_at DESC | Newest-first listing. | +| idx_documents_scheme_code | scheme_code | Look up documents by scheme. | + +## 2. pages + +Primary key: `id`. Unique: `(workflow_id, page_number)`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| id | INTEGER | PRIMARY KEY AUTOINCREMENT | Row ID. | +| workflow_id | TEXT | NOT NULL | Owning document (`documents.workflow_id`). | +| page_number | INTEGER | NOT NULL | 1-based page position in the document. | +| original_markdown | TEXT | | Raw OCR output for the page, kept unchanged. | +| edited_markdown | TEXT | | Reviewer-corrected OCR text. Takes precedence over the original when present. | +| is_reviewed | INTEGER | DEFAULT 0 | Whether the OCR text for this page has been reviewed. | +| reviewer_notes | TEXT | | Reviewer comments on the OCR text. | +| detected_language | TEXT | | Language detected for the page (via the `lang-detect` service). | +| translated_markdown | TEXT | | Machine translation of the page. | +| edited_translation | TEXT | | Reviewer-corrected translation. | +| translation_reviewed | INTEGER | DEFAULT 0 | Whether the translation has been reviewed. | +| translation_notes | TEXT | | Reviewer comments on the translation. | +| translation_provider | TEXT | | Provider that produced the translation. | +| translation_model | TEXT | | Model used for the translation. | +| translation_target_language | TEXT | | Language the page was translated into. | +| translated_at | TEXT | | When the translation was produced. | + +Indexes: `idx_pages_workflow (workflow_id)` — fetch all pages of a document. + +## 3. chunks + +Primary key: `id`. Unique: `(workflow_id, chunk_number)`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| id | INTEGER | PRIMARY KEY AUTOINCREMENT | Row ID. | +| workflow_id | TEXT | NOT NULL | Owning document (`documents.workflow_id`). | +| chunk_number | INTEGER | NOT NULL | Order of the chunk within the document. | +| original_text | TEXT | | Chunk text as produced by the chunker. | +| edited_text | TEXT | | Reviewer-edited chunk text. This is what gets indexed when present. | +| token_count | INTEGER | DEFAULT 0 | Token length of the chunk. | +| page_start | INTEGER | DEFAULT 1 | First page the chunk covers. | +| page_end | INTEGER | DEFAULT 1 | Last page the chunk covers. | +| source_page_numbers_json | TEXT | | JSON list of every page the chunk draws from. | +| source_spans_json | TEXT | | JSON list of text spans in the source pages, used to trace a chunk back to its provenance. | +| section_title | TEXT | | Heading of the section the chunk belongs to. | +| content_type | TEXT | | Kind of content in the chunk, for example `heading` or `body`. | +| is_reference | INTEGER | DEFAULT 0 | Marks reference material. Search can exclude these (`search_exclude_reference`). | +| chunking_provider | TEXT | | Chunking provider that produced the chunk. | +| chunking_model | TEXT | | Model used for chunking, if any. | +| chunking_config_json | TEXT | | JSON snapshot of the chunking settings used. | +| chunking_run_id | TEXT | | ID of the chunking run, to group chunks created together. | +| chunk_version | INTEGER | DEFAULT 1 | Version counter for the chunk. | +| is_reviewed | INTEGER | DEFAULT 0 | Whether the chunk has been reviewed. | +| is_excluded | INTEGER | DEFAULT 0 | Reviewer excluded the chunk. Excluded chunks are not indexed. | +| reviewer_notes | TEXT | | Reviewer comments on the chunk. | + +Indexes: `idx_chunks_workflow (workflow_id)` — fetch all chunks of a document. + +## 4. audit_logs + +Primary key: `id`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| id | INTEGER | PRIMARY KEY AUTOINCREMENT | Row ID. | +| workflow_id | TEXT | NOT NULL | Document the action applied to. | +| document_id | TEXT | NOT NULL | Document identifier at the time of the action. | +| action_type | TEXT | NOT NULL | What happened, for example `promote_to_prod`. | +| entity_type | TEXT | | Kind of record changed, for example `document`, page or chunk. | +| entity_id | INTEGER | | ID or number of the record changed. | +| field_name | TEXT | | Field that was changed. | +| old_value | TEXT | | Value before the change. | +| new_value | TEXT | | Value after the change. | +| metadata | TEXT | | Extra context, stored as JSON text. | +| timestamp | TEXT | NOT NULL | When the action happened. | +| actor_user_id | TEXT | | Keycloak user ID of whoever acted. | +| actor_username | TEXT | | Username of the actor. | +| actor_email | TEXT | | Email of the actor. | +| actor_roles | TEXT | | Roles of the actor at the time. | + +Indexes: + +| Index | Columns | Purpose | +|-------|---------|---------| +| idx_audit_workflow | workflow_id | Per-document audit trail. | +| idx_audit_timestamp | timestamp DESC | Newest-first global audit log. | + +## 5. settings + +Primary key: `key`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| key | TEXT | PRIMARY KEY | Setting name. | +| value | TEXT | NOT NULL | Setting value, always stored as text. | +| description | TEXT | | What the setting does. | +| updated_at | TEXT | NOT NULL | When the setting last changed. | + +Default rows (`INSERT OR IGNORE`): + +| key | default value | Purpose | +|-----|---------------|---------| +| search_method | HYBRID | Search mode: TENSOR, LEXICAL or HYBRID. | +| search_limit | 12 | Number of search results to return. | +| search_alpha | 0.6 | Hybrid weighting: 0 is lexical only, 1 is semantic only. | +| search_ranking_method | rrf | Hybrid ranking: `rrf` or `normalize_linear`. | +| search_show_highlights | true | Show highlighted matches in results. | +| search_ef_search | 256 | HNSW search accuracy parameter. | +| search_index_name | documents-index | Default vector DB collection name. | +| search_candidate_cap | 120 | Cap on the candidate retrieval pool. | +| search_candidate_multiplier | 10 | Candidate pool multiplier before the final cut. | +| search_max_chunks_per_doc | 2 | Result diversity cap per document. | +| search_use_e5_prefix | true | Prefix queries with `query:` for e5 embedding models. | +| search_exclude_reference | true | Exclude reference chunks when the index supports it. | +| search_query_expansion_profile | gu-v1 | Query expansion profile. | +| search_rerank_mode | none | Post-search reranking mode. | +| search_hybrid_rrfk | 60 | RRF tuning parameter for hybrid search. | + +## 6. document_jobs + +Primary key: `id`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| id | INTEGER | PRIMARY KEY AUTOINCREMENT | Job ID. Referenced by `documents.latest_job_id` and `document_artifacts.job_id`. | +| workflow_id | TEXT | NOT NULL | Document the job ran for. | +| job_type | TEXT | NOT NULL | Kind of run: `ocr_only`, `ocr_retry`, `translation_retry`, `chunking_retry`, `reingest`, `promote_prod`, plus the main pipeline run. | +| temporal_workflow_id | TEXT | | Temporal workflow ID, to find the run in the Temporal UI. | +| temporal_run_id | TEXT | | Temporal run ID. | +| status | TEXT | NOT NULL DEFAULT 'running' | Job state. | +| current_stage | TEXT | | Stage the job is in right now. | +| started_at | TEXT | NOT NULL | When the job started. | +| completed_at | TEXT | | When the job ended. | +| error_message | TEXT | | Failure reason. | +| config_json | TEXT | | JSON snapshot of the settings the job ran with. | + +Indexes: `idx_document_jobs_workflow (workflow_id, started_at DESC)` — latest jobs for a document. + +## 7. document_artifacts + +Primary key: `id`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| id | INTEGER | PRIMARY KEY AUTOINCREMENT | Artifact ID. Referenced by `documents.original_artifact_id` and `normalized_artifact_id`. | +| workflow_id | TEXT | NOT NULL | Owning document. | +| job_id | INTEGER | | Job that produced the artifact (`document_jobs.id`). | +| artifact_type | TEXT | NOT NULL | Kind of output, for example `original_upload`, `normalized_pdf`, `normalized_spreadsheet`, `ocr_pages_json`, `translation_pages_json`, `chunk_json_export`, `vector_payload_export`. | +| stage | TEXT | | Pipeline stage that created it. | +| storage_uri | TEXT | NOT NULL | MinIO location of the blob. | +| mime_type | TEXT | | MIME type of the blob. | +| filename | TEXT | | File name of the blob. | +| size_bytes | INTEGER | | Blob size in bytes. | +| metadata_json | TEXT | | Extra details as JSON, for example the source path or input type. | +| created_at | TEXT | NOT NULL | When the artifact was recorded. | + +Indexes: `idx_document_artifacts_workflow (workflow_id, created_at DESC)` — artifacts of a document, newest first. + +## 8. document_index_status + +Primary key: `(workflow_id, index_name)`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| workflow_id | TEXT | NOT NULL | Document that was indexed. | +| index_name | TEXT | NOT NULL | Vector index or collection (for example DEV or PROD). One row per document and index. | +| doc_id | TEXT | | Document ID as stored in the vector index. Replaces legacy `marqo_doc_id`. | +| chunk_count_indexed | INTEGER | DEFAULT 0 | Number of chunks currently in the index. | +| last_indexed_at | TEXT | | When the document was last written to the index. | +| last_verified_at | TEXT | | When the index state was last checked against the vector store. | +| schema_version | TEXT | | Index payload schema version, for spotting stale payloads. | +| status | TEXT | NOT NULL DEFAULT 'unknown' | Index state, for example `indexed` or `unknown`. | +| details_json | TEXT | | Extra details as JSON. | + +Older DBs may still carry `marqo_doc_id` until a cleanup script is run. Startup backfills `doc_id` from it when present. + +## 9. document_manifest_entries + +Primary key: `id`. Unique: `manifest_filename`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| id | INTEGER | PRIMARY KEY AUTOINCREMENT | Row ID. | +| manifest_filename | TEXT | NOT NULL UNIQUE | Filename the entry describes. Matches `documents.source_manifest_name`. | +| title_en | TEXT | | English title. | +| title_gu | TEXT | | Gujarati title. | +| doc_language | TEXT | | Language of the document. | +| category_tags | TEXT | | Category tags for the document. | +| description | TEXT | | Short description. | +| quality_score | TEXT | | Quality rating from the manifest. | +| priority_rank | TEXT | | Ingestion priority from the manifest. | +| ingestion_status | TEXT | | Ingestion status recorded in the manifest. | +| feedback | TEXT | | Reviewer feedback from the manifest. | +| source_csv_path | TEXT | | CSV file the row was imported from. | +| imported_at | TEXT | NOT NULL | When the row was imported. | + +Indexes: `idx_manifest_entries_filename (manifest_filename)` — look up by filename. + +## 10. scheme_catalog_meta + +Single-row table, enforced by `CHECK (id = 1)`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| id | INTEGER | PRIMARY KEY, CHECK (id = 1) | Fixed to 1 so only one row can exist. | +| version | INTEGER | NOT NULL DEFAULT 0 | Catalog version counter. Bumped on catalog changes so pull clients can detect updates. | +| updated_at | TEXT | | When the version last changed. | +| notes | TEXT | | Reason for the last bump. | + +## 11. scheme_catalog_entries + +Primary key: `scheme_code`. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| scheme_code | TEXT | PRIMARY KEY | Scheme identifier (for example `pmkisan`). | +| scheme_name | TEXT | | Human-readable scheme name. | +| scheme_aliases_json | TEXT | | JSON list of alternate names for matching queries. | +| instances_json | TEXT | | JSON list of instances (tenants) that have this scheme. | +| workflow_ids_json | TEXT | | JSON list of `documents.workflow_id` values that back the scheme. | +| collection_name | TEXT | | Vector collection holding the scheme's chunks. | +| chunk_count | INTEGER | DEFAULT 0 | Number of chunks across the scheme's documents. | +| status | TEXT | NOT NULL DEFAULT 'disabled' | Catalog state: `live`, `pending_prod`, `pending_reindex` or `disabled`. Only live schemes are served as vector schemes. | +| network_visible | INTEGER | DEFAULT 0 | Whether the scheme is exposed on the network. | +| promoted_at | TEXT | | When the scheme was promoted to PROD. | +| content_hash | TEXT | | Hash of code, name, aliases, visibility and status, used to detect changes. | +| source | TEXT | | Where the entry came from: `pipeline` (built from scheme documents) or `import` (bootstrapped). | +| updated_at | TEXT | | When the entry last changed. | + +Indexes: `idx_scheme_catalog_status (status)` — filter entries by status. + +## 12. email_otps + +Primary key: `email`. One live code per email, overwritten on resend. + +| Column | Type | Constraints / Default | Purpose | +|--------|------|----------------------|---------| +| email | TEXT | PRIMARY KEY | Email the code was sent to. | +| code_hash | TEXT | NOT NULL | Hash of the one-time code. The plain code is never stored. | +| expires_at | TEXT | NOT NULL | Expiry time of the code. | +| attempts | INTEGER | NOT NULL DEFAULT 0 | Failed verification attempts. Reset to 0 on resend. | +| created_at | TEXT | NOT NULL | When the code was first issued. | +| last_sent_at | TEXT | NOT NULL | When the code was last sent. Used to throttle resends. | + +## Relationships (by convention, not enforced) + +| From | Column | To | +|------|--------|----| +| pages | workflow_id | documents.workflow_id | +| chunks | workflow_id | documents.workflow_id | +| audit_logs | workflow_id | documents.workflow_id | +| document_jobs | workflow_id | documents.workflow_id | +| document_artifacts | workflow_id | documents.workflow_id | +| document_artifacts | job_id | document_jobs.id | +| document_index_status | workflow_id | documents.workflow_id | +| documents | original_artifact_id, normalized_artifact_id | document_artifacts.id | +| documents | latest_job_id | document_jobs.id | +| documents | source_manifest_name | document_manifest_entries.manifest_filename | +| documents | scheme_code | scheme_catalog_entries.scheme_code | +| scheme_catalog_entries | workflow_ids_json (JSON list) | documents.workflow_id | diff --git a/migrations/0001_initial_schema.sql b/migrations/0001_initial_schema.sql new file mode 100644 index 0000000..3fc75d0 --- /dev/null +++ b/migrations/0001_initial_schema.sql @@ -0,0 +1,292 @@ +CREATE EXTENSION IF NOT EXISTS pg_trgm; + +CREATE FUNCTION set_updated_at() RETURNS trigger AS $$ +BEGIN + NEW.updated_at := now(); + RETURN NEW; +END $$ LANGUAGE plpgsql; + +CREATE TYPE workflow_stage AS ENUM ( + 'registered', 'ocr_processing', 'ocr_review', 'translation_processing', + 'translation_review', 'chunking', 'chunk_review', 'ready_for_ingestion', + 'ingesting', 'publishing_to_network', 'approval_for_prod', 'ingesting_prod', + 'completed', 'failed' +); +CREATE TYPE workflow_type_enum AS ENUM ( + 'pipeline', 'ocr_only', 'ocr_retry', 'translation_retry', 'chunking_retry', + 'reingest', 'promote_prod' +); +CREATE TYPE workflow_status_enum AS ENUM ('running', 'waiting_review', 'completed', 'failed'); +CREATE TYPE audit_entity_type AS ENUM ('document', 'page', 'chunk', 'setting', 'catalog'); + +-- 1. users +CREATE TABLE users ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + keycloak_user_id TEXT NOT NULL UNIQUE, + username TEXT, + email TEXT, + roles TEXT[] NOT NULL DEFAULT '{}', + first_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(), + last_seen_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE UNIQUE INDEX users_email_lower_key ON users (lower(email)) WHERE email IS NOT NULL; + +INSERT INTO users (keycloak_user_id, username, roles) +VALUES ('local-admin', 'local-admin', '{superadmin}'); + +-- 2. catalog_entries +CREATE TABLE catalog_entries ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + type TEXT NOT NULL CHECK (type ~ '^[a-z0-9]+([_-][a-z0-9]+)*$'), + code TEXT NOT NULL CHECK (code ~ '^[a-z0-9]+(-[a-z0-9]+)*$'), + name TEXT NOT NULL, + aliases TEXT[] NOT NULL DEFAULT '{}' CHECK (array_position(aliases, '') IS NULL), + metadata JSONB NOT NULL DEFAULT '{"source": "pipeline"}' CHECK (jsonb_typeof(metadata) = 'object'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (type, code), + UNIQUE (id, type) +); +CREATE TRIGGER trg_catalog_entries_updated_at BEFORE UPDATE ON catalog_entries + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); + +-- 3. documents +CREATE TABLE documents ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + instance TEXT NOT NULL, + source JSONB NOT NULL + CHECK (jsonb_typeof(source) = 'object') + CHECK (source->>'fingerprint' ~ '^[0-9a-f]{32}$'), + document_kind TEXT NOT NULL DEFAULT 'document' CHECK (document_kind ~ '^[a-z0-9]+([_-][a-z0-9]+)*$'), + catalog_entry_id UUID, + valid_from DATE, + valid_to DATE CHECK (valid_from <= valid_to), + is_disabled BOOLEAN NOT NULL DEFAULT FALSE, + index_status JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(index_status) = 'object'), + metadata JSONB NOT NULL + DEFAULT '{"catalog_visible": true, "network_visible": true, "is_demo": false, "chunk_count": 0, "reindex_required": false}' + CHECK (jsonb_typeof(metadata) = 'object') + CHECK ((metadata->>'chunk_count')::int >= 0), + uploaded_by_user_id UUID REFERENCES users(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + FOREIGN KEY (catalog_entry_id, document_kind) REFERENCES catalog_entries (id, type) + ON UPDATE CASCADE ON DELETE RESTRICT +); +CREATE INDEX documents_instance_idx ON documents (instance); +CREATE INDEX documents_created_at_idx ON documents (created_at DESC); +CREATE INDEX documents_catalog_entry_idx ON documents (catalog_entry_id) WHERE catalog_entry_id IS NOT NULL; +CREATE INDEX documents_fingerprint_idx ON documents (instance, (source->>'fingerprint')); +CREATE INDEX documents_is_demo_idx ON documents (((metadata->>'is_demo')::boolean)); +CREATE TRIGGER trg_documents_updated_at BEFORE UPDATE ON documents + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); + +-- 4. workflows +CREATE TABLE workflows ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + workflow_type workflow_type_enum NOT NULL, + temporal_workflow_id TEXT NOT NULL, + status workflow_status_enum NOT NULL DEFAULT 'running', + current_stage workflow_stage NOT NULL DEFAULT 'registered', + stage_timestamps JSONB NOT NULL DEFAULT '{}' + CHECK (jsonb_typeof(stage_timestamps) = 'object' + AND (stage_timestamps - ARRAY['ocr', 'translation', 'chunks']) = '{}'::jsonb), + stop_after_ocr BOOLEAN NOT NULL DEFAULT FALSE, + error_message TEXT, + config JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(config) = 'object'), + started_by_user_id UUID REFERENCES users(id) ON DELETE SET NULL, + started_at TIMESTAMPTZ NOT NULL DEFAULT now(), + completed_at TIMESTAMPTZ, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX workflows_document_started_idx ON workflows (document_id, started_at DESC); +CREATE INDEX workflows_status_started_idx ON workflows (status, started_at DESC); +CREATE INDEX workflows_temporal_id_idx ON workflows (temporal_workflow_id); +CREATE TRIGGER trg_workflows_updated_at BEFORE UPDATE ON workflows + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); + +CREATE VIEW document_current_workflow AS +SELECT DISTINCT ON (document_id) * +FROM workflows +ORDER BY document_id, started_at DESC, id DESC; + +-- 5. workflow_artifacts +CREATE TABLE workflow_artifacts ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + workflow_id UUID NOT NULL REFERENCES workflows(id) ON DELETE CASCADE, + artifact_type TEXT NOT NULL CHECK (btrim(artifact_type) <> ''), + stage workflow_stage, + storage_uri TEXT NOT NULL, + mime_type TEXT, + filename TEXT, + size_bytes BIGINT CHECK (size_bytes >= 0), + metadata JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(metadata) = 'object'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX workflow_artifacts_workflow_idx ON workflow_artifacts (workflow_id, created_at DESC); +CREATE INDEX workflow_artifacts_type_idx ON workflow_artifacts (workflow_id, artifact_type, created_at DESC); + +-- 6. pages +CREATE TABLE pages ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + page_number INTEGER NOT NULL CHECK (page_number >= 1), + original_markdown TEXT, + edited_markdown TEXT, + translations JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(translations) = 'object'), + metadata JSONB NOT NULL DEFAULT '{"is_reviewed": false}' CHECK (jsonb_typeof(metadata) = 'object'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (document_id, page_number) +); +CREATE TRIGGER trg_pages_updated_at BEFORE UPDATE ON pages + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); + +-- 7. chunks +CREATE TABLE chunks ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE, + chunk_number INTEGER NOT NULL CHECK (chunk_number >= 1), + original_text TEXT NOT NULL, + edited_text TEXT, + token_count INTEGER NOT NULL DEFAULT 0 CHECK (token_count >= 0), + page_start INTEGER NOT NULL DEFAULT 1 CHECK (page_start >= 1), + page_end INTEGER NOT NULL DEFAULT 1, + source_spans JSONB NOT NULL DEFAULT '[]' CHECK (jsonb_typeof(source_spans) = 'array'), + section_title TEXT, + content_type TEXT, + is_reference BOOLEAN NOT NULL DEFAULT FALSE, + is_reviewed BOOLEAN NOT NULL DEFAULT FALSE, + is_excluded BOOLEAN NOT NULL DEFAULT FALSE, + reviewer_notes TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (document_id, chunk_number), + CHECK (page_end >= page_start) +); +CREATE INDEX chunks_text_trgm_idx ON chunks + USING gin (lower(coalesce(edited_text, original_text)) gin_trgm_ops); +CREATE TRIGGER trg_chunks_updated_at BEFORE UPDATE ON chunks + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); + +-- 8. audit_logs +CREATE TABLE audit_logs ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + document_id UUID REFERENCES documents(id) ON DELETE SET NULL, + workflow_id TEXT, + instance TEXT, + action_type TEXT NOT NULL CHECK (btrim(action_type) <> ''), + entity_type audit_entity_type, + entity_number INTEGER, + field_name TEXT, + old_value TEXT, + new_value TEXT, + metadata JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(metadata) = 'object'), + actor_user_id UUID REFERENCES users(id) ON DELETE SET NULL, + actor JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(actor) = 'object'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CHECK (entity_type IN ('setting', 'catalog') OR workflow_id IS NOT NULL) +); +CREATE INDEX audit_logs_document_idx ON audit_logs (document_id, created_at DESC); +CREATE INDEX audit_logs_created_idx ON audit_logs (created_at DESC); +CREATE INDEX audit_logs_action_idx ON audit_logs (action_type, created_at DESC); +CREATE INDEX audit_logs_settings_idx ON audit_logs (created_at DESC) WHERE entity_type = 'setting'; + +-- 9. search_settings +CREATE TABLE search_settings ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + singleton BOOLEAN NOT NULL DEFAULT TRUE UNIQUE CHECK (singleton), + retrieval JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(retrieval) = 'object'), + candidates JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(candidates) = 'object'), + results JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(results) = 'object'), + query JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(query) = 'object'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE TRIGGER trg_search_settings_updated_at BEFORE UPDATE ON search_settings + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); +INSERT INTO search_settings DEFAULT VALUES; + +-- 10. email_otps +CREATE TABLE email_otps ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + email TEXT NOT NULL UNIQUE CHECK (email = lower(email)), + code_hash TEXT NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0), + last_sent_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX email_otps_expires_idx ON email_otps (expires_at); + +-- webhook_records +CREATE TABLE webhook_records ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + source TEXT NOT NULL DEFAULT 'webhook', + external_id TEXT NOT NULL, + title TEXT, + slug TEXT, + status TEXT, + is_published BOOLEAN, + category TEXT, + payload JSONB NOT NULL CHECK (jsonb_typeof(payload) = 'object'), + payload_hash TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (source, external_id) +); +CREATE INDEX webhook_records_source_status_idx ON webhook_records (source, status); + +-- catalog view +CREATE VIEW catalog AS +SELECT + e.id AS entry_id, + e.type, + e.code, + e.name, + e.aliases, + e.metadata->>'source' AS source, + CASE + WHEN COALESCE(a.has_pending_reindex, FALSE) THEN 'pending_reindex' + WHEN COALESCE(a.live_count, 0) > 0 THEN 'live' + WHEN COALESCE(a.pending_prod_count, 0) > 0 THEN 'pending_prod' + WHEN COALESCE(a.doc_count, 0) = 0 AND e.metadata->>'source' = 'bootstrap' THEN 'live' + ELSE 'disabled' + END AS status, + CASE WHEN COALESCE(cardinality(a.instances), 0) = 0 THEN ARRAY['default']::TEXT[] ELSE a.instances END AS instances, + COALESCE(a.workflow_ids, '{}'::TEXT[]) AS workflow_ids, + COALESCE(a.chunk_count, 0)::INTEGER AS chunk_count, + CASE + WHEN COALESCE(a.doc_count, 0) = 0 THEN COALESCE(e.metadata->>'source' = 'bootstrap', FALSE) + ELSE COALESCE(a.network_visible, FALSE) + END AS network_visible, + a.promoted_at +FROM catalog_entries e +LEFT JOIN LATERAL ( + SELECT + COUNT(*) AS doc_count, + COUNT(*) FILTER (WHERE x.is_live) AS live_count, + BOOL_OR(x.is_live AND COALESCE((x.metadata->>'reindex_required')::boolean, FALSE)) AS has_pending_reindex, + COUNT(*) FILTER (WHERE x.current_stage = 'approval_for_prod' AND NOT x.is_disabled) AS pending_prod_count, + ARRAY(SELECT DISTINCT d2.instance FROM documents d2 + WHERE d2.catalog_entry_id = e.id ORDER BY d2.instance) AS instances, + ARRAY(SELECT w.temporal_workflow_id FROM documents d3 + JOIN LATERAL ( + SELECT w2.temporal_workflow_id FROM workflows w2 + WHERE w2.document_id = d3.id AND w2.workflow_type = 'pipeline' + ORDER BY w2.started_at ASC, w2.id ASC LIMIT 1 + ) w ON TRUE + WHERE d3.catalog_entry_id = e.id + ORDER BY d3.created_at, d3.id) AS workflow_ids, + COALESCE(SUM((x.metadata->>'chunk_count')::int), 0) AS chunk_count, + BOOL_OR(COALESCE((x.metadata->>'network_visible')::boolean, TRUE)) AS network_visible, + MAX((x.index_status->'prod'->>'last_indexed_at')::timestamptz) FILTER (WHERE x.is_live) AS promoted_at + FROM ( + SELECT d.*, cw.current_stage, + (cw.current_stage = 'completed' AND NOT d.is_disabled + AND COALESCE((d.metadata->>'catalog_visible')::boolean, TRUE)) AS is_live + FROM documents d + LEFT JOIN document_current_workflow cw ON cw.document_id = d.id + WHERE d.catalog_entry_id = e.id + ) x +) a ON TRUE; diff --git a/pipeline/migrate.py b/pipeline/migrate.py new file mode 100644 index 0000000..fe1a870 --- /dev/null +++ b/pipeline/migrate.py @@ -0,0 +1,76 @@ +"""Versioned SQL migration runner. + +Applies ``migrations/NNNN_.sql`` files in order, once each, recording +them in ``schema_migrations``. A session-level advisory lock serialises +concurrent api/worker startups so only one process applies a given file. +""" + +import logging +import re +from pathlib import Path + +import psycopg + +logger = logging.getLogger(__name__) + +MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations" +_MIGRATION_FILE_RE = re.compile(r"^(\d{4})_[a-z0-9_]+\.sql$") +_ADVISORY_LOCK_KEY = 7_204_911_338_105_221 + + +def list_migrations(directory: Path = MIGRATIONS_DIR) -> list[tuple[str, Path]]: + found = [] + for path in sorted(directory.glob("*.sql")): + match = _MIGRATION_FILE_RE.match(path.name) + if not match: + raise ValueError(f"Migration file name must be NNNN_name.sql: {path.name}") + found.append((match.group(1), path)) + return found + + +def apply_migrations(conninfo: str, directory: Path = MIGRATIONS_DIR) -> list[str]: + """Apply every pending migration. Returns the versions applied by this call.""" + applied: list[str] = [] + with psycopg.connect(conninfo, autocommit=True) as conn: + conn.execute("SELECT pg_advisory_lock(%s)", (_ADVISORY_LOCK_KEY,)) + try: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")} + for version, path in list_migrations(directory): + if version in done: + continue + with conn.transaction(): + conn.execute(path.read_text(encoding="utf-8")) + conn.execute( + "INSERT INTO schema_migrations (version, name) VALUES (%s, %s)", + (version, path.name), + ) + logger.info("applied migration %s", path.name) + applied.append(version) + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (_ADVISORY_LOCK_KEY,)) + return applied + + +def current_version(conninfo: str) -> str | None: + with psycopg.connect(conninfo, autocommit=True) as conn: + exists = conn.execute("SELECT to_regclass('schema_migrations')").fetchone()[0] + if not exists: + return None + row = conn.execute("SELECT max(version) FROM schema_migrations").fetchone() + return row[0] + + +def is_current(conninfo: str, directory: Path = MIGRATIONS_DIR) -> bool: + migrations = list_migrations(directory) + if not migrations: + return True + return current_version(conninfo) == migrations[-1][0] From 71414ea766efe7c6d2339179b060a32c97c1409b Mon Sep 17 00:00:00 2001 From: Naveen Date: Mon, 26 Oct 2026 13:18:48 +0530 Subject: [PATCH 30/53] feat(db): replace SQLite store with Postgres Rewrite pipeline/db.py on a psycopg connection pool, splitting documents from workflows. Functions keep their names and flat dict shapes and accept the document UUID or any legacy identifier (doc-, temporal workflow id, fingerprint, filename). Artifact and job ids are now UUIDs. Scheme catalog is served from catalog_entries and the catalog view. Authenticated callers are upserted into users. Tests run against a real Postgres (TEST_DATABASE_URL or testcontainers). --- pipeline/activities.py | 16 +- pipeline/api/api.py | 135 +- pipeline/api/documents.py | 21 +- pipeline/api/upload.py | 16 +- pipeline/auth/deps.py | 36 +- pipeline/db.py | 4302 +++++++++++++++-------------- pipeline/models.py | 136 +- pipeline/scheme_catalog.py | 258 +- pipeline/worker.py | 4 +- pipeline/workflows.py | 2 +- tests/conftest.py | 58 +- tests/test_activities.py | 2 +- tests/test_api.py | 142 +- tests/test_db.py | 680 ++++- tests/test_document_repository.py | 2 +- tests/test_provenance.py | 2 +- tests/test_scheme_catalog.py | 23 +- tests/test_tenancy.py | 14 +- tests/test_webhook.py | 6 +- ui/src/views/DocumentOpsView.jsx | 8 +- 20 files changed, 3443 insertions(+), 2420 deletions(-) diff --git a/pipeline/activities.py b/pipeline/activities.py index af719f6..d5a2cdb 100644 --- a/pipeline/activities.py +++ b/pipeline/activities.py @@ -929,7 +929,7 @@ async def run_ocr(filepath: str) -> list[dict]: @activity.defn async def run_ocr_and_store(workflow_id: str, filepath: str) -> dict: - """Run OCR and persist pages to SQLite to avoid large Temporal payloads.""" + """Run OCR and persist pages to Postgres to avoid large Temporal payloads.""" from . import db local_path, cleanup_local = _resolve_local_path(filepath) @@ -1111,7 +1111,7 @@ async def create_chunks_from_db( chunk_overlap: int = 128, min_tokens: int = 100, ) -> dict: - """Create chunks from persisted pages and persist chunks in SQLite.""" + """Create chunks from persisted pages and persist chunks in Postgres.""" from . import db pages = db.get_pages(workflow_id) @@ -1209,7 +1209,7 @@ async def _persist_chunking_progress(event: dict) -> None: "stats": result.stats, }, ) - # Reconcile the document row immediately after chunk persistence so SQLite + # Reconcile the document row immediately after chunk persistence so Postgres # remains truthful even if the workflow fails before its final state update. db.reconcile_materialized_state(workflow_id) await _persist_chunking_progress( @@ -1290,7 +1290,7 @@ async def promote_document_to_prod_qdrant( filename: str, batch_size: int = 10, ) -> dict: - """Re-upsert document chunks into the PROD Qdrant collection (from SQLite).""" + """Re-upsert document chunks into the PROD Qdrant collection (from Postgres).""" from . import db from .vector_store.qdrant_store import QdrantVectorStore, get_qdrant_client @@ -1413,7 +1413,7 @@ async def ingest_document_from_db( index_name: str = "documents-index", batch_size: int = 10, ) -> dict: - """Prepare and ingest chunks directly from SQLite by workflow_id.""" + """Prepare and ingest chunks directly from Postgres by workflow_id.""" from . import db chunks = db.get_chunks(workflow_id, include_excluded=True) @@ -1566,7 +1566,7 @@ async def update_document_state( chunk_count: int = 0, error_message: str = None, ) -> dict: - """Update document state in SQLite.""" + """Update document state in Postgres.""" from . import db activity.logger.info(f"Updating state for {workflow_id}: stage={stage}") @@ -1603,7 +1603,7 @@ async def update_document_state( @activity.defn async def persist_document_content(workflow_id: str, pages: list[dict], chunks: list[dict]) -> dict: - """Persist pages and chunks to SQLite.""" + """Persist pages and chunks to Postgres.""" from . import db activity.logger.info(f"Persisting content for {workflow_id}: {len(pages)} pages, {len(chunks)} chunks") @@ -1636,7 +1636,7 @@ async def detect_and_translate_pages_from_db( target_language: str = "en", source_language: str = None, ) -> dict: - """Detect and translate pages loaded from SQLite; persist updated pages back to SQLite.""" + """Detect and translate pages loaded from Postgres; persist updated pages back to Postgres.""" from . import db pages = db.get_pages(workflow_id) diff --git a/pipeline/api/api.py b/pipeline/api/api.py index 64416e3..9507152 100644 --- a/pipeline/api/api.py +++ b/pipeline/api/api.py @@ -153,8 +153,7 @@ async def lifespan(app: FastAPI): auth_cfg.keycloak_jwks_url, ) - # Initialize SQLite database - print("Initializing SQLite database...") + print("Initializing database...") db.init_db() # Temporal @@ -249,6 +248,41 @@ async def lifespan(app: FastAPI): # own much tighter budget than the general default. RATE_LIMIT_OTP = os.environ.get("RATE_LIMIT_OTP", "10/minute") +_DOCUMENT_PATH_RE = re.compile(r"^/documents/([^/]+)(/.*)?$") +_static_document_segments: Optional[set[str]] = None + + +def _static_document_path_segments() -> set[str]: + """Literal second segments of ``/documents/...`` routes (``upload``, ``batch``...).""" + global _static_document_segments + if _static_document_segments is None: + segments = set() + for route in app.routes: + match = _DOCUMENT_PATH_RE.match(getattr(route, "path", "")) + if match and "{" not in match.group(1): + segments.add(match.group(1)) + _static_document_segments = segments + return _static_document_segments + + +@app.middleware("http") +async def canonicalize_document_identifier(request: Request, call_next): + """Let ``/documents/{id}/...`` take the document UUID or any legacy identifier. + + Handlers address a document by its Temporal workflow id (they signal and + query Temporal with it), so the path is rewritten to that id before routing. + An identifier that matches no document is left alone and 404s in the handler. + """ + match = _DOCUMENT_PATH_RE.match(request.scope["path"]) + if match and match.group(1) not in _static_document_path_segments(): + document = await asyncio.to_thread(db.find_document_by_doc_identifier, match.group(1)) + if document and document["workflow_id"] and document["workflow_id"] != match.group(1): + new_path = f"/documents/{document['workflow_id']}{match.group(2) or ''}" + request.scope["path"] = new_path + request.scope["raw_path"] = quote(new_path, safe="/:").encode("ascii") + return await call_next(request) + + limiter = Limiter(key_func=get_remote_address) app.state.limiter = limiter app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) @@ -347,7 +381,7 @@ def _instance_scope_for_user(user: AuthUser) -> Optional[list[str]]: def _document_cohorts_payload(summary: dict, by_instance: Optional[list[dict]] = None) -> dict: - """Shape SQLite aggregate counts for DocumentCohortsResponse (all ints).""" + """Shape Postgres aggregate counts for DocumentCohortsResponse (all ints).""" def _i(key: str, default: int = 0) -> int: val = summary.get(key, default) return int(val or 0) @@ -432,6 +466,7 @@ def _actor_from_user(user) -> dict: def _document_summary_from_row(doc: dict, current_job: Optional[dict] = None) -> DocumentSummary: return DocumentSummary( + id=doc.get("id"), document_id=doc["document_id"], canonical_document_id=doc.get("canonical_document_id"), workflow_id=doc["workflow_id"], @@ -841,7 +876,7 @@ async def list_runs( @app.get("/runs/{job_id}") -async def get_run(job_id: int, user: RequireSearch): +async def get_run(job_id: str, user: RequireSearch): """Get a specific document job/run (404 if missing or outside tenant scope).""" run = db.get_document_job(job_id) if not run: @@ -891,6 +926,7 @@ def _build_document_detail(doc: dict) -> DocumentDetail: current_job = db.get_latest_document_job(workflow_id) artifacts = db.list_document_artifacts(workflow_id) return DocumentDetail( + id=doc.get("id"), document_id=doc["document_id"], canonical_document_id=doc.get("canonical_document_id"), workflow_id=workflow_id, @@ -1154,7 +1190,7 @@ async def get_workflow_error_details(workflow_id: str, user: RequireSearch): This endpoint queries Temporal directly to get the most detailed error information available, which may be more complete than what's - stored in SQLite. + stored in Postgres. """ import traceback @@ -1232,7 +1268,7 @@ async def get_workflow_error_details(workflow_id: str, user: RequireSearch): @app.get("/documents/{workflow_id}/runtime") async def get_document_runtime(workflow_id: str, user: RequireSearch): - """Return live runtime status by combining SQLite state and Temporal workflow state.""" + """Return live runtime status by combining Postgres state and Temporal workflow state.""" doc = _require_document_for_user(workflow_id, user) return await _get_runtime_payload(workflow_id, doc=doc) @@ -1244,7 +1280,7 @@ async def list_document_artifacts(workflow_id: str, user: RequireSearch): @app.get("/documents/{workflow_id}/artifacts/{artifact_id}") -async def get_document_artifact(workflow_id: str, user: RequireSearch, artifact_id: int): +async def get_document_artifact(workflow_id: str, user: RequireSearch, artifact_id: str): _require_document_for_user(workflow_id, user) artifact = db.get_document_artifact(workflow_id, artifact_id) if not artifact: @@ -1253,7 +1289,7 @@ async def get_document_artifact(workflow_id: str, user: RequireSearch, artifact_ @app.get("/documents/{workflow_id}/artifacts/{artifact_id}/content") -async def get_document_artifact_content(workflow_id: str, user: RequireSearch, artifact_id: int): +async def get_document_artifact_content(workflow_id: str, user: RequireSearch, artifact_id: str): _require_document_for_user(workflow_id, user) artifact = db.get_document_artifact(workflow_id, artifact_id) if not artifact: @@ -1323,7 +1359,7 @@ async def get_document_graph(workflow_id: str, user: RequireSearch): def _delete_artifacts_from_minio(workflow_id: str) -> dict: """Remove every MinIO object backing a document's artifacts. - Artifacts are the only part of a document that lives outside SQLite and the + Artifacts are the only part of a document that lives outside Postgres and the vector store — the original upload, the normalized PDF, and the OCR / translation / chunk JSON snapshots. Nothing else reads them once the document is gone, so a hard delete has to sweep them explicitly. @@ -1371,15 +1407,15 @@ async def disable_document( - Remove all chunks from the DEV vector index - Delete PROD Qdrant points too, if this document was ever promoted (schemes-index for scheme documents, documents-index otherwise) - - For scheme documents, also rebuild the legacy SQLite scheme catalog + - For scheme documents, also rebuild the scheme catalog - Stop the workflow if it is still running - Soft delete (default) then marks the document disabled in SQLite and keeps + Soft delete (default) then marks the document disabled in Postgres and keeps every row and artifact, so ``POST /documents/{id}/restore`` can bring it back. Use X-Include-Disabled: true in list_documents to see it. ``purge=true`` additionally deletes the MinIO artifacts (original upload, - normalized PDF, OCR/translation/chunk JSON) and every SQLite row (pages, + normalized PDF, OCR/translation/chunk JSON) and every Postgres row (pages, chunks, artifacts, jobs, index status, the document itself). This is irreversible — restore has nothing left to restore. Audit rows are kept unless ``keep_audit=false``, since they record the deletion itself. @@ -1387,7 +1423,7 @@ async def disable_document( Args: workflow_id: The document workflow ID remove_from_search: If True (default), removes chunks from the index - purge: If True, hard delete — also drops MinIO artifacts and SQLite rows + purge: If True, hard delete — also drops MinIO artifacts and Postgres rows keep_audit: If False (and purge=True), also deletes the audit trail Requires ``admin`` (super admin — any document), or ``delete_own`` @@ -1440,7 +1476,7 @@ async def disable_document( except Exception: pass # Workflow already completed/cancelled - # Mark as disabled in SQLite (a purge deletes the row outright below, but + # Mark as disabled in Postgres (a purge deletes the row outright below, but # this keeps the document hidden if a later step fails partway through) db.set_document_disabled(workflow_id, True) @@ -1467,7 +1503,7 @@ async def disable_document( else: result["prod_qdrant_deleted"] = _delete_document_points_from_prod(doc) - # Hard delete: drop the artifacts and the SQLite rows themselves. Ordered + # Hard delete: drop the artifacts and the Postgres rows themselves. Ordered # storage-then-database so a failure mid-way leaves rows pointing at # already-deleted objects (visible, fixable) rather than orphaned objects # no row references any more (invisible, unreclaimable). @@ -1586,7 +1622,6 @@ async def patch_scheme_metadata( async def catalog_version(request: Request, user: CurrentUser): """Cheap poll for cache warmers: { version, updated_at }.""" _require_catalog_read(request, user) - scheme_catalog.ensure_catalog_schema() scheme_catalog.bootstrap_catalog_if_empty() meta = db.get_catalog_meta() return { @@ -1672,7 +1707,6 @@ async def catalog_rebuild(request: Request, user: RequireAdmin): # Admin JWT always OK; service key also accepted for automation if not _service_key_ok(request) and not user.has_permission(Permission.ADMIN): raise HTTPException(403, "Rebuild requires admin") - scheme_catalog.ensure_catalog_schema() # Ensure bootstrap seeds exist before rebuild merge scheme_catalog.bootstrap_catalog_if_empty() version = scheme_catalog.rebuild_catalog_from_documents(reason="admin_rebuild") @@ -1704,7 +1738,7 @@ async def restore_document(workflow_id: str, user: RequireAdmin): """ Restore a soft-deleted (disabled) document. - Note: This only restores the document in SQLite. Chunks that were removed + Note: This only restores the document in Postgres. Chunks that were removed from the vector index will NOT be automatically re-indexed. To re-index, you would need to re-run the ingestion process. """ @@ -1734,12 +1768,12 @@ async def reingest_document( Use this to re-ingest documents that completed but weren't properly indexed (e.g., due to index schema changes). This starts a lightweight - workflow that uses chunks already stored in SQLite. + workflow that uses chunks already stored in Postgres. - The document must have chunks stored in SQLite (typically from a + The document must have chunks stored in Postgres (typically from a completed or previously ingested document). """ - # Get document from SQLite + # Get document from Postgres doc = _require_document_for_user(workflow_id, user) # Reingest skips the entire review pipeline (including ready_for_ingestion, @@ -1760,7 +1794,7 @@ async def reingest_document( "review stage entirely and would otherwise ingest with no scheme metadata.", ) - # Get chunks from SQLite + # Get chunks from Postgres chunks = db.get_chunks(workflow_id, include_excluded=False) if not chunks: raise HTTPException(400, "No chunks found for document. The document may need to be reprocessed from scratch.") @@ -1779,7 +1813,7 @@ async def reingest_document( args=[ document_id, filename, - workflow_id, # original workflow_id for SQLite updates + workflow_id, # original workflow_id for Postgres updates page_count, len(chunks), index_name @@ -1794,6 +1828,7 @@ async def reingest_document( status="running", current_stage="ingesting", config={"index_name": index_name, "chunk_count": len(chunks)}, + actor=_actor_from_user(user), ) # Log audit @@ -1841,15 +1876,16 @@ async def retry_ocr(workflow_id: str, user: RequirePipeline): id=temporal_workflow_id, task_queue=TASK_QUEUE, ) - job_id = db.create_document_job( + db.create_document_job( workflow_id=workflow_id, job_type="ocr_retry", temporal_workflow_id=temporal_workflow_id, status="running", current_stage="ocr_processing", config={"source": "api_retry_ocr"}, + actor=_actor_from_user(user), ) - db.update_document_fields(workflow_id, latest_job_id=job_id, error_message=None) + db.update_document_fields(workflow_id, error_message=None) _log_audit( workflow_id=workflow_id, action_type="retry_ocr", @@ -1873,15 +1909,16 @@ async def retry_translation(workflow_id: str, user: RequirePipeline): id=temporal_workflow_id, task_queue=TASK_QUEUE, ) - job_id = db.create_document_job( + db.create_document_job( workflow_id=workflow_id, job_type="translation_retry", temporal_workflow_id=temporal_workflow_id, status="running", current_stage="translation_processing", config={"source": "api_retry_translation"}, + actor=_actor_from_user(user), ) - db.update_document_fields(workflow_id, latest_job_id=job_id, error_message=None) + db.update_document_fields(workflow_id, error_message=None) _log_audit( workflow_id=workflow_id, action_type="retry_translation", @@ -1919,7 +1956,7 @@ async def retry_chunking( id=temporal_workflow_id, task_queue=TASK_QUEUE, ) - job_id = db.create_document_job( + db.create_document_job( workflow_id=workflow_id, job_type="chunking_retry", temporal_workflow_id=temporal_workflow_id, @@ -1931,8 +1968,9 @@ async def retry_chunking( "chunk_overlap": chunk_overlap, "min_tokens": min_tokens, }, + actor=_actor_from_user(user), ) - db.update_document_fields(workflow_id, latest_job_id=job_id, error_message=None) + db.update_document_fields(workflow_id, error_message=None) _log_audit( workflow_id=workflow_id, action_type="retry_chunking", @@ -2075,7 +2113,7 @@ async def _reconcile_single_document(doc: dict) -> dict: @app.post("/documents/{workflow_id}/reconcile") async def reconcile_single_document(workflow_id: str, user: RequirePipeline): - """Reconcile SQLite stage with Temporal state for one document.""" + """Reconcile Postgres stage with Temporal state for one document.""" doc = _require_document_for_user(workflow_id, user) return await _reconcile_single_document(doc) @@ -2113,7 +2151,7 @@ async def _validate_approval_stage(workflow_id: str, expected_stage: str): except HTTPException: raise except Exception: - # Try SQLite fallback to check if workflow exists but is completed/failed + # Try Postgres fallback to check if workflow exists but is completed/failed doc = db.get_document(workflow_id) if doc: current = doc.get("stage") @@ -2453,6 +2491,7 @@ async def approve_prod( temporal_workflow_id=promote_workflow_id, status="running", current_stage="ingesting_prod", + actor=_actor_from_user(user), ) db.update_document_stage( workflow_id=workflow_id, @@ -2516,7 +2555,7 @@ def _log_audit( metadata: Additional context as dict user: Optional AuthUser whose email/roles are stored on the audit row """ - # Get document_id from SQLite + # Get document_id from Postgres doc = db.get_document(workflow_id) document_id = doc["document_id"] if doc else workflow_id @@ -2628,7 +2667,7 @@ async def get_document_audit_log( @app.get("/documents/{workflow_id}/pages") async def list_pages(workflow_id: str, user: RequireSearch): - """Get all pages for a document. SQLite-first for speed.""" + """Get all pages for a document. Postgres-first for speed.""" # Enforce tenant scope up front (404 hides other tenants' documents). _require_document_for_user(workflow_id, user) return db.get_pages(workflow_id) @@ -2636,7 +2675,7 @@ async def list_pages(workflow_id: str, user: RequireSearch): @app.get("/documents/{workflow_id}/pages/{page_num}") async def get_page(workflow_id: str, user: RequireSearch, page_num: int = PathParam(..., ge=1, le=10000, description="Page number (1-indexed)")): - """Get a specific page. SQLite-first for speed.""" + """Get a specific page. Postgres-first for speed.""" _require_document_for_user(workflow_id, user) page = db.get_page(workflow_id, page_num) if page: @@ -2833,14 +2872,14 @@ async def search_chunks_across_documents( @app.get("/documents/{workflow_id}/chunks") async def list_chunks(workflow_id: str, user: RequireSearch, include_excluded: bool = False): - """Get all chunks for a document. SQLite-first for speed.""" + """Get all chunks for a document. Postgres-first for speed.""" _require_document_for_user(workflow_id, user) return db.get_chunks(workflow_id, include_excluded=include_excluded) @app.get("/documents/{workflow_id}/chunks/{chunk_num}") async def get_chunk(workflow_id: str, user: RequireSearch, chunk_num: int = PathParam(..., ge=1, le=10000, description="Chunk number (1-indexed)")): - """Get a specific chunk. SQLite-first for speed.""" + """Get a specific chunk. Postgres-first for speed.""" _require_document_for_user(workflow_id, user) chunk = db.get_chunk(workflow_id, chunk_num) if chunk: @@ -3059,9 +3098,9 @@ def _inline_content_disposition(filename: str) -> str: async def get_document_pdf(workflow_id: str, user: RequireSearch): """ Get the original PDF file for a document. - Returns the PDF as a streaming response. SQLite-first for speed. + Returns the PDF as a streaming response. Postgres-first for speed. """ - # SQLite-first - instant lookup, tenant-scoped (404 hides other tenants). + # Postgres-first - instant lookup, tenant-scoped (404 hides other tenants). doc = _require_document_for_user(workflow_id, user) filepath = doc.get("filepath", "") @@ -3117,7 +3156,7 @@ def file_iterator(): async def resolve_provenance_chunk( request: Request, user: RequireSearch, - doc_id: Optional[str] = Query(None, description="workflow slug, SQLite document_id, or legacy vector-index doc_id"), + doc_id: Optional[str] = Query(None, description="workflow slug, Postgres document_id, or legacy vector-index doc_id"), chunk_num: Optional[int] = Query(None, alias="chunk_num"), ): """ @@ -3184,14 +3223,14 @@ async def get_document_qdrant_status( ) normalized_hit.setdefault("chunk_number", chunk_num) hits.append(normalized_hit) - sqlite_chunks = db.get_chunks(workflow_id, include_excluded=True) + db_chunks = db.get_chunks(workflow_id, include_excluded=True) status = { "workflow_id": workflow_id, "index_name": resolved_index, "backend": backend, "index_doc_id": index_doc_id, "doc_id": index_doc_id, - "sqlite_chunk_count": len([c for c in sqlite_chunks if not c.get("is_excluded")]), + "sqlite_chunk_count": len([c for c in db_chunks if not c.get("is_excluded")]), "indexed_chunk_count": len(hits), "status": "indexed" if hits else "missing", "hits": hits, @@ -3244,7 +3283,7 @@ async def get_vector_indexes_summary( x_include_demo: Optional[str] = Header(None, alias="X-Include-Demo"), x_include_disabled: Optional[str] = Header(None, alias="X-Include-Disabled"), ): - """Summarize index coverage from SQLite-backed index status plus live vector-store stats.""" + """Summarize index coverage from Postgres-backed index status plus live vector-store stats.""" from ..vector_store import get_vector_store include_demo = x_include_demo and x_include_demo.lower() == "true" @@ -3287,7 +3326,8 @@ async def health(): """Health check.""" return { "status": "ok", - "temporal_connected": temporal_client is not None + "temporal_connected": temporal_client is not None, + "database": await asyncio.to_thread(db.check_health), } @@ -3429,10 +3469,10 @@ async def get_ingest_info(user: RequireAdmin): @app.post("/documents/reconcile") async def reconcile_document_states(user: RequirePipeline): """ - Reconcile SQLite document states with Temporal workflow states. + Reconcile Postgres document states with Temporal workflow states. This endpoint checks all documents in processing/review stages and updates - SQLite if the Temporal workflow has terminated or failed. This fixes + Postgres if the Temporal workflow has terminated or failed. This fixes inconsistencies caused by external workflow termination or worker crashes. Returns a summary of documents checked and updated. @@ -3526,7 +3566,10 @@ async def update_search_settings_endpoint(settings: SearchSettingsUpdate, user: if not updates: raise HTTPException(400, "No settings provided to update") - return db.update_search_settings(updates) + try: + return db.update_search_settings(updates) + except ValueError as exc: + raise HTTPException(400, str(exc)) from exc @app.get("/settings/search/audit", response_model=SettingsAuditResponse) diff --git a/pipeline/api/documents.py b/pipeline/api/documents.py index 5eb43ef..2c65771 100644 --- a/pipeline/api/documents.py +++ b/pipeline/api/documents.py @@ -62,8 +62,8 @@ async def start_document_workflow( workflow_id = _api.get_workflow_id(str(filepath)) document_id = canonical_document_id - # Reuse only when SQLite still tracks this workflow. - # If SQLite was purged, avoid returning stale Temporal state and create a fresh run ID. + # Reuse only when Postgres still tracks this workflow. + # If Postgres was purged, avoid returning stale Temporal state and create a fresh run ID. existing_doc = db.get_document(workflow_id) if existing_doc: # Same fingerprint/path must not leak or restart another tenant's doc. @@ -112,13 +112,13 @@ async def start_document_workflow( task_queue=_api.TASK_QUEUE, ) - # Save to SQLite for visibility during processing + # Save to Postgres for visibility during processing actor = _api._actor_from_user(user) # The period this document is searchable in starts here, at the one moment # that knows the upload day. A reviewer confirms or moves it later, on the # form that sets the document type. validity = document_validity.default_period() - db.upsert_document( + document_uuid = db.upsert_document( workflow_id=workflow_id, document_id=document_id, canonical_document_id=canonical_document_id, @@ -135,9 +135,10 @@ async def start_document_workflow( uploaded_by_roles=actor.get("roles_csv") or None, valid_from=validity.start_date, valid_to=validity.end_date, + with_workflow=False, ) - job_id = db.create_document_job( - workflow_id=workflow_id, + db.create_document_job( + workflow_id=document_uuid, job_type="ocr_only" if stop_after_ocr else "pipeline", temporal_workflow_id=workflow_id, status="running", @@ -151,8 +152,8 @@ async def start_document_workflow( "stop_after_ocr": stop_after_ocr, "uploaded_by": actor, }, + actor=actor, ) - db.update_document_fields(workflow_id, latest_job_id=job_id) _api._log_audit( workflow_id=workflow_id, action_type="document_upload", @@ -249,7 +250,7 @@ async def list_documents( """ List all document workflows. - Uses SQLite for fast listing (no Temporal queries for performance). + Uses Postgres for fast listing (no Temporal queries for performance). Demo documents are excluded by default - use X-Include-Demo: true header to show them. Disabled (soft-deleted) documents are excluded by default - use X-Include-Disabled: true to show them. Results are limited to instances the caller can access. @@ -262,7 +263,7 @@ async def list_documents( include_demo = x_include_demo and x_include_demo.lower() == "true" include_disabled = x_include_disabled and x_include_disabled.lower() == "true" - # Use SQLite only for fast listing - no Temporal queries + # Use Postgres only for fast listing - no Temporal queries docs = db.list_documents( stage=stage_filter, limit=limit, @@ -281,7 +282,7 @@ async def get_documents_summary( x_include_demo: Optional[str] = Header(None, alias="X-Include-Demo"), x_include_disabled: Optional[str] = Header(None, alias="X-Include-Disabled") ): - """Return aggregate SQLite counts for dashboard totals and migration planning.""" + """Return aggregate Postgres counts for dashboard totals and migration planning.""" include_demo = x_include_demo and x_include_demo.lower() == "true" include_disabled = x_include_disabled and x_include_disabled.lower() == "true" instance_scope = _api._instance_scope_for_user(user) diff --git a/pipeline/api/upload.py b/pipeline/api/upload.py index 543d4a7..320e08d 100644 --- a/pipeline/api/upload.py +++ b/pipeline/api/upload.py @@ -82,8 +82,8 @@ async def upload_and_process( document_id = file_hash canonical_document_id = file_hash - # Reuse only when SQLite still tracks this workflow. - # If SQLite was purged, avoid returning stale Temporal state and create a fresh run ID. + # Reuse only when Postgres still tracks this workflow. + # If Postgres was purged, avoid returning stale Temporal state and create a fresh run ID. existing_doc = db.get_document(workflow_id) if existing_doc: existing_doc = assert_document_instance_access(user, existing_doc) @@ -131,11 +131,11 @@ async def upload_and_process( task_queue=_api.TASK_QUEUE, ) - # Save to SQLite for visibility during processing + # Save to Postgres for visibility during processing actor = _api._actor_from_user(user) # See the manifest upload path above: the period starts at the upload. validity = document_validity.default_period() - db.upsert_document( + document_uuid = db.upsert_document( workflow_id=workflow_id, document_id=document_id, canonical_document_id=canonical_document_id, @@ -153,9 +153,10 @@ async def upload_and_process( uploaded_by_roles=actor.get("roles_csv") or None, valid_from=validity.start_date, valid_to=validity.end_date, + with_workflow=False, ) job_id = db.create_document_job( - workflow_id=workflow_id, + workflow_id=document_uuid, job_type="ocr_only" if stop_after_ocr else "pipeline", temporal_workflow_id=workflow_id, status="running", @@ -169,8 +170,9 @@ async def upload_and_process( "stop_after_ocr": stop_after_ocr, "uploaded_by": actor, }, + actor=actor, ) - original_artifact_id = db.add_document_artifact( + db.add_document_artifact( workflow_id=workflow_id, job_id=job_id, artifact_type="original_upload", @@ -199,8 +201,6 @@ async def upload_and_process( canonical_input_type = "spreadsheet" if suffix in {".csv", ".xlsx"} else "pdf" db.update_document_fields( workflow_id, - latest_job_id=job_id, - original_artifact_id=original_artifact_id, source_type=source_type, canonical_input_type=canonical_input_type, stop_after_ocr=1 if stop_after_ocr else 0, diff --git a/pipeline/auth/deps.py b/pipeline/auth/deps.py index c09dae9..4b192f3 100644 --- a/pipeline/auth/deps.py +++ b/pipeline/auth/deps.py @@ -5,10 +5,13 @@ import asyncio import base64 import json +import logging +import time from typing import Annotated, Any, Callable from fastapi import Depends, HTTPException, Request +from .. import db from .config import load_auth_config from .jwt import claims_to_user, decode_and_validate_token from .models import AuthUser, local_bypass_user @@ -72,6 +75,25 @@ def _bypass_user_with_optional_jwt(token: str | None) -> AuthUser: ) +_USER_SYNC_INTERVAL_SECONDS = 300 +_user_synced_at: dict[str, float] = {} +_WRITE_METHODS = {"POST", "PUT", "PATCH", "DELETE"} + + +def _sync_user_row(user: AuthUser) -> None: + """Mirror the caller into ``users`` (at most once per interval per user).""" + now = time.monotonic() + last = _user_synced_at.get(user.user_id) + if last is not None and now - last < _USER_SYNC_INTERVAL_SECONDS: + return + try: + db.upsert_user(user.user_id, user.username, user.email, user.roles) + except Exception: + logging.warning("users upsert failed for %s", user.user_id, exc_info=True) + return + _user_synced_at[user.user_id] = now + + async def get_current_user(request: Request) -> AuthUser: """Resolve the caller. When AUTH_DISABLED=true, returns a local bypass user. @@ -83,12 +105,16 @@ async def get_current_user(request: Request) -> AuthUser: token = _bearer_token(request, required=False) if config.disabled: - return _bypass_user_with_optional_jwt(token) + user = _bypass_user_with_optional_jwt(token) + else: + if not token: + raise HTTPException(401, "Missing Bearer token") + # JWKS fetch/refresh is sync and can block; keep the event loop free. + user = await asyncio.to_thread(decode_and_validate_token, token, config) - if not token: - raise HTTPException(401, "Missing Bearer token") - # JWKS fetch/refresh is sync and can block; keep the event loop free. - return await asyncio.to_thread(decode_and_validate_token, token, config) + if request.method in _WRITE_METHODS and user.user_id: + await asyncio.to_thread(_sync_user_row, user) + return user def require_permission(permission: Permission | str) -> Callable[..., AuthUser]: diff --git a/pipeline/db.py b/pipeline/db.py index 22fd9bc..0799b44 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -1,414 +1,483 @@ """ -SQLite state persistence for document visibility. +Postgres persistence for documents, workflows, pages, chunks, audit and settings. -This provides a fallback when Temporal workflow queries fail during -long-running activities, ensuring the dashboard always shows document status. +Schema: ``migrations/`` (see docs/POSTGRES_SCHEMA.md). One function per query. + +A document is content and metadata; a workflow is one processing run of a +document. Callers still address a document by ``workflow_id`` - the Temporal +workflow id of its main run (``doc-``) - and every function here also +accepts the document UUID, the fingerprint or the filename (see +``find_document_by_doc_identifier``). Documents, jobs and artifacts come back +as flat dicts so call sites need not know how the tables are split; ``id`` +carries the UUID. """ import hashlib import json +import logging import os -import sqlite3 +import re +import threading import uuid from contextlib import contextmanager -from datetime import datetime -from pathlib import Path -from threading import Lock -from typing import Optional +from datetime import date, datetime, timezone +from typing import Any, Optional + +import psycopg +from psycopg.rows import dict_row +from psycopg.types.json import Jsonb +from psycopg_pool import ConnectionPool + +from . import migrate +from .models import ( + CandidateSettings, + ChunkingRun, + DocumentMetadata, + DocumentSource, + IndexState, + PageMetadata, + PageTranslation, + QuerySettings, + ResultSettings, + RetrievalSettings, +) -# Database path - can be configured via environment -DB_PATH = os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db") +logger = logging.getLogger(__name__) -# Lock for thread-safe operations -_db_lock = Lock() +DEFAULT_DATABASE_URL = "postgresql://postgres:postgres@localhost:5432/knowledge_provider" +_pool: Optional[ConnectionPool] = None +_pool_url: Optional[str] = None +_pool_lock = threading.Lock() -def _add_column_if_missing(conn: sqlite3.Connection, table: str, column: str, definition: str) -> None: - """Best-effort SQLite migration helper.""" - try: - conn.execute(f"ALTER TABLE {table} ADD COLUMN {column} {definition}") - except sqlite3.OperationalError: - pass + +def database_url() -> str: + return os.environ.get("DATABASE_URL") or DEFAULT_DATABASE_URL -def _ensure_db_dir(): - """Ensure the database directory exists.""" - db_dir = Path(DB_PATH).parent - if not db_dir.exists(): - db_dir.mkdir(parents=True, exist_ok=True) +def get_pool() -> ConnectionPool: + global _pool, _pool_url + url = database_url() + if _pool is not None and _pool_url == url: + return _pool + with _pool_lock: + if _pool is not None and _pool_url != url: + _pool.close() + _pool = None + if _pool is None: + pool = ConnectionPool( + conninfo=url, + min_size=int(os.environ.get("DB_POOL_MIN_SIZE", "1")), + max_size=int(os.environ.get("DB_POOL_MAX_SIZE", "10")), + timeout=float(os.environ.get("DB_POOL_TIMEOUT_SECONDS", "30")), + kwargs={"row_factory": dict_row, "options": "-c timezone=UTC"}, + open=False, + ) + pool.open(wait=True) + _pool = pool + _pool_url = url + logger.info("db pool opened: %s", pool.get_stats()) + return _pool + + +def close_pool() -> None: + global _pool, _pool_url + with _pool_lock: + if _pool is not None: + _pool.close() + _pool = None + _pool_url = None @contextmanager def get_connection(): - """Get a database connection with proper cleanup and isolation.""" - _ensure_db_dir() - conn = sqlite3.connect( - DB_PATH, - check_same_thread=False, - isolation_level="IMMEDIATE", # Acquire lock immediately on write - timeout=30.0 # Wait up to 30s for locks - ) - conn.row_factory = sqlite3.Row - # Enable WAL mode for better concurrent access - conn.execute("PRAGMA journal_mode=WAL") - # Ensure foreign keys are enforced - conn.execute("PRAGMA foreign_keys=ON") - try: + """A pooled connection; the transaction commits on exit and rolls back on error.""" + with get_pool().connection() as conn: yield conn - finally: - conn.close() -def init_db(): - """Initialize the database schema.""" - with _db_lock: +def init_db() -> None: + """Apply pending migrations and open the connection pool.""" + applied = migrate.apply_migrations(database_url()) + current = migrate.current_version(database_url()) + logger.info("db migrations current=%s applied_now=%s", current, applied) + get_pool() + + +def check_health() -> dict: + """Reachability and migration state, for the /health endpoint.""" + try: with get_connection() as conn: - conn.execute(""" - CREATE TABLE IF NOT EXISTS documents ( - workflow_id TEXT PRIMARY KEY, - document_id TEXT NOT NULL, - canonical_document_id TEXT, - filename TEXT NOT NULL, - display_name TEXT, - source_filename TEXT, - source_manifest_name TEXT, - source_file_fingerprint TEXT, - filepath TEXT NOT NULL, - stage TEXT NOT NULL DEFAULT 'registered', - page_count INTEGER DEFAULT 0, - chunk_count INTEGER DEFAULT 0, - error_message TEXT, - is_demo INTEGER DEFAULT 0, - created_at TEXT, - updated_at TEXT, - ocr_completed_at TEXT, - translation_completed_at TEXT, - chunks_completed_at TEXT, - ingested_at TEXT, - source_type TEXT, - canonical_input_type TEXT, - stop_after_ocr INTEGER DEFAULT 0, - reindex_required INTEGER DEFAULT 0, - reindex_reason TEXT, - original_artifact_id INTEGER, - normalized_artifact_id INTEGER, - latest_job_id INTEGER - ) - """) - _add_column_if_missing(conn, "documents", "is_demo", "INTEGER DEFAULT 0") - _add_column_if_missing(conn, "documents", "is_disabled", "INTEGER DEFAULT 0") - _add_column_if_missing(conn, "documents", "display_name", "TEXT") - _add_column_if_missing(conn, "documents", "canonical_document_id", "TEXT") - _add_column_if_missing(conn, "documents", "source_filename", "TEXT") - _add_column_if_missing(conn, "documents", "source_manifest_name", "TEXT") - _add_column_if_missing(conn, "documents", "source_file_fingerprint", "TEXT") - _add_column_if_missing(conn, "documents", "translation_completed_at", "TEXT") - _add_column_if_missing(conn, "documents", "source_type", "TEXT") - _add_column_if_missing(conn, "documents", "canonical_input_type", "TEXT") - _add_column_if_missing(conn, "documents", "stop_after_ocr", "INTEGER DEFAULT 0") - _add_column_if_missing(conn, "documents", "reindex_required", "INTEGER DEFAULT 0") - _add_column_if_missing(conn, "documents", "reindex_reason", "TEXT") - _add_column_if_missing(conn, "documents", "original_artifact_id", "INTEGER") - _add_column_if_missing(conn, "documents", "normalized_artifact_id", "INTEGER") - _add_column_if_missing(conn, "documents", "latest_job_id", "INTEGER") - _add_column_if_missing(conn, "documents", "instance", "TEXT") - _add_column_if_missing(conn, "documents", "uploaded_by_user_id", "TEXT") - _add_column_if_missing(conn, "documents", "uploaded_by_username", "TEXT") - _add_column_if_missing(conn, "documents", "uploaded_by_email", "TEXT") - _add_column_if_missing(conn, "documents", "uploaded_by_roles", "TEXT") - _add_column_if_missing(conn, "documents", "prod_ready_requested_at", "TEXT") - _add_column_if_missing(conn, "documents", "prod_ready_requested_by_user_id", "TEXT") - _add_column_if_missing(conn, "documents", "prod_ready_requested_by_username", "TEXT") - # Period this document's chunks are searchable in, stamped on - # upload and editable by the approver. NULL on documents uploaded - # before validity existed - search reads that as always current - # (see pipeline/document_validity.period_from_row). - _add_column_if_missing(conn, "documents", "valid_from", "TEXT") - _add_column_if_missing(conn, "documents", "valid_to", "TEXT") - # Stamp NULL/empty rows with the configured default so list filters - # (which coalesce to DEFAULT_INSTANCE) match migrated data. - default_instance = ( - (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" - ) - conn.execute( - """ - UPDATE documents - SET instance = ? - WHERE instance IS NULL OR trim(instance) = '' - """, - (default_instance,), - ) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_documents_stage - ON documents(stage) - """) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_documents_instance - ON documents(instance) - """) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_documents_created - ON documents(created_at DESC) - """) - # Audit logs table - conn.execute(""" - CREATE TABLE IF NOT EXISTS audit_logs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - workflow_id TEXT NOT NULL, - document_id TEXT NOT NULL, - action_type TEXT NOT NULL, - entity_type TEXT, - entity_id INTEGER, - field_name TEXT, - old_value TEXT, - new_value TEXT, - metadata TEXT, - timestamp TEXT NOT NULL, - actor_user_id TEXT, - actor_username TEXT, - actor_email TEXT, - actor_roles TEXT - ) - """) - _add_column_if_missing(conn, "audit_logs", "actor_user_id", "TEXT") - _add_column_if_missing(conn, "audit_logs", "actor_username", "TEXT") - _add_column_if_missing(conn, "audit_logs", "actor_email", "TEXT") - _add_column_if_missing(conn, "audit_logs", "actor_roles", "TEXT") - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_audit_workflow - ON audit_logs(workflow_id) - """) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_audit_timestamp - ON audit_logs(timestamp DESC) - """) - # Pages table for persistence after workflow completion - conn.execute(""" - CREATE TABLE IF NOT EXISTS pages ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - workflow_id TEXT NOT NULL, - page_number INTEGER NOT NULL, - original_markdown TEXT, - edited_markdown TEXT, - is_reviewed INTEGER DEFAULT 0, - reviewer_notes TEXT, - detected_language TEXT, - translated_markdown TEXT, - edited_translation TEXT, - translation_reviewed INTEGER DEFAULT 0, - translation_notes TEXT, - translation_provider TEXT, - translation_model TEXT, - translation_target_language TEXT, - translated_at TEXT, - UNIQUE(workflow_id, page_number) - ) - """) - _add_column_if_missing(conn, "pages", "translation_provider", "TEXT") - _add_column_if_missing(conn, "pages", "translation_model", "TEXT") - _add_column_if_missing(conn, "pages", "translation_target_language", "TEXT") - _add_column_if_missing(conn, "pages", "translated_at", "TEXT") - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_pages_workflow - ON pages(workflow_id) - """) - # Chunks table for persistence after workflow completion - conn.execute(""" - CREATE TABLE IF NOT EXISTS chunks ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - workflow_id TEXT NOT NULL, - chunk_number INTEGER NOT NULL, - original_text TEXT, - edited_text TEXT, - token_count INTEGER DEFAULT 0, - page_start INTEGER DEFAULT 1, - page_end INTEGER DEFAULT 1, - source_page_numbers_json TEXT, - source_spans_json TEXT, - section_title TEXT, - content_type TEXT, - is_reference INTEGER DEFAULT 0, - chunking_provider TEXT, - chunking_model TEXT, - chunking_config_json TEXT, - chunking_run_id TEXT, - chunk_version INTEGER DEFAULT 1, - is_reviewed INTEGER DEFAULT 0, - is_excluded INTEGER DEFAULT 0, - reviewer_notes TEXT, - UNIQUE(workflow_id, chunk_number) - ) - """) - _add_column_if_missing(conn, "chunks", "source_page_numbers_json", "TEXT") - _add_column_if_missing(conn, "chunks", "source_spans_json", "TEXT") - _add_column_if_missing(conn, "chunks", "section_title", "TEXT") - _add_column_if_missing(conn, "chunks", "content_type", "TEXT") - _add_column_if_missing(conn, "chunks", "is_reference", "INTEGER DEFAULT 0") - _add_column_if_missing(conn, "chunks", "chunking_provider", "TEXT") - _add_column_if_missing(conn, "chunks", "chunking_model", "TEXT") - _add_column_if_missing(conn, "chunks", "chunking_config_json", "TEXT") - _add_column_if_missing(conn, "chunks", "chunking_run_id", "TEXT") - _add_column_if_missing(conn, "chunks", "chunk_version", "INTEGER DEFAULT 1") - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_chunks_workflow - ON chunks(workflow_id) - """) - # Settings table for application configuration - conn.execute(""" - CREATE TABLE IF NOT EXISTS settings ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL, - description TEXT, - updated_at TEXT NOT NULL - ) - """) - conn.execute(""" - CREATE TABLE IF NOT EXISTS document_jobs ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - workflow_id TEXT NOT NULL, - job_type TEXT NOT NULL, - temporal_workflow_id TEXT, - temporal_run_id TEXT, - status TEXT NOT NULL DEFAULT 'running', - current_stage TEXT, - started_at TEXT NOT NULL, - completed_at TEXT, - error_message TEXT, - config_json TEXT - ) - """) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_document_jobs_workflow - ON document_jobs(workflow_id, started_at DESC) - """) - conn.execute(""" - CREATE TABLE IF NOT EXISTS document_artifacts ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - workflow_id TEXT NOT NULL, - job_id INTEGER, - artifact_type TEXT NOT NULL, - stage TEXT, - storage_uri TEXT NOT NULL, - mime_type TEXT, - filename TEXT, - size_bytes INTEGER, - metadata_json TEXT, - created_at TEXT NOT NULL - ) - """) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_document_artifacts_workflow - ON document_artifacts(workflow_id, created_at DESC) - """) - conn.execute(""" - CREATE TABLE IF NOT EXISTS document_index_status ( - workflow_id TEXT NOT NULL, - index_name TEXT NOT NULL, - doc_id TEXT, - chunk_count_indexed INTEGER DEFAULT 0, - last_indexed_at TEXT, - last_verified_at TEXT, - schema_version TEXT, - status TEXT NOT NULL DEFAULT 'unknown', - details_json TEXT, - PRIMARY KEY (workflow_id, index_name) - ) - """) - # doc_id supersedes the legacy marqo_doc_id column (generic - # naming, no DB name baked into the schema) on databases that - # predate this change. marqo_doc_id is left in place, unwritten, - # until scripts/drop_legacy_marqo_columns.py is run — so this - # backfill is best-effort: a fresh database, or one that's already - # had the column dropped, simply has nothing to migrate. - _add_column_if_missing(conn, "document_index_status", "doc_id", "TEXT") - try: - conn.execute(""" - UPDATE document_index_status - SET doc_id = marqo_doc_id - WHERE doc_id IS NULL AND marqo_doc_id IS NOT NULL - """) - except sqlite3.OperationalError: - pass - conn.execute(""" - CREATE TABLE IF NOT EXISTS document_manifest_entries ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - manifest_filename TEXT NOT NULL UNIQUE, - title_en TEXT, - title_gu TEXT, - doc_language TEXT, - category_tags TEXT, - description TEXT, - quality_score TEXT, - priority_rank TEXT, - ingestion_status TEXT, - feedback TEXT, - source_csv_path TEXT, - imported_at TEXT NOT NULL - ) - """) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_manifest_entries_filename - ON document_manifest_entries(manifest_filename) - """) - # Insert default search settings if not exists - default_settings = [ - ("search_method", "HYBRID", "Search method: TENSOR, LEXICAL, or HYBRID"), - ("search_limit", "12", "Number of search results to return"), - ("search_alpha", "0.6", "Hybrid search alpha: 0=lexical, 1=semantic"), - ("search_ranking_method", "rrf", "Hybrid ranking: rrf or normalize_linear"), - ("search_show_highlights", "true", "Show highlighted matches in results"), - ("search_ef_search", "256", "HNSW search accuracy parameter"), - ("search_index_name", "documents-index", "Default vector DB collection name"), - ("search_candidate_cap", "120", "Candidate retrieval pool cap"), - ("search_candidate_multiplier", "10", "Candidate pool multiplier before final cut"), - ("search_max_chunks_per_doc", "2", "Final result diversity cap per document"), - ("search_use_e5_prefix", "true", "Prefix search queries with e5 query:"), - ("search_exclude_reference", "true", "Exclude reference chunks when index supports it"), - ("search_query_expansion_profile", "gu-v1", "Query expansion profile"), - ("search_rerank_mode", "none", "Post-search reranking mode"), - ("search_hybrid_rrfk", "60", "RRF tuning parameter for hybrid search"), - ] - for key, value, description in default_settings: - conn.execute(""" - INSERT OR IGNORE INTO settings (key, value, description, updated_at) - VALUES (?, ?, ?, ?) - """, (key, value, description, datetime.utcnow().isoformat())) - # Scheme metadata + master catalog tables (AI tool sync) - init_scheme_catalog_schema(conn=conn) - # Email-OTP login codes (one live code per email; overwritten on resend). - conn.execute(""" - CREATE TABLE IF NOT EXISTS email_otps ( - email TEXT PRIMARY KEY, - code_hash TEXT NOT NULL, - expires_at TEXT NOT NULL, - attempts INTEGER NOT NULL DEFAULT 0, - created_at TEXT NOT NULL, - last_sent_at TEXT NOT NULL - ) - """) - # Generic inbound webhook records (full body kept as JSON text in payload). - conn.execute(""" - CREATE TABLE IF NOT EXISTS webhook_records ( - id TEXT PRIMARY KEY, -- uuid4 - source TEXT NOT NULL DEFAULT 'webhook', - external_id TEXT NOT NULL, - title TEXT, - slug TEXT, - status TEXT, - is_published INTEGER, - category TEXT, - payload TEXT NOT NULL, - payload_hash TEXT, - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL, - UNIQUE (source, external_id) - ) - """) + conn.execute("SELECT 1") + return {"reachable": True, "migrations_current": migrate.is_current(database_url())} + except Exception as exc: + return {"reachable": False, "migrations_current": False, "error": str(exc)} + + +# ============================================================================= +# Helpers +# ============================================================================= + +_UUID_RE = re.compile( + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" +) +_FINGERPRINT_RE = re.compile(r"^[0-9a-f]{32}$") + +_REVIEW_STAGES = { + "ocr_review", + "translation_review", + "chunk_review", + "ready_for_ingestion", + "approval_for_prod", +} + +PROD_INDEX_KEY = "prod" +DEV_INDEX_KEY = "dev" + + +def _default_instance() -> str: + return (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + + +def _normalize_instances(instances: list[str]) -> list[str]: + default = _default_instance() + return sorted({(i or "").strip().lower() or default for i in instances}) + + +def _iso_now() -> str: + return datetime.now(timezone.utc).replace(tzinfo=None).isoformat() + + +def _is_uuid(value: Any) -> bool: + return isinstance(value, str) and bool(_UUID_RE.match(value)) + + +def _clean(value: Any) -> Any: + if isinstance(value, uuid.UUID): + return str(value) + if isinstance(value, datetime): + if value.tzinfo is not None: + value = value.astimezone(timezone.utc).replace(tzinfo=None) + return value.isoformat() + if isinstance(value, date): + return value.isoformat() + return value + + +def _clean_row(row: Optional[dict]) -> Optional[dict]: + if row is None: + return None + return {k: _clean(v) for k, v in row.items()} + + +def _csv_roles(roles: Any) -> list[str]: + if roles is None: + return [] + if isinstance(roles, (list, tuple, set)): + return [str(r).strip() for r in roles if str(r).strip()] + return [part.strip() for part in str(roles).split(",") if part.strip()] + + +def _fingerprint_for( + source_file_fingerprint: Optional[str], + document_id: Optional[str], + workflow_id: Optional[str], +) -> str: + for candidate in (source_file_fingerprint, document_id): + if candidate and _FINGERPRINT_RE.match(candidate): + return candidate + return hashlib.md5((document_id or workflow_id or "").encode()).hexdigest() + + +def _prod_collection_names() -> set[str]: + names = { + os.environ.get("PROD_VECTOR_DB_COLLECTION_NAME"), + os.environ.get("PROD_SCHEME_VECTOR_DB_COLLECTION_NAME"), + } + return {n.strip() for n in names if n and n.strip()} + + +def _index_key_for(index_name: str) -> str: + return PROD_INDEX_KEY if index_name in _prod_collection_names() else DEV_INDEX_KEY + + +# ============================================================================= +# Identifier resolution +# ============================================================================= + +def _resolve_document_uuid(conn, identifier: Optional[str]) -> Optional[str]: + """Document UUID for a UUID, Temporal workflow id, fingerprint or filename.""" + if not identifier: + return None + identifier = str(identifier) + + if _is_uuid(identifier): + row = conn.execute( + "SELECT id FROM documents WHERE id = %s::uuid", (identifier,) + ).fetchone() + if row: + return str(row["id"]) + + row = conn.execute( + """ + SELECT document_id FROM workflows + WHERE temporal_workflow_id = %s + ORDER BY started_at DESC LIMIT 1 + """, + (identifier,), + ).fetchone() + if row: + return str(row["document_id"]) + + row = conn.execute( + """ + SELECT id FROM documents + WHERE source->>'fingerprint' = %s OR source->>'original_filename' = %s + ORDER BY created_at DESC LIMIT 1 + """, + (identifier, identifier), + ).fetchone() + if row: + return str(row["id"]) + + row = conn.execute( + """ + SELECT id FROM documents d + WHERE EXISTS ( + SELECT 1 FROM jsonb_each(d.index_status) e WHERE e.value->>'doc_id' = %s + ) OR md5(d.source->>'fingerprint') = %s + ORDER BY created_at DESC LIMIT 1 + """, + (identifier, identifier), + ).fetchone() + return str(row["id"]) if row else None + + +def _require_document_uuid(conn, identifier: str) -> str: + doc_uuid = _resolve_document_uuid(conn, identifier) + if doc_uuid is None: + raise LookupError(f"Document not found: {identifier}") + return doc_uuid + + +# ============================================================================= +# Document row <-> flat dict +# ============================================================================= + +_DOC_SELECT = """ + SELECT + d.id, d.instance, d.source, d.document_kind, d.catalog_entry_id, + d.valid_from, d.valid_to, d.is_disabled, d.index_status, d.metadata, + d.created_at, d.updated_at, + root.temporal_workflow_id AS root_workflow_id, + root.stop_after_ocr AS root_stop_after_ocr, + cw.id AS current_workflow_pk, + cw.current_stage AS current_stage, + cw.error_message AS current_error_message, + ce.code AS entry_code, ce.name AS entry_name, ce.aliases AS entry_aliases, + u.keycloak_user_id AS up_user_id, u.username AS up_username, + u.email AS up_email, u.roles AS up_roles, + ts.ocr_completed_at, ts.translation_completed_at, ts.chunks_completed_at, + ts.ingested_at, + req.started_at AS prod_ready_requested_at, + ru.keycloak_user_id AS prod_ready_requested_by_user_id, + ru.username AS prod_ready_requested_by_username, + oa.id AS original_artifact_id, + na.id AS normalized_artifact_id + FROM documents d + LEFT JOIN document_current_workflow cw ON cw.document_id = d.id + LEFT JOIN LATERAL ( + SELECT w.temporal_workflow_id, w.stop_after_ocr FROM workflows w + WHERE w.document_id = d.id + ORDER BY (w.workflow_type IN ('pipeline', 'ocr_only')) DESC, w.started_at ASC, w.id ASC + LIMIT 1 + ) root ON TRUE + LEFT JOIN catalog_entries ce ON ce.id = d.catalog_entry_id + LEFT JOIN users u ON u.id = d.uploaded_by_user_id + LEFT JOIN LATERAL ( + SELECT min(w.stage_timestamps->>'ocr') AS ocr_completed_at, + min(w.stage_timestamps->>'translation') AS translation_completed_at, + min(w.stage_timestamps->>'chunks') AS chunks_completed_at, + max(COALESCE(w.completed_at, w.updated_at)) + FILTER (WHERE w.current_stage = 'completed') AS ingested_at + FROM workflows w WHERE w.document_id = d.id + ) ts ON TRUE + LEFT JOIN LATERAL ( + SELECT w.started_at, w.started_by_user_id FROM workflows w + WHERE w.document_id = d.id AND w.workflow_type = 'promote_prod' + AND w.status = 'waiting_review' + ORDER BY w.started_at DESC LIMIT 1 + ) req ON TRUE + LEFT JOIN users ru ON ru.id = req.started_by_user_id + LEFT JOIN LATERAL ( + SELECT a.id FROM workflow_artifacts a JOIN workflows w ON w.id = a.workflow_id + WHERE w.document_id = d.id AND a.artifact_type = 'original_upload' + ORDER BY a.created_at DESC, a.id DESC LIMIT 1 + ) oa ON TRUE + LEFT JOIN LATERAL ( + SELECT a.id FROM workflow_artifacts a JOIN workflows w ON w.id = a.workflow_id + WHERE w.document_id = d.id + AND a.artifact_type IN ('normalized_pdf', 'normalized_spreadsheet') + ORDER BY a.created_at DESC, a.id DESC LIMIT 1 + ) na ON TRUE +""" + + +def _document_from_row(row: Optional[dict]) -> Optional[dict]: + if row is None: + return None + source = row["source"] or {} + meta = row["metadata"] or {} + fingerprint = source.get("fingerprint") + roles = row["up_roles"] or [] + entry_aliases = row["entry_aliases"] + doc = { + "id": row["id"], + "workflow_id": row["root_workflow_id"], + "document_id": fingerprint, + "canonical_document_id": fingerprint, + "filename": source.get("original_filename") or "", + "display_name": meta.get("display_name"), + "source_filename": source.get("original_filename"), + "source_manifest_name": None, + "source_file_fingerprint": fingerprint, + "filepath": source.get("storage_uri") or "", + "instance": row["instance"], + "stage": row["current_stage"] or "registered", + "page_count": source.get("page_count") or 0, + "chunk_count": meta.get("chunk_count") or 0, + "error_message": row["current_error_message"], + "is_demo": bool(meta.get("is_demo")), + "is_disabled": bool(row["is_disabled"]), + "source_type": source.get("source_type"), + "canonical_input_type": source.get("canonical_input_type"), + "stop_after_ocr": bool(row["root_stop_after_ocr"]), + "reindex_required": bool(meta.get("reindex_required")), + "reindex_reason": meta.get("reindex_reason"), + "document_kind": row["document_kind"], + "scheme_code": row["entry_code"], + "scheme_name": row["entry_name"], + "scheme_aliases_json": json.dumps(list(entry_aliases)) if entry_aliases is not None else None, + "tool_routing": "qdrant" if row["entry_code"] else None, + "catalog_visible": bool(meta.get("catalog_visible", True)), + "network_visible": bool(meta.get("network_visible", True)), + "valid_from": row["valid_from"], + "valid_to": row["valid_to"], + "uploaded_by_user_id": row["up_user_id"], + "uploaded_by_username": row["up_username"], + "uploaded_by_email": row["up_email"], + "uploaded_by_roles": ",".join(roles) if roles else None, + "prod_ready_requested_at": row["prod_ready_requested_at"], + "prod_ready_requested_by_user_id": row["prod_ready_requested_by_user_id"], + "prod_ready_requested_by_username": row["prod_ready_requested_by_username"], + "ocr_completed_at": row["ocr_completed_at"], + "translation_completed_at": row["translation_completed_at"], + "chunks_completed_at": row["chunks_completed_at"], + "ingested_at": row["ingested_at"], + "original_artifact_id": row["original_artifact_id"], + "normalized_artifact_id": row["normalized_artifact_id"], + "latest_job_id": row["current_workflow_pk"], + "catalog_entry_id": row["catalog_entry_id"], + "index_status": row["index_status"] or {}, + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } + return {k: _clean(v) for k, v in doc.items()} + + +def _fetch_document(conn, doc_uuid: Optional[str]) -> Optional[dict]: + if doc_uuid is None: + return None + row = conn.execute(_DOC_SELECT + " WHERE d.id = %s::uuid", (doc_uuid,)).fetchone() + return _document_from_row(row) + + +def _document_filters( + include_demo: bool, + include_disabled: bool, + instances: Optional[list[str]], + alias: str = "d", +) -> tuple[list[str], list]: + clauses: list[str] = [] + params: list = [] + if not include_demo: + clauses.append(f"NOT COALESCE(({alias}.metadata->>'is_demo')::boolean, FALSE)") + if not include_disabled: + clauses.append(f"NOT {alias}.is_disabled") + if instances is not None: + clauses.append(f"lower({alias}.instance) = ANY(%s)") + params.append(_normalize_instances(instances)) + return clauses, params + + +# ============================================================================= +# Users +# ============================================================================= + +def upsert_user( + keycloak_user_id: str, + username: Optional[str] = None, + email: Optional[str] = None, + roles: Any = None, + conn=None, +) -> Optional[str]: + """Create or refresh the ``users`` row for a Keycloak subject; returns its UUID.""" + keycloak_user_id = (keycloak_user_id or "").strip() + if not keycloak_user_id: + return None + if conn is None: + with get_connection() as own: + return upsert_user(keycloak_user_id, username, email, roles, conn=own) + + role_list = _csv_roles(roles) + email = (email or "").strip() or None + username = (username or "").strip() or None + for candidate_email in ([email, None] if email else [None]): + try: + with conn.transaction(): + row = conn.execute( + """ + INSERT INTO users (keycloak_user_id, username, email, roles) + VALUES (%s, %s, %s, %s) + ON CONFLICT (keycloak_user_id) DO UPDATE SET + username = COALESCE(EXCLUDED.username, users.username), + email = COALESCE(EXCLUDED.email, users.email), + roles = CASE WHEN cardinality(EXCLUDED.roles) > 0 + THEN EXCLUDED.roles ELSE users.roles END, + last_seen_at = now() + RETURNING id + """, + (keycloak_user_id, username, candidate_email, role_list), + ).fetchone() + return str(row["id"]) + except psycopg.errors.UniqueViolation: + continue + return None + + +def get_user_by_keycloak_id(keycloak_user_id: str) -> Optional[dict]: + with get_connection() as conn: + return _clean_row( conn.execute( - "CREATE INDEX IF NOT EXISTS idx_webhook_records_source_status " - "ON webhook_records (source, status)" - ) - conn.commit() + "SELECT * FROM users WHERE keycloak_user_id = %s", (keycloak_user_id,) + ).fetchone() + ) + + +# ============================================================================= +# Webhook records +# ============================================================================= + +def _parse_timestamp(value: Any) -> Optional[datetime]: + if not value: + return None + try: + parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed def upsert_webhook_records(source: str, records: list) -> int: @@ -417,13 +486,13 @@ def upsert_webhook_records(source: str, records: list) -> int: external_id comes from contentID, else id, else slug, else a hash of the payload. Unchanged payloads (same payload_hash) are not rewritten but still counted. """ - now = datetime.utcnow().isoformat() + now = datetime.now(timezone.utc) rows = [] for rec in records: if not isinstance(rec, dict): raise ValueError("Each record must be a JSON object") - payload = json.dumps(rec, sort_keys=True, ensure_ascii=False, default=str) - payload_hash = hashlib.sha256(payload.encode("utf-8")).hexdigest() + payload_text = json.dumps(rec, sort_keys=True, ensure_ascii=False, default=str) + payload_hash = hashlib.sha256(payload_text.encode("utf-8")).hexdigest() ext = rec.get("contentID") if ext in (None, ""): ext = rec.get("id") @@ -433,296 +502,296 @@ def upsert_webhook_records(source: str, records: list) -> int: ext = payload_hash pub = rec.get("isPublished") rows.append(( - str(uuid.uuid4()), source, str(ext), rec.get("title"), rec.get("slug"), rec.get("status"), - int(pub) if isinstance(pub, (bool, int)) else None, - rec.get("domainName"), payload, payload_hash, - rec.get("createdAt") or now, rec.get("updatedAt") or now, + source, str(ext), rec.get("title"), rec.get("slug"), rec.get("status"), + bool(pub) if isinstance(pub, (bool, int)) else None, + rec.get("domainName"), Jsonb(json.loads(payload_text)), payload_hash, + _parse_timestamp(rec.get("createdAt")) or now, + _parse_timestamp(rec.get("updatedAt")) or now, )) - with _db_lock: - with get_connection() as conn: - for r in rows: - conn.execute( - """ - INSERT INTO webhook_records - (id, source, external_id, title, slug, status, is_published, category, - payload, payload_hash, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(source, external_id) DO UPDATE SET - title = excluded.title, - slug = excluded.slug, - status = excluded.status, - is_published = excluded.is_published, - category = excluded.category, - payload = excluded.payload, - payload_hash = excluded.payload_hash, - created_at = excluded.created_at, - updated_at = excluded.updated_at - WHERE webhook_records.payload_hash IS NOT excluded.payload_hash - """, - r, - ) - conn.commit() + with get_connection() as conn: + for r in rows: + conn.execute( + """ + INSERT INTO webhook_records + (source, external_id, title, slug, status, is_published, category, + payload, payload_hash, created_at, updated_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (source, external_id) DO UPDATE SET + title = EXCLUDED.title, + slug = EXCLUDED.slug, + status = EXCLUDED.status, + is_published = EXCLUDED.is_published, + category = EXCLUDED.category, + payload = EXCLUDED.payload, + payload_hash = EXCLUDED.payload_hash, + created_at = EXCLUDED.created_at, + updated_at = EXCLUDED.updated_at + WHERE webhook_records.payload_hash IS DISTINCT FROM EXCLUDED.payload_hash + """, + r, + ) return len(rows) +# ============================================================================= +# Email OTPs +# ============================================================================= + def store_email_otp( *, email: str, code_hash: str, expires_at: str, created_at: str, last_sent_at: str ) -> None: """Upsert the one live OTP for an email, resetting attempts on resend.""" - with _db_lock: - with get_connection() as conn: - conn.execute( - """ - INSERT INTO email_otps (email, code_hash, expires_at, attempts, created_at, last_sent_at) - VALUES (?, ?, ?, 0, ?, ?) - ON CONFLICT(email) DO UPDATE SET - code_hash = excluded.code_hash, - expires_at = excluded.expires_at, - attempts = 0, - created_at = excluded.created_at, - last_sent_at = excluded.last_sent_at - """, - (email, code_hash, expires_at, created_at, last_sent_at), - ) - conn.commit() + with get_connection() as conn: + conn.execute( + """ + INSERT INTO email_otps (email, code_hash, expires_at, attempts, last_sent_at) + VALUES (%s, %s, %s, 0, %s) + ON CONFLICT (email) DO UPDATE SET + code_hash = EXCLUDED.code_hash, + expires_at = EXCLUDED.expires_at, + attempts = 0, + last_sent_at = EXCLUDED.last_sent_at + """, + (email.strip().lower(), code_hash, expires_at, last_sent_at), + ) -def get_email_otp(email: str) -> Optional[sqlite3.Row]: +def get_email_otp(email: str) -> Optional[dict]: + """The live OTP row; timestamps are timezone-aware ISO strings (UTC).""" with get_connection() as conn: - return conn.execute( - "SELECT * FROM email_otps WHERE email = ?", (email,) + row = conn.execute( + "SELECT * FROM email_otps WHERE email = %s", (email.strip().lower(),) ).fetchone() + if row is None: + return None + row = _clean_row(row) | { + "expires_at": row["expires_at"].astimezone(timezone.utc).isoformat(), + "last_sent_at": row["last_sent_at"].astimezone(timezone.utc).isoformat(), + } + return row -def increment_email_otp_attempts(email: str) -> None: - with _db_lock: - with get_connection() as conn: - conn.execute( - "UPDATE email_otps SET attempts = attempts + 1 WHERE email = ?", (email,) - ) - conn.commit() +def increment_email_otp_attempts(email: str) -> Optional[int]: + """Atomically count one failed attempt; returns the new total.""" + with get_connection() as conn: + row = conn.execute( + "UPDATE email_otps SET attempts = attempts + 1 WHERE email = %s RETURNING attempts", + (email.strip().lower(),), + ).fetchone() + return int(row["attempts"]) if row else None def delete_email_otp(email: str) -> None: - with _db_lock: - with get_connection() as conn: - conn.execute("DELETE FROM email_otps WHERE email = ?", (email,)) - conn.commit() - - -def init_scheme_catalog_schema(conn: Optional[sqlite3.Connection] = None) -> None: - """Idempotent: scheme document columns + scheme_catalog_* tables.""" - - def _apply(c: sqlite3.Connection) -> None: - for column, definition in ( - ("document_kind", "TEXT DEFAULT 'document'"), - ("scheme_code", "TEXT"), - ("scheme_name", "TEXT"), - ("scheme_aliases_json", "TEXT"), - ("tool_routing", "TEXT"), - ("catalog_visible", "INTEGER DEFAULT 1"), - ("network_visible", "INTEGER DEFAULT 1"), - ): - _add_column_if_missing(c, "documents", column, definition) - - c.execute(""" - CREATE TABLE IF NOT EXISTS scheme_catalog_meta ( - id INTEGER PRIMARY KEY CHECK (id = 1), - version INTEGER NOT NULL DEFAULT 0, - updated_at TEXT, - notes TEXT - ) - """) - c.execute(""" - INSERT OR IGNORE INTO scheme_catalog_meta (id, version, updated_at, notes) - VALUES (1, 0, ?, ?) - """, (datetime.utcnow().isoformat(), "init")) - - c.execute(""" - CREATE TABLE IF NOT EXISTS scheme_catalog_entries ( - scheme_code TEXT PRIMARY KEY, - scheme_name TEXT, - scheme_aliases_json TEXT, - instances_json TEXT, - workflow_ids_json TEXT, - collection_name TEXT, - chunk_count INTEGER DEFAULT 0, - status TEXT NOT NULL DEFAULT 'disabled', - network_visible INTEGER DEFAULT 0, - promoted_at TEXT, - content_hash TEXT, - source TEXT, - updated_at TEXT - ) - """) - c.execute(""" - CREATE INDEX IF NOT EXISTS idx_scheme_catalog_status - ON scheme_catalog_entries(status) - """) - c.execute(""" - CREATE INDEX IF NOT EXISTS idx_documents_scheme_code - ON documents(scheme_code) - """) - - if conn is not None: - _apply(conn) - return - - with _db_lock: - with get_connection() as c: - _apply(c) - c.commit() + with get_connection() as conn: + conn.execute("DELETE FROM email_otps WHERE email = %s", (email.strip().lower(),)) + + +# ============================================================================= +# Catalog +# ============================================================================= + +CATALOG_VERSION_ACTION = "catalog_version_bump" def get_catalog_meta() -> dict: - init_scheme_catalog_schema() + """Catalog version: the number of version bumps recorded in ``audit_logs``.""" with get_connection() as conn: - row = conn.execute("SELECT * FROM scheme_catalog_meta WHERE id = 1").fetchone() - if not row: - return {"id": 1, "version": 0, "updated_at": None, "notes": None} - return dict(row) + row = conn.execute( + """ + SELECT count(*) AS version, max(created_at) AS updated_at, + (array_agg(metadata->>'reason' ORDER BY created_at DESC))[1] AS notes + FROM audit_logs + WHERE entity_type = 'catalog' AND action_type = %s + """, + (CATALOG_VERSION_ACTION,), + ).fetchone() + return { + "id": 1, + "version": int(row["version"] or 0), + "updated_at": _clean(row["updated_at"]), + "notes": row["notes"], + } def bump_catalog_version(reason: str = "") -> int: """Monotonic catalog version bump. Returns new version.""" - now = datetime.utcnow().isoformat() - init_scheme_catalog_schema() - with _db_lock: - with get_connection() as conn: - conn.execute( - """ - UPDATE scheme_catalog_meta - SET version = version + 1, updated_at = ?, notes = ? - WHERE id = 1 - """, - (now, (reason or "")[:500]), - ) - conn.commit() - row = conn.execute( - "SELECT version FROM scheme_catalog_meta WHERE id = 1" - ).fetchone() - return int(row["version"]) if row else 0 + with get_connection() as conn: + conn.execute( + """ + INSERT INTO audit_logs (action_type, entity_type, metadata) + VALUES (%s, 'catalog', %s) + """, + (CATALOG_VERSION_ACTION, Jsonb({"reason": (reason or "")[:500]})), + ) + row = conn.execute( + "SELECT count(*) AS version FROM audit_logs " + "WHERE entity_type = 'catalog' AND action_type = %s", + (CATALOG_VERSION_ACTION,), + ).fetchone() + return int(row["version"]) def upsert_catalog_entry( *, - scheme_code: str, - scheme_name: str, - scheme_aliases: list, - instances: list, - workflow_ids: list, - collection_name: str, - chunk_count: int, - status: str, - network_visible: int, - promoted_at: Optional[str], - content_hash: str, - source: str, - updated_at: str, -) -> None: - init_scheme_catalog_schema() - with _db_lock: - with get_connection() as conn: - conn.execute( - """ - INSERT INTO scheme_catalog_entries ( - scheme_code, scheme_name, scheme_aliases_json, instances_json, - workflow_ids_json, collection_name, chunk_count, status, - network_visible, promoted_at, content_hash, source, updated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(scheme_code) DO UPDATE SET - scheme_name = excluded.scheme_name, - scheme_aliases_json = excluded.scheme_aliases_json, - instances_json = excluded.instances_json, - workflow_ids_json = excluded.workflow_ids_json, - collection_name = excluded.collection_name, - chunk_count = excluded.chunk_count, - status = excluded.status, - network_visible = excluded.network_visible, - promoted_at = excluded.promoted_at, - content_hash = excluded.content_hash, - source = excluded.source, - updated_at = excluded.updated_at - """, - ( - scheme_code, - scheme_name, - json.dumps(scheme_aliases or []), - json.dumps(instances or []), - json.dumps(workflow_ids or []), - collection_name, - int(chunk_count or 0), - status, - int(network_visible or 0), - promoted_at, - content_hash, - source, - updated_at, - ), + type: str, + code: str, + name: Optional[str] = None, + aliases: Optional[list[str]] = None, + source: Optional[str] = None, + conn=None, +) -> dict: + """Insert or update one ``catalog_entries`` row. ``None`` fields keep stored values.""" + if conn is None: + with get_connection() as own: + return upsert_catalog_entry( + type=type, code=code, name=name, aliases=aliases, source=source, conn=own ) - conn.commit() + alias_list = [a for a in (aliases or []) if a] if aliases is not None else None + row = conn.execute( + """ + INSERT INTO catalog_entries (type, code, name, aliases, metadata) + VALUES (%s, %s, COALESCE(%s, %s), COALESCE(%s, '{}'::text[]), %s) + ON CONFLICT (type, code) DO UPDATE SET + name = COALESCE(%s, catalog_entries.name), + aliases = COALESCE(%s, catalog_entries.aliases), + metadata = CASE WHEN %s::text IS NULL THEN catalog_entries.metadata + ELSE catalog_entries.metadata || jsonb_build_object('source', %s::text) END + RETURNING id, type, code, name, aliases, metadata + """, + ( + type, code, name, code, alias_list, Jsonb({"source": source or "pipeline"}), + name, alias_list, source, source, + ), + ).fetchone() + return _clean_row(row) + + +def list_catalog_entry_codes(type: str) -> set[str]: + with get_connection() as conn: + rows = conn.execute("SELECT code FROM catalog_entries WHERE type = %s", (type,)).fetchall() + return {r["code"] for r in rows} -def list_catalog_entries(status: Optional[str] = None) -> list[dict]: - init_scheme_catalog_schema() +def _catalog_row(row: dict) -> dict: + cleaned = _clean_row(row) + cleaned["aliases"] = list(row["aliases"] or []) + cleaned["instances"] = list(row["instances"] or []) + cleaned["workflow_ids"] = list(row["workflow_ids"] or []) + return cleaned + + +def list_catalog_entries(status: Optional[str] = None, type: str = "scheme") -> list[dict]: + """Rows of the ``catalog`` view for one entry type.""" with get_connection() as conn: if status: rows = conn.execute( - "SELECT * FROM scheme_catalog_entries WHERE status = ? ORDER BY scheme_code", - (status,), + "SELECT * FROM catalog WHERE type = %s AND status = %s ORDER BY code", + (type, status), ).fetchall() else: rows = conn.execute( - "SELECT * FROM scheme_catalog_entries ORDER BY scheme_code" + "SELECT * FROM catalog WHERE type = %s ORDER BY code", (type,) ).fetchall() - return [dict(r) for r in rows] + return [_catalog_row(r) for r in rows] -def get_catalog_entry(scheme_code: str) -> Optional[dict]: - init_scheme_catalog_schema() +def get_catalog_entry(code: str, type: str = "scheme") -> Optional[dict]: with get_connection() as conn: row = conn.execute( - "SELECT * FROM scheme_catalog_entries WHERE scheme_code = ?", - (scheme_code,), + "SELECT * FROM catalog WHERE type = %s AND code = %s", (type, code) ).fetchone() - return dict(row) if row else None + return _catalog_row(row) if row else None -def list_scheme_documents_for_catalog() -> list[dict]: - """All documents that look scheme-related (for rebuild).""" - init_scheme_catalog_schema() +def list_used_scheme_codes(exclude_workflow_id: Optional[str] = None) -> set[str]: + """Every scheme code already taken - the collision domain for auto-generated codes. + + The entry the excluded document alone holds is not counted, so re-deriving a + document's own code does not collide with itself. + """ with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, exclude_workflow_id) if exclude_workflow_id else None rows = conn.execute( """ - SELECT * FROM documents - WHERE document_kind = 'scheme' - OR (scheme_code IS NOT NULL AND trim(scheme_code) != '') - """ + SELECT e.code FROM catalog_entries e + WHERE e.type = 'scheme' + AND NOT ( + %s::uuid IS NOT NULL + AND e.metadata->>'source' <> 'bootstrap' + AND EXISTS (SELECT 1 FROM documents d + WHERE d.catalog_entry_id = e.id AND d.id = %s::uuid) + AND NOT EXISTS (SELECT 1 FROM documents d + WHERE d.catalog_entry_id = e.id AND d.id <> %s::uuid) + ) + """, + (doc_uuid, doc_uuid, doc_uuid), ).fetchall() - return [dict(r) for r in rows] + return {r["code"] for r in rows} -def list_used_scheme_codes(exclude_workflow_id: Optional[str] = None) -> set[str]: - """Every scheme_code currently in use, across both the live documents table - and the scheme_catalog_entries registry (PROD-published codes may outlive - the document row that first defined them) — the full collision domain for - auto-generating a new code from a title.""" - init_scheme_catalog_schema() +def list_scheme_catalog_entries_for_workflow(workflow_id: str) -> list[dict]: + """Catalog rows whose documents still include this workflow.""" with get_connection() as conn: - doc_rows = conn.execute( - "SELECT scheme_code, workflow_id FROM documents " - "WHERE scheme_code IS NOT NULL AND trim(scheme_code) != ''" + rows = conn.execute( + "SELECT code, workflow_ids FROM catalog WHERE type = 'scheme' AND %s = ANY(workflow_ids)", + (workflow_id,), ).fetchall() - entry_rows = conn.execute("SELECT scheme_code FROM scheme_catalog_entries").fetchall() - codes = { - row["scheme_code"].strip().lower() - for row in doc_rows - if row["workflow_id"] != exclude_workflow_id - } - codes.update(row["scheme_code"].strip().lower() for row in entry_rows) - return codes + return [{"scheme_code": r["code"], "workflow_ids": list(r["workflow_ids"])} for r in rows] + + +def _link_catalog_entry( + conn, + doc_uuid: str, + updates: dict, +) -> dict[str, Any]: + """Resolve document_kind / scheme_* updates into documents column changes. + + Returns the extra ``documents`` assignments (``document_kind``, + ``catalog_entry_id``) so the kind and its entry change in one statement, + which the composite foreign key requires. + """ + current = conn.execute( + """ + SELECT d.document_kind, d.catalog_entry_id, ce.type AS entry_type, + ce.code AS entry_code, ce.name AS entry_name, ce.aliases AS entry_aliases + FROM documents d LEFT JOIN catalog_entries ce ON ce.id = d.catalog_entry_id + WHERE d.id = %s::uuid + """, + (doc_uuid,), + ).fetchone() + kind = updates.get("document_kind") or current["document_kind"] + code = updates["scheme_code"] if "scheme_code" in updates else current["entry_code"] + name = updates["scheme_name"] if "scheme_name" in updates else None + aliases = None + if "scheme_aliases_json" in updates: + raw = updates["scheme_aliases_json"] + aliases = json.loads(raw) if isinstance(raw, str) else list(raw or []) + + changes: dict[str, Any] = {} + if "document_kind" in updates: + changes["document_kind"] = kind + + current_entry_id = str(current["catalog_entry_id"]) if current["catalog_entry_id"] else None + new_entry_id = None + if kind != "document" and code: + moved = current["entry_type"] != kind or current["entry_code"] != code + if moved: + if name is None: + name = current["entry_name"] + if aliases is None and current["entry_aliases"] is not None: + aliases = list(current["entry_aliases"]) + new_entry_id = upsert_catalog_entry( + type=kind, code=code, name=name, aliases=aliases, source="pipeline", conn=conn + )["id"] + if new_entry_id != current_entry_id: + changes["catalog_entry_id"] = new_entry_id + return changes +# ============================================================================= +# Documents +# ============================================================================= + def upsert_document( workflow_id: str, document_id: str, @@ -744,9 +813,9 @@ def upsert_document( source_type: Optional[str] = None, canonical_input_type: Optional[str] = None, stop_after_ocr: bool = False, - original_artifact_id: Optional[int] = None, - normalized_artifact_id: Optional[int] = None, - latest_job_id: Optional[int] = None, + original_artifact_id: Optional[str] = None, + normalized_artifact_id: Optional[str] = None, + latest_job_id: Optional[str] = None, instance: Optional[str] = None, uploaded_by_user_id: Optional[str] = None, uploaded_by_username: Optional[str] = None, @@ -754,8 +823,9 @@ def upsert_document( uploaded_by_roles: Optional[str] = None, valid_from: Optional[str] = None, valid_to: Optional[str] = None, -): - """Insert or update a document record. + with_workflow: bool = True, +) -> str: + """Insert or update a document record; returns the document UUID. ``instance`` is applied on INSERT only. Updates leave the existing tenant stamp alone so a re-upload / restart cannot silently reassign tenants. @@ -766,94 +836,100 @@ def upsert_document( restart that re-registers the same workflow. What the period should default to is the caller's call (see ``pipeline/document_validity.py``); omitting both stores NULL, which search reads as "no stated period". - """ - now = datetime.utcnow().isoformat() - instance_value = (instance or os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" - with _db_lock: - with get_connection() as conn: - # Check if exists - row = conn.execute( - "SELECT created_at FROM documents WHERE workflow_id = ?", - (workflow_id,) + A new document also gets its first workflow row (``temporal_workflow_id`` = + ``workflow_id``) unless ``with_workflow`` is False, for callers that create + it themselves with ``create_document_job`` (addressing the document by the + returned UUID, since no workflow id resolves to it yet). + """ + instance_value = (instance or _default_instance()).strip().lower() or _default_instance() + fingerprint = _fingerprint_for( + source_file_fingerprint or canonical_document_id, document_id, workflow_id + ) + source_patch = { + "fingerprint": fingerprint, + "original_filename": filename, + "storage_uri": filepath, + "source_type": source_type, + "canonical_input_type": canonical_input_type, + "page_count": page_count, + } + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) if workflow_id else None + if doc_uuid: + patch = {k: v for k, v in source_patch.items() if v is not None} + DocumentSource.model_validate({**_load(conn, doc_uuid, "source"), **patch}) + metadata_patch: dict[str, Any] = {"chunk_count": chunk_count} + if display_name: + metadata_patch["display_name"] = display_name + uploader_uuid = ( + upsert_user(uploaded_by_user_id, uploaded_by_username, uploaded_by_email, + uploaded_by_roles, conn=conn) + if uploaded_by_user_id else None + ) + conn.execute( + """ + UPDATE documents SET + source = source || %s::jsonb, + metadata = metadata || %s::jsonb, + uploaded_by_user_id = COALESCE(uploaded_by_user_id, %s::uuid) + WHERE id = %s::uuid + """, + (Jsonb(patch), Jsonb(metadata_patch), uploader_uuid, doc_uuid), + ) + cw = conn.execute( + "SELECT id FROM document_current_workflow WHERE document_id = %s::uuid", (doc_uuid,) ).fetchone() + if cw: + conn.execute( + """ + UPDATE workflows SET current_stage = %s::workflow_stage, error_message = %s, + stop_after_ocr = %s + WHERE id = %s::uuid + """, + (stage, error_message, bool(stop_after_ocr), str(cw["id"])), + ) + return doc_uuid + + source = DocumentSource.model_validate( + {k: v for k, v in source_patch.items() if v is not None} + ).model_dump(exclude_none=True) + metadata = DocumentMetadata( + display_name=display_name, chunk_count=chunk_count + ).model_dump(exclude_none=True) + user_uuid = ( + upsert_user(uploaded_by_user_id, uploaded_by_username, uploaded_by_email, + uploaded_by_roles, conn=conn) + if uploaded_by_user_id else None + ) + row = conn.execute( + """ + INSERT INTO documents (instance, source, metadata, valid_from, valid_to, uploaded_by_user_id) + VALUES (%s, %s, %s, %s::date, %s::date, %s::uuid) + RETURNING id + """, + (instance_value, Jsonb(source), Jsonb(metadata), valid_from, valid_to, user_uuid), + ).fetchone() + if with_workflow: + conn.execute( + """ + INSERT INTO workflows (document_id, workflow_type, temporal_workflow_id, + current_stage, error_message, stop_after_ocr, + started_by_user_id) + VALUES (%s::uuid, %s::workflow_type_enum, %s, %s::workflow_stage, %s, %s, %s::uuid) + """, + ( + str(row["id"]), + "ocr_only" if stop_after_ocr else "pipeline", + workflow_id, stage, error_message, bool(stop_after_ocr), user_uuid, + ), + ) + return str(row["id"]) - if row: - # Update existing — do not overwrite instance (tenant ownership). - conn.execute(""" - UPDATE documents SET - document_id = ?, - canonical_document_id = COALESCE(?, canonical_document_id), - filename = ?, - display_name = COALESCE(?, display_name), - source_filename = COALESCE(?, source_filename), - source_manifest_name = COALESCE(?, source_manifest_name), - source_file_fingerprint = COALESCE(?, source_file_fingerprint), - filepath = ?, - stage = ?, - page_count = ?, - chunk_count = ?, - error_message = ?, - updated_at = ?, - ocr_completed_at = COALESCE(?, ocr_completed_at), - translation_completed_at = COALESCE(?, translation_completed_at), - chunks_completed_at = COALESCE(?, chunks_completed_at), - ingested_at = COALESCE(?, ingested_at), - source_type = COALESCE(?, source_type), - canonical_input_type = COALESCE(?, canonical_input_type), - stop_after_ocr = COALESCE(?, stop_after_ocr), - original_artifact_id = COALESCE(?, original_artifact_id), - normalized_artifact_id = COALESCE(?, normalized_artifact_id), - latest_job_id = COALESCE(?, latest_job_id), - uploaded_by_user_id = COALESCE(uploaded_by_user_id, ?), - uploaded_by_username = COALESCE(uploaded_by_username, ?), - uploaded_by_email = COALESCE(uploaded_by_email, ?), - uploaded_by_roles = COALESCE(uploaded_by_roles, ?) - WHERE workflow_id = ? - """, ( - document_id, canonical_document_id, filename, display_name, - source_filename, source_manifest_name, source_file_fingerprint, - filepath, stage, - page_count, chunk_count, error_message, now, - ocr_completed_at, translation_completed_at, chunks_completed_at, ingested_at, - source_type, canonical_input_type, 1 if stop_after_ocr else 0, - original_artifact_id, normalized_artifact_id, latest_job_id, - uploaded_by_user_id, uploaded_by_username, uploaded_by_email, uploaded_by_roles, - workflow_id - )) - else: - # Insert new - conn.execute(""" - INSERT INTO documents ( - workflow_id, document_id, filename, filepath, - canonical_document_id, display_name, source_filename, source_manifest_name, - source_file_fingerprint, - stage, page_count, chunk_count, error_message, - created_at, updated_at, - ocr_completed_at, translation_completed_at, chunks_completed_at, ingested_at, - source_type, canonical_input_type, stop_after_ocr, - reindex_required, reindex_reason, - original_artifact_id, normalized_artifact_id, latest_job_id, - instance, - uploaded_by_user_id, uploaded_by_username, uploaded_by_email, uploaded_by_roles, - valid_from, valid_to - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, ( - workflow_id, document_id, filename, filepath, - canonical_document_id, display_name, source_filename, source_manifest_name, - source_file_fingerprint, - stage, page_count, chunk_count, error_message, - now, now, - ocr_completed_at, translation_completed_at, chunks_completed_at, ingested_at, - source_type, canonical_input_type, 1 if stop_after_ocr else 0, - 0, None, - original_artifact_id, normalized_artifact_id, latest_job_id, - instance_value, - uploaded_by_user_id, uploaded_by_username, uploaded_by_email, uploaded_by_roles, - valid_from, valid_to, - )) - - conn.commit() + +def _load(conn, doc_uuid: str, column: str) -> dict: + row = conn.execute(f"SELECT {column} FROM documents WHERE id = %s::uuid", (doc_uuid,)).fetchone() + return dict(row[column]) if row else {} def update_document_stage( @@ -863,64 +939,61 @@ def update_document_stage( chunk_count: Optional[int] = None, error_message: Optional[str] = None ): - """Update just the stage and counts for a document.""" - now = datetime.utcnow().isoformat() - - with _db_lock: - with get_connection() as conn: - # Get previous state for audit logging - row = conn.execute( - "SELECT stage, document_id FROM documents WHERE workflow_id = ?", - (workflow_id,) - ).fetchone() - old_stage = row["stage"] if row else None - document_id = row["document_id"] if row else workflow_id - - # Build dynamic update - updates = ["stage = ?", "updated_at = ?"] - values = [stage, now] - - if page_count is not None: - updates.append("page_count = ?") - values.append(page_count) - - if chunk_count is not None: - updates.append("chunk_count = ?") - values.append(chunk_count) - - if error_message is not None: - updates.append("error_message = ?") - values.append(error_message) - - # Set timestamp based on stage - if stage == "ocr_review": - updates.append("ocr_completed_at = ?") - values.append(now) - elif stage == "translation_review": - updates.append("translation_completed_at = ?") - values.append(now) - elif stage == "chunk_review": - updates.append("chunks_completed_at = ?") - values.append(now) - elif stage == "completed": - updates.append("ingested_at = ?") - values.append(now) - - values.append(workflow_id) - - # Note: updates list contains only hardcoded field names, not user input - # Values are properly parameterized via ? - conn.execute( - f"UPDATE documents SET {', '.join(updates)} WHERE workflow_id = ?", - values + """Update the stage (of the document's current workflow) and counts.""" + old_stage = None + audit_workflow_id = workflow_id + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return + cw = conn.execute( + "SELECT id, current_stage FROM document_current_workflow WHERE document_id = %s::uuid", + (doc_uuid,), + ).fetchone() + if cw is None: + return + old_stage = cw["current_stage"] + + assignments = ["current_stage = %s::workflow_stage"] + params: list = [stage] + if error_message is not None: + assignments.append("error_message = %s") + params.append(error_message) + stamp_key = { + "ocr_review": "ocr", + "translation_review": "translation", + "chunk_review": "chunks", + }.get(stage) + if stamp_key: + assignments.append( + "stage_timestamps = jsonb_build_object(%s::text, %s::text) || stage_timestamps" ) - conn.commit() + params.extend([stamp_key, _iso_now()]) + if stage == "completed": + assignments.append("completed_at = now()") + params.append(str(cw["id"])) + conn.execute( + f"UPDATE workflows SET {', '.join(assignments)} WHERE id = %s::uuid", params + ) + + doc_assignments = ["updated_at = now()"] + doc_params: list = [] + if page_count is not None: + doc_assignments.append("source = source || jsonb_build_object('page_count', %s::int)") + doc_params.append(page_count) + if chunk_count is not None: + doc_assignments.append("metadata = metadata || jsonb_build_object('chunk_count', %s::int)") + doc_params.append(chunk_count) + doc_params.append(doc_uuid) + conn.execute( + f"UPDATE documents SET {', '.join(doc_assignments)} WHERE id = %s::uuid", doc_params + ) + audit_workflow_id = _root_workflow_id(conn, doc_uuid) or workflow_id - # Log stage transition if stage changed (outside lock to avoid deadlock) if old_stage and old_stage != stage: log_audit( - workflow_id=workflow_id, - document_id=document_id, + workflow_id=audit_workflow_id, + document_id=audit_workflow_id, action_type="stage_change", field_name="stage", old_value=old_stage, @@ -929,6 +1002,18 @@ def update_document_stage( ) +def _root_workflow_id(conn, doc_uuid: str) -> Optional[str]: + row = conn.execute( + """ + SELECT temporal_workflow_id FROM workflows WHERE document_id = %s::uuid + ORDER BY (workflow_type IN ('pipeline', 'ocr_only')) DESC, started_at ASC, id ASC + LIMIT 1 + """, + (doc_uuid,), + ).fetchone() + return row["temporal_workflow_id"] if row else None + + def reconcile_materialized_state(workflow_id: str) -> Optional[dict]: """ Reconcile a document row against already-persisted page/chunk content. @@ -939,23 +1024,19 @@ def reconcile_materialized_state(workflow_id: str) -> Optional[dict]: doc = get_document(workflow_id) if not doc: return None + workflow_id = doc["workflow_id"] with get_connection() as conn: page_stats = conn.execute( """ SELECT COUNT(*) AS page_rows, COALESCE(MAX(page_number), 0) AS max_page_number - FROM pages - WHERE workflow_id = ? + FROM pages WHERE document_id = %s::uuid """, - (workflow_id,), + (doc["id"],), ).fetchone() chunk_stats = conn.execute( - """ - SELECT COUNT(*) AS chunk_rows - FROM chunks - WHERE workflow_id = ? - """, - (workflow_id,), + "SELECT COUNT(*) AS chunk_rows FROM chunks WHERE document_id = %s::uuid", + (doc["id"],), ).fetchone() page_count = int((page_stats["page_rows"] if page_stats else 0) or 0) @@ -966,7 +1047,7 @@ def reconcile_materialized_state(workflow_id: str) -> Optional[dict]: current_stage = doc.get("stage") latest_job = get_latest_document_job(workflow_id) updated = False - now = datetime.utcnow().isoformat() + now = _iso_now() # Highest-confidence reconciliation first: if chunks exist, the document has # already reached chunk review materiality even if the workflow stalled. @@ -1049,14 +1130,9 @@ def reconcile_materialized_state(workflow_id: str) -> Optional[dict]: def get_document(workflow_id: str) -> Optional[dict]: - """Get a single document by workflow ID.""" + """Get a single document by workflow id, UUID or any other identifier.""" with get_connection() as conn: - row = conn.execute( - "SELECT * FROM documents WHERE workflow_id = ?", - (workflow_id,) - ).fetchone() - - return dict(row) if row else None + return _fetch_document(conn, _resolve_document_uuid(conn, workflow_id)) def list_documents( @@ -1076,43 +1152,40 @@ def list_documents( include_demo: If False (default), excludes demo documents from results include_disabled: If False (default), excludes soft-deleted documents from results instances: If provided, only return docs whose instance is in this list - (NULL/empty instance treated as DEFAULT_INSTANCE / 'default') """ - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + if instances is not None and not _normalize_instances(instances): + return [] + clauses, params = _document_filters(include_demo, include_disabled, instances) + if stage: + clauses.append("cw.current_stage = %s::workflow_stage") + params.append(stage) + where = f"WHERE {' AND '.join(clauses)}" if clauses else "" with get_connection() as conn: - # Note: filters are hardcoded SQL fragments based on boolean flags, not user input - demo_filter = "" if include_demo else "AND (is_demo = 0 OR is_demo IS NULL)" - disabled_filter = "" if include_disabled else "AND (is_disabled = 0 OR is_disabled IS NULL)" - instance_filter = "" - params: list = [] - - if instances is not None: - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) - if not normalized: - return [] - placeholders = ",".join("?" for _ in normalized) - instance_filter = ( - f"AND lower(COALESCE(NULLIF(trim(instance), ''), ?)) IN ({placeholders})" - ) - params.append(default_instance) - params.extend(normalized) - - if stage: - rows = conn.execute(f""" - SELECT * FROM documents - WHERE stage = ? {demo_filter} {disabled_filter} {instance_filter} - ORDER BY created_at DESC - LIMIT ? OFFSET ? - """, (stage, *params, limit, offset)).fetchall() - else: - rows = conn.execute(f""" - SELECT * FROM documents - WHERE 1=1 {demo_filter} {disabled_filter} {instance_filter} - ORDER BY created_at DESC - LIMIT ? OFFSET ? - """, (*params, limit, offset)).fetchall() - - return [dict(row) for row in rows] + rows = conn.execute( + f"{_DOC_SELECT} {where} ORDER BY d.created_at DESC, d.id LIMIT %s OFFSET %s", + (*params, limit, offset), + ).fetchall() + return [_document_from_row(r) for r in rows] + + +_REVIEW_QUEUE_STAGES = ("ocr_review", "translation_review", "chunk_review", "approval_for_prod") + +_SUMMARY_KEYS = ( + "total_documents", + "authoritative_documents", + "legacy_documents", + "completed_documents", + "review_queue", + "ocr_review_documents", + "translation_review_documents", + "chunk_review_documents", + "translation_processing_documents", + "chunking_documents", + "ready_for_ingestion_documents", + "failed_documents", + "needs_reindex", + "running_jobs", +) def get_document_summary_counts( @@ -1121,90 +1194,37 @@ def get_document_summary_counts( instances: Optional[list[str]] = None, ) -> dict: """Return aggregate document counts for dashboard and migration planning.""" - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + if instances is not None and not _normalize_instances(instances): + return {key: 0 for key in _SUMMARY_KEYS} + clauses, params = _document_filters(include_demo, include_disabled, instances) + where = f"WHERE {' AND '.join(clauses)}" if clauses else "" with get_connection() as conn: - demo_filter = "" if include_demo else "AND (is_demo = 0 OR is_demo IS NULL)" - disabled_filter = "" if include_disabled else "AND (is_disabled = 0 OR is_disabled IS NULL)" - job_demo_filter = "" if include_demo else "AND (d.is_demo = 0 OR d.is_demo IS NULL)" - job_disabled_filter = "" if include_disabled else "AND (d.is_disabled = 0 OR d.is_disabled IS NULL)" - instance_filter = "" - job_instance_filter = "" - params: list = [] - job_params: list = [] - - if instances is not None: - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) - if not normalized: - return { - "total_documents": 0, - "authoritative_documents": 0, - "legacy_documents": 0, - "completed_documents": 0, - "review_queue": 0, - "ocr_review_documents": 0, - "translation_review_documents": 0, - "chunk_review_documents": 0, - "translation_processing_documents": 0, - "chunking_documents": 0, - "ready_for_ingestion_documents": 0, - "failed_documents": 0, - "needs_reindex": 0, - "running_jobs": 0, - } - placeholders = ",".join("?" for _ in normalized) - instance_filter = ( - f"AND lower(COALESCE(NULLIF(trim(instance), ''), ?)) IN ({placeholders})" - ) - job_instance_filter = ( - f"AND lower(COALESCE(NULLIF(trim(d.instance), ''), ?)) IN ({placeholders})" - ) - params = [default_instance, *normalized] - job_params = [default_instance, *normalized] - - row = conn.execute(f""" + row = conn.execute( + f""" SELECT COUNT(*) AS total_documents, - -- SQLite SUM() returns NULL when no rows match; COALESCE keeps response ints valid. - COALESCE(SUM(CASE WHEN source_manifest_name IS NOT NULL THEN 1 ELSE 0 END), 0) AS authoritative_documents, - COALESCE(SUM(CASE WHEN source_manifest_name IS NULL THEN 1 ELSE 0 END), 0) AS legacy_documents, - COALESCE(SUM(CASE WHEN stage = 'completed' THEN 1 ELSE 0 END), 0) AS completed_documents, - COALESCE(SUM(CASE WHEN stage IN ('ocr_review', 'translation_review', 'chunk_review', 'approval_for_prod') THEN 1 ELSE 0 END), 0) AS review_queue, - COALESCE(SUM(CASE WHEN stage = 'ocr_review' THEN 1 ELSE 0 END), 0) AS ocr_review_documents, - COALESCE(SUM(CASE WHEN stage = 'translation_review' THEN 1 ELSE 0 END), 0) AS translation_review_documents, - COALESCE(SUM(CASE WHEN stage = 'chunk_review' THEN 1 ELSE 0 END), 0) AS chunk_review_documents, - COALESCE(SUM(CASE WHEN stage = 'translation_processing' THEN 1 ELSE 0 END), 0) AS translation_processing_documents, - COALESCE(SUM(CASE WHEN stage = 'chunking' THEN 1 ELSE 0 END), 0) AS chunking_documents, - COALESCE(SUM(CASE WHEN stage = 'ready_for_ingestion' THEN 1 ELSE 0 END), 0) AS ready_for_ingestion_documents, - COALESCE(SUM(CASE WHEN stage = 'failed' THEN 1 ELSE 0 END), 0) AS failed_documents, - COALESCE(SUM(CASE WHEN reindex_required = 1 THEN 1 ELSE 0 END), 0) AS needs_reindex, - ( - SELECT COUNT(*) - FROM document_jobs j - JOIN documents d ON d.workflow_id = j.workflow_id - WHERE j.status = 'running' {job_demo_filter} {job_disabled_filter} {job_instance_filter} - ) AS running_jobs - FROM documents - WHERE 1=1 {demo_filter} {disabled_filter} {instance_filter} - """, (*job_params, *params)).fetchone() - result = dict(row) if row else {} - # Defensive: every count field must be an int for DocumentCohortsResponse. - int_keys = ( - "total_documents", - "authoritative_documents", - "legacy_documents", - "completed_documents", - "review_queue", - "ocr_review_documents", - "translation_review_documents", - "chunk_review_documents", - "translation_processing_documents", - "chunking_documents", - "ready_for_ingestion_documents", - "failed_documents", - "needs_reindex", - "running_jobs", - ) - return {key: int(result.get(key) or 0) for key in int_keys} + 0 AS authoritative_documents, + COUNT(*) AS legacy_documents, + COUNT(*) FILTER (WHERE cw.current_stage = 'completed') AS completed_documents, + COUNT(*) FILTER (WHERE cw.current_stage IN + ('ocr_review', 'translation_review', 'chunk_review', 'approval_for_prod')) AS review_queue, + COUNT(*) FILTER (WHERE cw.current_stage = 'ocr_review') AS ocr_review_documents, + COUNT(*) FILTER (WHERE cw.current_stage = 'translation_review') AS translation_review_documents, + COUNT(*) FILTER (WHERE cw.current_stage = 'chunk_review') AS chunk_review_documents, + COUNT(*) FILTER (WHERE cw.current_stage = 'translation_processing') AS translation_processing_documents, + COUNT(*) FILTER (WHERE cw.current_stage = 'chunking') AS chunking_documents, + COUNT(*) FILTER (WHERE cw.current_stage = 'ready_for_ingestion') AS ready_for_ingestion_documents, + COUNT(*) FILTER (WHERE cw.current_stage = 'failed') AS failed_documents, + COUNT(*) FILTER (WHERE COALESCE((d.metadata->>'reindex_required')::boolean, FALSE)) AS needs_reindex, + COALESCE(SUM((SELECT COUNT(*) FROM workflows j + WHERE j.document_id = d.id AND j.status = 'running')), 0) AS running_jobs + FROM documents d + LEFT JOIN document_current_workflow cw ON cw.document_id = d.id + {where} + """, + params, + ).fetchone() + return {key: int(row[key] or 0) for key in _SUMMARY_KEYS} def get_document_counts_by_instance( @@ -1213,57 +1233,50 @@ def get_document_counts_by_instance( instances: Optional[list[str]] = None, ) -> list[dict]: """Return document counts grouped by instance (state), for dashboard breakdowns.""" - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + if instances is not None and not _normalize_instances(instances): + return [] + clauses, params = _document_filters(include_demo, include_disabled, instances) + where = f"WHERE {' AND '.join(clauses)}" if clauses else "" with get_connection() as conn: - demo_filter = "" if include_demo else "AND (is_demo = 0 OR is_demo IS NULL)" - disabled_filter = "" if include_disabled else "AND (is_disabled = 0 OR is_disabled IS NULL)" - instance_filter = "" - params: list = [default_instance] - - if instances is not None: - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) - if not normalized: - return [] - placeholders = ",".join("?" for _ in normalized) - instance_filter = ( - f"AND lower(COALESCE(NULLIF(trim(instance), ''), ?)) IN ({placeholders})" - ) - params.append(default_instance) - params.extend(normalized) - - rows = conn.execute(f""" + rows = conn.execute( + f""" SELECT - lower(COALESCE(NULLIF(trim(instance), ''), ?)) AS instance, + lower(d.instance) AS instance, COUNT(*) AS count, - COALESCE(SUM(CASE WHEN stage = 'completed' THEN 1 ELSE 0 END), 0) AS success, - COALESCE(SUM(CASE WHEN stage = 'failed' THEN 1 ELSE 0 END), 0) AS failed, - COALESCE(SUM(CASE WHEN stage IN ('ocr_review', 'translation_review', 'chunk_review', 'approval_for_prod') THEN 1 ELSE 0 END), 0) AS dev_approval - FROM documents - WHERE 1=1 {demo_filter} {disabled_filter} {instance_filter} - GROUP BY instance + COUNT(*) FILTER (WHERE cw.current_stage = 'completed') AS success, + COUNT(*) FILTER (WHERE cw.current_stage = 'failed') AS failed, + COUNT(*) FILTER (WHERE cw.current_stage IN + ('ocr_review', 'translation_review', 'chunk_review', 'approval_for_prod')) AS dev_approval + FROM documents d + LEFT JOIN document_current_workflow cw ON cw.document_id = d.id + {where} + GROUP BY lower(d.instance) ORDER BY count DESC, instance ASC - """, params).fetchall() - return [ - { - "instance": row["instance"], - "count": int(row["count"] or 0), - "success": int(row["success"] or 0), - "failed": int(row["failed"] or 0), - "dev_approval": int(row["dev_approval"] or 0), - } - for row in rows - ] + """, + params, + ).fetchall() + return [ + { + "instance": row["instance"], + "count": int(row["count"] or 0), + "success": int(row["success"] or 0), + "failed": int(row["failed"] or 0), + "dev_approval": int(row["dev_approval"] or 0), + } + for row in rows + ] def set_document_demo(workflow_id: str, is_demo: bool = True): """Mark a document as demo (filtered from UI by default).""" - with _db_lock: - with get_connection() as conn: + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid: conn.execute( - "UPDATE documents SET is_demo = ? WHERE workflow_id = ?", - (1 if is_demo else 0, workflow_id) + "UPDATE documents SET metadata = metadata || jsonb_build_object('is_demo', %s::boolean) " + "WHERE id = %s::uuid", + (bool(is_demo), doc_uuid), ) - conn.commit() def set_document_disabled(workflow_id: str, is_disabled: bool = True): @@ -1272,98 +1285,67 @@ def set_document_disabled(workflow_id: str, is_disabled: bool = True): Unlike hard delete, this preserves the document record for audit purposes. Use X-Include-Disabled: true header in API to see disabled documents. """ - with _db_lock: - with get_connection() as conn: + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid: conn.execute( - "UPDATE documents SET is_disabled = ?, updated_at = ? WHERE workflow_id = ?", - (1 if is_disabled else 0, datetime.utcnow().isoformat(), workflow_id) + "UPDATE documents SET is_disabled = %s WHERE id = %s::uuid", + (bool(is_disabled), doc_uuid), ) - conn.commit() def delete_document(workflow_id: str): - """Delete a document record.""" - with _db_lock: - with get_connection() as conn: - conn.execute( - "DELETE FROM documents WHERE workflow_id = ?", - (workflow_id,) - ) - conn.commit() - - -# Every table holding per-document rows, child-first so a partial failure never -# strands parent rows. `chunk_tags` is legacy — created by an older tagging -# build and absent from fresh databases — so it is probed rather than assumed. -_PURGE_TABLES = ( - "chunk_tags", - "chunks", - "pages", - "document_artifacts", - "document_index_status", - "document_jobs", - "documents", -) + """Delete a document record and everything that cascades from it.""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid: + conn.execute("DELETE FROM documents WHERE id = %s::uuid", (doc_uuid,)) def purge_document(workflow_id: str, keep_audit: bool = True) -> dict: - """Hard delete every SQLite row belonging to a document. + """Hard delete every row belonging to a document. - Unlike :func:`set_document_disabled` this is irreversible — there is + Unlike :func:`set_document_disabled` this is irreversible - there is nothing left for ``POST /documents/{id}/restore`` to bring back. Audit rows are kept by default: they are the record that the deletion - happened, not part of the document itself. + happened, not part of the document itself. Their ``document_id`` becomes + NULL; the ``workflow_id`` and ``instance`` snapshots stay. Returns a per-table count of deleted rows. """ deleted: dict[str, int] = {} - with _db_lock: - with get_connection() as conn: - existing = { - row[0] - for row in conn.execute( - "SELECT name FROM sqlite_master WHERE type = 'table'" - ) - } - for table in _PURGE_TABLES: - if table not in existing: - continue - columns = {r[1] for r in conn.execute(f"PRAGMA table_info([{table}])")} - if "workflow_id" not in columns: - continue - cursor = conn.execute( - f"DELETE FROM [{table}] WHERE workflow_id = ?", (workflow_id,) - ) - if cursor.rowcount > 0: - deleted[table] = cursor.rowcount - - if not keep_audit and "audit_logs" in existing: - cursor = conn.execute( - "DELETE FROM audit_logs WHERE workflow_id = ?", (workflow_id,) - ) - if cursor.rowcount > 0: - deleted["audit_logs"] = cursor.rowcount - - conn.commit() - return deleted - - -def list_scheme_catalog_entries_for_workflow(workflow_id: str) -> list[dict]: - """Scheme catalog rows that still reference this workflow.""" with get_connection() as conn: - rows = conn.execute( - "SELECT scheme_code, workflow_ids_json FROM scheme_catalog_entries" - ).fetchall() - matches = [] - for row in rows: - try: - ids = json.loads(row["workflow_ids_json"] or "[]") - except (ValueError, TypeError): - ids = [] - if workflow_id in ids: - matches.append({"scheme_code": row["scheme_code"], "workflow_ids": ids}) - return matches + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return deleted + root = _root_workflow_id(conn, doc_uuid) or workflow_id + + def _count(sql: str) -> int: + return int(conn.execute(sql, (doc_uuid,)).fetchone()["n"]) + + counts = { + "chunks": _count("SELECT count(*) AS n FROM chunks WHERE document_id = %s::uuid"), + "pages": _count("SELECT count(*) AS n FROM pages WHERE document_id = %s::uuid"), + "document_artifacts": _count( + "SELECT count(*) AS n FROM workflow_artifacts a JOIN workflows w ON w.id = a.workflow_id " + "WHERE w.document_id = %s::uuid" + ), + "document_jobs": _count("SELECT count(*) AS n FROM workflows WHERE document_id = %s::uuid"), + } + if not keep_audit: + row = conn.execute( + "DELETE FROM audit_logs WHERE document_id = %s::uuid OR workflow_id = %s", + (doc_uuid, root), + ) + if row.rowcount > 0: + deleted["audit_logs"] = row.rowcount + conn.execute("DELETE FROM documents WHERE id = %s::uuid", (doc_uuid,)) + for table, count in counts.items(): + if count > 0: + deleted[table] = count + deleted["documents"] = 1 + return deleted def get_document_count(stage: Optional[str] = None) -> int: @@ -1371,121 +1353,242 @@ def get_document_count(stage: Optional[str] = None) -> int: with get_connection() as conn: if stage: row = conn.execute( - "SELECT COUNT(*) as cnt FROM documents WHERE stage = ?", - (stage,) + """ + SELECT COUNT(*) AS cnt FROM documents d + JOIN document_current_workflow cw ON cw.document_id = d.id + WHERE cw.current_stage = %s::workflow_stage + """, + (stage,), ).fetchone() else: - row = conn.execute("SELECT COUNT(*) as cnt FROM documents").fetchone() - - return row["cnt"] if row else 0 - - -def upsert_manifest_entry( - manifest_filename: str, - title_en: Optional[str] = None, - title_gu: Optional[str] = None, - doc_language: Optional[str] = None, - category_tags: Optional[str] = None, - description: Optional[str] = None, - quality_score: Optional[str] = None, - priority_rank: Optional[str] = None, - ingestion_status: Optional[str] = None, - feedback: Optional[str] = None, - source_csv_path: Optional[str] = None, -) -> dict: - imported_at = datetime.utcnow().isoformat() - with _db_lock: - with get_connection() as conn: - conn.execute(""" - INSERT INTO document_manifest_entries ( - manifest_filename, title_en, title_gu, doc_language, - category_tags, description, quality_score, priority_rank, - ingestion_status, feedback, source_csv_path, imported_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(manifest_filename) DO UPDATE SET - title_en = COALESCE(excluded.title_en, document_manifest_entries.title_en), - title_gu = COALESCE(excluded.title_gu, document_manifest_entries.title_gu), - doc_language = COALESCE(excluded.doc_language, document_manifest_entries.doc_language), - category_tags = COALESCE(excluded.category_tags, document_manifest_entries.category_tags), - description = COALESCE(excluded.description, document_manifest_entries.description), - quality_score = COALESCE(excluded.quality_score, document_manifest_entries.quality_score), - priority_rank = COALESCE(excluded.priority_rank, document_manifest_entries.priority_rank), - ingestion_status = COALESCE(excluded.ingestion_status, document_manifest_entries.ingestion_status), - feedback = COALESCE(excluded.feedback, document_manifest_entries.feedback), - source_csv_path = COALESCE(excluded.source_csv_path, document_manifest_entries.source_csv_path), - imported_at = excluded.imported_at - """, ( - manifest_filename, title_en, title_gu, doc_language, - category_tags, description, quality_score, priority_rank, - ingestion_status, feedback, source_csv_path, imported_at - )) - conn.commit() - return get_manifest_entry(manifest_filename) or {} - - -def get_manifest_entry(manifest_filename: str) -> Optional[dict]: + row = conn.execute("SELECT COUNT(*) AS cnt FROM documents").fetchone() + return int(row["cnt"]) if row else 0 + + +def update_document_fields(workflow_id: str, **updates: object) -> Optional[dict]: + """Update selected document fields and return the document. + + Legacy flat names are mapped onto the tables: ``stage``/``error_message`` act + on the current workflow, ``scheme_*`` on the catalog entry, counts and + flags on the JSONB columns. Names derived from other rows (artifact and job + pointers, completion timestamps) are accepted and ignored. + """ with get_connection() as conn: - row = conn.execute( - "SELECT * FROM document_manifest_entries WHERE manifest_filename = ?", - (manifest_filename,), - ).fetchone() - return dict(row) if row else None + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + if not updates: + return _fetch_document(conn, doc_uuid) + + doc_sets: list[str] = [] + doc_params: list = [] + source_patch: dict[str, Any] = {} + meta_patch: dict[str, Any] = {} + meta_remove: list[str] = [] + wf_sets: list[str] = [] + wf_params: list = [] + + for key, value in updates.items(): + if key == "stage": + wf_sets.append("current_stage = %s::workflow_stage") + wf_params.append(value) + elif key == "error_message": + wf_sets.append("error_message = %s") + wf_params.append(value) + elif key == "page_count": + source_patch["page_count"] = value + elif key == "chunk_count": + meta_patch["chunk_count"] = value + elif key == "source_type": + source_patch["source_type"] = value + elif key == "canonical_input_type": + source_patch["canonical_input_type"] = value + elif key == "filepath": + source_patch["storage_uri"] = value + elif key in ("filename", "source_filename"): + if key == "filename" or "filename" not in updates: + source_patch["original_filename"] = value + elif key in ("document_id", "canonical_document_id", "source_file_fingerprint"): + if isinstance(value, str) and _FINGERPRINT_RE.match(value): + source_patch["fingerprint"] = value + elif key == "display_name": + if value: + meta_patch["display_name"] = value + else: + meta_remove.append("display_name") + elif key in ("reindex_required", "catalog_visible", "network_visible"): + meta_patch[key] = bool(value) + elif key == "reindex_reason": + if value: + meta_patch["reindex_reason"] = value + else: + meta_remove.append("reindex_reason") + elif key == "stop_after_ocr": + conn.execute( + "UPDATE workflows SET stop_after_ocr = %s WHERE id = (" + "SELECT id FROM workflows WHERE document_id = %s::uuid " + "ORDER BY (workflow_type IN ('pipeline', 'ocr_only')) DESC, started_at ASC, id ASC LIMIT 1)", + (bool(value), doc_uuid), + ) + elif key in ("valid_from", "valid_to"): + doc_sets.append(f"{key} = %s::date") + doc_params.append(value) + elif key == "uploaded_by_user_id" and value: + doc_sets.append("uploaded_by_user_id = COALESCE(uploaded_by_user_id, %s::uuid)") + doc_params.append( + upsert_user( + str(value), + updates.get("uploaded_by_username"), + updates.get("uploaded_by_email"), + updates.get("uploaded_by_roles"), + conn=conn, + ) + ) + + if any(k in updates for k in ("document_kind", "scheme_code", "scheme_name", "scheme_aliases_json")): + for column, new_value in _link_catalog_entry(conn, doc_uuid, updates).items(): + doc_sets.append(f"{column} = %s" + ("::uuid" if column == "catalog_entry_id" else "")) + doc_params.append(new_value) + + if source_patch: + doc_sets.append("source = source || %s::jsonb") + doc_params.append(Jsonb({k: v for k, v in source_patch.items() if v is not None})) + if meta_patch or meta_remove: + expr = "metadata" + for removed in meta_remove: + expr = f"({expr} - '{removed}')" + if meta_patch: + expr = f"{expr} || %s::jsonb" + doc_params.append(Jsonb(meta_patch)) + doc_sets.append(f"metadata = {expr}") + + if wf_sets: + conn.execute( + f"UPDATE workflows SET {', '.join(wf_sets)} WHERE id = (" + "SELECT id FROM document_current_workflow WHERE document_id = %s::uuid)", + (*wf_params, doc_uuid), + ) + if doc_sets: + conn.execute( + f"UPDATE documents SET {', '.join(doc_sets)} WHERE id = %s::uuid", + (*doc_params, doc_uuid), + ) + elif wf_sets: + conn.execute("UPDATE documents SET updated_at = now() WHERE id = %s::uuid", (doc_uuid,)) + return _fetch_document(conn, doc_uuid) + + +def mark_document_reindex_required( + workflow_id: str, + required: bool = True, + reason: Optional[str] = None, +) -> Optional[dict]: + return update_document_fields( + workflow_id, + reindex_required=bool(required), + reindex_reason=reason if required else None, + ) -def list_manifest_entries(limit: int = 1000, offset: int = 0) -> list[dict]: +def set_document_validity( + workflow_id: str, + valid_from: str, + valid_to: str, +) -> Optional[dict]: + """Record the period this document's chunks are searchable in. Both ends + are `YYYY-MM-DD`; validation belongs to the caller + (`pipeline/document_validity.parse_period`), not to this SQL layer.""" + return update_document_fields(workflow_id, valid_from=valid_from, valid_to=valid_to) + + +def mark_prod_ready_requested( + workflow_id: str, + user_id: Optional[str] = None, + username: Optional[str] = None, +) -> Optional[dict]: + """Flag that a state_admin has asked a super_admin to review this document + for prod promotion. Purely informational - does not fire the approve_prod + signal itself; the super_admin still makes that call separately. + + Recorded as a ``promote_prod`` workflow in ``waiting_review``.""" with get_connection() as conn: - rows = conn.execute(""" - SELECT * FROM document_manifest_entries - ORDER BY manifest_filename - LIMIT ? OFFSET ? - """, (limit, offset)).fetchall() - return [dict(row) for row in rows] + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + root = _root_workflow_id(conn, doc_uuid) or workflow_id + user_uuid = upsert_user(user_id, username, None, None, conn=conn) if user_id else None + conn.execute( + """ + UPDATE workflows SET status = 'running' + WHERE document_id = %s::uuid AND workflow_type = 'promote_prod' AND status = 'waiting_review' + """, + (doc_uuid,), + ) + conn.execute( + """ + INSERT INTO workflows (document_id, workflow_type, temporal_workflow_id, status, + current_stage, started_by_user_id) + VALUES (%s::uuid, 'promote_prod', %s, 'waiting_review', 'approval_for_prod', %s::uuid) + """, + (doc_uuid, f"{root}-prod-request", user_uuid), + ) + return _fetch_document(conn, doc_uuid) -def update_document_fields(workflow_id: str, **updates: object) -> Optional[dict]: - """Update selected document fields and return the document.""" - if not updates: - return get_document(workflow_id) +def clear_prod_ready_requested(workflow_id: str) -> Optional[dict]: + """Reset the request flag once prod approval actually happens (or the + document leaves the approval_for_prod gate for any other reason).""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + conn.execute( + """ + UPDATE workflows SET status = 'running' + WHERE document_id = %s::uuid AND workflow_type = 'promote_prod' AND status = 'waiting_review' + """, + (doc_uuid,), + ) + return _fetch_document(conn, doc_uuid) - allowed_fields = { - "stage", "page_count", "chunk_count", "error_message", "updated_at", - "ocr_completed_at", "translation_completed_at", "chunks_completed_at", "ingested_at", - "source_type", "canonical_input_type", "original_artifact_id", "normalized_artifact_id", - "latest_job_id", "filepath", "filename", "display_name", "document_id", - "canonical_document_id", "source_filename", "source_manifest_name", - "source_file_fingerprint", "stop_after_ocr", "reindex_required", "reindex_reason", - "uploaded_by_user_id", "uploaded_by_username", "uploaded_by_email", "uploaded_by_roles", - # Scheme / master catalog fields - "document_kind", "scheme_code", "scheme_name", "scheme_aliases_json", - "tool_routing", "catalog_visible", "network_visible", - "prod_ready_requested_at", "prod_ready_requested_by_user_id", "prod_ready_requested_by_username", - # Document validity period (searchable-from / searchable-to) - "valid_from", "valid_to", - } - set_clauses = [] - values: list[object] = [] - for key, value in updates.items(): - if key not in allowed_fields: - continue - set_clauses.append(f"{key} = ?") - values.append(value) - if not set_clauses: - return get_document(workflow_id) +# ============================================================================= +# Workflows (exposed as "jobs") +# ============================================================================= - if "updated_at" not in updates: - set_clauses.append("updated_at = ?") - values.append(datetime.utcnow().isoformat()) +_JOB_SELECT = """ + SELECT w.id, w.document_id, w.workflow_type, w.temporal_workflow_id, w.status, + w.current_stage, w.config, w.error_message, w.started_at, w.completed_at, + root.temporal_workflow_id AS doc_workflow_id + FROM workflows w + LEFT JOIN LATERAL ( + SELECT r.temporal_workflow_id FROM workflows r + WHERE r.document_id = w.document_id + ORDER BY (r.workflow_type IN ('pipeline', 'ocr_only')) DESC, r.started_at ASC, r.id ASC + LIMIT 1 + ) root ON TRUE +""" - values.append(workflow_id) - with _db_lock: - with get_connection() as conn: - conn.execute( - f"UPDATE documents SET {', '.join(set_clauses)} WHERE workflow_id = ?", - values, - ) - conn.commit() - return get_document(workflow_id) + +def _job_from_row(row: Optional[dict]) -> Optional[dict]: + if row is None: + return None + config = row["config"] or {} + job = { + "id": row["id"], + "workflow_id": row["doc_workflow_id"], + "job_type": row["workflow_type"], + "temporal_workflow_id": row["temporal_workflow_id"], + "temporal_run_id": config.get("temporal_run_id"), + "status": row["status"], + "current_stage": row["current_stage"], + "started_at": row["started_at"], + "completed_at": row["completed_at"], + "error_message": row["error_message"], + "config": config, + "config_json": json.dumps(config) if config else None, + } + return {k: _clean(v) for k, v in job.items()} def create_document_job( @@ -1496,32 +1599,65 @@ def create_document_job( status: str = "running", current_stage: Optional[str] = None, config: Optional[dict] = None, -) -> int: - now = datetime.utcnow().isoformat() - config_json = json.dumps(config) if config else None - with _db_lock: - with get_connection() as conn: - cursor = conn.execute(""" - INSERT INTO document_jobs ( - workflow_id, job_type, temporal_workflow_id, temporal_run_id, - status, current_stage, started_at, config_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?) - """, ( - workflow_id, job_type, temporal_workflow_id, temporal_run_id, - status, current_stage, now, config_json - )) - job_id = cursor.lastrowid - conn.execute( - "UPDATE documents SET latest_job_id = ?, updated_at = ? WHERE workflow_id = ?", - (job_id, now, workflow_id), + actor: Optional[dict] = None, +) -> str: + """Record a workflow run for a document; returns the workflow UUID. + + A ``promote_prod`` run adopts the document's pending request row (see + ``mark_prod_ready_requested``) instead of adding a second one. + """ + cfg = dict(config or {}) + if temporal_run_id: + cfg["temporal_run_id"] = temporal_run_id + with get_connection() as conn: + doc_uuid = _require_document_uuid(conn, workflow_id) + root = _root_workflow_id(conn, doc_uuid) or workflow_id + user_uuid = None + if actor and actor.get("user_id"): + user_uuid = upsert_user( + actor["user_id"], actor.get("username"), actor.get("email"), actor.get("roles"), + conn=conn, ) - conn.commit() - return job_id + if job_type == "promote_prod": + adopted = conn.execute( + """ + UPDATE workflows SET temporal_workflow_id = %s, status = %s::workflow_status_enum, + current_stage = COALESCE(%s::workflow_stage, current_stage), + config = config || %s::jsonb + WHERE id = ( + SELECT id FROM workflows + WHERE document_id = %s::uuid AND workflow_type = 'promote_prod' + AND status = 'waiting_review' + ORDER BY started_at DESC LIMIT 1) + RETURNING id + """, + (temporal_workflow_id or root, status, current_stage, Jsonb(cfg), doc_uuid), + ).fetchone() + if adopted: + conn.execute("UPDATE documents SET updated_at = now() WHERE id = %s::uuid", (doc_uuid,)) + return str(adopted["id"]) + row = conn.execute( + """ + INSERT INTO workflows (document_id, workflow_type, temporal_workflow_id, status, + current_stage, stop_after_ocr, config, started_by_user_id) + VALUES (%s::uuid, %s::workflow_type_enum, %s, %s::workflow_status_enum, + COALESCE(%s, 'registered')::workflow_stage, %s, %s, %s::uuid) + RETURNING id + """, + ( + doc_uuid, job_type, temporal_workflow_id or root, status, current_stage, + bool(cfg.get("stop_after_ocr")), Jsonb(cfg), user_uuid, + ), + ).fetchone() + conn.execute("UPDATE documents SET updated_at = now() WHERE id = %s::uuid", (doc_uuid,)) + return str(row["id"]) -def update_document_job(job_id: int, **updates: object) -> Optional[dict]: +def update_document_job(job_id: str, **updates: object) -> Optional[dict]: if not updates: return get_document_job(job_id) + if not _is_uuid(str(job_id)): + return None allowed_fields = { "status", "current_stage", "completed_at", "error_message", "temporal_run_id", "config_json" } @@ -1530,52 +1666,62 @@ def update_document_job(job_id: int, **updates: object) -> Optional[dict]: for key, value in updates.items(): if key not in allowed_fields: continue - if key == "config_json" and isinstance(value, dict): - value = json.dumps(value) - set_clauses.append(f"{key} = ?") + if key == "status": + set_clauses.append("status = %s::workflow_status_enum") + elif key == "current_stage": + set_clauses.append("current_stage = %s::workflow_stage") + elif key == "completed_at": + set_clauses.append("completed_at = %s::timestamptz") + elif key == "temporal_run_id": + set_clauses.append("config = config || jsonb_build_object('temporal_run_id', %s::text)") + elif key == "config_json": + if isinstance(value, str): + value = json.loads(value) + set_clauses.append("config = config || %s::jsonb") + value = Jsonb(value or {}) + else: + set_clauses.append(f"{key} = %s") values.append(value) if not set_clauses: return get_document_job(job_id) - values.append(job_id) - with _db_lock: - with get_connection() as conn: - conn.execute( - f"UPDATE document_jobs SET {', '.join(set_clauses)} WHERE id = ?", - values, - ) - conn.commit() + values.append(str(job_id)) + with get_connection() as conn: + conn.execute(f"UPDATE workflows SET {', '.join(set_clauses)} WHERE id = %s::uuid", values) return get_document_job(job_id) -def get_document_job(job_id: int) -> Optional[dict]: +def get_document_job(job_id: str) -> Optional[dict]: + if not _is_uuid(str(job_id)): + return None with get_connection() as conn: - row = conn.execute( - "SELECT * FROM document_jobs WHERE id = ?", - (job_id,), - ).fetchone() - return dict(row) if row else None + return _job_from_row( + conn.execute(_JOB_SELECT + " WHERE w.id = %s::uuid", (str(job_id),)).fetchone() + ) def get_latest_document_job(workflow_id: str) -> Optional[dict]: with get_connection() as conn: - row = conn.execute(""" - SELECT * FROM document_jobs - WHERE workflow_id = ? - ORDER BY started_at DESC, id DESC - LIMIT 1 - """, (workflow_id,)).fetchone() - return dict(row) if row else None + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + return _job_from_row( + conn.execute( + _JOB_SELECT + " WHERE w.document_id = %s::uuid ORDER BY w.started_at DESC, w.id DESC LIMIT 1", + (doc_uuid,), + ).fetchone() + ) def list_document_jobs(workflow_id: str, limit: int = 50) -> list[dict]: with get_connection() as conn: - rows = conn.execute(""" - SELECT * FROM document_jobs - WHERE workflow_id = ? - ORDER BY started_at DESC, id DESC - LIMIT ? - """, (workflow_id, limit)).fetchall() - return [dict(row) for row in rows] + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return [] + rows = conn.execute( + _JOB_SELECT + " WHERE w.document_id = %s::uuid ORDER BY w.started_at DESC, w.id DESC LIMIT %s", + (doc_uuid, limit), + ).fetchall() + return [_job_from_row(r) for r in rows] def list_operations_queue( @@ -1588,64 +1734,51 @@ def list_operations_queue( """List documents needing action, optionally scoped to tenant instances. ``instances``: - - ``None`` — unrestricted (super admin / bypass) - - ``[]`` — restricted user with no states → empty result - - non-empty — only those document.instance values + - ``None`` - unrestricted (super admin / bypass) + - ``[]`` - restricted user with no states -> empty result + - non-empty - only those document.instance values + """ + if instances is not None and not _normalize_instances(instances): + return [], 0 + clauses, params = _document_filters(include_demo, include_disabled, instances) + clauses.append( + "(cw.current_stage IN ('ocr_review', 'translation_review', 'chunk_review', " + "'ready_for_ingestion', 'approval_for_prod', 'failed') " + "OR COALESCE((d.metadata->>'reindex_required')::boolean, FALSE) " + "OR cw.status = 'running')" + ) + where = f"WHERE {' AND '.join(clauses)}" + from_sql = """ + FROM documents d + JOIN document_current_workflow cw ON cw.document_id = d.id """ - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" with get_connection() as conn: - demo_filter = "" if include_demo else "AND (d.is_demo = 0 OR d.is_demo IS NULL)" - disabled_filter = "" if include_disabled else "AND (d.is_disabled = 0 OR d.is_disabled IS NULL)" - instance_filter = "" - instance_params: list = [] - if instances is not None: - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) - if not normalized: - return [], 0 - placeholders = ",".join("?" for _ in normalized) - instance_filter = ( - f"AND lower(COALESCE(NULLIF(trim(d.instance), ''), ?)) IN ({placeholders})" - ) - instance_params = [default_instance, *normalized] - where_clause = f""" - ( - d.stage IN ('ocr_review', 'translation_review', 'chunk_review', 'ready_for_ingestion', 'approval_for_prod', 'failed') - OR d.reindex_required = 1 - OR (j.status = 'running') - ) {demo_filter} {disabled_filter} {instance_filter} - """ - total_row = conn.execute( - f""" - SELECT COUNT(*) AS count - FROM documents d - LEFT JOIN document_jobs j ON j.id = d.latest_job_id - WHERE {where_clause} - """, - instance_params, - ).fetchone() + total_row = conn.execute(f"SELECT COUNT(*) AS count {from_sql} {where}", params).fetchone() rows = conn.execute( f""" SELECT - d.workflow_id, - d.filename, - d.stage, + d.id AS id, + (SELECT r.temporal_workflow_id FROM workflows r WHERE r.document_id = d.id + ORDER BY (r.workflow_type IN ('pipeline', 'ocr_only')) DESC, r.started_at ASC, r.id ASC + LIMIT 1) AS workflow_id, + d.source->>'original_filename' AS filename, + cw.current_stage AS stage, d.instance, - d.error_message, - d.reindex_required, - d.reindex_reason, - j.id AS job_id, - j.job_type, - j.status AS job_status, - j.started_at - FROM documents d - LEFT JOIN document_jobs j ON j.id = d.latest_job_id - WHERE {where_clause} - ORDER BY COALESCE(j.started_at, d.updated_at, d.created_at) DESC - LIMIT ? OFFSET ? + cw.error_message, + COALESCE((d.metadata->>'reindex_required')::boolean, FALSE) AS reindex_required, + d.metadata->>'reindex_reason' AS reindex_reason, + cw.id AS job_id, + cw.workflow_type AS job_type, + cw.status AS job_status, + cw.started_at AS started_at + {from_sql} + {where} + ORDER BY cw.started_at DESC, d.id + LIMIT %s OFFSET %s """, - (*instance_params, limit, offset), + (*params, limit, offset), ).fetchall() - return [dict(row) for row in rows], (total_row["count"] if total_row else 0) + return [_clean_row(r) for r in rows], int(total_row["count"] if total_row else 0) def list_runs( @@ -1654,190 +1787,89 @@ def list_runs( status: Optional[str] = None, instances: Optional[list[str]] = None, ) -> list[dict]: - """List jobs with document title fields for the Runs UI. - - Joins ``documents`` so the UI can show filename / display_name instead of - falling back to "Untitled document". + """List workflows with document title fields for the Runs UI. Args: - instances: If provided, only return jobs whose document instance is in - this list (NULL/empty instance treated as DEFAULT_INSTANCE). - Empty list returns no rows. None = unrestricted (all tenants). + instances: If provided, only return runs whose document instance is in + this list. Empty list returns no rows. None = unrestricted. """ - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + where_parts: list[str] = [] + params: list = [] + if status: + where_parts.append("w.status = %s::workflow_status_enum") + params.append(status) + if instances is not None: + normalized = _normalize_instances(instances) + if not normalized: + return [] + where_parts.append("lower(d.instance) = ANY(%s)") + params.append(normalized) + where_sql = f"WHERE {' AND '.join(where_parts)}" if where_parts else "" with get_connection() as conn: - # Prefer document identity columns for list labels. - select_sql = """ - SELECT - j.*, - d.filename AS filename, - d.display_name AS display_name, - d.source_filename AS source_filename, - d.stage AS document_stage, - d.instance AS instance - FROM document_jobs j - LEFT JOIN documents d ON d.workflow_id = j.workflow_id - """ - where_parts: list[str] = [] - params: list = [] - - if status: - where_parts.append("j.status = ?") - params.append(status) - - if instances is not None: - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) - if not normalized: - return [] - placeholders = ",".join("?" for _ in normalized) - # Scope via owning document.instance (legacy NULL/empty → default). - where_parts.append( - f"lower(COALESCE(NULLIF(trim(d.instance), ''), ?)) IN ({placeholders})" - ) - params.append(default_instance) - params.extend(normalized) - - where_sql = f"WHERE {' AND '.join(where_parts)}" if where_parts else "" rows = conn.execute( - select_sql - + f""" + f""" + SELECT w.id, w.document_id, w.workflow_type, w.temporal_workflow_id, w.status, + w.current_stage, w.config, w.error_message, w.started_at, w.completed_at, + (SELECT r.temporal_workflow_id FROM workflows r WHERE r.document_id = w.document_id + ORDER BY (r.workflow_type IN ('pipeline', 'ocr_only')) DESC, r.started_at ASC, r.id ASC + LIMIT 1) AS doc_workflow_id, + d.source->>'original_filename' AS filename, + d.metadata->>'display_name' AS display_name, + d.source->>'original_filename' AS source_filename, + cw.current_stage AS document_stage, + d.instance AS instance + FROM workflows w + JOIN documents d ON d.id = w.document_id + LEFT JOIN document_current_workflow cw ON cw.document_id = d.id {where_sql} - ORDER BY j.started_at DESC, j.id DESC - LIMIT ? OFFSET ? + ORDER BY w.started_at DESC, w.id DESC + LIMIT %s OFFSET %s """, (*params, limit, offset), ).fetchall() - return [dict(row) for row in rows] - - -def list_index_summaries(include_demo: bool = False, include_disabled: bool = False) -> list[dict]: - with get_connection() as conn: - demo_filter = "" if include_demo else "AND (d.is_demo = 0 OR d.is_demo IS NULL)" - disabled_filter = "" if include_disabled else "AND (d.is_disabled = 0 OR d.is_disabled IS NULL)" - rows = conn.execute(f""" - SELECT - s.index_name, - COUNT(*) AS documents, - SUM(COALESCE(s.chunk_count_indexed, 0)) AS indexed_chunks, - SUM(CASE WHEN d.reindex_required = 1 THEN 1 ELSE 0 END) AS stale_documents, - MAX(s.last_indexed_at) AS last_indexed_at, - GROUP_CONCAT(DISTINCT s.status) AS statuses - FROM document_index_status s - JOIN documents d ON d.workflow_id = s.workflow_id - WHERE 1=1 {demo_filter} {disabled_filter} - GROUP BY s.index_name - ORDER BY s.index_name - """).fetchall() - result: list[dict] = [] - for row in rows: - item = dict(row) - item["statuses"] = [part for part in (item.get("statuses") or "").split(",") if part] - result.append(item) - return result - - -def mark_document_reindex_required( - workflow_id: str, - required: bool = True, - reason: Optional[str] = None, -) -> Optional[dict]: - with _db_lock: - with get_connection() as conn: - conn.execute( - """ - UPDATE documents - SET reindex_required = ?, reindex_reason = ?, updated_at = ? - WHERE workflow_id = ? - """, - ( - 1 if required else 0, - reason if required else None, - datetime.utcnow().isoformat(), - workflow_id, - ), - ) - conn.commit() - return get_document(workflow_id) + result = [] + for row in rows: + job = _job_from_row(row) + job["document_uuid"] = str(row["document_id"]) + for extra in ("filename", "display_name", "source_filename", "document_stage", "instance"): + job[extra] = row[extra] + result.append(job) + return result -def mark_prod_ready_requested( - workflow_id: str, - user_id: Optional[str] = None, - username: Optional[str] = None, -) -> Optional[dict]: - """Flag that a state_admin has asked a super_admin to review this document - for prod promotion. Purely informational — does not fire the approve_prod - signal itself; the super_admin still makes that call separately.""" - with _db_lock: - with get_connection() as conn: - conn.execute( - """ - UPDATE documents - SET prod_ready_requested_at = ?, - prod_ready_requested_by_user_id = ?, - prod_ready_requested_by_username = ?, - updated_at = ? - WHERE workflow_id = ? - """, - ( - datetime.utcnow().isoformat(), - user_id, - username, - datetime.utcnow().isoformat(), - workflow_id, - ), - ) - conn.commit() - return get_document(workflow_id) - +# ============================================================================= +# Artifacts +# ============================================================================= -def set_document_validity( - workflow_id: str, - valid_from: str, - valid_to: str, -) -> Optional[dict]: - """Record the period this document's chunks are searchable in. Both ends - are `YYYY-MM-DD`; validation belongs to the caller - (`pipeline/document_validity.parse_period`), not to this SQL layer.""" - with _db_lock: - with get_connection() as conn: - conn.execute( - """ - UPDATE documents - SET valid_from = ?, - valid_to = ?, - updated_at = ? - WHERE workflow_id = ? - """, - ( - valid_from, - valid_to, - datetime.utcnow().isoformat(), - workflow_id, - ), - ) - conn.commit() - return get_document(workflow_id) +def _artifact_from_row(row: Optional[dict]) -> Optional[dict]: + if row is None: + return None + metadata = row["metadata"] or {} + artifact = { + "id": row["id"], + "workflow_id": row["doc_workflow_id"], + "job_id": row["workflow_pk"], + "artifact_type": row["artifact_type"], + "stage": row["stage"], + "storage_uri": row["storage_uri"], + "mime_type": row["mime_type"], + "filename": row["filename"], + "size_bytes": row["size_bytes"], + "metadata_json": json.dumps(metadata) if metadata else None, + "created_at": row["created_at"], + } + return {k: _clean(v) for k, v in artifact.items()} -def clear_prod_ready_requested(workflow_id: str) -> Optional[dict]: - """Reset the request flag once prod approval actually happens (or the - document leaves the approval_for_prod gate for any other reason).""" - with _db_lock: - with get_connection() as conn: - conn.execute( - """ - UPDATE documents - SET prod_ready_requested_at = NULL, - prod_ready_requested_by_user_id = NULL, - prod_ready_requested_by_username = NULL, - updated_at = ? - WHERE workflow_id = ? - """, - (datetime.utcnow().isoformat(), workflow_id), - ) - conn.commit() - return get_document(workflow_id) +_ARTIFACT_SELECT = """ + SELECT a.id, a.workflow_id AS workflow_pk, a.artifact_type, a.stage, a.storage_uri, + a.mime_type, a.filename, a.size_bytes, a.metadata, a.created_at, + (SELECT r.temporal_workflow_id FROM workflows r WHERE r.document_id = w.document_id + ORDER BY (r.workflow_type IN ('pipeline', 'ocr_only')) DESC, r.started_at ASC, r.id ASC + LIMIT 1) AS doc_workflow_id + FROM workflow_artifacts a + JOIN workflows w ON w.id = a.workflow_id +""" def add_document_artifact( @@ -1845,51 +1877,97 @@ def add_document_artifact( artifact_type: str, storage_uri: str, stage: Optional[str] = None, - job_id: Optional[int] = None, + job_id: Optional[str] = None, mime_type: Optional[str] = None, filename: Optional[str] = None, size_bytes: Optional[int] = None, metadata: Optional[dict] = None, -) -> int: - created_at = datetime.utcnow().isoformat() - metadata_json = json.dumps(metadata) if metadata else None - with _db_lock: - with get_connection() as conn: - cursor = conn.execute(""" - INSERT INTO document_artifacts ( - workflow_id, job_id, artifact_type, stage, storage_uri, - mime_type, filename, size_bytes, metadata_json, created_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, ( - workflow_id, job_id, artifact_type, stage, storage_uri, - mime_type, filename, size_bytes, metadata_json, created_at - )) - artifact_id = cursor.lastrowid - if artifact_type == "original_upload": - conn.execute("UPDATE documents SET original_artifact_id = ? WHERE workflow_id = ?", (artifact_id, workflow_id)) - elif artifact_type in {"normalized_pdf", "normalized_spreadsheet"}: - conn.execute("UPDATE documents SET normalized_artifact_id = ? WHERE workflow_id = ?", (artifact_id, workflow_id)) - conn.commit() - return artifact_id +) -> str: + """Record a file produced by a workflow; returns the artifact UUID. + + Belongs to ``job_id`` when given, else to the document's current workflow. + """ + with get_connection() as conn: + doc_uuid = _require_document_uuid(conn, workflow_id) + wf_uuid = None + if job_id and _is_uuid(str(job_id)): + row = conn.execute( + "SELECT id FROM workflows WHERE id = %s::uuid AND document_id = %s::uuid", + (str(job_id), doc_uuid), + ).fetchone() + wf_uuid = str(row["id"]) if row else None + if wf_uuid is None: + row = conn.execute( + "SELECT id FROM document_current_workflow WHERE document_id = %s::uuid", (doc_uuid,) + ).fetchone() + if row is None: + row = conn.execute( + """ + INSERT INTO workflows (document_id, workflow_type, temporal_workflow_id) + VALUES (%s::uuid, 'pipeline', %s) RETURNING id + """, + (doc_uuid, workflow_id), + ).fetchone() + wf_uuid = str(row["id"]) + created = conn.execute( + """ + INSERT INTO workflow_artifacts (workflow_id, artifact_type, stage, storage_uri, + mime_type, filename, size_bytes, metadata) + VALUES (%s::uuid, %s, %s::workflow_stage, %s, %s, %s, %s, %s) + RETURNING id + """, + ( + wf_uuid, artifact_type, stage, storage_uri, mime_type, filename, size_bytes, + Jsonb(metadata or {}), + ), + ).fetchone() + return str(created["id"]) def list_document_artifacts(workflow_id: str) -> list[dict]: with get_connection() as conn: - rows = conn.execute(""" - SELECT * FROM document_artifacts - WHERE workflow_id = ? - ORDER BY created_at DESC, id DESC - """, (workflow_id,)).fetchall() - return [dict(row) for row in rows] + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return [] + rows = conn.execute( + _ARTIFACT_SELECT + " WHERE w.document_id = %s::uuid ORDER BY a.created_at DESC, a.id DESC", + (doc_uuid,), + ).fetchall() + return [_artifact_from_row(r) for r in rows] -def get_document_artifact(workflow_id: str, artifact_id: int) -> Optional[dict]: +def get_document_artifact(workflow_id: str, artifact_id: str) -> Optional[dict]: + if not _is_uuid(str(artifact_id)): + return None with get_connection() as conn: - row = conn.execute(""" - SELECT * FROM document_artifacts - WHERE workflow_id = ? AND id = ? - """, (workflow_id, artifact_id)).fetchone() - return dict(row) if row else None + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + return _artifact_from_row( + conn.execute( + _ARTIFACT_SELECT + " WHERE w.document_id = %s::uuid AND a.id = %s::uuid", + (doc_uuid, str(artifact_id)), + ).fetchone() + ) + + +# ============================================================================= +# Vector index state (documents.index_status) +# ============================================================================= + +def _index_entry_to_legacy(workflow_id: str, key: str, entry: dict) -> dict: + details = entry.get("details") or {} + return { + "workflow_id": workflow_id, + "index_name": entry.get("index_name") or key, + "doc_id": entry.get("doc_id"), + "chunk_count_indexed": entry.get("chunk_count_indexed") or 0, + "last_indexed_at": entry.get("last_indexed_at"), + "last_verified_at": entry.get("last_verified_at"), + "schema_version": entry.get("schema_version"), + "status": entry.get("status") or "unknown", + "details_json": json.dumps(details) if details else None, + } def upsert_document_index_status( @@ -1903,111 +1981,102 @@ def upsert_document_index_status( status: str = "unknown", details: Optional[dict] = None, ) -> dict: - details_json = json.dumps(details) if details else None - with _db_lock: - with get_connection() as conn: - conn.execute(""" - INSERT INTO document_index_status ( - workflow_id, index_name, doc_id, chunk_count_indexed, - last_indexed_at, last_verified_at, schema_version, status, details_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(workflow_id, index_name) DO UPDATE SET - doc_id = COALESCE(excluded.doc_id, document_index_status.doc_id), - chunk_count_indexed = COALESCE(excluded.chunk_count_indexed, document_index_status.chunk_count_indexed), - last_indexed_at = COALESCE(excluded.last_indexed_at, document_index_status.last_indexed_at), - last_verified_at = COALESCE(excluded.last_verified_at, document_index_status.last_verified_at), - schema_version = COALESCE(excluded.schema_version, document_index_status.schema_version), - status = excluded.status, - details_json = COALESCE(excluded.details_json, document_index_status.details_json) - """, ( - workflow_id, index_name, doc_id, chunk_count_indexed, - last_indexed_at, last_verified_at, schema_version, status, details_json - )) - conn.commit() - return get_document_index_status(workflow_id, index_name) or {} - + """Merge one index's state into ``documents.index_status``. -def get_document_index_status(workflow_id: str, index_name: str) -> Optional[dict]: + The key is ``prod`` for the PROD collection names and ``dev`` otherwise; + the collection name is kept inside the entry. ``None`` arguments leave the + stored value alone, except ``status`` which always replaces it. + """ + key = _index_key_for(index_name) with get_connection() as conn: - row = conn.execute(""" - SELECT * FROM document_index_status - WHERE workflow_id = ? AND index_name = ? - """, (workflow_id, index_name)).fetchone() - return dict(row) if row else None + doc_uuid = _require_document_uuid(conn, workflow_id) + row = conn.execute( + "SELECT index_status->%s::text AS entry FROM documents WHERE id = %s::uuid FOR UPDATE", + (key, doc_uuid), + ).fetchone() + existing = dict(row["entry"] or {}) + incoming = { + "index_name": index_name, + "doc_id": doc_id, + "chunk_count_indexed": chunk_count_indexed, + "last_indexed_at": last_indexed_at, + "last_verified_at": last_verified_at, + "schema_version": schema_version, + "details": details or None, + } + merged = {**existing, **{k: v for k, v in incoming.items() if v is not None}, "status": status} + validated = IndexState.model_validate(merged).model_dump() + conn.execute( + "UPDATE documents SET index_status = index_status || jsonb_build_object(%s::text, %s::jsonb) " + "WHERE id = %s::uuid", + (key, Jsonb(validated), doc_uuid), + ) + root = _root_workflow_id(conn, doc_uuid) or workflow_id + return _index_entry_to_legacy(root, key, validated) def list_document_index_status(workflow_id: str) -> list[dict]: - with get_connection() as conn: - rows = conn.execute(""" - SELECT * FROM document_index_status - WHERE workflow_id = ? - ORDER BY last_verified_at DESC, last_indexed_at DESC - """, (workflow_id,)).fetchall() - return [dict(row) for row in rows] - - -def find_document_by_doc_identifier(identifier: str) -> Optional[dict]: - """Resolve a vector-store/chat doc identifier to a SQLite document row.""" - if not identifier: - return None + doc = get_document(workflow_id) + if not doc: + return [] + entries = [ + _index_entry_to_legacy(doc["workflow_id"], key, entry) + for key, entry in doc["index_status"].items() + ] + entries.sort( + key=lambda e: (e["last_verified_at"] or "", e["last_indexed_at"] or ""), reverse=True + ) + return entries - with get_connection() as conn: - row = conn.execute( - "SELECT * FROM documents WHERE workflow_id = ?", - (identifier,), - ).fetchone() - if row: - return dict(row) - row = conn.execute( - "SELECT * FROM documents WHERE document_id = ?", - (identifier,), - ).fetchone() - if row: - return dict(row) +def get_document_index_status(workflow_id: str, index_name: str) -> Optional[dict]: + for entry in list_document_index_status(workflow_id): + if entry["index_name"] == index_name: + return entry + return None - row = conn.execute( - "SELECT * FROM documents WHERE filename = ?", - (identifier,), - ).fetchone() - if row: - return dict(row) - row = conn.execute( - """ - SELECT d.* - FROM document_index_status s - JOIN documents d ON d.workflow_id = s.workflow_id - WHERE s.doc_id = ? - LIMIT 1 +def list_index_summaries(include_demo: bool = False, include_disabled: bool = False) -> list[dict]: + clauses, params = _document_filters(include_demo, include_disabled, None) + where = f"AND {' AND '.join(clauses)}" if clauses else "" + with get_connection() as conn: + rows = conn.execute( + f""" + SELECT + COALESCE(e.value->>'index_name', e.key) AS index_name, + COUNT(*) AS documents, + SUM(COALESCE((e.value->>'chunk_count_indexed')::int, 0)) AS indexed_chunks, + SUM(CASE WHEN COALESCE((d.metadata->>'reindex_required')::boolean, FALSE) + THEN 1 ELSE 0 END) AS stale_documents, + MAX(e.value->>'last_indexed_at') AS last_indexed_at, + array_agg(DISTINCT COALESCE(e.value->>'status', 'unknown')) AS statuses + FROM documents d, jsonb_each(d.index_status) e + WHERE TRUE {where} + GROUP BY COALESCE(e.value->>'index_name', e.key) + ORDER BY index_name """, - (identifier,), - ).fetchone() - if row: - return dict(row) + params, + ).fetchall() + result = [] + for row in rows: + item = _clean_row(row) + item["indexed_chunks"] = int(item["indexed_chunks"] or 0) + item["stale_documents"] = int(item["stale_documents"] or 0) + item["statuses"] = sorted(s for s in row["statuses"] if s) + result.append(item) + return result - legacy_doc_id_hash = hashlib.md5(identifier.encode()).hexdigest() - if legacy_doc_id_hash != identifier: - row = conn.execute( - """ - SELECT d.* - FROM document_index_status s - JOIN documents d ON d.workflow_id = s.workflow_id - WHERE s.doc_id = ? - LIMIT 1 - """, - (legacy_doc_id_hash,), - ).fetchone() - if row: - return dict(row) - rows = conn.execute("SELECT * FROM documents").fetchall() - for candidate in rows: - document_id = candidate["document_id"] - if hashlib.md5(document_id.encode()).hexdigest() == identifier: - return dict(candidate) +def find_document_by_doc_identifier(identifier: str) -> Optional[dict]: + """Resolve a vector-store/chat doc identifier to a document. - return None + Matches the document UUID, a Temporal workflow id, the fingerprint, the + filename, an index ``doc_id``, or the legacy ``md5(fingerprint)`` doc id. + """ + if not identifier: + return None + with get_connection() as conn: + return _fetch_document(conn, _resolve_document_uuid(conn, identifier)) def resolve_chunk_provenance( @@ -2052,6 +2121,9 @@ def resolve_chunk_provenance( # Audit Log Functions # ============================================================================= +_AUDIT_ENTITY_TYPES = {"document", "page", "chunk", "setting", "catalog"} + + def log_audit( workflow_id: str, document_id: str, @@ -2066,50 +2138,68 @@ def log_audit( actor_username: Optional[str] = None, actor_email: Optional[str] = None, actor_roles: Optional[str] = None, -) -> int: +) -> str: """ Log an audit entry. Args: - workflow_id: The Temporal workflow ID - document_id: The document hash ID + workflow_id: The Temporal workflow ID (or document identifier). Ids that + start with ``__`` mark system-level entries and are not stored. + document_id: Unused; kept for call-site compatibility. The document is + resolved from ``workflow_id``. action_type: Type of action (stage_change, page_edit, chunk_edit, approval, reset) - entity_type: Type of entity (page, chunk, document) - entity_id: ID of the entity (page_number or chunk_number) + entity_type: Type of entity (document, page, chunk, setting, catalog) + entity_id: Page number or chunk number the action applied to field_name: Name of the field that changed old_value: Previous value (as string or JSON) new_value: New value (as string or JSON) - metadata: Additional context as dict (will be JSON serialized) + metadata: Additional context as dict actor_*: Identity of the human/system that performed the action Returns: - The ID of the created audit log entry + The UUID of the created audit log entry """ - now = datetime.utcnow().isoformat() - metadata_json = json.dumps(metadata) if metadata else None - - with _db_lock: - with get_connection() as conn: - cursor = conn.execute(""" - INSERT INTO audit_logs ( - workflow_id, document_id, action_type, - entity_type, entity_id, field_name, - old_value, new_value, metadata, timestamp, - actor_user_id, actor_username, actor_email, actor_roles - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, ( - workflow_id, document_id, action_type, - entity_type, entity_id, field_name, - old_value, new_value, metadata_json, now, - actor_user_id, actor_username, actor_email, actor_roles, - )) - conn.commit() - return cursor.lastrowid + entity = entity_type if entity_type in _AUDIT_ENTITY_TYPES else None + system_entry = (workflow_id or "").startswith("__") + roles = _csv_roles(actor_roles) + with get_connection() as conn: + doc_uuid = None if system_entry else _resolve_document_uuid(conn, workflow_id) + instance = None + stored_workflow_id = None if system_entry else workflow_id + if doc_uuid: + doc_row = conn.execute( + "SELECT instance FROM documents WHERE id = %s::uuid", (doc_uuid,) + ).fetchone() + instance = doc_row["instance"] + if _is_uuid(workflow_id): + stored_workflow_id = _root_workflow_id(conn, doc_uuid) or workflow_id + if stored_workflow_id is None and entity not in ("setting", "catalog"): + stored_workflow_id = workflow_id + actor_uuid = ( + upsert_user(actor_user_id, actor_username, actor_email, roles, conn=conn) + if actor_user_id else None + ) + row = conn.execute( + """ + INSERT INTO audit_logs (document_id, workflow_id, instance, action_type, entity_type, + entity_number, field_name, old_value, new_value, metadata, + actor_user_id, actor) + VALUES (%s::uuid, %s, %s, %s, %s::audit_entity_type, %s, %s, %s, %s, %s, %s::uuid, %s) + RETURNING id + """, + ( + doc_uuid, stored_workflow_id, instance, action_type, entity, entity_id, + field_name, old_value, new_value, Jsonb(metadata or {}), actor_uuid, + Jsonb({"username": actor_username or None, "email": actor_email or None, "roles": roles}) + if (actor_user_id or actor_username or actor_email or roles) else Jsonb({}), + ), + ).fetchone() + return str(row["id"]) # Keycloak attaches these to every account regardless of what the user can # actually do here. Rendering them made the actor column a wall of noise -# ("…manage-account,manage-account-links,offline_access,uma_authorization…") +# ("...manage-account,manage-account-links,offline_access,uma_authorization...") # that buried the one role a reviewer cares about. _NOISE_ROLES = { "manage-account", @@ -2168,6 +2258,67 @@ def _normalize_audit_row(row) -> dict: return entry +_AUDIT_SELECT = """ + SELECT a.id, a.workflow_id, a.action_type, a.entity_type, a.entity_number, a.field_name, + a.old_value, a.new_value, a.metadata, a.created_at, a.actor, a.instance AS audit_instance, + au.keycloak_user_id AS actor_keycloak_id, + d.source->>'fingerprint' AS doc_fingerprint, + d.source->>'original_filename' AS filename, + d.metadata->>'display_name' AS display_name, + d.instance AS doc_instance, + uu.username AS uploaded_by_username, uu.email AS uploaded_by_email + FROM audit_logs a + LEFT JOIN documents d ON d.id = a.document_id + LEFT JOIN users au ON au.id = a.actor_user_id + LEFT JOIN users uu ON uu.id = d.uploaded_by_user_id +""" + + +def _audit_row_to_legacy(row: dict) -> dict: + actor = row["actor"] or {} + metadata = row["metadata"] or {} + entity_type = row["entity_type"] + if entity_type == "setting": + workflow_id, document_id = "__system__", "__settings__" + elif entity_type == "catalog" and not row["workflow_id"]: + workflow_id, document_id = "__catalog__", "__catalog__" + else: + workflow_id = row["workflow_id"] + document_id = row["doc_fingerprint"] or workflow_id + roles = actor.get("roles") or [] + entry = { + "id": row["id"], + "workflow_id": workflow_id, + "document_id": document_id, + "action_type": row["action_type"], + "entity_type": entity_type, + "entity_id": row["entity_number"], + "field_name": row["field_name"], + "old_value": row["old_value"], + "new_value": row["new_value"], + "metadata": json.dumps(metadata) if metadata else None, + "timestamp": row["created_at"], + "actor_user_id": row["actor_keycloak_id"], + "actor_username": actor.get("username"), + "actor_email": actor.get("email"), + "actor_roles": ",".join(roles) if roles else None, + "filename": row["filename"], + "display_name": row["display_name"], + "instance": row["doc_instance"] or row["audit_instance"], + "uploaded_by_username": row["uploaded_by_username"], + "uploaded_by_email": row["uploaded_by_email"], + } + return _normalize_audit_row({k: _clean(v) for k, v in entry.items()}) + + +def _audit_scope(workflow_id: str, conn) -> tuple[str, list]: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid: + root = _root_workflow_id(conn, doc_uuid) or workflow_id + return "(a.document_id = %s::uuid OR a.workflow_id = %s)", [doc_uuid, root] + return "a.workflow_id = %s", [workflow_id] + + def get_audit_logs( workflow_id: str, action_type: Optional[str] = None, @@ -2178,7 +2329,7 @@ def get_audit_logs( Get audit logs for a document. Args: - workflow_id: The Temporal workflow ID + workflow_id: The Temporal workflow ID (or document UUID) action_type: Optional filter by action type limit: Maximum number of entries to return offset: Offset for pagination @@ -2187,39 +2338,40 @@ def get_audit_logs( List of audit log entries as dicts """ with get_connection() as conn: + where, params = _audit_scope(workflow_id, conn) if action_type: - rows = conn.execute(""" - SELECT * FROM audit_logs - WHERE workflow_id = ? AND action_type = ? - ORDER BY timestamp DESC - LIMIT ? OFFSET ? - """, (workflow_id, action_type, limit, offset)).fetchall() - else: - rows = conn.execute(""" - SELECT * FROM audit_logs - WHERE workflow_id = ? - ORDER BY timestamp DESC - LIMIT ? OFFSET ? - """, (workflow_id, limit, offset)).fetchall() - - return [_normalize_audit_row(row) for row in rows] + where += " AND a.action_type = %s" + params.append(action_type) + rows = conn.execute( + f"{_AUDIT_SELECT} WHERE {where} ORDER BY a.created_at DESC, a.id LIMIT %s OFFSET %s", + (*params, limit, offset), + ).fetchall() + return [_audit_row_to_legacy(r) for r in rows] def get_audit_log_count(workflow_id: str, action_type: Optional[str] = None) -> int: """Get count of audit logs for a document.""" with get_connection() as conn: + where, params = _audit_scope(workflow_id, conn) if action_type: - row = conn.execute( - "SELECT COUNT(*) as cnt FROM audit_logs WHERE workflow_id = ? AND action_type = ?", - (workflow_id, action_type) - ).fetchone() - else: - row = conn.execute( - "SELECT COUNT(*) as cnt FROM audit_logs WHERE workflow_id = ?", - (workflow_id,) - ).fetchone() - - return row["cnt"] if row else 0 + where += " AND a.action_type = %s" + params.append(action_type) + row = conn.execute(f"SELECT COUNT(*) AS cnt FROM audit_logs a WHERE {where}", params).fetchone() + return int(row["cnt"]) if row else 0 + + +def _global_audit_filters( + action_type: Optional[str], instances: Optional[list[str]] +) -> tuple[list[str], list]: + where_parts: list[str] = ["a.action_type <> %s"] + params: list = [CATALOG_VERSION_ACTION] + if action_type: + where_parts.append("a.action_type = %s") + params.append(action_type) + if instances is not None: + where_parts.append("lower(COALESCE(a.instance, d.instance, %s)) = ANY(%s)") + params.extend([_default_instance(), _normalize_instances(instances)]) + return where_parts, params def get_all_audit_logs( @@ -2240,38 +2392,16 @@ def get_all_audit_logs( Returns: List of audit log entries as dicts, with document filename included """ - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + if instances is not None and not _normalize_instances(instances): + return [] + where_parts, params = _global_audit_filters(action_type, instances) with get_connection() as conn: - where_parts: list[str] = [] - params: list = [] - if action_type: - where_parts.append("a.action_type = ?") - params.append(action_type) - if instances is not None: - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) - if not normalized: - return [] - placeholders = ",".join("?" for _ in normalized) - where_parts.append( - f"lower(COALESCE(NULLIF(trim(d.instance), ''), ?)) IN ({placeholders})" - ) - params.append(default_instance) - params.extend(normalized) - where_sql = f"WHERE {' AND '.join(where_parts)}" if where_parts else "" rows = conn.execute( - f""" - SELECT a.*, d.filename, d.display_name, d.instance, - d.uploaded_by_username, d.uploaded_by_email - FROM audit_logs a - LEFT JOIN documents d ON a.workflow_id = d.workflow_id - {where_sql} - ORDER BY a.timestamp DESC - LIMIT ? OFFSET ? - """, + f"{_AUDIT_SELECT} WHERE {' AND '.join(where_parts)} " + "ORDER BY a.created_at DESC, a.id LIMIT %s OFFSET %s", (*params, limit, offset), ).fetchall() - - return [_normalize_audit_row(row) for row in rows] + return [_audit_row_to_legacy(r) for r in rows] def get_all_audit_log_count( @@ -2279,85 +2409,153 @@ def get_all_audit_log_count( instances: Optional[list[str]] = None, ) -> int: """Get total count of all audit logs (optionally tenant-scoped).""" - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + if instances is not None and not _normalize_instances(instances): + return 0 + where_parts, params = _global_audit_filters(action_type, instances) with get_connection() as conn: - where_parts: list[str] = [] - params: list = [] - if action_type: - where_parts.append("a.action_type = ?") - params.append(action_type) - if instances is not None: - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) - if not normalized: - return 0 - placeholders = ",".join("?" for _ in normalized) - where_parts.append( - f"lower(COALESCE(NULLIF(trim(d.instance), ''), ?)) IN ({placeholders})" - ) - params.append(default_instance) - params.extend(normalized) - where_sql = f"WHERE {' AND '.join(where_parts)}" if where_parts else "" - # Join documents when instance-scoped so tenant filter applies. - if instances is not None: - row = conn.execute( - f""" - SELECT COUNT(*) as cnt - FROM audit_logs a - LEFT JOIN documents d ON a.workflow_id = d.workflow_id - {where_sql} - """, - params, - ).fetchone() - elif action_type: - row = conn.execute( - "SELECT COUNT(*) as cnt FROM audit_logs a WHERE a.action_type = ?", - (action_type,), - ).fetchone() - else: - row = conn.execute("SELECT COUNT(*) as cnt FROM audit_logs").fetchone() - - return row["cnt"] if row else 0 + row = conn.execute( + f"SELECT COUNT(*) AS cnt FROM audit_logs a LEFT JOIN documents d ON d.id = a.document_id " + f"WHERE {' AND '.join(where_parts)}", + params, + ).fetchone() + return int(row["cnt"]) if row else 0 # ============================================================================= -# Page Functions (for persistence after workflow completion) +# Page Functions # ============================================================================= +_PAGE_SELECT = """ + SELECT p.id, p.document_id, p.page_number, p.original_markdown, p.edited_markdown, + p.translations, p.metadata, p.created_at, p.updated_at + FROM pages p +""" + + +def _primary_translation(translations: dict) -> tuple[Optional[str], dict]: + if not translations: + return None, {} + lang = max( + translations, + key=lambda k: (translations[k].get("translated_at") or "", k == "en"), + ) + return lang, translations[lang] + + +def _page_from_row(row: Optional[dict], workflow_id: str) -> Optional[dict]: + if row is None: + return None + meta = row["metadata"] or {} + translations = row["translations"] or {} + lang, tr = _primary_translation(translations) + page = { + "id": row["id"], + "workflow_id": workflow_id, + "page_number": row["page_number"], + "original_markdown": row["original_markdown"], + "edited_markdown": row["edited_markdown"], + "is_reviewed": bool(meta.get("is_reviewed")), + "reviewer_notes": meta.get("reviewer_notes"), + "detected_language": meta.get("detected_language"), + "translated_markdown": tr.get("translated_markdown"), + "edited_translation": tr.get("edited_translation"), + "translation_reviewed": bool(tr.get("is_reviewed")), + "translation_notes": tr.get("reviewer_notes"), + "translation_provider": tr.get("provider"), + "translation_model": tr.get("model"), + "translation_target_language": lang, + "translated_at": tr.get("translated_at"), + "translations": translations, + "metadata": meta, + } + return {k: _clean(v) for k, v in page.items()} + + +_TRANSLATION_FLAT_KEYS = ( + "translated_markdown", "edited_translation", "translation_notes", + "translation_provider", "translation_model", "translated_at", +) + + +def _translation_from_flat(page: dict) -> Optional[tuple[str, dict]]: + if not any(page.get(k) not in (None, "") for k in _TRANSLATION_FLAT_KEYS) and not page.get( + "translation_reviewed" + ): + return None + lang = page.get("translation_target_language") or "en" + entry = PageTranslation( + translated_markdown=page.get("translated_markdown"), + edited_translation=page.get("edited_translation"), + is_reviewed=bool(page.get("translation_reviewed")), + reviewer_notes=page.get("translation_notes"), + provider=page.get("translation_provider"), + model=page.get("translation_model"), + translated_at=page.get("translated_at"), + ).model_dump() + return lang, entry + + def save_pages(workflow_id: str, pages: list[dict]): """ Save all pages for a document (bulk upsert). - Called when workflow completes to persist data. + + Each call replaces the whole row, so the caller passes every field. Page + ``translations`` are kept as stored unless the page carries ``translations`` + (replaces them) or flat ``translated_*`` fields (merged in under + ``translation_target_language``); a page whose OCR text changed loses its + translations because they no longer describe it. """ - with _db_lock: - with get_connection() as conn: - for page in pages: - conn.execute(""" - INSERT OR REPLACE INTO pages ( - workflow_id, page_number, original_markdown, edited_markdown, - is_reviewed, reviewer_notes, detected_language, - translated_markdown, edited_translation, - translation_reviewed, translation_notes, - translation_provider, translation_model, - translation_target_language, translated_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, ( - workflow_id, - page.get("page_number"), - page.get("original_markdown"), - page.get("edited_markdown"), - 1 if page.get("is_reviewed") else 0, - page.get("reviewer_notes"), - page.get("detected_language"), - page.get("translated_markdown"), - page.get("edited_translation"), - 1 if page.get("translation_reviewed") else 0, - page.get("translation_notes"), - page.get("translation_provider"), - page.get("translation_model"), - page.get("translation_target_language"), - page.get("translated_at"), - )) - conn.commit() + with get_connection() as conn: + doc_uuid = _require_document_uuid(conn, workflow_id) + existing = { + r["page_number"]: r + for r in conn.execute( + "SELECT page_number, original_markdown, translations FROM pages " + "WHERE document_id = %s::uuid", + (doc_uuid,), + ).fetchall() + } + for page in pages: + number = page.get("page_number") + current = existing.get(number) + meta = dict(page.get("metadata") or {}) + for flat_key in ("is_reviewed", "reviewer_notes", "detected_language"): + if flat_key in page: + meta[flat_key] = page[flat_key] + meta["is_reviewed"] = bool(meta.get("is_reviewed")) + page_meta = PageMetadata.model_validate( + {k: v for k, v in meta.items() if v is not None} + ).model_dump(exclude_none=True) + + translations = dict(current["translations"]) if current else {} + if current and current["original_markdown"] != page.get("original_markdown"): + translations = {} + if isinstance(page.get("translations"), dict): + translations = { + lang: PageTranslation.model_validate(entry).model_dump() + for lang, entry in page["translations"].items() + } + else: + flat = _translation_from_flat(page) + if flat: + translations[flat[0]] = flat[1] + + conn.execute( + """ + INSERT INTO pages (document_id, page_number, original_markdown, edited_markdown, + translations, metadata) + VALUES (%s::uuid, %s, %s, %s, %s, %s) + ON CONFLICT (document_id, page_number) DO UPDATE SET + original_markdown = EXCLUDED.original_markdown, + edited_markdown = EXCLUDED.edited_markdown, + translations = EXCLUDED.translations, + metadata = EXCLUDED.metadata + """, + ( + doc_uuid, number, page.get("original_markdown"), page.get("edited_markdown"), + Jsonb(translations), Jsonb(page_meta), + ), + ) def persist_document_content(workflow_id: str, pages: list[dict], chunks: list[dict]): @@ -2367,71 +2565,64 @@ def persist_document_content(workflow_id: str, pages: list[dict], chunks: list[d def get_pages(workflow_id: str) -> list[dict]: - """Get all pages for a document from SQLite.""" + """Get all pages for a document.""" with get_connection() as conn: - rows = conn.execute(""" - SELECT * FROM pages - WHERE workflow_id = ? - ORDER BY page_number - """, (workflow_id,)).fetchall() - - pages = [] - for row in rows: - page = dict(row) - # Convert SQLite integers back to booleans - page["is_reviewed"] = bool(page.get("is_reviewed")) - page["translation_reviewed"] = bool(page.get("translation_reviewed")) - pages.append(page) - return pages + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return [] + root = _root_workflow_id(conn, doc_uuid) or workflow_id + rows = conn.execute( + _PAGE_SELECT + " WHERE p.document_id = %s::uuid ORDER BY p.page_number", (doc_uuid,) + ).fetchall() + return [_page_from_row(r, root) for r in rows] def count_translated_pages(workflow_id: str) -> int: """Count pages with translation text without loading full page bodies.""" with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return 0 row = conn.execute( """ - SELECT COUNT(*) AS n - FROM pages - WHERE workflow_id = ? - AND ( - (translated_markdown IS NOT NULL AND TRIM(translated_markdown) != '') - OR (edited_translation IS NOT NULL AND TRIM(edited_translation) != '') + SELECT COUNT(*) AS n FROM pages p + WHERE p.document_id = %s::uuid + AND EXISTS ( + SELECT 1 FROM jsonb_each(p.translations) t + WHERE nullif(btrim(t.value->>'translated_markdown'), '') IS NOT NULL + OR nullif(btrim(t.value->>'edited_translation'), '') IS NOT NULL ) """, - (workflow_id,), + (doc_uuid,), ).fetchone() - return int(row["n"] if row else 0) + return int(row["n"] if row else 0) def get_saved_page_numbers(workflow_id: str) -> list[int]: """Get saved page numbers for a document without loading full page content.""" with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return [] rows = conn.execute( - """ - SELECT page_number - FROM pages - WHERE workflow_id = ? - ORDER BY page_number - """, - (workflow_id,), + "SELECT page_number FROM pages WHERE document_id = %s::uuid ORDER BY page_number", + (doc_uuid,), ).fetchall() - return [int(row["page_number"]) for row in rows] + return [int(row["page_number"]) for row in rows] def get_page(workflow_id: str, page_num: int) -> Optional[dict]: - """Get a specific page from SQLite.""" + """Get a specific page.""" with get_connection() as conn: - row = conn.execute(""" - SELECT * FROM pages - WHERE workflow_id = ? AND page_number = ? - """, (workflow_id, page_num)).fetchone() - - if row: - page = dict(row) - page["is_reviewed"] = bool(page.get("is_reviewed")) - page["translation_reviewed"] = bool(page.get("translation_reviewed")) - return page - return None + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + root = _root_workflow_id(conn, doc_uuid) or workflow_id + row = conn.execute( + _PAGE_SELECT + " WHERE p.document_id = %s::uuid AND p.page_number = %s", + (doc_uuid, page_num), + ).fetchone() + return _page_from_row(row, root) def update_page( @@ -2444,152 +2635,210 @@ def update_page( translation_reviewed: Optional[bool] = None, translation_notes: Optional[str] = None, ) -> Optional[dict]: - """Update a page in SQLite. Returns updated page or None if not found.""" - with _db_lock: - with get_connection() as conn: - # Check if page exists - existing = conn.execute( - "SELECT id FROM pages WHERE workflow_id = ? AND page_number = ?", - (workflow_id, page_num) - ).fetchone() - - if not existing: - return None - - # Build dynamic update - updates = [] - values = [] - - if edited_markdown is not None: - updates.append("edited_markdown = ?") - values.append(edited_markdown) - - if is_reviewed is not None: - updates.append("is_reviewed = ?") - values.append(1 if is_reviewed else 0) - - if reviewer_notes is not None: - updates.append("reviewer_notes = ?") - values.append(reviewer_notes) - - if edited_translation is not None: - updates.append("edited_translation = ?") - values.append(edited_translation) - - if translation_reviewed is not None: - updates.append("translation_reviewed = ?") - values.append(1 if translation_reviewed else 0) - - if translation_notes is not None: - updates.append("translation_notes = ?") - values.append(translation_notes) + """Update a page. Returns updated page or None if not found.""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + existing = conn.execute( + "SELECT translations FROM pages WHERE document_id = %s::uuid AND page_number = %s FOR UPDATE", + (doc_uuid, page_num), + ).fetchone() + if not existing: + return None - if updates: - values.extend([workflow_id, page_num]) - # Note: updates list contains only hardcoded field names, not user input - conn.execute( - f"UPDATE pages SET {', '.join(updates)} WHERE workflow_id = ? AND page_number = ?", - values - ) - conn.commit() + assignments: list[str] = [] + params: list = [] + if edited_markdown is not None: + assignments.append("edited_markdown = %s") + params.append(edited_markdown) + + meta_patch: dict[str, Any] = {} + if is_reviewed is not None: + meta_patch["is_reviewed"] = bool(is_reviewed) + if reviewer_notes is not None: + meta_patch["reviewer_notes"] = reviewer_notes + if meta_patch: + assignments.append("metadata = metadata || %s::jsonb") + params.append(Jsonb(meta_patch)) + + tr_patch: dict[str, Any] = {} + if edited_translation is not None: + tr_patch["edited_translation"] = edited_translation + if translation_reviewed is not None: + tr_patch["is_reviewed"] = bool(translation_reviewed) + if translation_notes is not None: + tr_patch["reviewer_notes"] = translation_notes + if tr_patch: + lang, _ = _primary_translation(existing["translations"] or {}) + lang = lang or "en" + assignments.append( + "translations = jsonb_set(translations, ARRAY[%s::text], " + "COALESCE(translations->%s::text, '{}'::jsonb) || %s::jsonb, TRUE)" + ) + params.extend([lang, lang, Jsonb(tr_patch)]) + if assignments: + conn.execute( + f"UPDATE pages SET {', '.join(assignments)} " + "WHERE document_id = %s::uuid AND page_number = %s", + (*params, doc_uuid, page_num), + ) return get_page(workflow_id, page_num) def reset_page(workflow_id: str, page_num: int) -> Optional[dict]: - """Reset a page to original markdown in SQLite.""" - with _db_lock: - with get_connection() as conn: - conn.execute(""" + """Reset a page to its original OCR markdown.""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is not None: + conn.execute( + """ UPDATE pages SET edited_markdown = NULL, - is_reviewed = 0, - reviewer_notes = NULL - WHERE workflow_id = ? AND page_number = ? - """, (workflow_id, page_num)) - conn.commit() - + metadata = (metadata - 'reviewer_notes') || '{"is_reviewed": false}'::jsonb + WHERE document_id = %s::uuid AND page_number = %s + """, + (doc_uuid, page_num), + ) return get_page(workflow_id, page_num) # ============================================================================= -# Chunk Functions (for persistence after workflow completion) +# Chunk Functions # ============================================================================= +_CHUNK_SELECT = """ + SELECT c.id, c.document_id, c.chunk_number, c.original_text, c.edited_text, c.token_count, + c.page_start, c.page_end, c.source_spans, c.section_title, c.content_type, + c.is_reference, c.is_reviewed, c.is_excluded, c.reviewer_notes, + c.created_at, c.updated_at, d.metadata->'chunking' AS chunking + FROM chunks c + JOIN documents d ON d.id = c.document_id +""" + + +def _chunk_from_row(row: Optional[dict], workflow_id: str) -> Optional[dict]: + if row is None: + return None + spans = row["source_spans"] or [] + page_numbers = sorted({int(s["page_number"]) for s in spans if "page_number" in s}) or list( + range(int(row["page_start"]), int(row["page_end"]) + 1) + ) + run = row.get("chunking") or {} + chunk = { + "id": row["id"], + "workflow_id": workflow_id, + "chunk_number": row["chunk_number"], + "original_text": row["original_text"], + "edited_text": row["edited_text"], + "token_count": row["token_count"], + "page_start": row["page_start"], + "page_end": row["page_end"], + "source_page_numbers_json": json.dumps(page_numbers), + "source_spans_json": json.dumps(spans), + "section_title": row["section_title"], + "content_type": row["content_type"], + "is_reference": bool(row["is_reference"]), + "chunking_provider": run.get("provider"), + "chunking_model": run.get("model"), + "chunking_config_json": json.dumps(run["config"]) if run.get("config") else None, + "chunking_run_id": f"run-{run['run_number']}" if run.get("run_number") else None, + "chunk_version": run.get("run_number") or 1, + "is_reviewed": bool(row["is_reviewed"]), + "is_excluded": bool(row["is_excluded"]), + "reviewer_notes": row["reviewer_notes"], + } + return {k: _clean(v) for k, v in chunk.items()} + + +def _load_json_value(raw: Any, default: Any) -> Any: + if raw is None or raw == "": + return default + if isinstance(raw, str): + return json.loads(raw) + return raw + + def save_chunks(workflow_id: str, chunks: list[dict]): """ - Save all chunks for a document (bulk upsert). - Called when workflow completes to persist data. + Save all chunks for a document. + + Re-chunking runs in one transaction: the document's chunks are deleted, the + new ones inserted, and ``metadata.chunk_count`` and ``metadata.chunking`` + (with ``run_number`` incremented) updated together. """ - with _db_lock: - with get_connection() as conn: - existing_version = conn.execute( - "SELECT COALESCE(MAX(chunk_version), 0) AS max_version FROM chunks WHERE workflow_id = ?", - (workflow_id,), - ).fetchone() - next_version = int(existing_version["max_version"] or 0) + 1 - conn.execute("DELETE FROM chunks WHERE workflow_id = ?", (workflow_id,)) - for chunk in chunks: - chunk_version = int(chunk.get("chunk_version") or next_version) - conn.execute(""" - INSERT INTO chunks ( - workflow_id, chunk_number, original_text, edited_text, - token_count, page_start, page_end, - source_page_numbers_json, source_spans_json, - section_title, content_type, is_reference, - chunking_provider, chunking_model, chunking_config_json, - chunking_run_id, chunk_version, - is_reviewed, is_excluded, reviewer_notes - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - """, ( - workflow_id, + with get_connection() as conn: + doc_uuid = _require_document_uuid(conn, workflow_id) + previous = conn.execute( + "SELECT metadata->'chunking'->>'run_number' AS run_number FROM documents " + "WHERE id = %s::uuid FOR UPDATE", + (doc_uuid,), + ).fetchone() + run_number = int(previous["run_number"] or 0) + 1 + conn.execute("DELETE FROM chunks WHERE document_id = %s::uuid", (doc_uuid,)) + for chunk in chunks: + spans = _load_json_value( + chunk.get("source_spans") if chunk.get("source_spans") is not None + else chunk.get("source_spans_json"), + [], + ) + page_start = int(chunk.get("page_start") or 1) + page_end = max(int(chunk.get("page_end") or page_start), page_start) + conn.execute( + """ + INSERT INTO chunks (document_id, chunk_number, original_text, edited_text, + token_count, page_start, page_end, source_spans, + section_title, content_type, is_reference, is_reviewed, + is_excluded, reviewer_notes) + VALUES (%s::uuid, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + """, + ( + doc_uuid, chunk.get("chunk_number"), - chunk.get("original_text"), + chunk.get("original_text") or "", chunk.get("edited_text"), - chunk.get("token_count", 0), - chunk.get("page_start", 1), - chunk.get("page_end", 1), - chunk.get("source_page_numbers_json"), - chunk.get("source_spans_json"), + chunk.get("token_count") or 0, + page_start, + page_end, + Jsonb(spans), chunk.get("section_title"), chunk.get("content_type"), - 1 if chunk.get("is_reference") else 0, - chunk.get("chunking_provider"), - chunk.get("chunking_model"), - chunk.get("chunking_config_json"), - chunk.get("chunking_run_id"), - chunk_version, - 1 if chunk.get("is_reviewed") else 0, - 1 if chunk.get("is_excluded") else 0, - chunk.get("reviewer_notes") - )) - conn.commit() + bool(chunk.get("is_reference")), + bool(chunk.get("is_reviewed")), + bool(chunk.get("is_excluded")), + chunk.get("reviewer_notes"), + ), + ) + meta_patch: dict[str, Any] = {"chunk_count": len(chunks)} + if chunks: + first = chunks[0] + meta_patch["chunking"] = ChunkingRun( + run_number=run_number, + provider=first.get("chunking_provider"), + model=first.get("chunking_model"), + config=_load_json_value(first.get("chunking_config_json"), {}), + created_at=_iso_now(), + ).model_dump() + conn.execute( + "UPDATE documents SET metadata = metadata || %s::jsonb WHERE id = %s::uuid", + (Jsonb(meta_patch), doc_uuid), + ) def get_chunks(workflow_id: str, include_excluded: bool = False) -> list[dict]: - """Get all chunks for a document from SQLite.""" + """Get all chunks for a document.""" with get_connection() as conn: - if include_excluded: - rows = conn.execute(""" - SELECT * FROM chunks - WHERE workflow_id = ? - ORDER BY chunk_number - """, (workflow_id,)).fetchall() - else: - rows = conn.execute(""" - SELECT * FROM chunks - WHERE workflow_id = ? AND is_excluded = 0 - ORDER BY chunk_number - """, (workflow_id,)).fetchall() - - chunks = [] - for row in rows: - chunk = dict(row) - chunk["is_reviewed"] = bool(chunk.get("is_reviewed")) - chunk["is_excluded"] = bool(chunk.get("is_excluded")) - chunk["is_reference"] = bool(chunk.get("is_reference")) - chunks.append(chunk) - return chunks + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return [] + root = _root_workflow_id(conn, doc_uuid) or workflow_id + excluded = "" if include_excluded else "AND NOT c.is_excluded" + rows = conn.execute( + f"{_CHUNK_SELECT} WHERE c.document_id = %s::uuid {excluded} ORDER BY c.chunk_number", + (doc_uuid,), + ).fetchall() + return [_chunk_from_row(r, root) for r in rows] def search_chunks( @@ -2601,92 +2850,83 @@ def search_chunks( stage: Optional[str] = None, instances: Optional[list[str]] = None, ) -> tuple[list[dict], int]: - """Search chunks across documents (SQLite-first) for maintainer workflows. + """Search chunks across documents for maintainer workflows. instances: If provided, only return chunks whose owning document's instance is in this list (tenant scoping). ``None`` means no instance restriction. """ - where_clauses = ["1=1"] + where_clauses = ["TRUE"] params: list = [] if not include_excluded: - where_clauses.append("c.is_excluded = 0") + where_clauses.append("NOT c.is_excluded") if stage: - where_clauses.append("d.stage = ?") + where_clauses.append("cw.current_stage = %s::workflow_stage") params.append(stage) if instances is not None: - default_instance = (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" - normalized = sorted({(i or "").strip().lower() or default_instance for i in instances}) + normalized = _normalize_instances(instances) if not normalized: - # Restricted caller with no instances: match nothing. - where_clauses.append("1 = 0") + where_clauses.append("FALSE") else: - placeholders = ",".join("?" for _ in normalized) - where_clauses.append( - f"LOWER(COALESCE(d.instance, ?)) IN ({placeholders})" - ) - params.append(default_instance) - params.extend(normalized) + where_clauses.append("lower(d.instance) = ANY(%s)") + params.append(normalized) if query and query.strip(): - where_clauses.append("LOWER(COALESCE(c.edited_text, c.original_text, '')) LIKE ?") + where_clauses.append("lower(coalesce(c.edited_text, c.original_text)) LIKE %s") params.append(f"%{query.strip().lower()}%") - where_sql = " AND ".join(where_clauses) base_from_sql = f""" FROM chunks c - JOIN documents d ON d.workflow_id = c.workflow_id - WHERE {where_sql} + JOIN documents d ON d.id = c.document_id + LEFT JOIN document_current_workflow cw ON cw.document_id = d.id + WHERE {' AND '.join(where_clauses)} """ - with get_connection() as conn: - total_row = conn.execute( - f"SELECT COUNT(*) AS cnt {base_from_sql}", - params, - ).fetchone() + total_row = conn.execute(f"SELECT COUNT(*) AS cnt {base_from_sql}", params).fetchone() total = int(total_row["cnt"] if total_row and total_row["cnt"] is not None else 0) - rows = conn.execute( f""" SELECT - c.*, - d.document_id, - d.filename, - d.display_name, - d.stage, - d.updated_at AS document_updated_at + c.id, c.chunk_number, c.original_text, c.edited_text, c.token_count, + c.page_start, c.page_end, c.source_spans, c.section_title, c.content_type, + c.is_reference, c.is_reviewed, c.is_excluded, c.reviewer_notes, + d.metadata->'chunking' AS chunking, + d.source->>'fingerprint' AS document_id, + d.source->>'original_filename' AS filename, + d.metadata->>'display_name' AS display_name, + cw.current_stage AS stage, + d.updated_at AS document_updated_at, + (SELECT r.temporal_workflow_id FROM workflows r WHERE r.document_id = d.id + ORDER BY (r.workflow_type IN ('pipeline', 'ocr_only')) DESC, r.started_at ASC, r.id ASC + LIMIT 1) AS root_workflow_id {base_from_sql} - ORDER BY d.updated_at DESC, c.workflow_id ASC, c.chunk_number ASC - LIMIT ? OFFSET ? + ORDER BY d.updated_at DESC, d.id, c.chunk_number + LIMIT %s OFFSET %s """, [*params, int(limit), int(offset)], ).fetchall() chunks: list[dict] = [] for row in rows: - chunk = dict(row) - chunk["is_reviewed"] = bool(chunk.get("is_reviewed")) - chunk["is_excluded"] = bool(chunk.get("is_excluded")) - chunk["is_reference"] = bool(chunk.get("is_reference")) + row = dict(row) + chunk = _chunk_from_row(row, row["root_workflow_id"]) + for extra in ("document_id", "filename", "display_name", "stage", "document_updated_at"): + chunk[extra] = _clean(row[extra]) chunks.append(chunk) - return chunks, total def get_chunk(workflow_id: str, chunk_num: int) -> Optional[dict]: - """Get a specific chunk from SQLite.""" + """Get a specific chunk.""" with get_connection() as conn: - row = conn.execute(""" - SELECT * FROM chunks - WHERE workflow_id = ? AND chunk_number = ? - """, (workflow_id, chunk_num)).fetchone() - - if row: - chunk = dict(row) - chunk["is_reviewed"] = bool(chunk.get("is_reviewed")) - chunk["is_excluded"] = bool(chunk.get("is_excluded")) - chunk["is_reference"] = bool(chunk.get("is_reference")) - return chunk - return None + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + root = _root_workflow_id(conn, doc_uuid) or workflow_id + row = conn.execute( + _CHUNK_SELECT + " WHERE c.document_id = %s::uuid AND c.chunk_number = %s", + (doc_uuid, chunk_num), + ).fetchone() + return _chunk_from_row(row, root) def update_chunk( @@ -2697,64 +2937,53 @@ def update_chunk( is_excluded: Optional[bool] = None, reviewer_notes: Optional[str] = None ) -> Optional[dict]: - """Update a chunk in SQLite. Returns updated chunk or None if not found.""" - with _db_lock: - with get_connection() as conn: - # Check if chunk exists - existing = conn.execute( - "SELECT id FROM chunks WHERE workflow_id = ? AND chunk_number = ?", - (workflow_id, chunk_num) - ).fetchone() - - if not existing: - return None - - # Build dynamic update - updates = [] - values = [] - - if edited_text is not None: - updates.append("edited_text = ?") - values.append(edited_text) - - if is_reviewed is not None: - updates.append("is_reviewed = ?") - values.append(1 if is_reviewed else 0) - - if is_excluded is not None: - updates.append("is_excluded = ?") - values.append(1 if is_excluded else 0) - - if reviewer_notes is not None: - updates.append("reviewer_notes = ?") - values.append(reviewer_notes) - - if updates: - values.extend([workflow_id, chunk_num]) - # Note: updates list contains only hardcoded field names, not user input - conn.execute( - f"UPDATE chunks SET {', '.join(updates)} WHERE workflow_id = ? AND chunk_number = ?", - values - ) - conn.commit() - + """Update a chunk. Returns updated chunk or None if not found.""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return None + existing = conn.execute( + "SELECT id FROM chunks WHERE document_id = %s::uuid AND chunk_number = %s", + (doc_uuid, chunk_num), + ).fetchone() + if not existing: + return None + + updates = [] + values: list = [] + if edited_text is not None: + updates.append("edited_text = %s") + values.append(edited_text) + if is_reviewed is not None: + updates.append("is_reviewed = %s") + values.append(bool(is_reviewed)) + if is_excluded is not None: + updates.append("is_excluded = %s") + values.append(bool(is_excluded)) + if reviewer_notes is not None: + updates.append("reviewer_notes = %s") + values.append(reviewer_notes) + if updates: + conn.execute( + f"UPDATE chunks SET {', '.join(updates)} WHERE document_id = %s::uuid AND chunk_number = %s", + (*values, doc_uuid, chunk_num), + ) return get_chunk(workflow_id, chunk_num) def reset_chunk(workflow_id: str, chunk_num: int) -> Optional[dict]: - """Reset a chunk to original text in SQLite.""" - with _db_lock: - with get_connection() as conn: - conn.execute(""" - UPDATE chunks SET - edited_text = NULL, - is_reviewed = 0, - is_excluded = 0, - reviewer_notes = NULL - WHERE workflow_id = ? AND chunk_number = ? - """, (workflow_id, chunk_num)) - conn.commit() - + """Reset a chunk to its original text.""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is not None: + conn.execute( + """ + UPDATE chunks SET edited_text = NULL, is_reviewed = FALSE, is_excluded = FALSE, + reviewer_notes = NULL + WHERE document_id = %s::uuid AND chunk_number = %s + """, + (doc_uuid, chunk_num), + ) return get_chunk(workflow_id, chunk_num) @@ -2762,88 +2991,57 @@ def reset_chunk(workflow_id: str, chunk_num: int) -> Optional[dict]: # Settings Functions # ============================================================================= -def get_setting(key: str) -> Optional[dict]: - """Get a single setting by key.""" - with get_connection() as conn: - row = conn.execute( - "SELECT * FROM settings WHERE key = ?", - (key,) - ).fetchone() - return dict(row) if row else None - - -def get_all_settings() -> dict: - """Get all settings as a dict of key -> value.""" - with get_connection() as conn: - rows = conn.execute("SELECT * FROM settings").fetchall() - return {row["key"]: dict(row) for row in rows} +_SETTING_GROUPS: dict[str, tuple[str, type]] = { + "searchMethod": ("retrieval", RetrievalSettings), + "alpha": ("retrieval", RetrievalSettings), + "rankingMethod": ("retrieval", RetrievalSettings), + "hybridRrfK": ("retrieval", RetrievalSettings), + "indexName": ("candidates", CandidateSettings), + "efSearch": ("candidates", CandidateSettings), + "candidateCap": ("candidates", CandidateSettings), + "candidateMultiplier": ("candidates", CandidateSettings), + "limit": ("results", ResultSettings), + "maxChunksPerDoc": ("results", ResultSettings), + "excludeReference": ("results", ResultSettings), + "showHighlights": ("results", ResultSettings), + "rerankMode": ("results", ResultSettings), + "useE5Prefix": ("query", QuerySettings), + "queryExpansionProfile": ("query", QuerySettings), +} +_GROUP_MODELS: dict[str, type] = {group: model for group, model in _SETTING_GROUPS.values()} -def get_search_settings() -> dict: - """Get all search-related settings as a simple dict.""" - all_settings = get_all_settings() - return { - "searchMethod": all_settings.get("search_method", {}).get("value", "HYBRID"), - "limit": int(all_settings.get("search_limit", {}).get("value", "12")), - "alpha": float(all_settings.get("search_alpha", {}).get("value", "0.6")), - "rankingMethod": all_settings.get("search_ranking_method", {}).get("value", "rrf"), - "showHighlights": all_settings.get("search_show_highlights", {}).get("value", "true") == "true", - "efSearch": int(all_settings.get("search_ef_search", {}).get("value", "256")), - "indexName": all_settings.get("search_index_name", {}).get("value", "documents-index"), - "candidateCap": int(all_settings.get("search_candidate_cap", {}).get("value", "120")), - "candidateMultiplier": int(all_settings.get("search_candidate_multiplier", {}).get("value", "10")), - "maxChunksPerDoc": int(all_settings.get("search_max_chunks_per_doc", {}).get("value", "2")), - "useE5Prefix": all_settings.get("search_use_e5_prefix", {}).get("value", "true") == "true", - "excludeReference": all_settings.get("search_exclude_reference", {}).get("value", "true") == "true", - "queryExpansionProfile": all_settings.get("search_query_expansion_profile", {}).get("value", "gu-v1"), - "rerankMode": all_settings.get("search_rerank_mode", {}).get("value", "none"), - "hybridRrfK": int(all_settings.get("search_hybrid_rrfk", {}).get("value", "60")), - } +_SETTINGS_ROW_SQL = ( + "SELECT retrieval, candidates, results, query FROM search_settings WHERE singleton" +) -def update_setting(key: str, value: str, log_change: bool = True) -> dict: - """ - Update a setting value. +def _settings_row(conn, lock: bool = False) -> dict: + sql = _SETTINGS_ROW_SQL + (" FOR UPDATE" if lock else "") + row = conn.execute(sql).fetchone() + if row is None: + conn.execute("INSERT INTO search_settings DEFAULT VALUES ON CONFLICT (singleton) DO NOTHING") + row = conn.execute(sql).fetchone() + return row - Args: - key: Setting key - value: New value (as string) - log_change: Whether to log to audit (default True) - Returns: - Updated setting dict - """ - now = datetime.utcnow().isoformat() - old_value = None +def _flatten_settings(row: dict) -> dict: + flat: dict[str, Any] = {} + for group, model in _GROUP_MODELS.items(): + flat.update(model.model_validate(row[group] or {}).model_dump()) + return flat - with _db_lock: - with get_connection() as conn: - # Get old value for audit - row = conn.execute( - "SELECT value FROM settings WHERE key = ?", - (key,) - ).fetchone() - old_value = row["value"] if row else None - conn.execute(""" - UPDATE settings SET value = ?, updated_at = ? - WHERE key = ? - """, (value, now, key)) - conn.commit() +def get_search_settings() -> dict: + """Get all search settings as a flat dict (stored values merged over defaults).""" + with get_connection() as conn: + return _flatten_settings(_settings_row(conn)) - # Log the change to audit - if log_change and old_value != value: - log_audit( - workflow_id="__system__", - document_id="__settings__", - action_type="settings_change", - entity_type="setting", - field_name=key, - old_value=old_value, - new_value=value - ) - return get_setting(key) +def _setting_text(value: Any) -> str: + if isinstance(value, bool): + return "true" if value else "false" + return str(value) def update_search_settings(settings: dict) -> dict: @@ -2855,55 +3053,61 @@ def update_search_settings(settings: dict) -> dict: Returns: Updated search settings - """ - # Map UI keys to DB keys - key_map = { - "searchMethod": "search_method", - "limit": "search_limit", - "alpha": "search_alpha", - "rankingMethod": "search_ranking_method", - "showHighlights": "search_show_highlights", - "efSearch": "search_ef_search", - "indexName": "search_index_name", - "candidateCap": "search_candidate_cap", - "candidateMultiplier": "search_candidate_multiplier", - "maxChunksPerDoc": "search_max_chunks_per_doc", - "useE5Prefix": "search_use_e5_prefix", - "excludeReference": "search_exclude_reference", - "queryExpansionProfile": "search_query_expansion_profile", - "rerankMode": "search_rerank_mode", - "hybridRrfK": "search_hybrid_rrfk", - } - - for ui_key, db_key in key_map.items(): - if ui_key in settings: - value = settings[ui_key] - # Convert to string for storage - if isinstance(value, bool): - value = "true" if value else "false" - else: - value = str(value) - update_setting(db_key, value) - return get_search_settings() + Raises: + ValueError: a value fails the setting's validation rules. + """ + unknown = sorted(set(settings) - set(_SETTING_GROUPS)) + if unknown: + raise ValueError(f"Unknown search settings: {', '.join(unknown)}") + changes: list[tuple[str, str, str]] = [] + with get_connection() as conn: + row = _settings_row(conn, lock=True) + before = _flatten_settings(row) + for group, model in _GROUP_MODELS.items(): + patch = {k: v for k, v in settings.items() if _SETTING_GROUPS[k][0] == group} + if not patch: + continue + try: + validated = model.model_validate({**model.model_validate(row[group] or {}).model_dump(), **patch}) + except Exception as exc: + raise ValueError(str(exc)) from exc + stored = {**(row[group] or {}), **{k: getattr(validated, k) for k in patch}} + conn.execute( + f"UPDATE search_settings SET {group} = {group} || %s::jsonb WHERE singleton", + (Jsonb({k: getattr(validated, k) for k in patch}),), + ) + for key in patch: + if before[key] != stored[key]: + changes.append((f"{group}.{key}", _setting_text(before[key]), _setting_text(stored[key]))) + for field_name, old_value, new_value in changes: + conn.execute( + """ + INSERT INTO audit_logs (action_type, entity_type, field_name, old_value, new_value) + VALUES ('settings_change', 'setting', %s, %s, %s) + """, + (field_name, old_value, new_value), + ) + return _flatten_settings( + conn.execute(_SETTINGS_ROW_SQL).fetchone() + ) def get_settings_audit_logs(limit: int = 50, offset: int = 0) -> list[dict]: """Get audit logs for settings changes.""" with get_connection() as conn: - rows = conn.execute(""" - SELECT * FROM audit_logs - WHERE document_id = '__settings__' - ORDER BY timestamp DESC - LIMIT ? OFFSET ? - """, (limit, offset)).fetchall() - return [dict(row) for row in rows] + rows = conn.execute( + f"{_AUDIT_SELECT} WHERE a.entity_type = 'setting' " + "ORDER BY a.created_at DESC, a.id LIMIT %s OFFSET %s", + (limit, offset), + ).fetchall() + return [_audit_row_to_legacy(r) for r in rows] def get_settings_audit_count() -> int: """Get count of settings audit logs.""" with get_connection() as conn: row = conn.execute( - "SELECT COUNT(*) as cnt FROM audit_logs WHERE document_id = '__settings__'" + "SELECT COUNT(*) AS cnt FROM audit_logs WHERE entity_type = 'setting'" ).fetchone() - return row["cnt"] if row else 0 + return int(row["cnt"]) if row else 0 diff --git a/pipeline/models.py b/pipeline/models.py index c4d5522..0746ef0 100644 --- a/pipeline/models.py +++ b/pipeline/models.py @@ -7,7 +7,7 @@ from enum import Enum from typing import Any, Optional -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict, Field class DocumentStage(str, Enum): @@ -180,6 +180,7 @@ class ApprovalRequest(BaseModel): class DocumentSummary(BaseModel): + id: Optional[str] = None # documents.id (UUID); canonical identifier in API paths document_id: str canonical_document_id: Optional[str] = None workflow_id: str # The Temporal workflow ID (use this for API calls) @@ -243,9 +244,9 @@ class SchemeMetadataUpdate(BaseModel): class DocumentArtifact(BaseModel): - id: int + id: str workflow_id: str - job_id: Optional[int] = None + job_id: Optional[str] = None artifact_type: str stage: Optional[str] = None storage_uri: str @@ -257,7 +258,7 @@ class DocumentArtifact(BaseModel): class DocumentJob(BaseModel): - id: int + id: str workflow_id: str job_type: str temporal_workflow_id: Optional[str] = None @@ -305,9 +306,9 @@ class DocumentDetail(DocumentSummary): source_type: Optional[str] = None canonical_input_type: Optional[str] = None stop_after_ocr: bool = False - original_artifact_id: Optional[int] = None - normalized_artifact_id: Optional[int] = None - latest_job_id: Optional[int] = None + original_artifact_id: Optional[str] = None + normalized_artifact_id: Optional[str] = None + latest_job_id: Optional[str] = None current_job: Optional[dict[str, Any]] = None artifacts: list[DocumentArtifact] = [] index_status: list[DocumentIndexStatus] = [] @@ -349,7 +350,7 @@ class OperationQueueEntry(BaseModel): workflow_id: str filename: str stage: str - job_id: Optional[int] = None + job_id: Optional[str] = None job_type: Optional[str] = None job_status: Optional[str] = None started_at: Optional[str] = None @@ -393,7 +394,7 @@ class ReindexStateRequest(BaseModel): class AuditLogEntry(BaseModel): """A single audit log entry.""" - id: int + id: str workflow_id: str document_id: str action_type: str # stage_change, page_edit, chunk_edit, approval, page_reset, chunk_reset @@ -438,7 +439,7 @@ class SearchSettings(BaseModel): """Search configuration settings.""" searchMethod: str = "HYBRID" # TENSOR, LEXICAL, HYBRID limit: int = 12 - alpha: float = 0.7 # 0=lexical, 1=semantic + alpha: float = 0.6 # 0=lexical, 1=semantic rankingMethod: str = "rrf" # rrf, normalize_linear showHighlights: bool = True efSearch: int = 256 @@ -486,3 +487,118 @@ class SettingsAuditResponse(BaseModel): total: int limit: int offset: int + + +# ============================================================================= +# JSONB column models +# +# The database only checks that these columns hold a JSON object; the value +# rules live here. ``db.py`` validates every value through these models before +# it writes one. +# ============================================================================= + +class DocumentSource(BaseModel): + """``documents.source``: facts about the uploaded file.""" + model_config = ConfigDict(extra="forbid") + + fingerprint: str = Field(pattern=r"^[0-9a-f]{32}$") + original_filename: Optional[str] = None + storage_uri: Optional[str] = None + source_type: Optional[str] = None # spreadsheet | document | unknown + canonical_input_type: Optional[str] = None # spreadsheet | pdf + page_count: Optional[int] = Field(default=None, ge=0) + + +class IndexState(BaseModel): + """One entry of ``documents.index_status`` (keys ``dev`` / ``prod``).""" + model_config = ConfigDict(extra="forbid") + + status: str = "unknown" # unknown | indexed | missing + index_name: Optional[str] = None + doc_id: Optional[str] = None + chunk_count_indexed: int = Field(default=0, ge=0) + last_indexed_at: Optional[str] = None + last_verified_at: Optional[str] = None + schema_version: Optional[str] = None + details: dict[str, Any] = Field(default_factory=dict) + + +class ChunkingRun(BaseModel): + model_config = ConfigDict(extra="forbid") + + run_number: int = Field(default=1, ge=1) + provider: Optional[str] = None + model: Optional[str] = None + config: dict[str, Any] = Field(default_factory=dict) + created_at: Optional[str] = None + + +class DocumentMetadata(BaseModel): + """``documents.metadata``: display and flag fields.""" + model_config = ConfigDict(extra="forbid") + + display_name: Optional[str] = None + catalog_visible: bool = True + network_visible: bool = True + is_demo: bool = False + chunk_count: int = Field(default=0, ge=0) + reindex_required: bool = False + reindex_reason: Optional[str] = None + chunking: Optional[ChunkingRun] = None + + +class PageMetadata(BaseModel): + """``pages.metadata``: OCR review and language fields.""" + model_config = ConfigDict(extra="forbid") + + is_reviewed: bool = False + reviewer_notes: Optional[str] = None + detected_language: Optional[str] = None + + +class PageTranslation(BaseModel): + """One language entry of ``pages.translations``.""" + model_config = ConfigDict(extra="forbid") + + translated_markdown: Optional[str] = None + edited_translation: Optional[str] = None + is_reviewed: bool = False + reviewer_notes: Optional[str] = None + provider: Optional[str] = None + model: Optional[str] = None + translated_at: Optional[str] = None + + +class RetrievalSettings(BaseModel): + model_config = ConfigDict(extra="forbid") + + searchMethod: str = Field(default="HYBRID", pattern=r"^(TENSOR|LEXICAL|HYBRID)$") + alpha: float = Field(default=0.6, ge=0, le=1) + rankingMethod: str = Field(default="rrf", pattern=r"^(rrf|normalize_linear)$") + hybridRrfK: int = Field(default=60, ge=1) + + +class CandidateSettings(BaseModel): + model_config = ConfigDict(extra="forbid") + + indexName: str = "documents-index" + efSearch: int = Field(default=256, ge=1) + candidateCap: int = Field(default=120, ge=1) + candidateMultiplier: int = Field(default=10, ge=1) + + +class ResultSettings(BaseModel): + model_config = ConfigDict(extra="forbid") + + limit: int = Field(default=12, ge=1) + maxChunksPerDoc: int = Field(default=2, ge=1) + excludeReference: bool = True + showHighlights: bool = True + rerankMode: str = "none" + + +class QuerySettings(BaseModel): + model_config = ConfigDict(extra="forbid") + + useE5Prefix: bool = True + queryExpansionProfile: str = "gu-v1" diff --git a/pipeline/scheme_catalog.py b/pipeline/scheme_catalog.py index c27d8a7..28c6c45 100644 --- a/pipeline/scheme_catalog.py +++ b/pipeline/scheme_catalog.py @@ -371,54 +371,26 @@ def resolve_scheme_aliases( return _dedupe_aliases(merged) -def _content_hash(code: str, name: str, aliases: list[str], network_visible: bool, status: str) -> str: - payload = json.dumps( - { - "code": code, - "name": name, - "aliases": sorted(aliases), - "network_visible": network_visible, - "status": status, - }, - sort_keys=True, +def _seed_bootstrap_entry(code: str, definition: dict[str, Any]) -> None: + db.upsert_catalog_entry( + type="scheme", + code=code, + name=definition["scheme_name"], + aliases=list(definition.get("scheme_aliases") or []), + source="bootstrap", ) - return hashlib.sha256(payload.encode()).hexdigest()[:16] - - -def ensure_catalog_schema() -> None: - """Create catalog tables and scheme metadata columns (idempotent).""" - db.init_scheme_catalog_schema() def bootstrap_catalog_if_empty(*, force: bool = False) -> dict[str, Any]: - """Seed bootstrap vector + legacy schemes when catalog is empty (or force).""" - ensure_catalog_schema() - meta = db.get_catalog_meta() - entries = db.list_catalog_entries() - if entries and not force: - return {"bootstrapped": False, "version": meta.get("version", 0), "entry_count": len(entries)} - - now = utc_now_iso() - collection = scheme_collection_name() - for code, definition in BOOTSTRAP_VECTOR_SCHEMES.items(): - aliases = list(definition.get("scheme_aliases") or []) - name = definition["scheme_name"] - status = "live" - db.upsert_catalog_entry( - scheme_code=code, - scheme_name=name, - scheme_aliases=aliases, - instances=["default"], - workflow_ids=[], - collection_name=collection, - chunk_count=0, - status=status, - network_visible=True, - promoted_at=None, - content_hash=_content_hash(code, name, aliases, True, status), - source="bootstrap", - updated_at=now, - ) + """Seed bootstrap vector schemes that have no catalog entry yet (all of them if force).""" + existing = db.list_catalog_entry_codes("scheme") + missing = {c: d for c, d in BOOTSTRAP_VECTOR_SCHEMES.items() if force or c not in existing} + if not missing: + meta = db.get_catalog_meta() + return {"bootstrapped": False, "version": meta.get("version", 0), "entry_count": len(existing)} + + for code, definition in missing.items(): + _seed_bootstrap_entry(code, definition) version = db.bump_catalog_version("bootstrap_vector_schemes") return { "bootstrapped": True, @@ -430,182 +402,16 @@ def bootstrap_catalog_if_empty(*, force: bool = False) -> dict[str, Any]: def rebuild_catalog_from_documents(*, reason: str = "rebuild") -> int: """ - Recompute scheme_catalog_entries from documents table, then bump version. + Re-seed bootstrap codes that have no entry, then bump the catalog version. - Pipeline-sourced live docs override bootstrap for the same scheme_code. - Bootstrap-only codes with no live docs remain live (migration phase A1). + Status, instances, chunk counts and the rest are computed by the ``catalog`` + view, so there is nothing to recompute here. Bootstrap-only codes with no + documents stay live. """ - ensure_catalog_schema() - now = utc_now_iso() - collection = scheme_collection_name() - - docs = db.list_scheme_documents_for_catalog() - by_code: dict[str, list[dict]] = {} - for doc in docs: - code = (doc.get("scheme_code") or "").strip().lower() - if not code: - continue - by_code.setdefault(code, []).append(doc) - - # Track which codes are fully owned by pipeline live docs - live_pipeline_codes: set[str] = set() - - for code, group in by_code.items(): - live_docs = [ - d - for d in group - if (d.get("stage") == "completed") - and not int(d.get("is_disabled") or 0) - and int(d.get("catalog_visible") if d.get("catalog_visible") is not None else 1) - and (d.get("document_kind") or "document") == "scheme" - ] - pending_reindex = any( - (d.get("stage") == "completed") - and not int(d.get("is_disabled") or 0) - and (d.get("reindex_required") or d.get("_pending_reindex")) - for d in group - ) - # Also treat reindex_reason containing scheme_code_rename as pending_reindex - for d in group: - reason_txt = (d.get("reindex_reason") or "") - if "scheme_code_rename" in reason_txt and d.get("stage") == "completed" and not int(d.get("is_disabled") or 0): - pending_reindex = True - - if pending_reindex and live_docs: - # If any live doc is waiting reindex after rename, publish as pending_reindex - # unless we only have non-rename live docs — keep simple: any reindex_required on scheme - if any(int(d.get("reindex_required") or 0) for d in live_docs): - status = "pending_reindex" - else: - status = "live" - elif live_docs: - status = "live" - live_pipeline_codes.add(code) - elif any(d.get("stage") == "approval_for_prod" and not int(d.get("is_disabled") or 0) for d in group): - status = "pending_prod" - else: - status = "disabled" - - # Prefer live docs for aggregation; else all group - use = live_docs or group - names = [d.get("scheme_name") for d in use if d.get("scheme_name")] - scheme_name = names[0] if names else code - aliases: list[str] = [] - seen_alias: set[str] = set() - for d in use: - for a in parse_aliases(d.get("scheme_aliases_json")): - key = a.lower() - if key not in seen_alias: - seen_alias.add(key) - aliases.append(a) - instances = sorted( - { - (d.get("instance") or "default").strip().lower() or "default" - for d in use - } - ) - workflow_ids = [d["workflow_id"] for d in use if d.get("workflow_id")] - chunk_count = sum(int(d.get("chunk_count") or 0) for d in use) - network_visible = any(int(d.get("network_visible") or 0) for d in use) # OR - promoted_times = [d.get("ingested_at") or d.get("updated_at") for d in live_docs if d.get("ingested_at") or d.get("updated_at")] - promoted_at = max(promoted_times) if promoted_times else None - source = "pipeline" if any((d.get("document_kind") or "") == "scheme" for d in group) else "import" - - # pending_reindex entries must not appear as live for consumers - if status == "pending_reindex": - # still store entry but status excludes from vector_schemes - pass - - db.upsert_catalog_entry( - scheme_code=code, - scheme_name=scheme_name, - scheme_aliases=aliases, - instances=instances, - workflow_ids=workflow_ids, - collection_name=collection, - chunk_count=chunk_count, - status=status if status != "disabled" or live_docs else "disabled", - network_visible=1 if network_visible else 0, - promoted_at=promoted_at, - content_hash=_content_hash(code, scheme_name, aliases, network_visible, status), - source=source, - updated_at=now, - ) - - # Preserve bootstrap live entries for codes with no document rows; - # disable orphan pipeline / import codes that have no docs left. - existing = {e["scheme_code"]: e for e in db.list_catalog_entries()} + existing = db.list_catalog_entry_codes("scheme") for code, definition in BOOTSTRAP_VECTOR_SCHEMES.items(): - if code in by_code: - continue - if code in existing and existing[code].get("source") == "pipeline": - # Was pipeline-owned but no longer eligible → mark disabled - if code not in live_pipeline_codes: - db.upsert_catalog_entry( - scheme_code=code, - scheme_name=existing[code].get("scheme_name") or definition["scheme_name"], - scheme_aliases=parse_aliases(existing[code].get("scheme_aliases_json")), - instances=parse_aliases(existing[code].get("instances_json")) or ["default"], - workflow_ids=parse_aliases(existing[code].get("workflow_ids_json")), - collection_name=collection, - chunk_count=int(existing[code].get("chunk_count") or 0), - status="disabled", - network_visible=int(existing[code].get("network_visible") or 0), - promoted_at=existing[code].get("promoted_at"), - content_hash=_content_hash( - code, - existing[code].get("scheme_name") or definition["scheme_name"], - parse_aliases(existing[code].get("scheme_aliases_json")), - bool(existing[code].get("network_visible")), - "disabled", - ), - source="pipeline", - updated_at=now, - ) - continue - # Keep or re-seed bootstrap - aliases = list(definition.get("scheme_aliases") or []) - name = definition["scheme_name"] - db.upsert_catalog_entry( - scheme_code=code, - scheme_name=name, - scheme_aliases=aliases, - instances=["default"], - workflow_ids=[], - collection_name=collection, - chunk_count=0, - status="live", - network_visible=1, - promoted_at=None, - content_hash=_content_hash(code, name, aliases, True, "live"), - source="bootstrap", - updated_at=now, - ) - - # Disable non-bootstrap entries that no longer appear in any document group - for code, entry in existing.items(): - if code in by_code or code in BOOTSTRAP_VECTOR_SCHEMES: - continue - if (entry.get("status") or "") == "disabled": - continue - name = entry.get("scheme_name") or code - aliases = parse_aliases(entry.get("scheme_aliases_json")) - db.upsert_catalog_entry( - scheme_code=code, - scheme_name=name, - scheme_aliases=aliases, - instances=parse_aliases(entry.get("instances_json")) or ["default"], - workflow_ids=[], - collection_name=collection, - chunk_count=int(entry.get("chunk_count") or 0), - status="disabled", - network_visible=int(entry.get("network_visible") or 0), - promoted_at=entry.get("promoted_at"), - content_hash=_content_hash(code, name, aliases, bool(entry.get("network_visible")), "disabled"), - source=entry.get("source") or "pipeline", - updated_at=now, - ) - + if code not in existing: + _seed_bootstrap_entry(code, definition) return db.bump_catalog_version(reason) @@ -668,7 +474,6 @@ def build_snapshot( include_pending: bool = False, network_visible: Optional[bool] = None, ) -> dict[str, Any]: - ensure_catalog_schema() meta = db.get_catalog_meta() version = int(meta.get("version") or 0) updated_at = meta.get("updated_at") or utc_now_iso() @@ -686,27 +491,25 @@ def build_snapshot( # Live snapshot for warmers: only status=live (pending excluded unless include_pending) if not include_pending and st != "live": continue - instances = parse_aliases(e.get("instances_json")) + instances = e["instances"] if instance: inst = instance.strip().lower() if inst not in {i.lower() for i in instances} and instances: - # bootstrap may have ["default"] only - if inst not in {i.lower() for i in instances}: - continue - nv = bool(int(e.get("network_visible") or 0)) + continue + nv = bool(e.get("network_visible")) if network_visible is not None and nv != network_visible: continue vector_schemes.append( { - "scheme_code": e["scheme_code"], - "scheme_name": e.get("scheme_name") or e["scheme_code"], - "scheme_aliases": parse_aliases(e.get("scheme_aliases_json")), + "scheme_code": e["code"], + "scheme_name": e.get("name") or e["code"], + "scheme_aliases": e["aliases"], "tool": "search_schemes", "tool_routing": "qdrant", "instances": instances or ["default"], "network_visible": nv, "chunk_count": int(e.get("chunk_count") or 0), - "workflow_ids": parse_aliases(e.get("workflow_ids_json")), + "workflow_ids": e["workflow_ids"], "promoted_at": e.get("promoted_at"), "source": e.get("source") or "bootstrap", "status": st, @@ -767,7 +570,6 @@ def apply_scheme_metadata( document has none - so a reviewer who only moves the end date does not have to restate the start. """ - ensure_catalog_schema() doc = db.get_document(workflow_id) if not doc: raise LookupError(f"Document not found: {workflow_id}") diff --git a/pipeline/worker.py b/pipeline/worker.py index 6cd996b..6101442 100644 --- a/pipeline/worker.py +++ b/pipeline/worker.py @@ -62,8 +62,8 @@ async def main(): print("Error: TRANSLATION_VLLM_BASE_URL not set") return - # Initialize SQLite database - print("Initializing SQLite database...") + # Initialize Postgres database + print("Initializing database...") db.init_db() temporal_host = os.environ.get("TEMPORAL_HOST", "localhost:7233") diff --git a/pipeline/workflows.py b/pipeline/workflows.py index 4765534..d6ab900 100644 --- a/pipeline/workflows.py +++ b/pipeline/workflows.py @@ -96,7 +96,7 @@ async def _mirror_state( error_message: Optional[str] = None, ) -> dict: """ - Mirror workflow state into SQLite as a local activity. + Mirror workflow state into Postgres as a local activity. Local activities avoid starving tiny state updates behind long-running OCR, translation, or chunking activities on the shared task queue. diff --git a/tests/conftest.py b/tests/conftest.py index fc77ac7..e023105 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,8 @@ Pytest configuration and fixtures for the document ingestion pipeline. Provides fixtures for: -- SQLite database (in-memory for isolation) +- Postgres database (a throwaway container, or the server named by + TEST_DATABASE_URL; migrations applied once, tables truncated per test) - FastAPI TestClient - Mock Temporal client - Mock MinIO client @@ -15,31 +16,58 @@ import pytest # Set test environment before imports -os.environ["DOCUMENT_DB_PATH"] = ":memory:" os.environ["MINIO_ACCESS_KEY"] = "test-access-key" os.environ["MINIO_SECRET_KEY"] = "test-secret-key" os.environ["TRANSLATION_VLLM_BASE_URL"] = "http://localhost:8000/v1" os.environ["AUTH_DISABLED"] = "true" -@pytest.fixture(scope="function") -def temp_db_path(tmp_path): - """Create a temporary database path for each test.""" - db_path = tmp_path / "test.db" - os.environ["DOCUMENT_DB_PATH"] = str(db_path) - yield str(db_path) - # Cleanup happens automatically with tmp_path +_TRUNCATE_SQL = """ + TRUNCATE documents, users, catalog_entries, audit_logs, email_otps, + webhook_records, search_settings CASCADE +""" -@pytest.fixture(scope="function") -def db_connection(temp_db_path): - """Initialize database and provide connection for testing.""" +@pytest.fixture(scope="session") +def postgres_url(): + """URL of the Postgres the tests run against, with migrations applied. + + Every table is truncated before each test, so TEST_DATABASE_URL must name a + scratch database, never a real one. + """ + url = os.environ.get("TEST_DATABASE_URL") + container = None + if not url: + from testcontainers.postgres import PostgresContainer + + container = PostgresContainer("postgres:15-alpine", driver=None) + container.start() + url = container.get_connection_url() + os.environ["DATABASE_URL"] = url from pipeline import db - db.DB_PATH = temp_db_path + + db.close_pool() db.init_db() + yield url + db.close_pool() + if container is not None: + container.stop() + + +@pytest.fixture(scope="function") +def db_connection(postgres_url): + """The db module on an empty database (seed rows restored after each truncate).""" + from pipeline import db + + os.environ["DATABASE_URL"] = postgres_url + with db.get_connection() as conn: + conn.execute(_TRUNCATE_SQL) + conn.execute( + "INSERT INTO users (keycloak_user_id, username, roles) " + "VALUES ('local-admin', 'local-admin', '{superadmin}')" + ) + conn.execute("INSERT INTO search_settings DEFAULT VALUES") yield db - # Reset for next test - db.DB_PATH = os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db") @pytest.fixture diff --git a/tests/test_activities.py b/tests/test_activities.py index 8291c26..d14097c 100644 --- a/tests/test_activities.py +++ b/tests/test_activities.py @@ -395,7 +395,7 @@ class TestUpdateDocumentState: @pytest.mark.unit def test_update_state(self, db_connection): - """Test updating document state in SQLite.""" + """Test updating document state in the database.""" from pipeline.activities import update_document_state workflow_id = "state-update-test" diff --git a/tests/test_api.py b/tests/test_api.py index 396d7a8..cba7a64 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -10,12 +10,12 @@ - Error handling """ +import hashlib import os import pytest # Set test environment -os.environ["DOCUMENT_DB_PATH"] = ":memory:" os.environ["MINIO_ACCESS_KEY"] = "test-key" os.environ["MINIO_SECRET_KEY"] = "test-secret" os.environ["ALLOWED_FILE_PATHS"] = "/app/books,/tmp" @@ -86,7 +86,7 @@ def test_list_documents_with_stage_filter(self, test_client, db_connection): @pytest.mark.api @pytest.mark.unit def test_get_document_from_db(self, test_client, db_connection): - """Test getting document details from SQLite fallback.""" + """Test getting document details from the database.""" workflow_id = "get-test-001" db_connection.upsert_document( workflow_id=workflow_id, @@ -109,6 +109,144 @@ def test_get_document_not_found(self, test_client): assert response.status_code == 404 +class TestDocumentIdentifiers: + """`/documents/{id}` takes the UUID (canonical) or any legacy identifier.""" + + FINGERPRINT = hashlib.md5(b"identifiers").hexdigest() + WORKFLOW_ID = f"doc-{FINGERPRINT}" + + def _document(self, db_connection): + db_connection.upsert_document( + workflow_id=self.WORKFLOW_ID, + document_id=self.FINGERPRINT, + filename="identifiers.pdf", + filepath="minio://documents/identifiers.pdf", + stage="ocr_review", + ) + db_connection.save_pages( + self.WORKFLOW_ID, [{"page_number": 1, "original_markdown": "page one"}] + ) + return db_connection.get_document(self.WORKFLOW_ID) + + @pytest.mark.api + @pytest.mark.unit + def test_the_response_carries_the_uuid_and_the_temporal_id(self, test_client, db_connection): + doc = self._document(db_connection) + body = test_client.get(f"/documents/{doc['id']}").json() + assert body["id"] == doc["id"] + assert body["workflow_id"] == self.WORKFLOW_ID + assert body["document_id"] == self.FINGERPRINT + + @pytest.mark.api + @pytest.mark.unit + @pytest.mark.parametrize("form", ["uuid", "workflow", "fingerprint", "filename"]) + def test_every_identifier_form_reaches_the_same_document(self, test_client, db_connection, form): + doc = self._document(db_connection) + identifier = { + "uuid": doc["id"], + "workflow": self.WORKFLOW_ID, + "fingerprint": self.FINGERPRINT, + "filename": "identifiers.pdf", + }[form] + detail = test_client.get(f"/documents/{identifier}") + pages = test_client.get(f"/documents/{identifier}/pages") + assert detail.status_code == 200 and pages.status_code == 200 + assert detail.json()["id"] == doc["id"] + assert [p["page_number"] for p in pages.json()] == [1] + + @pytest.mark.api + @pytest.mark.unit + def test_the_uuid_is_rewritten_to_the_temporal_id_before_handlers_run( + self, test_client, db_connection, mock_temporal_client + ): + doc = self._document(db_connection) + db_connection.update_document_stage(doc["id"], "approval_for_prod") + response = test_client.post(f"/documents/{doc['id']}/approve-prod") + assert response.status_code == 200 + queried = [call.args[0] for call in mock_temporal_client.get_workflow_handle.call_args_list] + assert queried and set(queried) == {self.WORKFLOW_ID} + + @pytest.mark.api + @pytest.mark.unit + def test_static_document_routes_are_not_mistaken_for_identifiers(self, test_client, db_connection): + db_connection.upsert_document( + workflow_id="summary", document_id=hashlib.md5(b"summary").hexdigest(), + filename="summary", filepath="/tmp/s.pdf", + ) + response = test_client.get("/documents/summary") + assert response.status_code == 200 + assert "total_documents" in response.json() + + @pytest.mark.api + @pytest.mark.unit + def test_an_unknown_identifier_is_404(self, test_client, db_connection): + assert test_client.get("/documents/00000000-0000-0000-0000-000000000000").status_code == 404 + + +class TestArtifactAndJobIdentifiers: + """Artifact and job ids are UUIDs.""" + + def _document(self, db_connection): + db_connection.upsert_document( + workflow_id="wf-art-api", document_id=hashlib.md5(b"art").hexdigest(), + filename="a.pdf", filepath="minio://documents/a.pdf", stage="ocr_review", + ) + artifact_id = db_connection.add_document_artifact( + "wf-art-api", "original_upload", "minio://documents/a.pdf", stage="registered", + filename="a.pdf", + ) + return artifact_id + + @pytest.mark.api + @pytest.mark.unit + def test_artifacts_are_addressed_by_uuid(self, test_client, db_connection): + artifact_id = self._document(db_connection) + listed = test_client.get("/documents/wf-art-api/artifacts").json() + assert [a["id"] for a in listed] == [artifact_id] + assert test_client.get(f"/documents/wf-art-api/artifacts/{artifact_id}").json()["id"] == artifact_id + assert test_client.get("/documents/wf-art-api/artifacts/7").status_code == 404 + + @pytest.mark.api + @pytest.mark.unit + def test_document_detail_derives_artifact_and_job_pointers(self, test_client, db_connection): + artifact_id = self._document(db_connection) + body = test_client.get("/documents/wf-art-api").json() + job = db_connection.get_latest_document_job("wf-art-api") + assert body["original_artifact_id"] == artifact_id + assert body["normalized_artifact_id"] is None + assert body["latest_job_id"] == job["id"] + assert body["artifacts"][0]["job_id"] == job["id"] + + @pytest.mark.api + @pytest.mark.unit + def test_runs_are_addressed_by_uuid(self, test_client, db_connection): + self._document(db_connection) + job = db_connection.get_latest_document_job("wf-art-api") + assert test_client.get(f"/runs/{job['id']}").json()["id"] == job["id"] + assert test_client.get("/runs/3").status_code == 404 + assert [r["id"] for r in test_client.get("/runs").json()] == [job["id"]] + + +class TestHealthAndSettings: + @pytest.mark.api + @pytest.mark.unit + def test_health_reports_the_database(self, test_client): + body = test_client.get("/health").json() + assert body["database"] == {"reachable": True, "migrations_current": True} + + @pytest.mark.api + @pytest.mark.unit + def test_settings_default_alpha_and_validation(self, test_client): + assert test_client.get("/settings/search").json()["alpha"] == 0.6 + assert test_client.put("/settings/search", json={"alpha": 2}).status_code == 400 + updated = test_client.put("/settings/search", json={"limit": 7}).json() + assert updated["limit"] == 7 + reset = test_client.post("/settings/search/reset").json() + assert reset["limit"] == 12 and reset["alpha"] == 0.6 + audit = test_client.get("/settings/search/audit").json() + assert audit["total"] == 2 and audit["logs"][0]["document_id"] == "__settings__" + + class TestProdApproval: """Promoting to PROD takes no body. diff --git a/tests/test_db.py b/tests/test_db.py index b771611..87e00ea 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -1,5 +1,5 @@ """ -Unit tests for pipeline/db.py - SQLite database operations. +Unit tests for pipeline/db.py against a real Postgres. Tests cover: - Document CRUD operations @@ -7,12 +7,20 @@ - Chunk operations - Audit logging - Settings management +- Schema constraints, JSONB merges, workflows, the catalog view, identifier + resolution and the migration runner """ +import hashlib +import psycopg import pytest +def _md5(text): + return hashlib.md5(text.encode()).hexdigest() + + class TestDocumentOperations: """Tests for document CRUD operations.""" @@ -21,7 +29,7 @@ class TestDocumentOperations: def test_upsert_document_creates_new(self, db_connection): """Test creating a new document.""" workflow_id = "test-wf-001" - document_id = "test-doc-001" + document_id = _md5("test-doc-001") db_connection.upsert_document( workflow_id=workflow_id, @@ -586,3 +594,671 @@ def test_settings_audit_log(self, db_connection): db_connection.update_search_settings({"limit": 50}) logs = db_connection.get_settings_audit_logs() assert len(logs) >= 1 + + +def _new_doc(db, workflow_id, **kwargs): + kwargs.setdefault("document_id", _md5(workflow_id)) + kwargs.setdefault("filename", f"{workflow_id}.pdf") + kwargs.setdefault("filepath", f"minio://documents/{workflow_id}.pdf") + return db.upsert_document(workflow_id=workflow_id, **kwargs) + + +class TestSchemaConstraints: + """Rules the database itself enforces.""" + + @pytest.mark.db + @pytest.mark.unit + def test_document_kind_must_match_its_catalog_entry_type(self, db_connection): + with db_connection.get_connection() as conn: + entry = db_connection.upsert_catalog_entry(type="scheme", code="x-scheme", name="X", conn=conn) + with pytest.raises(psycopg.errors.ForeignKeyViolation): + conn.execute( + "INSERT INTO documents (instance, source, document_kind, catalog_entry_id) " + "VALUES ('default', %s, 'advisory', %s::uuid)", + (psycopg.types.json.Jsonb({"fingerprint": _md5("k")}), entry["id"]), + ) + + @pytest.mark.db + @pytest.mark.unit + def test_source_requires_a_hex_fingerprint(self, db_connection): + with db_connection.get_connection() as conn: + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute( + "INSERT INTO documents (instance, source) VALUES ('default', %s)", + (psycopg.types.json.Jsonb({"fingerprint": "not-hex"}),), + ) + + @pytest.mark.db + @pytest.mark.unit + def test_stage_timestamps_only_allow_known_keys(self, db_connection): + _new_doc(db_connection, "wf-ts") + with db_connection.get_connection() as conn: + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute( + "UPDATE workflows SET stage_timestamps = '{\"bogus\": \"x\"}'::jsonb " + "WHERE temporal_workflow_id = 'wf-ts'" + ) + + @pytest.mark.db + @pytest.mark.unit + def test_validity_period_must_not_end_before_it_starts(self, db_connection): + with pytest.raises(psycopg.errors.CheckViolation): + _new_doc(db_connection, "wf-bad-period", valid_from="2026-02-01", valid_to="2026-01-01") + + @pytest.mark.db + @pytest.mark.unit + def test_chunk_page_range_must_not_run_backwards(self, db_connection): + _new_doc(db_connection, "wf-range") + with pytest.raises(psycopg.errors.CheckViolation): + with db_connection.get_connection() as conn: + conn.execute( + "INSERT INTO chunks (document_id, chunk_number, original_text, page_start, page_end) " + "SELECT id, 1, 't', 3, 2 FROM documents" + ) + + @pytest.mark.db + @pytest.mark.unit + def test_audit_entry_needs_a_workflow_unless_system_level(self, db_connection): + with db_connection.get_connection() as conn: + conn.execute("INSERT INTO audit_logs (action_type, entity_type) VALUES ('x', 'setting')") + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute("INSERT INTO audit_logs (action_type, entity_type) VALUES ('x', 'page')") + + @pytest.mark.db + @pytest.mark.unit + def test_search_settings_is_a_single_row(self, db_connection): + with db_connection.get_connection() as conn: + with pytest.raises(psycopg.errors.UniqueViolation): + conn.execute("INSERT INTO search_settings DEFAULT VALUES") + + +class TestWorkflows: + @pytest.mark.db + @pytest.mark.unit + def test_upsert_creates_the_main_workflow(self, db_connection): + _new_doc(db_connection, "wf-main", stage="ocr_processing") + jobs = db_connection.list_document_jobs("wf-main") + assert [(j["job_type"], j["temporal_workflow_id"]) for j in jobs] == [("pipeline", "wf-main")] + + @pytest.mark.db + @pytest.mark.unit + def test_current_workflow_is_the_latest_by_start_time(self, db_connection): + _new_doc(db_connection, "wf-cur", stage="completed") + job_id = db_connection.create_document_job( + "wf-cur", "chunking_retry", temporal_workflow_id="wf-cur-retry-chunking-1", + current_stage="chunking", + ) + doc = db_connection.get_document("wf-cur") + assert doc["stage"] == "chunking" + assert doc["latest_job_id"] == job_id + assert doc["workflow_id"] == "wf-cur" + + @pytest.mark.db + @pytest.mark.unit + def test_stage_updates_follow_the_current_workflow(self, db_connection): + _new_doc(db_connection, "wf-follow", stage="completed") + db_connection.create_document_job("wf-follow", "ocr_retry", current_stage="ocr_processing") + db_connection.update_document_stage("wf-follow", "ocr_review") + by_type = {j["job_type"]: j["current_stage"] for j in db_connection.list_document_jobs("wf-follow")} + assert by_type == {"pipeline": "completed", "ocr_retry": "ocr_review"} + + @pytest.mark.db + @pytest.mark.unit + def test_stage_timestamps_are_set_once(self, db_connection): + _new_doc(db_connection, "wf-once") + db_connection.update_document_stage("wf-once", "ocr_review") + first = db_connection.get_document("wf-once")["ocr_completed_at"] + assert first + db_connection.update_document_stage("wf-once", "ocr_processing") + db_connection.update_document_stage("wf-once", "ocr_review") + assert db_connection.get_document("wf-once")["ocr_completed_at"] == first + + @pytest.mark.db + @pytest.mark.unit + def test_prod_request_is_a_waiting_promote_workflow_that_approval_adopts(self, db_connection): + _new_doc(db_connection, "wf-prod", stage="approval_for_prod") + doc = db_connection.mark_prod_ready_requested("wf-prod", user_id="kc-1", username="reviewer1") + assert doc["prod_ready_requested_by_username"] == "reviewer1" + latest = db_connection.get_latest_document_job("wf-prod") + assert (latest["job_type"], latest["status"]) == ("promote_prod", "waiting_review") + + db_connection.create_document_job( + "wf-prod", "promote_prod", temporal_workflow_id="wf-prod-promote-prod-1", + current_stage="ingesting_prod", + ) + jobs = [j for j in db_connection.list_document_jobs("wf-prod") if j["job_type"] == "promote_prod"] + assert len(jobs) == 1 + assert jobs[0]["temporal_workflow_id"] == "wf-prod-promote-prod-1" + assert db_connection.get_document("wf-prod")["prod_ready_requested_at"] is None + + @pytest.mark.db + @pytest.mark.unit + def test_job_config_merges(self, db_connection): + _new_doc(db_connection, "wf-cfg") + job = db_connection.get_latest_document_job("wf-cfg") + db_connection.update_document_job(job["id"], config_json={"a": 1}) + db_connection.update_document_job(job["id"], config_json={"chunking_progress": {"percent": 5}}) + config = db_connection.get_document_job(job["id"])["config"] + assert config == {"a": 1, "chunking_progress": {"percent": 5}} + + @pytest.mark.db + @pytest.mark.unit + def test_artifacts_belong_to_their_workflow_and_derive_pointers(self, db_connection): + _new_doc(db_connection, "wf-art") + job = db_connection.get_latest_document_job("wf-art") + art_id = db_connection.add_document_artifact( + "wf-art", "original_upload", "minio://documents/a.pdf", stage="registered", job_id=job["id"], + ) + norm_id = db_connection.add_document_artifact( + "wf-art", "normalized_pdf", "minio://documents/n.pdf", stage="ocr_processing", + ) + doc = db_connection.get_document("wf-art") + assert doc["original_artifact_id"] == art_id + assert doc["normalized_artifact_id"] == norm_id + artifact = db_connection.get_document_artifact(doc["id"], art_id) + assert artifact["job_id"] == job["id"] + assert artifact["workflow_id"] == "wf-art" + assert db_connection.get_document_artifact("wf-art", "123") is None + + +class TestPageTranslations: + @pytest.mark.db + @pytest.mark.unit + def test_an_ocr_save_keeps_existing_translations(self, db_connection): + _new_doc(db_connection, "wf-tr") + db_connection.save_pages("wf-tr", [{ + "page_number": 1, "original_markdown": "namaste", "detected_language": "hi", + "translated_markdown": "hello", "translation_target_language": "en", + "translation_provider": "p", "translation_model": "m", + }]) + db_connection.save_pages("wf-tr", [{"page_number": 1, "original_markdown": "namaste"}]) + page = db_connection.get_page("wf-tr", 1) + assert page["translated_markdown"] == "hello" + assert page["translations"]["en"]["provider"] == "p" + + @pytest.mark.db + @pytest.mark.unit + def test_changed_ocr_text_drops_stale_translations(self, db_connection): + _new_doc(db_connection, "wf-tr2") + db_connection.save_pages("wf-tr2", [{ + "page_number": 1, "original_markdown": "old", "translated_markdown": "t", + "translation_target_language": "en", + }]) + db_connection.save_pages("wf-tr2", [{"page_number": 1, "original_markdown": "new"}]) + assert db_connection.get_page("wf-tr2", 1)["translations"] == {} + + @pytest.mark.db + @pytest.mark.unit + def test_reading_and_saving_pages_back_is_lossless(self, db_connection): + _new_doc(db_connection, "wf-rt") + db_connection.save_pages("wf-rt", [{ + "page_number": 1, "original_markdown": "a", "detected_language": "gu", + "translated_markdown": "b", "translation_target_language": "en", "reviewer_notes": "n", + }]) + before = db_connection.get_pages("wf-rt") + db_connection.save_pages("wf-rt", before) + assert db_connection.get_pages("wf-rt") == before + + @pytest.mark.db + @pytest.mark.unit + def test_translation_edits_target_one_language_entry(self, db_connection): + _new_doc(db_connection, "wf-edit") + db_connection.save_pages("wf-edit", [{ + "page_number": 1, "original_markdown": "a", "translated_markdown": "b", + "translation_target_language": "en", + }]) + db_connection.update_page( + "wf-edit", 1, edited_translation="fixed", translation_reviewed=True, is_reviewed=True, + ) + page = db_connection.get_page("wf-edit", 1) + assert page["edited_translation"] == "fixed" + assert page["translated_markdown"] == "b" + assert page["translation_reviewed"] is True + assert page["is_reviewed"] is True + + @pytest.mark.db + @pytest.mark.unit + def test_page_metadata_merges_instead_of_replacing(self, db_connection): + _new_doc(db_connection, "wf-meta") + db_connection.save_pages("wf-meta", [ + {"page_number": 1, "original_markdown": "a", "detected_language": "hi"} + ]) + db_connection.update_page("wf-meta", 1, reviewer_notes="ok") + page = db_connection.get_page("wf-meta", 1) + assert page["detected_language"] == "hi" + assert page["reviewer_notes"] == "ok" + + @pytest.mark.db + @pytest.mark.unit + def test_count_translated_pages(self, db_connection): + _new_doc(db_connection, "wf-count") + db_connection.save_pages("wf-count", [ + {"page_number": 1, "original_markdown": "a", "translated_markdown": "x", + "translation_target_language": "en"}, + {"page_number": 2, "original_markdown": "b"}, + ]) + assert db_connection.count_translated_pages("wf-count") == 1 + + +class TestChunkRuns: + @pytest.mark.db + @pytest.mark.unit + def test_rechunking_replaces_chunks_and_bumps_the_run(self, db_connection): + _new_doc(db_connection, "wf-run") + first = [{"chunk_number": 1, "original_text": "a", "chunking_provider": "p1"}] + db_connection.save_chunks("wf-run", first) + old_id = db_connection.get_chunk("wf-run", 1)["id"] + db_connection.save_chunks("wf-run", [ + {"chunk_number": 1, "original_text": "b", "chunking_provider": "p2", + "chunking_config_json": '{"size": 450}'}, + {"chunk_number": 2, "original_text": "c", "chunking_provider": "p2"}, + ]) + doc = db_connection.get_document("wf-run") + chunk = db_connection.get_chunk("wf-run", 1) + assert doc["chunk_count"] == 2 + assert chunk["id"] != old_id + assert chunk["chunk_version"] == 2 + assert chunk["chunking_provider"] == "p2" + assert chunk["chunking_config_json"] == '{"size": 450}' + + @pytest.mark.db + @pytest.mark.unit + def test_a_failed_rechunk_leaves_the_previous_chunks(self, db_connection): + _new_doc(db_connection, "wf-atomic") + db_connection.save_chunks("wf-atomic", [{"chunk_number": 1, "original_text": "keep"}]) + with pytest.raises(psycopg.errors.UniqueViolation): + db_connection.save_chunks("wf-atomic", [ + {"chunk_number": 1, "original_text": "x"}, + {"chunk_number": 1, "original_text": "dup"}, + ]) + assert db_connection.get_chunk("wf-atomic", 1)["original_text"] == "keep" + assert db_connection.get_document("wf-atomic")["chunk_count"] == 1 + + @pytest.mark.db + @pytest.mark.unit + def test_source_spans_round_trip_and_derive_page_numbers(self, db_connection): + _new_doc(db_connection, "wf-spans") + db_connection.save_chunks("wf-spans", [{ + "chunk_number": 1, "original_text": "a", "page_start": 2, "page_end": 3, + "source_spans_json": '[{"page_number": 2, "start_char": 0, "end_char": 1}, ' + '{"page_number": 3, "start_char": 0, "end_char": 1}]', + }]) + chunk = db_connection.get_chunk("wf-spans", 1) + assert chunk["source_page_numbers_json"] == "[2, 3]" + + @pytest.mark.db + @pytest.mark.unit + def test_search_chunks_matches_edited_text_first(self, db_connection): + _new_doc(db_connection, "wf-search", instance="mh") + db_connection.save_chunks("wf-search", [ + {"chunk_number": 1, "original_text": "alpha beta", "edited_text": "gamma delta"}, + ]) + hits, total = db_connection.search_chunks(query="gamma") + assert total == 1 and hits[0]["workflow_id"] == "wf-search" + assert db_connection.search_chunks(query="alpha")[1] == 0 + assert db_connection.search_chunks(query="gamma", instances=["up"])[1] == 0 + + +class TestIndexStatus: + @pytest.mark.db + @pytest.mark.unit + def test_entries_merge_and_dev_prod_are_separate_keys(self, db_connection, monkeypatch): + monkeypatch.setenv("PROD_VECTOR_DB_COLLECTION_NAME", "prod-docs") + _new_doc(db_connection, "wf-idx") + db_connection.upsert_document_index_status( + "wf-idx", "documents-index", doc_id="abc", chunk_count_indexed=4, + last_indexed_at="2026-01-01T00:00:00", status="indexed", + ) + db_connection.upsert_document_index_status( + "wf-idx", "documents-index", last_verified_at="2026-01-02T00:00:00", status="indexed", + ) + db_connection.upsert_document_index_status("wf-idx", "prod-docs", status="missing") + doc = db_connection.get_document("wf-idx") + assert set(doc["index_status"]) == {"dev", "prod"} + dev = db_connection.get_document_index_status("wf-idx", "documents-index") + assert dev["chunk_count_indexed"] == 4 + assert dev["last_indexed_at"] == "2026-01-01T00:00:00" + assert dev["last_verified_at"] == "2026-01-02T00:00:00" + summaries = {s["index_name"]: s for s in db_connection.list_index_summaries()} + assert summaries["documents-index"]["indexed_chunks"] == 4 + + +class TestIdentifierResolution: + @pytest.mark.db + @pytest.mark.unit + def test_every_documented_form_resolves(self, db_connection): + fingerprint = _md5("content") + _new_doc(db_connection, "doc-" + fingerprint, document_id=fingerprint, filename="pm.pdf") + db_connection.create_document_job( + "doc-" + fingerprint, "ocr_retry", temporal_workflow_id=f"doc-{fingerprint}-retry-ocr-9", + ) + doc = db_connection.get_document("doc-" + fingerprint) + for identifier in ( + doc["id"], + "doc-" + fingerprint, + f"doc-{fingerprint}-retry-ocr-9", + fingerprint, + "pm.pdf", + _md5(fingerprint), + ): + found = db_connection.find_document_by_doc_identifier(identifier) + assert found and found["id"] == doc["id"], identifier + assert found["workflow_id"] == "doc-" + fingerprint + assert db_connection.find_document_by_doc_identifier("nope") is None + assert db_connection.get_document("00000000-0000-0000-0000-000000000000") is None + + +class TestAuditTrail: + @pytest.mark.db + @pytest.mark.unit + def test_audit_outlives_the_document_and_keeps_tenant_scope(self, db_connection): + _new_doc(db_connection, "wf-aud", instance="mh") + db_connection.log_audit( + workflow_id="wf-aud", document_id="x", action_type="purge_document", + entity_type="document", actor_user_id="kc-9", actor_username="boss", + actor_email="b@x.in", actor_roles="superadmin", + ) + db_connection.purge_document("wf-aud") + + logs = db_connection.get_audit_logs("wf-aud") + assert [entry["action_type"] for entry in logs] == ["purge_document"] + assert logs[0]["actor"] == "b@x.in (superadmin)" + assert logs[0]["actor_user_id"] == "kc-9" + assert db_connection.get_all_audit_log_count(instances=["mh"]) == 1 + assert db_connection.get_all_audit_log_count(instances=["up"]) == 0 + + @pytest.mark.db + @pytest.mark.unit + def test_system_entries_need_no_document(self, db_connection): + db_connection.log_audit( + workflow_id="__catalog__", document_id="__catalog__", action_type="catalog_rebuild", + entity_type="catalog", + ) + logs = db_connection.get_all_audit_logs(action_type="catalog_rebuild") + assert logs[0]["workflow_id"] == "__catalog__" + + @pytest.mark.db + @pytest.mark.unit + def test_settings_changes_are_logged_per_key_with_group_names(self, db_connection): + db_connection.update_search_settings({"alpha": 0.2, "limit": 3}) + fields = {entry["field_name"] for entry in db_connection.get_settings_audit_logs()} + assert fields == {"retrieval.alpha", "results.limit"} + assert db_connection.get_settings_audit_count() == 2 + db_connection.update_search_settings({"alpha": 0.2}) + assert db_connection.get_settings_audit_count() == 2 + + +class TestSearchSettings: + @pytest.mark.db + @pytest.mark.unit + def test_defaults_come_from_the_models(self, db_connection): + settings = db_connection.get_search_settings() + assert settings["alpha"] == 0.6 + assert len(settings) == 15 + + @pytest.mark.db + @pytest.mark.unit + def test_values_are_validated(self, db_connection): + with pytest.raises(ValueError): + db_connection.update_search_settings({"alpha": 3}) + with pytest.raises(ValueError): + db_connection.update_search_settings({"searchMethod": "FUZZY"}) + with pytest.raises(ValueError): + db_connection.update_search_settings({"nonsense": 1}) + + @pytest.mark.db + @pytest.mark.unit + def test_a_partial_update_keeps_the_rest_of_its_group(self, db_connection): + db_connection.update_search_settings({"alpha": 0.3}) + db_connection.update_search_settings({"hybridRrfK": 10}) + settings = db_connection.get_search_settings() + assert (settings["alpha"], settings["hybridRrfK"]) == (0.3, 10) + + +class TestCatalogView: + @staticmethod + def _scheme_doc(db, workflow_id, code, stage, **kwargs): + _new_doc(db, workflow_id, stage=stage, **kwargs) + db.update_document_fields( + workflow_id, document_kind="scheme", scheme_code=code, scheme_name=code.upper(), + ) + + @pytest.mark.db + @pytest.mark.unit + def test_status_rules(self, db_connection): + db = db_connection + self._scheme_doc(db, "wf-live", "live-one", "completed") + self._scheme_doc(db, "wf-stale", "stale-one", "completed") + db.mark_document_reindex_required("wf-stale", True, "edit") + self._scheme_doc(db, "wf-gate", "gate-one", "approval_for_prod") + self._scheme_doc(db, "wf-wip", "wip-one", "chunk_review") + self._scheme_doc(db, "wf-off", "off-one", "completed") + db.set_document_disabled("wf-off", True) + db.upsert_catalog_entry(type="scheme", code="boot-one", name="Boot", source="bootstrap") + + status = {e["code"]: e["status"] for e in db.list_catalog_entries()} + assert status == { + "live-one": "live", + "stale-one": "pending_reindex", + "gate-one": "pending_prod", + "wip-one": "disabled", + "off-one": "disabled", + "boot-one": "live", + } + + @pytest.mark.db + @pytest.mark.unit + def test_computed_columns(self, db_connection): + db = db_connection + self._scheme_doc(db, "wf-a", "multi", "completed", instance="mh") + self._scheme_doc(db, "wf-b", "multi", "completed", instance="up") + db.save_chunks("wf-a", [{"chunk_number": 1, "original_text": "x"}]) + db.save_chunks("wf-b", [{"chunk_number": 1, "original_text": "y"}, + {"chunk_number": 2, "original_text": "z"}]) + db.update_document_fields("wf-b", network_visible=False) + db.upsert_document_index_status("wf-a", "documents-index", last_indexed_at="2026-05-01T00:00:00") + + entry = db.get_catalog_entry("multi") + assert entry["instances"] == ["mh", "up"] + assert sorted(entry["workflow_ids"]) == ["wf-a", "wf-b"] + assert entry["chunk_count"] == 3 + assert entry["network_visible"] is True + assert entry["promoted_at"] is None + + @pytest.mark.db + @pytest.mark.unit + def test_entries_without_documents_default_to_the_default_instance(self, db_connection): + db_connection.upsert_catalog_entry(type="scheme", code="lonely", name="Lonely", source="import") + entry = db_connection.get_catalog_entry("lonely") + assert entry["instances"] == ["default"] + assert entry["status"] == "disabled" + assert entry["network_visible"] is False + + @pytest.mark.db + @pytest.mark.unit + def test_other_entry_types_stay_out_of_the_scheme_listing(self, db_connection): + _new_doc(db_connection, "wf-adv", stage="completed") + db_connection.update_document_fields( + "wf-adv", document_kind="advisory", scheme_code="kharif", scheme_name="Kharif", + ) + assert db_connection.list_catalog_entries() == [] + assert [e["code"] for e in db_connection.list_catalog_entries(type="advisory")] == ["kharif"] + + @pytest.mark.db + @pytest.mark.unit + def test_changing_the_kind_moves_or_unlinks_the_entry(self, db_connection): + _new_doc(db_connection, "wf-move") + db_connection.update_document_fields( + "wf-move", document_kind="scheme", scheme_code="movable", scheme_name="Movable", + ) + db_connection.update_document_fields("wf-move", document_kind="advisory") + doc = db_connection.get_document("wf-move") + assert (doc["document_kind"], doc["scheme_code"], doc["scheme_name"]) == ( + "advisory", "movable", "Movable", + ) + db_connection.update_document_fields("wf-move", document_kind="document") + doc = db_connection.get_document("wf-move") + assert doc["scheme_code"] is None and doc["catalog_entry_id"] is None + + @pytest.mark.db + @pytest.mark.unit + def test_version_counts_bumps(self, db_connection): + assert db_connection.get_catalog_meta()["version"] == 0 + assert db_connection.bump_catalog_version("a") == 1 + assert db_connection.bump_catalog_version("b") == 2 + meta = db_connection.get_catalog_meta() + assert (meta["version"], meta["notes"]) == (2, "b") + + +class TestUsersAndOtps: + @pytest.mark.db + @pytest.mark.unit + def test_upsert_user_refreshes_and_survives_an_email_clash(self, db_connection): + first = db_connection.upsert_user("kc-a", "ann", "ann@x.in", ["reviewer"]) + again = db_connection.upsert_user("kc-a", "ann2", None, ["admin"]) + assert first == again + user = db_connection.get_user_by_keycloak_id("kc-a") + assert (user["username"], user["email"], user["roles"]) == ("ann2", "ann@x.in", ["admin"]) + other = db_connection.upsert_user("kc-b", "bob", "ANN@x.in", []) + assert other and other != first + assert db_connection.get_user_by_keycloak_id("kc-b")["email"] is None + + @pytest.mark.db + @pytest.mark.unit + def test_uploader_is_a_user_row(self, db_connection): + _new_doc(db_connection, "wf-up", uploaded_by_user_id="kc-u", uploaded_by_username="uma", + uploaded_by_email="u@x.in", uploaded_by_roles="contributor,reviewer") + doc = db_connection.get_document("wf-up") + assert (doc["uploaded_by_user_id"], doc["uploaded_by_username"]) == ("kc-u", "uma") + assert doc["uploaded_by_roles"] == "contributor,reviewer" + + @pytest.mark.db + @pytest.mark.unit + def test_otp_attempts_increment_atomically(self, db_connection): + db_connection.store_email_otp( + email="A@X.in", code_hash="h", expires_at="2099-01-01T00:00:00+00:00", + created_at="x", last_sent_at="2026-01-01T00:00:00+00:00", + ) + assert db_connection.increment_email_otp_attempts("a@x.in") == 1 + assert db_connection.increment_email_otp_attempts("a@x.in") == 2 + row = db_connection.get_email_otp("a@x.in") + assert row["attempts"] == 2 + assert row["expires_at"].endswith("+00:00") + db_connection.store_email_otp( + email="a@x.in", code_hash="h2", expires_at="2099-01-01T00:00:00+00:00", + created_at="x", last_sent_at="2026-01-02T00:00:00+00:00", + ) + assert db_connection.get_email_otp("a@x.in")["attempts"] == 0 + db_connection.delete_email_otp("a@x.in") + assert db_connection.get_email_otp("a@x.in") is None + + +class TestOperationsViews: + @pytest.mark.db + @pytest.mark.unit + def test_summary_counts_and_queue_use_the_current_workflow(self, db_connection): + _new_doc(db_connection, "wf-q1", stage="ocr_review", instance="mh") + _new_doc(db_connection, "wf-q2", stage="completed", instance="up") + db_connection.create_document_job("wf-q2", "reingest", current_stage="ingesting") + counts = db_connection.get_document_summary_counts() + assert counts["total_documents"] == 2 + assert counts["ocr_review_documents"] == 1 + assert counts["completed_documents"] == 0 + assert counts["running_jobs"] == 3 + assert db_connection.get_document_summary_counts(instances=[])["total_documents"] == 0 + by_instance = {r["instance"]: r for r in db_connection.get_document_counts_by_instance()} + assert by_instance["mh"]["dev_approval"] == 1 + + rows, total = db_connection.list_operations_queue() + assert total == 2 + assert {r["workflow_id"]: r["job_type"] for r in rows} == { + "wf-q1": "pipeline", "wf-q2": "reingest", + } + assert db_connection.list_operations_queue(instances=["up"])[1] == 1 + + @pytest.mark.db + @pytest.mark.unit + def test_demo_and_disabled_documents_are_hidden_by_default(self, db_connection): + _new_doc(db_connection, "wf-demo") + _new_doc(db_connection, "wf-off") + db_connection.set_document_demo("wf-demo", True) + db_connection.set_document_disabled("wf-off", True) + assert db_connection.list_documents() == [] + assert len(db_connection.list_documents(include_demo=True, include_disabled=True)) == 2 + + @pytest.mark.db + @pytest.mark.unit + def test_reconcile_moves_a_stalled_document_into_review(self, db_connection): + _new_doc(db_connection, "wf-rec", stage="ocr_processing") + db_connection.save_pages("wf-rec", [{"page_number": 1, "original_markdown": "a"}]) + result = db_connection.reconcile_materialized_state("wf-rec") + assert result["updated"] is True and result["stage"] == "ocr_review" + assert db_connection.get_latest_document_job("wf-rec")["status"] == "waiting_review" + + @pytest.mark.db + @pytest.mark.unit + def test_webhook_upsert_skips_unchanged_payloads(self, db_connection): + db = db_connection + assert db.upsert_webhook_records("webhook", [{"contentID": "c1", "title": "T", "isPublished": True}]) == 1 + assert db.upsert_webhook_records("webhook", [{"contentID": "c1", "title": "T", "isPublished": True}]) == 1 + with db.get_connection() as conn: + rows = conn.execute("SELECT title, is_published FROM webhook_records").fetchall() + assert rows == [{"title": "T", "is_published": True}] + + +class TestMigrationRunner: + @pytest.mark.db + @pytest.mark.unit + def test_applied_migrations_are_recorded_and_not_reapplied(self, db_connection, postgres_url): + from pipeline import migrate + + assert migrate.apply_migrations(postgres_url) == [] + assert migrate.is_current(postgres_url) + assert migrate.current_version(postgres_url) == migrate.list_migrations()[-1][0] + + @pytest.mark.db + @pytest.mark.unit + def test_migration_files_are_numbered_and_unversioned_by_schema_name(self): + from pipeline import migrate + + names = [path.name for _, path in migrate.list_migrations()] + assert names[0] == "0001_initial_schema.sql" + + @pytest.mark.db + @pytest.mark.unit + def test_a_new_migration_runs_once(self, db_connection, postgres_url, tmp_path): + from pipeline import migrate + + (tmp_path / "0001_initial_schema.sql").write_text("SELECT 1;") + (tmp_path / "0002_probe.sql").write_text("CREATE TABLE mig_probe (n int);") + try: + assert migrate.apply_migrations(postgres_url, tmp_path) == ["0002"] + assert migrate.apply_migrations(postgres_url, tmp_path) == [] + finally: + with db_connection.get_connection() as conn: + conn.execute("DROP TABLE IF EXISTS mig_probe") + conn.execute("DELETE FROM schema_migrations WHERE version = '0002'") + + @pytest.mark.db + @pytest.mark.unit + def test_a_failing_migration_applies_nothing_of_itself(self, db_connection, postgres_url, tmp_path): + from pipeline import migrate + + (tmp_path / "0001_initial_schema.sql").write_text("SELECT 1;") + (tmp_path / "0002_bad.sql").write_text("CREATE TABLE mig_half (n int); SELECT 1/0;") + with pytest.raises(psycopg.errors.DivisionByZero): + migrate.apply_migrations(postgres_url, tmp_path) + with db_connection.get_connection() as conn: + assert conn.execute("SELECT to_regclass('mig_half') AS t").fetchone()["t"] is None + assert conn.execute( + "SELECT count(*) AS n FROM schema_migrations WHERE version = '0002'" + ).fetchone()["n"] == 0 + + @pytest.mark.db + @pytest.mark.unit + def test_migration_file_names_are_validated(self, tmp_path): + from pipeline import migrate + + (tmp_path / "init.sql").write_text("SELECT 1;") + with pytest.raises(ValueError): + migrate.list_migrations(tmp_path) diff --git a/tests/test_document_repository.py b/tests/test_document_repository.py index ae6dba4..1b9673e 100644 --- a/tests/test_document_repository.py +++ b/tests/test_document_repository.py @@ -1,7 +1,7 @@ """Unit tests for DocumentRepository. Two styles on purpose: the `db_connection` fixture exercises the repository -against real SQLite (so a drifted db.py signature fails here), and the fake-db +against real the database (so a drifted db.py signature fails here), and the fake-db tests pin the None guards that are the repository's own behaviour. """ diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 898f51c..f7a1794 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -95,7 +95,7 @@ def test_find_document_by_legacy_doc_id_hash(db_connection): def test_provenance_chunk_endpoint(db_connection): db = db_connection workflow_id = "doc-provenance01" - document_id = "prov-doc-id-xyz" + document_id = hashlib.md5(b"prov-doc-id-xyz").hexdigest() db.upsert_document( workflow_id=workflow_id, document_id=document_id, diff --git a/tests/test_scheme_catalog.py b/tests/test_scheme_catalog.py index 2ffe77d..6ed3764 100644 --- a/tests/test_scheme_catalog.py +++ b/tests/test_scheme_catalog.py @@ -2,21 +2,14 @@ from __future__ import annotations -import json -import os - import pytest @pytest.fixture -def catalog_db(temp_db_path): - from pipeline import db, scheme_catalog +def catalog_db(db_connection): + from pipeline import scheme_catalog - db.DB_PATH = temp_db_path - db.init_db() - scheme_catalog.ensure_catalog_schema() - yield db, scheme_catalog - db.DB_PATH = os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db") + yield db_connection, scheme_catalog def test_bootstrap_seeds_13_vector_schemes(catalog_db): @@ -72,8 +65,7 @@ def test_scheme_metadata_and_promote_rebuild(catalog_db): assert entry is not None assert entry["status"] == "live" assert entry["source"] == "pipeline" - aliases = json.loads(entry["scheme_aliases_json"]) - assert "organic" in aliases or "PKVY" in aliases + assert "organic" in entry["aliases"] or "PKVY" in entry["aliases"] def test_code_rename_pending_reindex(catalog_db): @@ -237,20 +229,16 @@ def test_prepare_records_document_unchanged(): @pytest.fixture -def asgi_client(temp_db_path, mock_temporal_client, mock_minio_client): +def asgi_client(db_connection, mock_temporal_client, mock_minio_client): """HTTP client that does not run real Temporal/MinIO lifespan.""" from contextlib import asynccontextmanager from fastapi.testclient import TestClient - from pipeline import db, scheme_catalog from pipeline.api import api - db.DB_PATH = temp_db_path - db.init_db() api.temporal_client = mock_temporal_client api.minio_client = mock_minio_client - scheme_catalog.ensure_catalog_schema() @asynccontextmanager async def _noop_lifespan(_app): @@ -263,7 +251,6 @@ async def _noop_lifespan(_app): yield client finally: api.app.router.lifespan_context = original - db.DB_PATH = os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db") def test_catalog_api_snapshot(asgi_client): diff --git a/tests/test_tenancy.py b/tests/test_tenancy.py index 53a69c3..7141780 100644 --- a/tests/test_tenancy.py +++ b/tests/test_tenancy.py @@ -213,7 +213,8 @@ def test_summary_counts_honor_instance_filter(db_connection): def test_list_runs_filters_by_instance(db_connection): """Runs must honor the same tenant scope as documents (role-based states).""" db = db_connection - db.upsert_document( + doc_a = db.upsert_document( + with_workflow=False, workflow_id="wf-run-a", document_id="d-run-a", filename="a.pdf", @@ -221,7 +222,8 @@ def test_list_runs_filters_by_instance(db_connection): stage="completed", instance="tenant-a", ) - db.upsert_document( + doc_b = db.upsert_document( + with_workflow=False, workflow_id="wf-run-b", document_id="d-run-b", filename="b.pdf", @@ -229,8 +231,8 @@ def test_list_runs_filters_by_instance(db_connection): stage="completed", instance="tenant-b", ) - job_a = db.create_document_job("wf-run-a", "ocr", status="completed") - job_b = db.create_document_job("wf-run-b", "ocr", status="running") + job_a = db.create_document_job(doc_a, "ocr_only", status="completed") + job_b = db.create_document_job(doc_b, "ocr_only", status="running") tenant_a_only = db.list_runs(instances=["tenant-a"]) assert {r["id"] for r in tenant_a_only} == {job_a} @@ -271,8 +273,8 @@ def test_state_user_cannot_see_bv_portal_documents(db_connection): stage="ocr_review", instance="mh", ) - db.create_document_job("wf-bv", "ocr", status="running") - db.create_document_job("wf-mh", "ocr", status="running") + db.create_document_job("wf-bv", "ocr_only", status="running") + db.create_document_job("wf-mh", "ocr_only", status="running") # MH contributor from Keycloak group path mh_user = claims_to_user( diff --git a/tests/test_webhook.py b/tests/test_webhook.py index d218885..47177b8 100644 --- a/tests/test_webhook.py +++ b/tests/test_webhook.py @@ -1,7 +1,6 @@ """Tests for POST /webhook (x-api-key protected).""" import os -import uuid from unittest.mock import patch import pytest @@ -30,6 +29,7 @@ def test_client(db_connection): """TestClient without lifespan (no Temporal/MinIO needed for /webhook).""" from fastapi.testclient import TestClient + from pipeline import api return TestClient(api.app) @@ -87,9 +87,9 @@ def test_valid_payload(self, test_client, db_connection, key_env): rows = _rows(db_connection) assert len(rows) == 1 assert rows[0]["external_id"] == "101" - assert uuid.UUID(rows[0]["id"]).version == 4 + assert rows[0]["id"].version == 4 assert rows[0]["category"] == "Agriculture" - assert '"popList"' in rows[0]["payload"] + assert "popList" in rows[0]["payload"] def test_repost_is_idempotent(self, test_client, db_connection, key_env): test_client.post("/webhook", json=SAMPLE, headers=self.H) diff --git a/ui/src/views/DocumentOpsView.jsx b/ui/src/views/DocumentOpsView.jsx index 4543c1c..eba6575 100644 --- a/ui/src/views/DocumentOpsView.jsx +++ b/ui/src/views/DocumentOpsView.jsx @@ -373,7 +373,7 @@ const STAGE_POLL_INTERVAL_MS = 1500 const STAGE_POLL_MAX_ATTEMPTS = 40 // ~60s safety net so a stuck workflow doesn't wedge the button forever function stageWantsPages(stage, pageCount = 0) { - // Skip empty registered docs; otherwise pages are local SQLite (fast) and power OCR/translation. + // Skip empty registered docs; otherwise pages come from the database (fast) and power OCR/translation. if ((stage === 'registered' || stage === 'ocr_processing') && !pageCount) return false return true } @@ -569,7 +569,7 @@ export default function DocumentOpsView() { /** * Progressive load: * 1) document shell (unblocks UI immediately) - * 2) SQLite pages/chunks needed for the active review surface + * 2) Database pages/chunks needed for the active review surface * 3) heavy panels (Qdrant / Temporal / audit) only when tab or stage needs them */ const load = useCallback(async ({ soft = false, forcePanels = null } = {}) => { @@ -606,7 +606,7 @@ export default function DocumentOpsView() { attemptedPanelsRef.current.chunks = true localTasks.push(loadChunks(wid)) } - // Local SQLite panels in parallel; do not wait on remote index/Temporal for first paint. + // Database panels in parallel; do not wait on remote index/Temporal for first paint. await Promise.allSettled(localTasks) if (requestId !== requestIdRef.current) return @@ -1707,7 +1707,7 @@ export default function DocumentOpsView() {

Document State

- SQLite Stage + Database Stage
From cd080014b35199cd30b565c962d29565ea7d3728 Mon Sep 17 00:00:00 2001 From: Naveen Date: Mon, 26 Oct 2026 13:18:48 +0530 Subject: [PATCH 31/53] feat(scripts): add SQLite to Postgres migration and retire legacy scripts migrate_sqlite_to_postgres.py imports an existing documents.db (dry-run, row-count report, idempotent). Remove start_manifest_translation_batch.py and drop_legacy_marqo_columns.py, whose tables and columns no longer exist. Adapt the backup, failed-workflow, OCR-batch and validity e2e scripts to Postgres. --- scripts/backup_volumes.sh | 17 +- scripts/drop_legacy_marqo_columns.py | 89 -- scripts/list_failed_workflows.py | 52 +- scripts/migrate_sqlite_to_postgres.py | 848 ++++++++++++++++++++ scripts/start_manifest_translation_batch.py | 129 --- scripts/start_ocr_only_batch.py | 39 +- scripts/verify_document_validity_e2e.py | 21 +- tests/test_migrate_sqlite_to_postgres.py | 393 +++++++++ 8 files changed, 1297 insertions(+), 291 deletions(-) delete mode 100644 scripts/drop_legacy_marqo_columns.py create mode 100644 scripts/migrate_sqlite_to_postgres.py delete mode 100644 scripts/start_manifest_translation_batch.py create mode 100644 tests/test_migrate_sqlite_to_postgres.py diff --git a/scripts/backup_volumes.sh b/scripts/backup_volumes.sh index 4dc91c0..6a7b08e 100644 --- a/scripts/backup_volumes.sh +++ b/scripts/backup_volumes.sh @@ -4,7 +4,8 @@ set -euo pipefail # Backup all persistent data volumes used by the docs pipeline: # - Temporal Postgres database # - MinIO object storage -# - SQLite documents/chunks DB +# - Application Postgres volume (documents, workflows, pages, chunks, audit) +# (a logical pg_dump is also written when the postgres container is running) # # Usage: # ./scripts/backup_volumes.sh # writes to ./backups @@ -52,8 +53,18 @@ backup_volume "temporal-db" "temporal-db-data" # 2) MinIO object storage (all PDFs / objects) backup_volume "minio" "minio-data" -# 3) SQLite documents/chunks DB -backup_volume "sqlite" "sqlite-data" +# 3) Application Postgres volume +backup_volume "app-db" "app-db-data" + +# 4) Logical dump of the application database, consistent while the stack runs +PG_CONTAINER="$(docker ps --format '{{.Names}}' | grep 'postgres' | grep -v 'temporal\|keycloak' | head -n 1 || true)" +if [[ -n "$PG_CONTAINER" ]]; then + echo "Dumping application database from ${PG_CONTAINER}" + docker exec "$PG_CONTAINER" sh -c 'pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc' \ + > "${BACKUP_DIR}/app-db-${TIMESTAMP}.dump" +else + echo "WARNING: application postgres container not running; skipped pg_dump" >&2 +fi echo echo "Backup complete. Files in ${BACKUP_DIR}:" diff --git a/scripts/drop_legacy_marqo_columns.py b/scripts/drop_legacy_marqo_columns.py deleted file mode 100644 index f178e3f..0000000 --- a/scripts/drop_legacy_marqo_columns.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python3 -""" -Drop the legacy marqo_doc_id column from document_index_status. - -Context: the Marqo backend was removed and document_index_status.marqo_doc_id -was superseded by a generically-named doc_id column, backfilled once at -startup (see pipeline/db.py init_db()). marqo_doc_id has been left in -place, unwritten, so the schema stayed usable by the pre-removal code during -rollout. This script is the followup: run it manually, once you've confirmed -the app has been stable on the new code for a while — it's a one-way door -(SQLite's DROP COLUMN can't be undone short of restoring a backup). - -No other legacy Marqo-only tables/columns were found in this codebase as of -this writing (only this one column) — if a future migration needs the same -treatment, extend this script rather than writing a new one. - -Requires SQLite 3.35+ for ALTER TABLE ... DROP COLUMN (bundled with Python -3.9+ on most platforms; the script checks and aborts with a clear message on -older versions rather than attempting a table-rebuild workaround). - -Usage: - uv run python scripts/drop_legacy_marqo_columns.py [--yes] -""" -import argparse -import sqlite3 -import sys -from pathlib import Path - -project_root = Path(__file__).parent.parent -sys.path.insert(0, str(project_root)) - -from pipeline import db # noqa: E402 - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--yes", action="store_true", help="Skip the interactive confirmation prompt.") - args = parser.parse_args() - - if sqlite3.sqlite_version_info < (3, 35, 0): - print( - f"ERROR: SQLite {sqlite3.sqlite_version} does not support " - "ALTER TABLE ... DROP COLUMN (needs 3.35+). Aborting — this " - "script does not implement a table-rebuild fallback.", - file=sys.stderr, - ) - sys.exit(1) - - with db.get_connection() as conn: - columns = {row[1] for row in conn.execute("PRAGMA table_info(document_index_status)")} - if "marqo_doc_id" not in columns: - print("document_index_status.marqo_doc_id doesn't exist — nothing to drop.") - sys.exit(0) - - stale = conn.execute( - """ - SELECT COUNT(*) FROM document_index_status - WHERE doc_id IS NULL AND marqo_doc_id IS NOT NULL - """ - ).fetchone()[0] - if stale: - print( - f"ERROR: {stale} row(s) in document_index_status have " - "marqo_doc_id set but doc_id NULL — the backfill in " - "init_db() hasn't run against this database yet. Start the " - "app once (so init_db() runs) before dropping the column.", - file=sys.stderr, - ) - sys.exit(1) - - total = conn.execute("SELECT COUNT(*) FROM document_index_status").fetchone()[0] - print(f"document_index_status: {total} row(s), all backfilled to doc_id.") - - if not args.yes: - answer = input( - "Drop the legacy marqo_doc_id column now? This cannot be undone " - "short of restoring a backup. [y/N] " - ).strip().lower() - if answer != "y": - print("Aborted.") - sys.exit(0) - - conn.execute("ALTER TABLE document_index_status DROP COLUMN marqo_doc_id") - conn.commit() - print("Dropped document_index_status.marqo_doc_id.") - - -if __name__ == "__main__": - main() diff --git a/scripts/list_failed_workflows.py b/scripts/list_failed_workflows.py index 4978822..1a19271 100755 --- a/scripts/list_failed_workflows.py +++ b/scripts/list_failed_workflows.py @@ -3,7 +3,7 @@ Query failed workflows and write their IDs and error messages to a file. This script: -1. Queries SQLite database for all failed workflows +1. Queries the Postgres database (DATABASE_URL) for all failed workflows 2. Extracts workflow_id, filename, and error_message 3. Optionally queries Temporal for additional error details 4. Writes results to a CSV file @@ -12,7 +12,6 @@ import asyncio import csv import os -import sqlite3 import sys from pathlib import Path @@ -22,8 +21,7 @@ from temporalio.client import Client, WorkflowFailureError -# Database path - try environment variable first, then default -DB_PATH = os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db") +from pipeline import db # Configuration TEMPORAL_HOST = os.environ.get("TEMPORAL_HOST", "localhost:7233") @@ -94,38 +92,24 @@ async def get_temporal_error_details(workflow_id: str, client: Client) -> dict: } -def query_failed_workflows_from_db(db_path: str) -> list: - """Query SQLite database directly for failed workflows.""" - conn = sqlite3.connect(db_path) - conn.row_factory = sqlite3.Row - - try: - cursor = conn.execute(""" - SELECT workflow_id, document_id, filename, error_message, created_at - FROM documents - WHERE stage = 'failed' - ORDER BY created_at DESC - """) - rows = cursor.fetchall() - return [dict(row) for row in rows] - finally: - conn.close() +def query_failed_workflows_from_db() -> list: + """Query the database for failed workflows (including demo and disabled documents).""" + return db.list_documents( + stage="failed", limit=1_000_000, include_demo=True, include_disabled=True + ) async def main(): """Main function to query failed workflows and write to file.""" - print(f"Querying failed workflows from SQLite database at {DB_PATH}...") - - # Check if database exists - if not os.path.exists(DB_PATH): - print(f"Error: Database not found at {DB_PATH}") - print("Make sure you're running this script inside the Docker container or") - print("that the database path is correct.") + print(f"Querying failed workflows from the database at {db.database_url().rsplit('@', 1)[-1]}...") + + try: + failed_docs = query_failed_workflows_from_db() + except Exception as e: + print(f"Error: could not query the database: {e}") + print("Check DATABASE_URL, and run inside the Docker network if the database is not published.") sys.exit(1) - # Query SQLite for failed workflows - failed_docs = query_failed_workflows_from_db(DB_PATH) - print(f"Found {len(failed_docs)} failed workflows") # Prepare data for CSV @@ -139,7 +123,7 @@ async def main(): print("Connected to Temporal") except Exception as e: print(f"Warning: Could not connect to Temporal: {e}") - print("Will only use SQLite error messages") + print("Will only use database error messages") temporal_client = None # Process each failed workflow @@ -154,7 +138,7 @@ async def main(): "workflow_id": workflow_id, "document_id": document_id, "filename": filename, - "sqlite_error_message": error_message, + "db_error_message": error_message, "created_at": created_at, } @@ -180,7 +164,7 @@ async def main(): "document_id", "filename", "created_at", - "sqlite_error_message", + "db_error_message", "temporal_status", "temporal_error_message", "temporal_error_type", @@ -196,7 +180,7 @@ async def main(): print(f"✓ Successfully wrote {len(rows)} failed workflows to {OUTPUT_FILE}") print("\nSummary:") print(f" - Total failed workflows: {len(rows)}") - print(f" - With SQLite error messages: {sum(1 for r in rows if r.get('sqlite_error_message'))}") + print(f" - With database error messages: {sum(1 for r in rows if r.get('db_error_message'))}") if temporal_client: print(f" - With Temporal error messages: {sum(1 for r in rows if r.get('temporal_error_message'))}") else: diff --git a/scripts/migrate_sqlite_to_postgres.py b/scripts/migrate_sqlite_to_postgres.py new file mode 100644 index 0000000..dc583c0 --- /dev/null +++ b/scripts/migrate_sqlite_to_postgres.py @@ -0,0 +1,848 @@ +#!/usr/bin/env python3 +"""One-off migration of the legacy SQLite store into Postgres. + +Run in the maintenance window, after the Postgres migrations are applied and +with the api and worker stopped. The SQLite file is opened read-only and is +never modified, so rolling back is redeploying the previous images on it. + + python scripts/migrate_sqlite_to_postgres.py --dry-run + python scripts/migrate_sqlite_to_postgres.py + +Every migrated row gets a deterministic UUID derived from its legacy key, so a +second run finds the rows already there and changes nothing. A document is +migrated in one transaction. ``--dry-run`` runs everything and rolls it back. + +Mapping: one legacy ``documents`` row becomes one document plus its main +workflow (``temporal_workflow_id`` = the legacy ``workflow_id``); each +``document_jobs`` row becomes a workflow; ``scheme_catalog_*`` becomes +``catalog_entries``; ``document_manifest_entries`` is dropped. +""" + +import argparse +import json +import os +import re +import sqlite3 +import sys +import uuid +from collections import Counter +from datetime import date, datetime, timezone +from pathlib import Path +from typing import Any, Optional + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import psycopg # noqa: E402 +from psycopg.rows import dict_row # noqa: E402 +from psycopg.types.json import Jsonb # noqa: E402 + +from pipeline import db, migrate # noqa: E402 +from pipeline.models import ( # noqa: E402 + ChunkingRun, + DocumentMetadata, + DocumentSource, + IndexState, + PageMetadata, + PageTranslation, +) + +_NAMESPACE = uuid.UUID("3f2b6c0e-5d1a-4a53-9a77-0c6a5c1d7e10") +_FINGERPRINT_RE = re.compile(r"^[0-9a-f]{32}$") +_CODE_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") +_KIND_RE = re.compile(r"^[a-z0-9]+([_-][a-z0-9]+)*$") +_STAGES = { + "registered", "ocr_processing", "ocr_review", "translation_processing", + "translation_review", "chunking", "chunk_review", "ready_for_ingestion", + "ingesting", "publishing_to_network", "approval_for_prod", "ingesting_prod", + "completed", "failed", +} +_WORKFLOW_TYPES = { + "pipeline", "ocr_only", "ocr_retry", "translation_retry", "chunking_retry", + "reingest", "promote_prod", +} +_WORKFLOW_STATUSES = {"running", "waiting_review", "completed", "failed"} +_REVIEW_STAGES = { + "ocr_review", "translation_review", "chunk_review", "ready_for_ingestion", "approval_for_prod", +} +_ENTITY_TYPES = {"document", "page", "chunk", "setting", "catalog"} + +_LEGACY_SETTING_KEYS = { + "search_method": "searchMethod", + "search_limit": "limit", + "search_alpha": "alpha", + "search_ranking_method": "rankingMethod", + "search_show_highlights": "showHighlights", + "search_ef_search": "efSearch", + "search_index_name": "indexName", + "search_candidate_cap": "candidateCap", + "search_candidate_multiplier": "candidateMultiplier", + "search_max_chunks_per_doc": "maxChunksPerDoc", + "search_use_e5_prefix": "useE5Prefix", + "search_exclude_reference": "excludeReference", + "search_query_expansion_profile": "queryExpansionProfile", + "search_rerank_mode": "rerankMode", + "search_hybrid_rrfk": "hybridRrfK", +} +_BOOL_SETTINGS = {"showHighlights", "useE5Prefix", "excludeReference"} +_INT_SETTINGS = { + "limit", "efSearch", "candidateCap", "candidateMultiplier", "maxChunksPerDoc", "hybridRrfK", +} +_FLOAT_SETTINGS = {"alpha"} + + +class Report: + """Per-table counts: rows read from SQLite and what became of them.""" + + def __init__(self) -> None: + self.tables: dict[str, Counter] = {} + self.skip_reasons: list[dict[str, Any]] = [] + self.notes: list[str] = [] + + def count(self, table: str, outcome: str, n: int = 1) -> None: + self.tables.setdefault(table, Counter())[outcome] += n + + def skip(self, table: str, key: str, reason: str) -> None: + self.count(table, "skipped") + self.skip_reasons.append({"table": table, "key": key, "reason": reason}) + + def mismatches(self) -> list[str]: + problems = [] + for table, counts in sorted(self.tables.items()): + accounted = counts["inserted"] + counts["existing"] + counts["skipped"] + counts["dropped"] + if counts["read"] != accounted: + problems.append(f"{table}: read {counts['read']} but accounted for {accounted}") + return problems + + def as_dict(self) -> dict[str, Any]: + return { + "tables": {t: dict(c) for t, c in sorted(self.tables.items())}, + "skipped": self.skip_reasons, + "notes": self.notes, + "mismatches": self.mismatches(), + } + + def render(self) -> str: + lines = [f"{'table':<28}{'read':>8}{'inserted':>10}{'existing':>10}{'skipped':>9}{'dropped':>9}"] + for table, c in sorted(self.tables.items()): + lines.append( + f"{table:<28}{c['read']:>8}{c['inserted']:>10}{c['existing']:>10}" + f"{c['skipped']:>9}{c['dropped']:>9}" + ) + for item in self.skip_reasons: + lines.append(f" skipped {item['table']} {item['key']}: {item['reason']}") + lines.extend(f" note: {n}" for n in self.notes) + problems = self.mismatches() + lines.append("verification: OK" if not problems else "verification: FAILED") + lines.extend(f" {p}" for p in problems) + return "\n".join(lines) + + +def det_id(kind: str, key: str) -> str: + return str(uuid.uuid5(_NAMESPACE, f"{kind}:{key}")) + + +def parse_ts(value: Any) -> Optional[datetime]: + if not value: + return None + try: + parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def parse_date(value: Any) -> Optional[date]: + if not value: + return None + try: + return date.fromisoformat(str(value)[:10]) + except ValueError: + return None + + +def load_json(text: Any, default: Any) -> Any: + if text in (None, ""): + return default + try: + return json.loads(text) + except (TypeError, ValueError): + return default + + +def truthy(value: Any, default: bool = False) -> bool: + if value is None: + return default + return bool(int(value)) + + +def _default_instance() -> str: + return (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" + + +def _workflow_status_for_stage(stage: str) -> str: + if stage == "completed": + return "completed" + if stage == "failed": + return "failed" + return "waiting_review" if stage in _REVIEW_STAGES else "running" + + +def _index_key(index_name: str) -> str: + prod_names = { + n.strip() + for n in ( + os.environ.get("PROD_VECTOR_DB_COLLECTION_NAME"), + os.environ.get("PROD_SCHEME_VECTOR_DB_COLLECTION_NAME"), + ) + if n and n.strip() + } + return "prod" if index_name in prod_names else "dev" + + +def _has_table(sq: sqlite3.Connection, name: str) -> bool: + return sq.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", (name,) + ).fetchone() is not None + + +def _rows(sq: sqlite3.Connection, table: str, where: str = "", params: tuple = ()) -> list[sqlite3.Row]: + if not _has_table(sq, table): + return [] + return sq.execute(f"SELECT * FROM {table} {where}", params).fetchall() + + +def _get(row: sqlite3.Row, key: str, default: Any = None) -> Any: + return row[key] if key in row.keys() else default + + +# ----------------------------------------------------------------------------- +# Catalog +# ----------------------------------------------------------------------------- + +def migrate_catalog_registry(sq, conn, report: Report) -> None: + for row in _rows(sq, "scheme_catalog_entries"): + report.count("scheme_catalog_entries", "read") + code = (row["scheme_code"] or "").strip().lower() + if not _CODE_RE.match(code): + report.skip("scheme_catalog_entries", str(row["scheme_code"]), "code is not a valid slug") + continue + aliases = [a for a in load_json(_get(row, "scheme_aliases_json"), []) if isinstance(a, str) and a] + source = _get(row, "source") if _get(row, "source") in ("bootstrap", "pipeline", "import") else "pipeline" + inserted = conn.execute( + """ + INSERT INTO catalog_entries (id, type, code, name, aliases, metadata) + VALUES (%s::uuid, 'scheme', %s, %s, %s, %s) + ON CONFLICT (type, code) DO NOTHING RETURNING id + """, + (det_id("entry", f"scheme:{code}"), code, row["scheme_name"] or code, aliases, + Jsonb({"source": source})), + ).fetchone() + report.count("scheme_catalog_entries", "inserted" if inserted else "existing") + + +def migrate_catalog_version(sq, conn, report: Report) -> None: + rows = _rows(sq, "scheme_catalog_meta") + legacy_version = int(rows[0]["version"] or 0) if rows else 0 + if rows: + report.count("scheme_catalog_meta", "read") + report.count("scheme_catalog_meta", "inserted") + for n in range(1, legacy_version + 1): + conn.execute( + """ + INSERT INTO audit_logs (id, action_type, entity_type, metadata) + VALUES (%s::uuid, %s, 'catalog', %s) ON CONFLICT (id) DO NOTHING + """, + (det_id("catalog-bump", str(n)), db.CATALOG_VERSION_ACTION, + Jsonb({"reason": "sqlite_migration:carried_over"})), + ) + conn.execute( + """ + INSERT INTO audit_logs (id, action_type, entity_type, metadata) + VALUES (%s::uuid, %s, 'catalog', %s) ON CONFLICT (id) DO NOTHING + """, + (det_id("catalog-bump", "migration"), db.CATALOG_VERSION_ACTION, + Jsonb({"reason": "sqlite_migration"})), + ) + + +def _ensure_doc_catalog_entry(conn, kind: str, code: str, name: str, aliases: list[str]) -> str: + conn.execute( + """ + INSERT INTO catalog_entries (id, type, code, name, aliases, metadata) + VALUES (%s::uuid, %s, %s, %s, %s, %s) ON CONFLICT (type, code) DO NOTHING + """, + (det_id("entry", f"{kind}:{code}"), kind, code, name or code, aliases, + Jsonb({"source": "pipeline"})), + ) + return str( + conn.execute( + "SELECT id FROM catalog_entries WHERE type = %s AND code = %s", (kind, code) + ).fetchone()["id"] + ) + + +# ----------------------------------------------------------------------------- +# Documents +# ----------------------------------------------------------------------------- + +def _user_for(conn, user_id, username, email, roles) -> Optional[str]: + if not (user_id or "").strip(): + return None + return db.upsert_user(user_id, username, email, roles, conn=conn) + + +def _build_index_status(sq, workflow_id: str) -> dict[str, dict]: + entries: dict[str, dict] = {} + for row in _rows(sq, "document_index_status", "WHERE workflow_id = ?", (workflow_id,)): + key = _index_key(row["index_name"]) + details = load_json(row["details_json"], {}) + state = IndexState( + status=row["status"] if row["status"] in ("unknown", "indexed", "missing") else "unknown", + index_name=row["index_name"], + doc_id=_get(row, "doc_id") or _get(row, "marqo_doc_id"), + chunk_count_indexed=max(int(row["chunk_count_indexed"] or 0), 0), + last_indexed_at=row["last_indexed_at"], + last_verified_at=row["last_verified_at"], + schema_version=row["schema_version"], + details=details if isinstance(details, dict) else {}, + ).model_dump() + current = entries.get(key) + if current is None or (state["last_indexed_at"] or "") >= (current["last_indexed_at"] or ""): + entries[key] = state + return entries + + +def _build_workflows( + sq, doc: sqlite3.Row, started_by: Optional[str], report: Report +) -> tuple[list[dict], dict[Any, str]]: + workflow_id = doc["workflow_id"] + stage = doc["stage"] if doc["stage"] in _STAGES else "registered" + created_at = parse_ts(doc["created_at"]) or datetime.now(timezone.utc) + stamps = { + key: value + for key, value in ( + ("ocr", _get(doc, "ocr_completed_at")), + ("translation", _get(doc, "translation_completed_at")), + ("chunks", _get(doc, "chunks_completed_at")), + ) + if value + } + + workflows: list[dict] = [] + job_ids: dict[Any, str] = {} + has_root = False + for job in _rows(sq, "document_jobs", "WHERE workflow_id = ? ORDER BY started_at, id", (workflow_id,)): + job_type = job["job_type"] + if job_type not in _WORKFLOW_TYPES: + report.skip("document_jobs", f"{workflow_id}#{job['id']}", f"unknown job_type {job_type!r}") + continue + temporal_id = job["temporal_workflow_id"] or workflow_id + is_root = job_type in ("pipeline", "ocr_only") and temporal_id == workflow_id and not has_root + has_root = has_root or is_root + wf_id = det_id("workflow", f"{workflow_id}:job:{job['id']}") + job_ids[job["id"]] = wf_id + status = job["status"] if job["status"] in _WORKFLOW_STATUSES else "failed" + job_stage = job["current_stage"] if job["current_stage"] in _STAGES else stage + config = load_json(job["config_json"], {}) + config = config if isinstance(config, dict) else {} + if job["temporal_run_id"]: + config["temporal_run_id"] = job["temporal_run_id"] + workflows.append({ + "id": wf_id, "type": job_type, "temporal_id": temporal_id, "status": status, + "stage": job_stage, "stamps": stamps if is_root else {}, + "stop_after_ocr": truthy(_get(doc, "stop_after_ocr")) and is_root, + "error_message": job["error_message"], "config": config, + "started_by": started_by if is_root else None, + "started_at": parse_ts(job["started_at"]) or created_at, + "completed_at": parse_ts(job["completed_at"]), "job_id": job["id"], + "inserted_as": "document_jobs", + }) + + if not has_root: + workflows.append({ + "id": det_id("workflow", f"{workflow_id}:root"), + "type": "ocr_only" if truthy(_get(doc, "stop_after_ocr")) else "pipeline", + "temporal_id": workflow_id, "status": _workflow_status_for_stage(stage), "stage": stage, + "stamps": stamps, "stop_after_ocr": truthy(_get(doc, "stop_after_ocr")), + "error_message": doc["error_message"], "config": {}, "started_by": started_by, + "started_at": created_at, + "completed_at": parse_ts(_get(doc, "ingested_at")) if stage == "completed" else None, + "job_id": None, "inserted_as": None, + }) + + requested_at = parse_ts(_get(doc, "prod_ready_requested_at")) + if requested_at and stage not in ("completed", "ingesting_prod"): + workflows.append({ + "id": det_id("workflow", f"{workflow_id}:prod-request"), "type": "promote_prod", + "temporal_id": f"{workflow_id}-prod-request", "status": "waiting_review", + "stage": "approval_for_prod", "stamps": {}, "stop_after_ocr": False, + "error_message": None, "config": {}, "started_by": None, "started_at": requested_at, + "completed_at": None, "job_id": None, "inserted_as": None, + "requested_by": ( + _get(doc, "prod_ready_requested_by_user_id"), + _get(doc, "prod_ready_requested_by_username"), + ), + }) + + workflows.sort(key=lambda w: w["started_at"]) + latest = workflows[-1] + latest["stage"] = stage + latest["error_message"] = doc["error_message"] + return workflows, job_ids + + +def migrate_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workflow: dict[str, list]) -> bool: + workflow_id = doc["workflow_id"] + doc_uuid = det_id("document", workflow_id) + exists = conn.execute("SELECT 1 FROM documents WHERE id = %s::uuid", (doc_uuid,)).fetchone() + if exists: + report.count("documents", "existing") + for table, label in ( + ("document_jobs", "document_jobs"), ("document_artifacts", "document_artifacts"), + ("pages", "pages"), ("chunks", "chunks"), + ): + report.count(label, "existing", len(_rows(sq, table, "WHERE workflow_id = ?", (workflow_id,)))) + return False + + fingerprint = next( + ( + c for c in ( + _get(doc, "source_file_fingerprint"), _get(doc, "canonical_document_id"), doc["document_id"] + ) if c and _FINGERPRINT_RE.match(c) + ), + None, + ) + if fingerprint is None: + import hashlib + + fingerprint = hashlib.md5((doc["document_id"] or workflow_id).encode()).hexdigest() + report.notes.append(f"{workflow_id}: no hex fingerprint, derived md5 of document_id") + + page_count = max(int(doc["page_count"] or 0), 0) + chunk_rows = _rows(sq, "chunks", "WHERE workflow_id = ? ORDER BY chunk_number", (workflow_id,)) + kind = (_get(doc, "document_kind") or "document").strip().lower() + if not _KIND_RE.match(kind): + report.notes.append(f"{workflow_id}: invalid document_kind {kind!r}, stored as 'document'") + kind = "document" + + chunking = None + if chunk_rows: + first = chunk_rows[0] + chunking = ChunkingRun( + run_number=max(max(int(r["chunk_version"] or 1) for r in chunk_rows), 1), + provider=_get(first, "chunking_provider"), + model=_get(first, "chunking_model"), + config=load_json(_get(first, "chunking_config_json"), {}), + ) + metadata = DocumentMetadata( + display_name=_get(doc, "display_name") or None, + catalog_visible=truthy(_get(doc, "catalog_visible"), True), + network_visible=truthy(_get(doc, "network_visible"), True), + is_demo=truthy(_get(doc, "is_demo")), + chunk_count=len(chunk_rows) if chunk_rows else max(int(doc["chunk_count"] or 0), 0), + reindex_required=truthy(_get(doc, "reindex_required")), + reindex_reason=_get(doc, "reindex_reason") or None, + chunking=chunking, + ).model_dump(exclude_none=True) + source = DocumentSource( + fingerprint=fingerprint, + original_filename=_get(doc, "source_filename") or doc["filename"], + storage_uri=doc["filepath"], + source_type=_get(doc, "source_type"), + canonical_input_type=_get(doc, "canonical_input_type"), + page_count=page_count, + ).model_dump(exclude_none=True) + + valid_from, valid_to = parse_date(_get(doc, "valid_from")), parse_date(_get(doc, "valid_to")) + if valid_from and valid_to and valid_from > valid_to: + report.notes.append(f"{workflow_id}: valid_from after valid_to, valid_to cleared") + valid_to = None + + uploader = _user_for( + conn, _get(doc, "uploaded_by_user_id"), _get(doc, "uploaded_by_username"), + _get(doc, "uploaded_by_email"), _get(doc, "uploaded_by_roles"), + ) + + entry_id = None + code = (_get(doc, "scheme_code") or "").strip().lower() + if code and kind != "document": + if _CODE_RE.match(code): + aliases = [a for a in load_json(_get(doc, "scheme_aliases_json"), []) if isinstance(a, str) and a] + entry_id = _ensure_doc_catalog_entry( + conn, kind, code, _get(doc, "scheme_name") or code, aliases + ) + else: + report.notes.append(f"{workflow_id}: scheme_code {code!r} is not a valid slug, not linked") + elif code: + report.notes.append(f"{workflow_id}: scheme_code {code!r} on a plain document, not linked") + + created_at = parse_ts(doc["created_at"]) or datetime.now(timezone.utc) + updated_at = parse_ts(doc["updated_at"]) or created_at + conn.execute( + """ + INSERT INTO documents (id, instance, source, document_kind, catalog_entry_id, valid_from, + valid_to, is_disabled, index_status, metadata, uploaded_by_user_id, + created_at, updated_at) + VALUES (%s::uuid, %s, %s, %s, %s::uuid, %s, %s, %s, %s, %s, %s::uuid, %s, %s) + """, + ( + doc_uuid, (_get(doc, "instance") or _default_instance()).strip().lower() or _default_instance(), + Jsonb(source), kind, entry_id, valid_from, valid_to, truthy(_get(doc, "is_disabled")), + Jsonb(_build_index_status(sq, workflow_id)), Jsonb(metadata), uploader, created_at, + updated_at, + ), + ) + report.count("documents", "inserted") + + workflows, job_ids = _build_workflows(sq, doc, uploader, report) + root = next((w for w in workflows if w["type"] in ("pipeline", "ocr_only")), workflows[0]) + for w in workflows: + started_by = w["started_by"] + if w.get("requested_by"): + started_by = _user_for(conn, w["requested_by"][0], w["requested_by"][1], None, None) + conn.execute( + """ + INSERT INTO workflows (id, document_id, workflow_type, temporal_workflow_id, status, + current_stage, stage_timestamps, stop_after_ocr, error_message, + config, started_by_user_id, started_at, completed_at, updated_at) + VALUES (%s::uuid, %s::uuid, %s::workflow_type_enum, %s, %s::workflow_status_enum, + %s::workflow_stage, %s, %s, %s, %s, %s::uuid, %s, %s, %s) + """, + ( + w["id"], doc_uuid, w["type"], w["temporal_id"], w["status"], w["stage"], + Jsonb(w["stamps"]), w["stop_after_ocr"], w["error_message"], Jsonb(w["config"]), + started_by, w["started_at"], w["completed_at"], w["completed_at"] or w["started_at"], + ), + ) + if w["inserted_as"]: + report.count("document_jobs", "inserted") + + for art in _rows(sq, "document_artifacts", "WHERE workflow_id = ? ORDER BY id", (workflow_id,)): + size = art["size_bytes"] + art_metadata = load_json(art["metadata_json"], {}) + conn.execute( + """ + INSERT INTO workflow_artifacts (id, workflow_id, artifact_type, stage, storage_uri, + mime_type, filename, size_bytes, metadata, created_at) + VALUES (%s::uuid, %s::uuid, %s, %s::workflow_stage, %s, %s, %s, %s, %s, %s) + """, + ( + det_id("artifact", f"{workflow_id}:{art['id']}"), + job_ids.get(art["job_id"], root["id"]), + art["artifact_type"], art["stage"] if art["stage"] in _STAGES else None, + art["storage_uri"], art["mime_type"], art["filename"], + size if size is not None and size >= 0 else None, + Jsonb(art_metadata if isinstance(art_metadata, dict) else {}), + parse_ts(art["created_at"]) or created_at, + ), + ) + report.count("document_artifacts", "inserted") + + for page in _rows(sq, "pages", "WHERE workflow_id = ? ORDER BY page_number", (workflow_id,)): + if int(page["page_number"] or 0) < 1: + report.skip("pages", f"{workflow_id}#{page['page_number']}", "page_number < 1") + continue + translations = {} + if any( + _get(page, k) not in (None, "") + for k in ("translated_markdown", "edited_translation", "translation_notes") + ) or truthy(_get(page, "translation_reviewed")): + translations[_get(page, "translation_target_language") or "en"] = PageTranslation( + translated_markdown=_get(page, "translated_markdown"), + edited_translation=_get(page, "edited_translation"), + is_reviewed=truthy(_get(page, "translation_reviewed")), + reviewer_notes=_get(page, "translation_notes"), + provider=_get(page, "translation_provider"), + model=_get(page, "translation_model"), + translated_at=_get(page, "translated_at"), + ).model_dump() + page_meta = PageMetadata( + is_reviewed=truthy(_get(page, "is_reviewed")), + reviewer_notes=_get(page, "reviewer_notes"), + detected_language=_get(page, "detected_language"), + ).model_dump(exclude_none=True) + conn.execute( + """ + INSERT INTO pages (id, document_id, page_number, original_markdown, edited_markdown, + translations, metadata) + VALUES (%s::uuid, %s::uuid, %s, %s, %s, %s, %s) + """, + ( + det_id("page", f"{workflow_id}:{page['page_number']}"), doc_uuid, page["page_number"], + page["original_markdown"], page["edited_markdown"], Jsonb(translations), + Jsonb(page_meta), + ), + ) + report.count("pages", "inserted") + + for chunk in chunk_rows: + if int(chunk["chunk_number"] or 0) < 1: + report.skip("chunks", f"{workflow_id}#{chunk['chunk_number']}", "chunk_number < 1") + continue + page_start = max(int(chunk["page_start"] or 1), 1) + spans = load_json(_get(chunk, "source_spans_json"), []) + conn.execute( + """ + INSERT INTO chunks (id, document_id, chunk_number, original_text, edited_text, + token_count, page_start, page_end, source_spans, section_title, + content_type, is_reference, is_reviewed, is_excluded, reviewer_notes) + VALUES (%s::uuid, %s::uuid, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + """, + ( + det_id("chunk", f"{workflow_id}:{chunk['chunk_number']}"), doc_uuid, + chunk["chunk_number"], chunk["original_text"] or "", chunk["edited_text"], + max(int(chunk["token_count"] or 0), 0), page_start, + max(int(chunk["page_end"] or page_start), page_start), + Jsonb(spans if isinstance(spans, list) else []), chunk["section_title"], + chunk["content_type"], truthy(_get(chunk, "is_reference")), + truthy(_get(chunk, "is_reviewed")), truthy(_get(chunk, "is_excluded")), + chunk["reviewer_notes"], + ), + ) + report.count("chunks", "inserted") + + for audit in audit_by_workflow.pop(workflow_id, []): + insert_audit(conn, audit, doc_uuid, source_instance=(_get(doc, "instance") or _default_instance()), report=report) + return True + + +# ----------------------------------------------------------------------------- +# Audit, settings, OTPs, webhooks +# ----------------------------------------------------------------------------- + +def _legacy_audit_field(field_name: Optional[str], entity: Optional[str]) -> Optional[str]: + if entity == "setting" and field_name in _LEGACY_SETTING_KEYS: + flat = _LEGACY_SETTING_KEYS[field_name] + return f"{db._SETTING_GROUPS[flat][0]}.{flat}" + return field_name + + +def insert_audit(conn, audit: sqlite3.Row, doc_uuid: Optional[str], source_instance: Optional[str], report: Report) -> None: + entity = audit["entity_type"] if audit["entity_type"] in _ENTITY_TYPES else None + workflow_id = audit["workflow_id"] + if audit["document_id"] == "__settings__": + entity, workflow_id = "setting", None + elif workflow_id == "__catalog__": + entity, workflow_id = "catalog", None + metadata = load_json(audit["metadata"], None) + if not isinstance(metadata, dict): + metadata = {"raw": audit["metadata"]} if audit["metadata"] else {} + roles = [r.strip() for r in (_get(audit, "actor_roles") or "").split(",") if r.strip()] + actor_present = any((_get(audit, "actor_user_id"), _get(audit, "actor_username"), _get(audit, "actor_email"), roles)) + actor_uuid = _user_for( + conn, _get(audit, "actor_user_id"), _get(audit, "actor_username"), _get(audit, "actor_email"), roles + ) + inserted = conn.execute( + """ + INSERT INTO audit_logs (id, document_id, workflow_id, instance, action_type, entity_type, + entity_number, field_name, old_value, new_value, metadata, + actor_user_id, actor, created_at) + VALUES (%s::uuid, %s::uuid, %s, %s, %s, %s::audit_entity_type, %s, %s, %s, %s, %s, + %s::uuid, %s, %s) + ON CONFLICT (id) DO NOTHING RETURNING id + """, + ( + det_id("audit", str(audit["id"])), doc_uuid, workflow_id, + (source_instance or "").strip().lower() or None if doc_uuid else None, + audit["action_type"], entity, audit["entity_id"], + _legacy_audit_field(audit["field_name"], entity), audit["old_value"], audit["new_value"], + Jsonb(metadata), actor_uuid, + Jsonb({ + "username": _get(audit, "actor_username") or None, + "email": _get(audit, "actor_email") or None, + "roles": roles, + }) if actor_present else Jsonb({}), + parse_ts(audit["timestamp"]) or datetime.now(timezone.utc), + ), + ).fetchone() + report.count("audit_logs", "inserted" if inserted else "existing") + + +def migrate_settings(sq, conn, report: Report) -> None: + rows = _rows(sq, "settings") + for _ in rows: + report.count("settings", "read") + current = conn.execute( + "SELECT retrieval, candidates, results, query FROM search_settings WHERE singleton" + ).fetchone() + if current is None: + conn.execute("INSERT INTO search_settings DEFAULT VALUES") + current = {"retrieval": {}, "candidates": {}, "results": {}, "query": {}} + if any(current[g] for g in ("retrieval", "candidates", "results", "query")): + for _ in rows: + report.count("settings", "existing") + report.notes.append("search_settings already configured in Postgres, legacy settings not applied") + return + + groups: dict[str, dict[str, Any]] = {} + for row in rows: + flat = _LEGACY_SETTING_KEYS.get(row["key"]) + if flat is None: + report.skip("settings", row["key"], "unknown setting key") + continue + raw = row["value"] + try: + value: Any = ( + raw == "true" if flat in _BOOL_SETTINGS + else int(raw) if flat in _INT_SETTINGS + else float(raw) if flat in _FLOAT_SETTINGS + else raw + ) + except ValueError: + report.skip("settings", row["key"], f"unparseable value {raw!r}") + continue + group, model = db._SETTING_GROUPS[flat] + try: + model.model_validate({flat: value}) + except Exception as exc: + report.skip("settings", row["key"], f"invalid value: {exc}") + continue + groups.setdefault(group, {})[flat] = value + report.count("settings", "inserted") + for group, values in groups.items(): + conn.execute( + f"UPDATE search_settings SET {group} = {group} || %s::jsonb WHERE singleton", (Jsonb(values),) + ) + + +def migrate_email_otps(sq, conn, report: Report) -> None: + now = datetime.now(timezone.utc) + for row in _rows(sq, "email_otps"): + report.count("email_otps", "read") + expires = parse_ts(row["expires_at"]) + if expires is None or expires <= now: + report.count("email_otps", "dropped") + continue + inserted = conn.execute( + """ + INSERT INTO email_otps (email, code_hash, expires_at, attempts, last_sent_at) + VALUES (%s, %s, %s, %s, %s) ON CONFLICT (email) DO NOTHING RETURNING id + """, + (row["email"].strip().lower(), row["code_hash"], expires, max(int(row["attempts"] or 0), 0), + parse_ts(row["last_sent_at"]) or now), + ).fetchone() + report.count("email_otps", "inserted" if inserted else "existing") + + +def migrate_webhook_records(sq, conn, report: Report) -> None: + for row in _rows(sq, "webhook_records"): + report.count("webhook_records", "read") + payload = load_json(row["payload"], None) + if not isinstance(payload, dict): + report.skip("webhook_records", f"{row['source']}/{row['external_id']}", "payload is not a JSON object") + continue + try: + record_id = str(uuid.UUID(row["id"])) + except (ValueError, TypeError): + record_id = det_id("webhook", row["id"] or f"{row['source']}:{row['external_id']}") + published = row["is_published"] + inserted = conn.execute( + """ + INSERT INTO webhook_records (id, source, external_id, title, slug, status, is_published, + category, payload, payload_hash, created_at, updated_at) + VALUES (%s::uuid, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (source, external_id) DO NOTHING RETURNING id + """, + ( + record_id, row["source"], row["external_id"], row["title"], row["slug"], row["status"], + None if published is None else bool(published), row["category"], Jsonb(payload), + row["payload_hash"], parse_ts(row["created_at"]) or datetime.now(timezone.utc), + parse_ts(row["updated_at"]) or datetime.now(timezone.utc), + ), + ).fetchone() + report.count("webhook_records", "inserted" if inserted else "existing") + + +# ----------------------------------------------------------------------------- +# Driver +# ----------------------------------------------------------------------------- + +def run_migration(sqlite_path: str, database_url: str, dry_run: bool = False) -> Report: + report = Report() + sq = sqlite3.connect(f"file:{sqlite_path}?mode=ro", uri=True) + sq.row_factory = sqlite3.Row + try: + if not dry_run: + migrate.apply_migrations(database_url) + elif not migrate.is_current(database_url): + raise SystemExit("Postgres migrations are not applied; apply them before a dry run.") + + with psycopg.connect(database_url, autocommit=not dry_run, row_factory=dict_row) as conn: + conn.execute("SET timezone = 'UTC'") + documents = _rows(sq, "documents", "ORDER BY created_at, workflow_id") + known_workflows = {d["workflow_id"] for d in documents} + for table in ("document_jobs", "document_artifacts", "pages", "chunks"): + if _has_table(sq, table): + total = sq.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] + stray = sq.execute( + f"SELECT COUNT(*) FROM {table} WHERE workflow_id NOT IN " + "(SELECT workflow_id FROM documents)" + ).fetchone()[0] + report.count(table, "read", total) + if stray: + report.count(table, "skipped", stray) + report.notes.append(f"{table}: {stray} rows belong to no document and were not migrated") + + audit_by_workflow: dict[str, list] = {} + orphan_audit: list = [] + for audit in _rows(sq, "audit_logs", "ORDER BY id"): + report.count("audit_logs", "read") + if audit["workflow_id"] in known_workflows and audit["document_id"] != "__settings__": + audit_by_workflow.setdefault(audit["workflow_id"], []).append(audit) + else: + orphan_audit.append(audit) + + with conn.transaction(): + migrate_catalog_registry(sq, conn, report) + + for doc in documents: + report.count("documents", "read") + with conn.transaction(): + migrate_document(sq, conn, doc, report, audit_by_workflow) + + with conn.transaction(): + for audits in audit_by_workflow.values(): + orphan_audit.extend(audits) + for audit in orphan_audit: + insert_audit(conn, audit, None, None, report) + migrate_settings(sq, conn, report) + migrate_email_otps(sq, conn, report) + migrate_webhook_records(sq, conn, report) + migrate_catalog_version(sq, conn, report) + + manifest = _rows(sq, "document_manifest_entries") + if manifest: + report.count("document_manifest_entries", "read", len(manifest)) + report.count("document_manifest_entries", "dropped", len(manifest)) + report.notes.append("document_manifest_entries is not part of the schema and was not migrated") + + if dry_run: + conn.rollback() + finally: + sq.close() + return report + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--sqlite-path", default=os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db")) + parser.add_argument("--database-url", default=os.environ.get("DATABASE_URL", db.DEFAULT_DATABASE_URL)) + parser.add_argument("--dry-run", action="store_true", help="run everything, then roll back") + parser.add_argument("--report", help="write the report as JSON to this path") + args = parser.parse_args() + + if not os.path.exists(args.sqlite_path): + print(f"SQLite database not found at {args.sqlite_path}", file=sys.stderr) + return 2 + + report = run_migration(args.sqlite_path, args.database_url, dry_run=args.dry_run) + print(("DRY RUN - nothing was written\n" if args.dry_run else "") + report.render()) + if args.report: + Path(args.report).write_text(json.dumps(report.as_dict(), indent=2, default=str)) + return 1 if report.mismatches() else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/start_manifest_translation_batch.py b/scripts/start_manifest_translation_batch.py deleted file mode 100644 index 2c9c95c..0000000 --- a/scripts/start_manifest_translation_batch.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python3 -"""Start translation-only workflows for manifest-backed Gujarati/mixed documents in OCR review.""" - -from __future__ import annotations - -import argparse -import json -import os -import sqlite3 -import sys -import time -from datetime import datetime -from pathlib import Path - -from temporalio.client import Client - -PROJECT_ROOT = Path(__file__).resolve().parent.parent -sys.path.insert(0, str(PROJECT_ROOT)) - -from pipeline import db # noqa: E402 -from pipeline.workflows import TranslationOnlyWorkflow # noqa: E402 - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--temporal-host", default=os.environ.get("TEMPORAL_HOST", "localhost:7233")) - parser.add_argument("--task-queue", default=os.environ.get("TASK_QUEUE", "ocr-pipeline")) - parser.add_argument("--db-path", default=os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db")) - parser.add_argument("--limit", type=int, default=0) - parser.add_argument( - "--selection", - choices=["manifest_gujarati", "all_ocr_review"], - default="manifest_gujarati", - help="Choose whether to start only manifest rows marked Gujarati or any authoritative OCR-review docs.", - ) - parser.add_argument("--report-path", default="/tmp/start_manifest_translation_batch_report.json") - return parser.parse_args() - - -def eligible_documents(db_path: str, selection: str, limit: int = 0) -> list[dict]: - conn = sqlite3.connect(db_path) - conn.row_factory = sqlite3.Row - if selection == "all_ocr_review": - query = """ - SELECT d.workflow_id, d.document_id, d.filename, d.source_manifest_name, m.doc_language - FROM documents d - LEFT JOIN document_manifest_entries m - ON m.manifest_filename = d.source_manifest_name - WHERE d.source_manifest_name IS NOT NULL - AND d.stage = 'ocr_review' - AND d.translation_completed_at IS NULL - ORDER BY d.updated_at DESC, d.workflow_id - """ - else: - query = """ - SELECT d.workflow_id, d.document_id, d.filename, d.source_manifest_name, m.doc_language - FROM documents d - LEFT JOIN document_manifest_entries m - ON m.manifest_filename = d.source_manifest_name - WHERE d.source_manifest_name IS NOT NULL - AND d.stage = 'ocr_review' - AND d.translation_completed_at IS NULL - AND LOWER(COALESCE(m.doc_language, '')) LIKE '%gujarati%' - ORDER BY d.updated_at DESC, d.workflow_id - """ - rows = conn.execute(query).fetchall() - docs = [dict(r) for r in rows] - return docs[:limit] if limit > 0 else docs - - -async def main() -> int: - args = parse_args() - db.init_db() - client = await Client.connect(args.temporal_host) - - docs = eligible_documents(args.db_path, args.selection, args.limit) - report: dict[str, object] = { - "started_at": datetime.utcnow().isoformat(), - "temporal_host": args.temporal_host, - "task_queue": args.task_queue, - "selection": args.selection, - "translation_provider": os.environ.get("TRANSLATION_PROVIDER", "gemma_vllm"), - "translation_model": os.environ.get("TRANSLATION_MODEL", "gemma-4"), - "eligible_documents": len(docs), - "submitted": 0, - "skipped": 0, - "failed": [], - } - - for doc in docs: - temporal_workflow_id = f"{doc['workflow_id']}-translation-{int(time.time())}" - try: - await client.start_workflow( - TranslationOnlyWorkflow.run, - args=[doc["workflow_id"], doc["document_id"], doc["filename"]], - id=temporal_workflow_id, - task_queue=args.task_queue, - ) - job_id = db.create_document_job( - workflow_id=doc["workflow_id"], - job_type="translation_only", - temporal_workflow_id=temporal_workflow_id, - status="running", - current_stage="translation_processing", - config={ - "translation_provider": os.environ.get("TRANSLATION_PROVIDER", "gemma_vllm"), - "translation_model": os.environ.get("TRANSLATION_MODEL", "gemma-4"), - "source": "manifest_ocr_review", - }, - ) - db.update_document_fields(doc["workflow_id"], latest_job_id=job_id) - report["submitted"] = int(report["submitted"]) + 1 - except Exception as exc: - report["failed"].append( - { - "workflow_id": doc["workflow_id"], - "filename": doc["filename"], - "error": str(exc), - } - ) - - report["finished_at"] = datetime.utcnow().isoformat() - Path(args.report_path).write_text(json.dumps(report, indent=2), encoding="utf-8") - print(json.dumps(report, indent=2)) - return 1 if report["failed"] else 0 - - -if __name__ == "__main__": - raise SystemExit(__import__("asyncio").run(main())) diff --git a/scripts/start_ocr_only_batch.py b/scripts/start_ocr_only_batch.py index 70682c0..701b0c0 100644 --- a/scripts/start_ocr_only_batch.py +++ b/scripts/start_ocr_only_batch.py @@ -4,14 +4,16 @@ import argparse import hashlib import json -import os -import sqlite3 import sys import urllib.error import urllib.parse import urllib.request from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from pipeline import db # noqa: E402 + ALLOWED_EXTENSIONS = { ".pdf", ".doc", ".docx", ".ppt", ".pptx", ".xls", ".xlsx", ".csv", @@ -31,31 +33,16 @@ def get_workflow_id(filepath: str) -> str: return f"doc-{hashlib.md5(filepath.encode()).hexdigest()[:12]}" -def already_has_ocr(conn: sqlite3.Connection, filepath: str, fingerprint: str) -> bool: - workflow_id = get_workflow_id(filepath) - row = conn.execute( - """ - SELECT workflow_id, canonical_document_id, ocr_completed_at, stage - FROM documents - WHERE workflow_id = ? - OR canonical_document_id = ? - ORDER BY updated_at DESC, created_at DESC - LIMIT 1 - """, - (workflow_id, fingerprint), - ).fetchone() - if not row: +def already_has_ocr(filepath: str, fingerprint: str) -> bool: + doc = db.get_document(get_workflow_id(filepath)) or db.find_document_by_doc_identifier(fingerprint) + if not doc: return False - if row["ocr_completed_at"]: + if doc["ocr_completed_at"]: return True - if row["stage"] in {"ocr_review", "translation_processing", "translation_review", "chunking", + if doc["stage"] in {"ocr_review", "translation_processing", "translation_review", "chunking", "chunk_review", "ready_for_ingestion", "ingesting", "completed"}: return True - page_row = conn.execute( - "SELECT COUNT(*) AS cnt FROM pages WHERE workflow_id = ?", - (row["workflow_id"],), - ).fetchone() - return bool(page_row and page_row["cnt"] > 0) + return bool(db.get_saved_page_numbers(doc["workflow_id"])) def post_document(api_base: str, filepath: str) -> dict: @@ -76,7 +63,6 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("directory", help="Directory containing source docs") parser.add_argument("--api-base", default="http://localhost:8001", help="Docs-pipeline API base URL") - parser.add_argument("--db-path", default=os.environ.get("DOCUMENT_DB_PATH", "/data/documents.db")) parser.add_argument("--limit", type=int, default=0, help="Optional max files to submit") args = parser.parse_args() @@ -85,9 +71,6 @@ def main() -> int: print(f"Directory not found: {directory}", file=sys.stderr) return 1 - conn = sqlite3.connect(args.db_path) - conn.row_factory = sqlite3.Row - files = sorted( p for p in directory.iterdir() if p.is_file() and p.suffix.lower() in ALLOWED_EXTENSIONS @@ -101,7 +84,7 @@ def main() -> int: for path in files: fingerprint = compute_fingerprint(path) - if already_has_ocr(conn, str(path), fingerprint): + if already_has_ocr(str(path), fingerprint): skipped += 1 print(f"SKIP {path.name}: OCR already present") continue diff --git a/scripts/verify_document_validity_e2e.py b/scripts/verify_document_validity_e2e.py index e2c1b66..f249d58 100644 --- a/scripts/verify_document_validity_e2e.py +++ b/scripts/verify_document_validity_e2e.py @@ -2,11 +2,12 @@ End-to-end check of document validity against a live Qdrant. Run against a local stack: uv run python scripts/verify_document_validity_e2e.py -Needs Qdrant on :6333 and the small embedding model cached; it creates and -deletes its own scratch collection and a throwaway SQLite file, and touches -neither the real index nor the real db. +Needs Qdrant on :6333, the small embedding model cached, and a scratch Postgres +(E2E_DATABASE_URL, migrations are applied to it). It creates and deletes its own +scratch collection and four documents named wf-*, and touches neither the real +index nor the real db. -Real path, no stubs on the parts that matter: real SQLite rows, the real +Real path, no stubs on the parts that matter: real Postgres rows, the real _prepare_records + QdrantVectorStore.upsert the ingest activity calls, real embeddings, a real Qdrant collection, and the real search filter. Only the embedding *model* is swapped for a small cached one so this does not download @@ -15,15 +16,15 @@ import os import sys -import tempfile REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, REPO) -DB = os.path.join(tempfile.mkdtemp(), "e2e.db") COLLECTION = "e2e_validity_check" -os.environ["DOCUMENT_DB_PATH"] = DB +if not os.environ.get("E2E_DATABASE_URL"): + sys.exit("Set E2E_DATABASE_URL to a scratch Postgres database (it is never the real one).") +os.environ["DATABASE_URL"] = os.environ["E2E_DATABASE_URL"] os.environ["AUTH_DISABLED"] = "true" os.environ["VECTOR_DB_URL"] = "http://localhost:6333" os.environ["VECTOR_DB_COLLECTION_NAME"] = COLLECTION @@ -39,8 +40,10 @@ from pipeline.activities import _prepare_records, _validity_fields_from_doc # noqa: E402 from pipeline.vector_store.qdrant_store import QdrantVectorStore # noqa: E402 -db.DB_PATH = DB db.init_db() +SCRATCH_DOCUMENTS = ("wf-active", "wf-expired", "wf-future", "wf-legacy") +for scratch in SCRATCH_DOCUMENTS: + db.purge_document(scratch, keep_audit=False) PASS, FAIL = [], [] @@ -232,5 +235,7 @@ def search_docs(**kwargs): check("the shortened period takes it out of today's results", search_docs(), ["wf-legacy"]) store.client.delete_collection(COLLECTION) +for scratch in SCRATCH_DOCUMENTS: + db.purge_document(scratch, keep_audit=False) print(f"\n{len(PASS)} passed, {len(FAIL)} failed") sys.exit(1 if FAIL else 0) diff --git a/tests/test_migrate_sqlite_to_postgres.py b/tests/test_migrate_sqlite_to_postgres.py new file mode 100644 index 0000000..e963846 --- /dev/null +++ b/tests/test_migrate_sqlite_to_postgres.py @@ -0,0 +1,393 @@ +"""scripts/migrate_sqlite_to_postgres.py: a legacy SQLite file lands in Postgres. + +The fixture file is built with the legacy schema DDL, the way a production +database looked at cutover. +""" + +import hashlib +import importlib.util +import json +import sqlite3 +from pathlib import Path + +import pytest + +_SCRIPT = Path(__file__).resolve().parent.parent / "scripts" / "migrate_sqlite_to_postgres.py" + +LEGACY_DDL = """ +CREATE TABLE documents ( + workflow_id TEXT PRIMARY KEY, document_id TEXT NOT NULL, canonical_document_id TEXT, + filename TEXT NOT NULL, display_name TEXT, source_filename TEXT, source_manifest_name TEXT, + source_file_fingerprint TEXT, filepath TEXT NOT NULL, stage TEXT NOT NULL DEFAULT 'registered', + page_count INTEGER DEFAULT 0, chunk_count INTEGER DEFAULT 0, error_message TEXT, + is_demo INTEGER DEFAULT 0, created_at TEXT, updated_at TEXT, ocr_completed_at TEXT, + translation_completed_at TEXT, chunks_completed_at TEXT, ingested_at TEXT, source_type TEXT, + canonical_input_type TEXT, stop_after_ocr INTEGER DEFAULT 0, reindex_required INTEGER DEFAULT 0, + reindex_reason TEXT, original_artifact_id INTEGER, normalized_artifact_id INTEGER, + latest_job_id INTEGER, is_disabled INTEGER DEFAULT 0, instance TEXT, uploaded_by_user_id TEXT, + uploaded_by_username TEXT, uploaded_by_email TEXT, uploaded_by_roles TEXT, + prod_ready_requested_at TEXT, prod_ready_requested_by_user_id TEXT, + prod_ready_requested_by_username TEXT, valid_from TEXT, valid_to TEXT, + document_kind TEXT DEFAULT 'document', scheme_code TEXT, scheme_name TEXT, + scheme_aliases_json TEXT, tool_routing TEXT, catalog_visible INTEGER DEFAULT 1, + network_visible INTEGER DEFAULT 1 +); +CREATE TABLE audit_logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, workflow_id TEXT NOT NULL, document_id TEXT NOT NULL, + action_type TEXT NOT NULL, entity_type TEXT, entity_id INTEGER, field_name TEXT, old_value TEXT, + new_value TEXT, metadata TEXT, timestamp TEXT NOT NULL, actor_user_id TEXT, actor_username TEXT, + actor_email TEXT, actor_roles TEXT +); +CREATE TABLE pages ( + id INTEGER PRIMARY KEY AUTOINCREMENT, workflow_id TEXT NOT NULL, page_number INTEGER NOT NULL, + original_markdown TEXT, edited_markdown TEXT, is_reviewed INTEGER DEFAULT 0, reviewer_notes TEXT, + detected_language TEXT, translated_markdown TEXT, edited_translation TEXT, + translation_reviewed INTEGER DEFAULT 0, translation_notes TEXT, translation_provider TEXT, + translation_model TEXT, translation_target_language TEXT, translated_at TEXT, + UNIQUE(workflow_id, page_number) +); +CREATE TABLE chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, workflow_id TEXT NOT NULL, chunk_number INTEGER NOT NULL, + original_text TEXT, edited_text TEXT, token_count INTEGER DEFAULT 0, page_start INTEGER DEFAULT 1, + page_end INTEGER DEFAULT 1, source_page_numbers_json TEXT, source_spans_json TEXT, + section_title TEXT, content_type TEXT, is_reference INTEGER DEFAULT 0, chunking_provider TEXT, + chunking_model TEXT, chunking_config_json TEXT, chunking_run_id TEXT, chunk_version INTEGER DEFAULT 1, + is_reviewed INTEGER DEFAULT 0, is_excluded INTEGER DEFAULT 0, reviewer_notes TEXT, + UNIQUE(workflow_id, chunk_number) +); +CREATE TABLE settings (key TEXT PRIMARY KEY, value TEXT NOT NULL, description TEXT, updated_at TEXT NOT NULL); +CREATE TABLE document_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, workflow_id TEXT NOT NULL, job_type TEXT NOT NULL, + temporal_workflow_id TEXT, temporal_run_id TEXT, status TEXT NOT NULL DEFAULT 'running', + current_stage TEXT, started_at TEXT NOT NULL, completed_at TEXT, error_message TEXT, config_json TEXT +); +CREATE TABLE document_artifacts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, workflow_id TEXT NOT NULL, job_id INTEGER, + artifact_type TEXT NOT NULL, stage TEXT, storage_uri TEXT NOT NULL, mime_type TEXT, filename TEXT, + size_bytes INTEGER, metadata_json TEXT, created_at TEXT NOT NULL +); +CREATE TABLE document_index_status ( + workflow_id TEXT NOT NULL, index_name TEXT NOT NULL, doc_id TEXT, chunk_count_indexed INTEGER DEFAULT 0, + last_indexed_at TEXT, last_verified_at TEXT, schema_version TEXT, status TEXT NOT NULL DEFAULT 'unknown', + details_json TEXT, PRIMARY KEY (workflow_id, index_name) +); +CREATE TABLE document_manifest_entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, manifest_filename TEXT NOT NULL UNIQUE, title_en TEXT, + imported_at TEXT NOT NULL +); +CREATE TABLE email_otps ( + email TEXT PRIMARY KEY, code_hash TEXT NOT NULL, expires_at TEXT NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL, last_sent_at TEXT NOT NULL +); +CREATE TABLE webhook_records ( + id TEXT PRIMARY KEY, source TEXT NOT NULL DEFAULT 'webhook', external_id TEXT NOT NULL, title TEXT, + slug TEXT, status TEXT, is_published INTEGER, category TEXT, payload TEXT NOT NULL, payload_hash TEXT, + created_at TEXT NOT NULL, updated_at TEXT NOT NULL, UNIQUE (source, external_id) +); +CREATE TABLE scheme_catalog_meta ( + id INTEGER PRIMARY KEY CHECK (id = 1), version INTEGER NOT NULL DEFAULT 0, updated_at TEXT, notes TEXT +); +CREATE TABLE scheme_catalog_entries ( + scheme_code TEXT PRIMARY KEY, scheme_name TEXT, scheme_aliases_json TEXT, instances_json TEXT, + workflow_ids_json TEXT, collection_name TEXT, chunk_count INTEGER DEFAULT 0, + status TEXT NOT NULL DEFAULT 'disabled', network_visible INTEGER DEFAULT 0, promoted_at TEXT, + content_hash TEXT, source TEXT, updated_at TEXT +); +""" + +FP1 = hashlib.md5(b"scheme guideline").hexdigest() +FP2 = hashlib.md5(b"in flight").hexdigest() +WF1 = f"doc-{FP1}" +WF2 = f"doc-{FP2}" +WF3 = "doc-broken" +FUTURE = "2099-01-01T00:00:00" + + +def _load_script(): + spec = importlib.util.spec_from_file_location("migrate_sqlite_to_postgres", _SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _insert(conn, table, **values): + keys = ", ".join(values) + marks = ", ".join("?" for _ in values) + conn.execute(f"INSERT INTO {table} ({keys}) VALUES ({marks})", tuple(values.values())) + + +@pytest.fixture +def legacy_db(tmp_path): + path = tmp_path / "documents.db" + conn = sqlite3.connect(path) + conn.executescript(LEGACY_DDL) + + _insert( + conn, "documents", workflow_id=WF1, document_id=FP1, canonical_document_id=FP1, + filename="guideline.pdf", display_name="Guideline", source_filename="guideline.pdf", + source_file_fingerprint=FP1, filepath="minio://documents/guideline.pdf", stage="completed", + page_count=2, chunk_count=2, created_at="2026-03-01T10:00:00", updated_at="2026-03-05T10:00:00", + ocr_completed_at="2026-03-01T11:00:00", translation_completed_at="2026-03-02T11:00:00", + chunks_completed_at="2026-03-03T11:00:00", ingested_at="2026-03-04T11:00:00", + source_type="document", canonical_input_type="pdf", instance="mh", uploaded_by_user_id="kc-up", + uploaded_by_username="uma", uploaded_by_email="uma@x.in", uploaded_by_roles="contributor,reviewer", + valid_from="2026-03-01", valid_to="2027-03-01", document_kind="scheme", scheme_code="pm-kmy", + scheme_name="PM Kisan Maandhan", scheme_aliases_json=json.dumps(["PMKMY", "kmy"]), + catalog_visible=1, network_visible=0, reindex_required=1, reindex_reason="edit", + ) + _insert( + conn, "documents", workflow_id=WF2, document_id=FP2, filename="inflight.pdf", + filepath="/data/documents/inflight.pdf", stage="approval_for_prod", page_count=1, + created_at="2026-04-01T10:00:00", updated_at="2026-04-01T12:00:00", instance="up", + is_demo=1, prod_ready_requested_at="2026-04-02T09:00:00", + prod_ready_requested_by_user_id="kc-req", prod_ready_requested_by_username="rita", + ocr_completed_at="2026-04-01T11:00:00", + ) + _insert( + conn, "documents", workflow_id=WF3, document_id="not-a-hash", filename="broken.pdf", + filepath="/tmp/broken.pdf", stage="failed", error_message="OCR blew up", + created_at="2026-05-01T10:00:00", updated_at="2026-05-01T10:30:00", instance=" ", + document_kind="document", scheme_code="Bad Code", valid_from="2026-06-01", valid_to="2026-01-01", + ) + + _insert(conn, "document_jobs", id=1, workflow_id=WF1, job_type="pipeline", temporal_workflow_id=WF1, + status="completed", current_stage="completed", started_at="2026-03-01T10:00:00", + completed_at="2026-03-04T11:00:00", config_json=json.dumps({"auto_approve": False}), + temporal_run_id="run-1") + _insert(conn, "document_jobs", id=2, workflow_id=WF1, job_type="reingest", + temporal_workflow_id=f"{WF1}-reingest-1", status="completed", current_stage="ingesting", + started_at="2026-03-10T10:00:00", completed_at="2026-03-10T10:05:00") + _insert(conn, "document_jobs", id=3, workflow_id=WF1, job_type="mystery", + started_at="2026-03-11T10:00:00") + + _insert(conn, "document_artifacts", workflow_id=WF1, job_id=1, artifact_type="original_upload", + stage="registered", storage_uri="minio://documents/guideline.pdf", size_bytes=100, + metadata_json=json.dumps({"uploaded_via": "upload_endpoint"}), created_at="2026-03-01T10:00:01") + _insert(conn, "document_artifacts", workflow_id=WF1, job_id=None, artifact_type="normalized_pdf", + stage="ocr_processing", storage_uri="minio://documents/n.pdf", size_bytes=-1, + created_at="2026-03-01T10:30:00") + _insert(conn, "document_artifacts", workflow_id=WF1, job_id=2, artifact_type="vector_payload_export", + stage="ingesting", storage_uri="minio://documents/v.json", created_at="2026-03-10T10:01:00") + + _insert(conn, "pages", workflow_id=WF1, page_number=1, original_markdown="namaste", is_reviewed=1, + detected_language="hi", translated_markdown="hello", translation_provider="azure", + translation_model="gpt", translation_target_language="en", translated_at="2026-03-02T10:00:00") + _insert(conn, "pages", workflow_id=WF1, page_number=2, original_markdown="english page", + detected_language="en", reviewer_notes="fine") + _insert(conn, "pages", workflow_id="doc-purged", page_number=1, original_markdown="stray") + + _insert(conn, "chunks", workflow_id=WF1, chunk_number=1, original_text="chunk one", token_count=5, + page_start=1, page_end=2, source_spans_json=json.dumps([{"page_number": 1, "start_char": 0, "end_char": 5}]), + chunking_provider="recursive", chunking_model="rs", chunking_config_json='{"size": 450}', + chunk_version=3, is_reviewed=1) + _insert(conn, "chunks", workflow_id=WF1, chunk_number=2, original_text="chunk two", edited_text="two", + page_start=2, page_end=1, is_excluded=1) + + _insert(conn, "document_index_status", workflow_id=WF1, index_name="documents-index", doc_id="abc", + chunk_count_indexed=1, last_indexed_at="2026-03-04T11:00:00", status="indexed", + details_json='{"backend": "qdrant"}') + _insert(conn, "document_index_status", workflow_id=WF1, index_name="schemes-index", doc_id="abc", + chunk_count_indexed=2, last_indexed_at="2026-03-10T10:05:00", status="indexed") + + _insert(conn, "audit_logs", workflow_id=WF1, document_id=FP1, action_type="stage_change", + entity_type="document", field_name="stage", old_value="chunk_review", new_value="completed", + metadata='{"page_count": 2}', timestamp="2026-03-04T11:00:00") + _insert(conn, "audit_logs", workflow_id=WF1, document_id=FP1, action_type="page_edit", + entity_type="page", entity_id=1, metadata="not json", timestamp="2026-03-02T12:00:00", + actor_user_id="kc-rev", actor_username="ravi", actor_email="ravi@x.in", actor_roles="reviewer") + _insert(conn, "audit_logs", workflow_id="doc-purged", document_id="gone", action_type="purge_document", + entity_type="document", timestamp="2026-02-01T00:00:00") + _insert(conn, "audit_logs", workflow_id="__system__", document_id="__settings__", + action_type="settings_change", entity_type="setting", field_name="search_alpha", + old_value="0.7", new_value="0.6", timestamp="2026-01-01T00:00:00") + _insert(conn, "audit_logs", workflow_id="__catalog__", document_id="__catalog__", + action_type="catalog_rebuild", entity_type="catalog", timestamp="2026-01-02T00:00:00") + + for key, value in ( + ("search_alpha", "0.4"), ("search_limit", "20"), ("search_show_highlights", "false"), + ("search_method", "LEXICAL"), ("search_index_name", "documents-index"), ("search_mystery", "x"), + ("search_ef_search", "lots"), + ): + _insert(conn, "settings", key=key, value=value, updated_at="2026-01-01T00:00:00") + + _insert(conn, "email_otps", email="old@x.in", code_hash="h", expires_at="2020-01-01T00:00:00+00:00", + created_at="2020-01-01T00:00:00+00:00", last_sent_at="2020-01-01T00:00:00+00:00") + _insert(conn, "email_otps", email="LIVE@x.in", code_hash="h2", expires_at=FUTURE + "+00:00", + attempts=2, created_at="2026-01-01T00:00:00+00:00", last_sent_at="2026-01-01T00:00:00+00:00") + + _insert(conn, "webhook_records", id="3c8a2f0e-6b1d-4f6e-9d0a-5f8f0b2e7a11", external_id="c-1", + title="T", slug="t", status="published", is_published=1, category="Agri", + payload='{"contentID": "c-1"}', payload_hash="h", created_at="2026-01-01T00:00:00", + updated_at="2026-01-02T00:00:00") + + _insert(conn, "scheme_catalog_meta", id=1, version=5, updated_at="2026-02-01T00:00:00", notes="x") + _insert(conn, "scheme_catalog_entries", scheme_code="cdp", scheme_name="Crop Diversification Programme", + scheme_aliases_json='["CDP"]', status="live", source="bootstrap") + _insert(conn, "scheme_catalog_entries", scheme_code="pm-kmy", scheme_name="Registry name", + scheme_aliases_json='["registry"]', status="live", source="pipeline") + _insert(conn, "document_manifest_entries", manifest_filename="a.csv", imported_at="2026-01-01T00:00:00") + _insert(conn, "document_manifest_entries", manifest_filename="b.csv", imported_at="2026-01-01T00:00:00") + conn.commit() + conn.close() + return str(path) + + +def _counts(db): + with db.get_connection() as conn: + return { + table: conn.execute(f"SELECT count(*) AS n FROM {table}").fetchone()["n"] + for table in ( + "documents", "workflows", "workflow_artifacts", "pages", "chunks", "audit_logs", + "users", "catalog_entries", "email_otps", "webhook_records", + ) + } + + +@pytest.mark.db +@pytest.mark.unit +class TestSqliteToPostgresMigration: + def test_dry_run_writes_nothing_but_reports_everything(self, db_connection, postgres_url, legacy_db): + script = _load_script() + before = _counts(db_connection) + report = script.run_migration(legacy_db, postgres_url, dry_run=True) + assert _counts(db_connection) == before + assert report.tables["documents"]["inserted"] == 3 + assert report.mismatches() == [] + + def test_a_full_run_maps_every_table(self, db_connection, postgres_url, legacy_db): + db = db_connection + report = _load_script().run_migration(legacy_db, postgres_url) + + assert report.mismatches() == [] + tables = report.as_dict()["tables"] + assert tables["documents"]["inserted"] == 3 + assert tables["document_jobs"] == {"read": 3, "inserted": 2, "skipped": 1} + assert tables["pages"] == {"read": 3, "inserted": 2, "skipped": 1} + assert tables["chunks"]["inserted"] == 2 + assert tables["document_manifest_entries"] == {"read": 2, "dropped": 2} + assert tables["email_otps"] == {"read": 2, "inserted": 1, "dropped": 1} + assert tables["settings"]["skipped"] == 2 + + doc = db.get_document(WF1) + assert doc["stage"] == "completed" + assert doc["instance"] == "mh" + assert doc["document_id"] == FP1 + assert doc["display_name"] == "Guideline" + assert (doc["valid_from"], doc["valid_to"]) == ("2026-03-01", "2027-03-01") + assert (doc["scheme_code"], doc["document_kind"]) == ("pm-kmy", "scheme") + assert doc["network_visible"] is False and doc["reindex_required"] is True + assert doc["uploaded_by_username"] == "uma" + assert doc["uploaded_by_roles"] == "contributor,reviewer" + assert doc["created_at"] == "2026-03-01T10:00:00" + assert doc["ocr_completed_at"] == "2026-03-01T11:00:00" + assert doc["chunk_count"] == 2 + assert doc["page_count"] == 2 + assert set(doc["index_status"]) == {"dev"} + assert doc["index_status"]["dev"]["index_name"] == "schemes-index" + + jobs = {j["job_type"]: j for j in db.list_document_jobs(WF1)} + assert set(jobs) == {"pipeline", "reingest"} + assert jobs["pipeline"]["temporal_workflow_id"] == WF1 + assert jobs["pipeline"]["temporal_run_id"] == "run-1" + assert jobs["reingest"]["temporal_workflow_id"] == f"{WF1}-reingest-1" + assert doc["workflow_id"] == WF1 + + artifacts = {a["artifact_type"]: a for a in db.list_document_artifacts(WF1)} + assert artifacts["original_upload"]["job_id"] == jobs["pipeline"]["id"] + assert artifacts["vector_payload_export"]["job_id"] == jobs["reingest"]["id"] + assert artifacts["normalized_pdf"]["job_id"] == jobs["pipeline"]["id"] + assert artifacts["normalized_pdf"]["size_bytes"] is None + assert doc["original_artifact_id"] == artifacts["original_upload"]["id"] + + pages = db.get_pages(WF1) + assert pages[0]["translated_markdown"] == "hello" + assert pages[0]["translation_target_language"] == "en" + assert pages[0]["is_reviewed"] is True and pages[0]["detected_language"] == "hi" + assert pages[1]["translations"] == {} and pages[1]["reviewer_notes"] == "fine" + + chunks = db.get_chunks(WF1, include_excluded=True) + assert [c["chunk_version"] for c in chunks] == [3, 3] + assert chunks[0]["chunking_config_json"] == '{"size": 450}' + assert chunks[1]["page_end"] == 2 and chunks[1]["is_excluded"] is True + + entries = {e["code"]: e for e in db.list_catalog_entries()} + assert entries["cdp"]["source"] == "bootstrap" and entries["cdp"]["status"] == "live" + assert entries["pm-kmy"]["name"] == "Registry name" + assert entries["pm-kmy"]["status"] == "pending_reindex" + assert entries["pm-kmy"]["workflow_ids"] == [WF1] + assert db.get_catalog_meta()["version"] == 6 + + users = {u: db.get_user_by_keycloak_id(u) for u in ("kc-up", "kc-rev", "kc-req")} + assert all(users.values()) + + logs = db.get_audit_logs(WF1) + assert {e["action_type"] for e in logs} == {"stage_change", "page_edit"} + edit = next(e for e in logs if e["action_type"] == "page_edit") + assert edit["actor"] == "ravi@x.in (reviewer)" + assert edit["metadata"] == json.dumps({"raw": "not json"}) + purged = db.get_audit_logs("doc-purged") + assert [e["action_type"] for e in purged] == ["purge_document"] + settings_log = db.get_settings_audit_logs() + assert settings_log[0]["field_name"] == "retrieval.alpha" + + settings = db.get_search_settings() + assert settings["alpha"] == 0.4 and settings["limit"] == 20 + assert settings["showHighlights"] is False and settings["searchMethod"] == "LEXICAL" + assert settings["efSearch"] == 256 + + assert db.get_email_otp("live@x.in")["attempts"] == 2 + assert db.get_email_otp("old@x.in") is None + with db.get_connection() as conn: + hook = conn.execute("SELECT * FROM webhook_records").fetchone() + assert str(hook["id"]) == "3c8a2f0e-6b1d-4f6e-9d0a-5f8f0b2e7a11" + assert hook["is_published"] is True and hook["payload"] == {"contentID": "c-1"} + + def test_in_flight_documents_keep_their_stage_and_prod_request(self, db_connection, postgres_url, legacy_db): + db = db_connection + _load_script().run_migration(legacy_db, postgres_url) + + doc = db.get_document(WF2) + assert doc["stage"] == "approval_for_prod" + assert doc["is_demo"] is True + assert doc["prod_ready_requested_at"] == "2026-04-02T09:00:00" + assert doc["prod_ready_requested_by_username"] == "rita" + assert doc["instance"] == "up" + assert doc["ocr_completed_at"] == "2026-04-01T11:00:00" + types = sorted(j["job_type"] for j in db.list_document_jobs(WF2)) + assert types == ["pipeline", "promote_prod"] + assert db.get_latest_document_job(WF2)["status"] == "waiting_review" + assert db.get_document(doc["id"])["workflow_id"] == WF2 + assert db.get_document(FP2)["id"] == doc["id"] + + def test_unusable_values_are_repaired_and_noted(self, db_connection, postgres_url, legacy_db): + db = db_connection + report = _load_script().run_migration(legacy_db, postgres_url) + + doc = db.get_document(WF3) + assert doc["stage"] == "failed" and doc["error_message"] == "OCR blew up" + assert doc["document_id"] == hashlib.md5(b"not-a-hash").hexdigest() + assert doc["instance"] == "default" + assert doc["scheme_code"] is None + assert doc["valid_from"] == "2026-06-01" and doc["valid_to"] is None + notes = " ".join(report.notes) + assert "valid_to cleared" in notes and "not a valid slug" in notes or "plain document" in notes + + def test_running_it_twice_changes_nothing(self, db_connection, postgres_url, legacy_db): + script = _load_script() + script.run_migration(legacy_db, postgres_url) + first = _counts(db_connection) + + report = script.run_migration(legacy_db, postgres_url) + + assert _counts(db_connection) == first + assert report.tables["documents"] == {"read": 3, "existing": 3} + assert report.tables["audit_logs"]["inserted"] == 0 + assert report.tables["webhook_records"]["existing"] == 1 + assert report.mismatches() == [] + assert db_connection.get_catalog_meta()["version"] == 6 + + def test_legacy_file_is_opened_read_only(self, postgres_url, legacy_db): + script = _load_script() + before = Path(legacy_db).read_bytes() + script.run_migration(legacy_db, postgres_url) + assert Path(legacy_db).read_bytes() == before From b1f7f2d42e76222562c9050332ed2fdac0e63c25 Mon Sep 17 00:00:00 2001 From: Naveen Date: Mon, 26 Oct 2026 13:18:48 +0530 Subject: [PATCH 32/53] docs: describe Postgres as the canonical store --- AGENTS.md | 10 ++--- CONTEXT.md | 2 +- README.md | 16 +++---- docs/SYSTEM_DESIGN.md | 68 +++++++++++++++--------------- docs/architecture-pipeline-rbac.md | 20 ++++----- docs/openapi-search.yaml | 8 ++-- 6 files changed, 62 insertions(+), 62 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b50a7bb..03b5f33 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## What this is -A review-driven document ingestion pipeline (product name "Bharat Vistaar Docs Pipeline"): Temporal workflows + FastAPI + SQLite + MinIO + Qdrant. Documents move through explicit stages (OCR → review → translate → review → chunk → review → ingest DEV → Super Admin promote to PROD), with every major output persisted as an inspectable artifact rather than hidden. +A review-driven document ingestion pipeline (product name "Bharat Vistaar Docs Pipeline"): Temporal workflows + FastAPI + Postgres + MinIO + Qdrant. Documents move through explicit stages (OCR → review → translate → review → chunk → review → ingest DEV → Super Admin promote to PROD), with every major output persisted as an inspectable artifact rather than hidden. Full narrative docs (read these before deep architectural changes, don't re-derive): - `README.md` — what the system does, API surface, supported inputs, auth model @@ -68,7 +68,7 @@ Core modules (all under `pipeline/`): - `workflows.py` — Temporal workflow definitions (`DocumentPipelineWorkflow` is the main one; also `ReingestionWorkflow`, `PromoteToProdWorkflow`, `TranslationOnlyWorkflow`, `OcrOnlyWorkflow`, `ChunkingOnlyWorkflow`) - `activities.py` — Temporal activities invoked by the workflows (OCR, translate, chunk, ingest, state updates) - `worker.py` — Temporal worker process; registers workflows/activities on task queue `ocr-pipeline` -- `db.py` — SQLite access layer; SQLite is the **canonical** metadata/review-state store (documents, pages, chunks, jobs, artifacts, audit log), not just a cache +- `db.py` — Postgres access layer (psycopg3 sync connection pool); Postgres is the **canonical** metadata/review-state store (documents, workflows, pages, chunks, artifacts, audit log, settings), not just a cache. Documents and jobs come back as flat dicts keyed by `workflow_id` (the document's main Temporal workflow id); every function also accepts the document UUID. Schema: `migrations/` (applied at api/worker startup by `migrate.py`), described in `docs/POSTGRES_SCHEMA.md` - `document_repository.py` — `DocumentRepository`, a domain layer over `db.py` for document reads; `db.py` stays one-function-per-query with no domain knowledge - `network_constants.py` — fixed values sent on the network (catalog/resource ids, topics, languages, Beckn version). The file v2 edits when the AI layer starts deriving them - `catalog_builder.py` — pure builder mapping a document's knowledge kind (`advisory`/`scheme`) to the single `OnDemand` Beckn catalog announced for that kind; takes the resolved provider and prefixes catalog/resource ids for a Group; returns `None` for any other kind. No env, no I/O @@ -78,7 +78,7 @@ Core modules (all under `pipeline/`): - `models.py` — Pydantic models, including `DocumentStage` enum and `PIPELINE_STAGES` (the stepper-UI stage list) - `config.py` — `Config` dataclass reading env vars, with defaults - `instances.py` — instance/tenant (state) code ↔ human name mapping; `bv` = Bharat Vistaar platform-wide, others are state codes matching Keycloak group paths (`/states/MH/...` → `mh`) -- `scheme_catalog.py` — SQLite-backed scheme catalog behind the pull-based `/catalog/v1/*` API +- `scheme_catalog.py` — scheme catalog behind the pull-based `/catalog/v1/*` API, served from the `catalog_entries` table and the computed `catalog` view - `ocr/`, `translation/`, `chunking/`, `vector_store/` — pluggable provider packages, each with a `base.py` protocol/ABC and a `service.py` (or equivalent) that dispatches to the configured provider - `auth/` — Keycloak JWT validation, group-based tenancy/permissions, email-OTP flow, admin user management @@ -104,7 +104,7 @@ registered → ocr_processing → ocr_review → translation_processing → tran ### Storage responsibilities (don't blur these) -- **SQLite** — canonical metadata/content/review state (documents, pages, chunks, jobs, artifacts, audit, search settings). Source of truth for anything editable. +- **Postgres** — canonical metadata/content/review state (documents, workflows, pages, chunks, artifacts, audit, search settings). Source of truth for anything editable. - **MinIO** — blob storage for original uploads, normalized files, and stage artifacts (OCR/translation/chunk exports, vector-index payload snapshots). - **Temporal** — orchestration/retries/review-gate signaling only; not a content store. - **Qdrant** — downstream search projections of approved chunk state; never the place to edit content. @@ -119,4 +119,4 @@ React 18 + Vite + Tailwind v4 + Radix UI operator console (not a plain upload fo ### Scripts (`scripts/`) -Operational tools, not one-off hacks — e.g. `list_failed_workflows.py`, `terminate_stuck_workflows.py`, `keycloak_bootstrap_docs_pipeline.py`, `drop_legacy_marqo_columns.py`, `backfill_provider_id.py` (sets `provider_id` on existing DEV/PROD chunks; dry run unless `--apply`). Check here before writing a new maintenance script — an equivalent likely exists. +Operational tools, not one-off hacks — e.g. `list_failed_workflows.py`, `terminate_stuck_workflows.py`, `keycloak_bootstrap_docs_pipeline.py`, `drop_legacy_marqo_columns.py`, `backfill_provider_id.py` (sets `provider_id` on existing DEV/PROD chunks; dry run unless `--apply`), `migrate_sqlite_to_postgres.py`. Check here before writing a new maintenance script — an equivalent likely exists. diff --git a/CONTEXT.md b/CONTEXT.md index e5bfa79..504d8dd 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -13,7 +13,7 @@ The pipeline stage that runs only after a document's PROD promotion succeeds, an _Avoid_: "publish" alone — ambiguous with Publish to Dev. **Scheme Catalog**: -The SQLite-backed, scheme-only registry that aggregates one or more documents sharing a `scheme_code`. Backs the pull-based `GET /catalog/v1/*` API that other BAPs poll. +The scheme-only registry (`catalog_entries` rows plus the computed `catalog` view) that aggregates one or more documents sharing a `scheme_code`. Backs the pull-based `GET /catalog/v1/*` API that other BAPs poll. _Avoid_: "catalog" alone — ambiguous with the Network Catalog Envelope. **Network Catalog Envelope**: diff --git a/README.md b/README.md index 2eab93b..81d055f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![codecov](https://codecov.io/gh/OpenAgriNet/knowledge-provider/graph/badge.svg?token=2TKM2CR2AC)](https://codecov.io/gh/OpenAgriNet/knowledge-provider) -This repository contains a review-driven document ingestion pipeline built around Temporal workflows, FastAPI, SQLite, MinIO, and Qdrant. It is designed for teams that need to normalize heterogeneous files, extract structured text, review and correct outputs, generate chunks, and publish searchable records into a vector index. +This repository contains a review-driven document ingestion pipeline built around Temporal workflows, FastAPI, Postgres, MinIO, and Qdrant. It is designed for teams that need to normalize heterogeneous files, extract structured text, review and correct outputs, generate chunks, and publish searchable records into a vector index. The system is intentionally operational, not just algorithmic. Documents move through explicit stages, every major output can be persisted as an artifact, and the operator UI is designed to inspect and manage the pipeline rather than hide it. @@ -63,7 +63,7 @@ The platform is composed of six main services: | | | | v v v v +--------+---------+ +--------+--+ +---+---------------+ +------------------+ -| MinIO | | SQLite | | Worker | | Lang Detect | +| MinIO | | Postgres | | Worker | | Lang Detect | | originals, | | canonical | | normalize, OCR, | | language hints | | normalized files, | | metadata, | | translate, chunk, | | before | | artifacts | | pages, | | ingest | | translation | @@ -90,7 +90,7 @@ The document has been accepted into the system and assigned a workflow identifie Typical outputs: -- document row in SQLite +- document row in Postgres - original file reference - initial job record @@ -191,16 +191,16 @@ Document-level view of what has been pushed to the vector index. ## Storage Responsibilities -### SQLite +### Postgres -SQLite is the canonical metadata and review-state store. +Postgres is the canonical metadata and review-state store. Its schema is in `migrations/` and described in `docs/POSTGRES_SCHEMA.md`; the api and worker apply pending migrations at startup. Moving an existing SQLite deployment over is a one-off run of `scripts/migrate_sqlite_to_postgres.py` (see the `migrate-sqlite` compose profile). It owns: - document rows +- workflow rows (one per processing run, shown as jobs) - page rows - chunk rows -- jobs - artifact metadata - audit logs - search settings @@ -320,7 +320,7 @@ Default local ports from `docker-compose.yml`: - Docker and Docker Compose - an OCR provider API key exposed as `MISTRAL_API_KEY` -- enough local disk for SQLite, MinIO artifacts, and Qdrant state +- enough local disk for Postgres, MinIO artifacts, and Qdrant state ### Start @@ -355,7 +355,7 @@ Important runtime variables include: - `MINIO_ACCESS_KEY` - `MINIO_SECRET_KEY` - `MINIO_BUCKET` -- `DOCUMENT_DB_PATH` +- `DATABASE_URL` - `TRANSLATION_PROVIDER` - `TRANSLATION_MODEL` - `TRANSLATION_PAGE_CONCURRENCY` diff --git a/docs/SYSTEM_DESIGN.md b/docs/SYSTEM_DESIGN.md index d659603..6cb6229 100644 --- a/docs/SYSTEM_DESIGN.md +++ b/docs/SYSTEM_DESIGN.md @@ -39,7 +39,7 @@ gate before **PROD**. | Multi-state tenancy | `documents.instance` + JWT groups; lists/search scoped | | DEV before PROD | Stage 9 writes DEV index; stage 10 waits; Super Admin promotes to PROD | | Operability | Runs, queue, audit, reconcile, artifacts UI | -| Durable long work | Temporal retries + SQLite state mirror | +| Durable long work | Temporal retries + Postgres state mirror | ### Non-goals - Real-time chat / low-latency LLM product surface @@ -98,7 +98,7 @@ flowchart TB end subgraph Data - SQLITE[("SQLite WAL
documents.db")] + PGDB[("Postgres
documents, workflows, pages, chunks")] MINIO[("MinIO
objects")] QDEV[("Qdrant DEV index
vector + lexical")] QPROD[("Qdrant PROD
production vectors")] @@ -111,13 +111,13 @@ flowchart TB UI -->|"/api + Bearer JWT"| API UI -.->|OIDC PKCE| KC API -->|JWKS| KC - API --> SQLITE + API --> PGDB API --> MINIO API --> QDEV API -->|start / signal| TEMP TEMP --> TEMPDB TEMP --> WORKER - WORKER --> SQLITE + WORKER --> PGDB WORKER --> MINIO WORKER --> QDEV WORKER -->|promote_document_to_prod_qdrant| QPROD @@ -136,10 +136,10 @@ flowchart TB | **minio** | official | Source PDF + stage artifacts | 9000 | | **qdrant** | official image | DEV search index | 6333 | | **qdrant (prod)** | external / env | PROD vectors after Super Admin promote | `PROD_QDRANT_URL` | -| **sqlite** | volume file | Canonical document/page/chunk/job/audit state | path | +| **postgres** | official image | Canonical document/workflow/page/chunk/audit state | `DATABASE_URL` | | **keycloak** | official | Identity, groups `/states/{ST}/role`, `/global/super-admin` | 8082 | -API and worker share one **SQLite volume** and the same env for providers so +API and worker share one **Postgres database** and the same env for providers so read and write paths stay consistent. --- @@ -161,7 +161,7 @@ read and write paths stay consistent. │ │ ┌───────────────▼──────────────┐ ┌──────────▼────────────────────────────┐ │ ORCHESTRATION LAYER │ │ DOMAIN STATE LAYER │ -│ Temporal workflows │ │ SQLite (SoT for content) │ +│ Temporal workflows │ │ Postgres (SoT for content) │ │ + review signals │ │ MinIO (binaries) │ │ + activity retries │ │ Jobs / audit / index status │ └───────────────┬──────────────┘ └───────────────────────────────────────┘ @@ -245,36 +245,36 @@ Supporting workflows: `OcrOnly`, `TranslationOnly`, `ChunkingOnly`, | Store | System of record for | Not for | |---|---|---| -| **SQLite** | Document stage, pages, chunks, tags, jobs, audit, uploader, `instance` | Binary blobs | +| **Postgres** | Document stage, workflows, pages, chunks, artifacts, audit, uploader, `instance` | Binary blobs | | **MinIO** | PDF originals, exports, payloads | AuthZ decisions | | **Temporal DB** | Workflow history & timers | Business content | | **Qdrant DEV** | Searchable DEV vectors | Authoritative text | | **Qdrant PROD** | Production vectors | Draft/unapproved content | -### 7.2 Key entities (SQLite) +### 7.2 Key entities (Postgres) ```text -documents (workflow_id PK) +documents (id UUID PK) ├── instance # tenant / state code - ├── stage # DocumentStage enum - ├── uploaded_by_* # actor provenance - ├── page_count / chunk_count - ├── latest_job_id - ├── reindex_required - └── is_disabled / is_demo + ├── source (JSONB) # fingerprint, filename, storage_uri, page_count + ├── index_status (JSONB) # dev / prod vector index state + ├── metadata (JSONB) # chunk_count, reindex_required, is_demo, ... + ├── uploaded_by_user_id # → users + └── is_disabled / catalog_entry_id -pages (workflow_id, page_number) - └── OCR + translation fields + review flags +workflows (one per processing run; the latest gives the document's stage) + └── workflow_type, status, current_stage, temporal_workflow_id → Runs UI -chunks (workflow_id, chunk_num) - └── text, lineage, exclusion, provenance - └── chunk_tags[] +workflow_artifacts → files each workflow produced (MinIO) + +pages (document_id, page_number) + └── OCR text + translations (JSONB) + review flags -document_jobs - └── job_type, status, temporal ids → Runs UI +chunks (document_id, chunk_number) + └── text, lineage, exclusion, provenance audit_logs - └── approvals, edits, promote_to_prod + └── approvals, edits, promote_to_prod (outlives the document) ``` ### 7.3 Tenant rule @@ -337,7 +337,7 @@ UI always calls API via `/api` proxy — no direct Temporal/MinIO from browser. | Activity | Purpose | Failure handling | |---|---|---| -| `run_ocr_and_store` | Extract pages → SQLite + MinIO | OCR_RETRY | +| `run_ocr_and_store` | Extract pages → Postgres + MinIO | OCR_RETRY | | `detect_and_translate_pages_from_db` | Lang detect + translate | TRANSLATION_RETRY | | `create_chunks_from_db` | Chunk reviewed text | CHUNK_RETRY | | `auto_tag_chunks_from_db` | Domain tags | CHUNK_RETRY | @@ -353,12 +353,12 @@ Providers are **config-driven** (env URLs/models) so OCR/vLLM backends can chang ```mermaid flowchart LR - SQLITE[(SQLite chunks)] + PGDB[(Postgres chunks)] DEV[(DEV index
Qdrant)] PROD[(PROD Qdrant)] - SQLITE -->|approve_ingestion
ingest_document_from_db| DEV - SQLITE -->|approve_prod Super Admin
promote_document_to_prod_qdrant| PROD + PGDB -->|approve_ingestion
ingest_document_from_db| DEV + PGDB -->|approve_prod Super Admin
promote_document_to_prod_qdrant| PROD DEV -.->|not automatic| PROD ``` @@ -399,8 +399,8 @@ Routes: Dashboard, Documents, New Document, Document Ops, Runs, Search, Users (S ### Runtime properties - API + worker: **same image**, different entrypoint -- Shared SQLite volume (single-writer lock + WAL) -- Horizontal scale: Temporal workers scale out; SQLite remains single-primary (acceptable for ops volume; upgrade path = Postgres if needed) +- Shared Postgres database (pooled connections per process; schema applied from `migrations/` at startup) +- Horizontal scale: Temporal workers and API replicas scale out against the same database --- @@ -408,7 +408,7 @@ Routes: Dashboard, Documents, New Document, Document Ops, Runs, Search, Users (S | Concern | Design | |---|---| -| **Consistency** | SQLite SoT; Temporal for execution; search is eventual projection | +| **Consistency** | Postgres SoT; Temporal for execution; search is eventual projection | | **Idempotency** | Workflow ids; reingest overwrites by document id | | **Observability** | Runs UI, Temporal UI, audit_logs, stage mirror | | **Security** | JWT, least privilege, tenant 404, Super Admin prod gate | @@ -421,7 +421,7 @@ Routes: Dashboard, Documents, New Document, Document Ops, Runs, Search, Users (S ## 15. Threat & trust boundaries ```text -[ Browser ] --JWT--> [ API ] --internal net--> [ Worker / Temporal / MinIO / SQLite ] +[ Browser ] --JWT--> [ API ] --internal net--> [ Worker / Temporal / MinIO / Postgres ] | +--JWKS--> [ Keycloak ] +--HTTP--> [ Qdrant DEV ] @@ -439,7 +439,7 @@ Routes: Dashboard, Documents, New Document, Document Ops, Runs, Search, Users (S | Attribute | Target approach | |---|---| | Availability | Compose restart policies; Temporal retries | -| Durability | Workflow history + SQLite + MinIO objects | +| Durability | Workflow history + Postgres + MinIO objects | | Latency | Search interactive; OCR batch (minutes) | | Isolation | Per-state `instance` + Keycloak groups | | Auditability | append-only audit + Temporal history | @@ -450,7 +450,7 @@ Routes: Dashboard, Documents, New Document, Document Ops, Runs, Search, Users (S ## 17. Evolution / extension points 1. Explicit **“Request PROD approval”** action (today: auto-enter queue after DEV). -2. SQLite → Postgres if multi-writer scale is needed. +2. Moving `chunk_count` / `is_demo` out of `documents.metadata` into columns if list queries slow down. 3. Stricter per-state role on every mutation (not only global permission). 4. Automated quality checks before Super Admin sees queue. diff --git a/docs/architecture-pipeline-rbac.md b/docs/architecture-pipeline-rbac.md index 4495e9e..8721c8e 100644 --- a/docs/architecture-pipeline-rbac.md +++ b/docs/architecture-pipeline-rbac.md @@ -43,7 +43,7 @@ flowchart TB end subgraph DataPlane - SQLite["SQLite
documents, pages, chunks,
jobs, audit"] + Postgres["Postgres
documents, pages, chunks,
jobs, audit"] MinIO["MinIO / S3
source PDFs"] QdrantDev["Qdrant DEV
search index"] QdrantProd["Qdrant PROD
search index"] @@ -59,11 +59,11 @@ flowchart TB Browser -->|"OIDC PKCE login"| KC Browser -->|"Bearer JWT + /api/*"| API API -->|"validate JWT / JWKS"| KC - API --> SQLite + API --> Postgres API --> MinIO API -->|"start / signal workflows"| Temporal Temporal --> Worker - Worker --> SQLite + Worker --> Postgres Worker --> MinIO Worker --> OCR Worker --> TR @@ -81,7 +81,7 @@ flowchart TB | **Keycloak** | Source of truth for *who* is Super Admin vs which **state + role** a user has | | **JWT → API** | Map groups/roles → `permissions`, `instances`, `state_roles`; enforce tenancy + capability | | **Temporal workflow** | Durable pipeline with human gates (signals) | -| **SQLite** | System of record for document stage, pages, chunks, uploader, `instance` | +| **Postgres** | System of record for document stage, pages, chunks, uploader, `instance` | | **Qdrant DEV** | Searchable vectors after state-approved ingestion | | **Qdrant PROD** | Production vectors only after Super Admin `approve_prod` | @@ -213,8 +213,8 @@ stateDiagram-v2 | Stage | Who advances it | Effect | |---|---|---| -| Upload → Registered | Contributor / Super Admin (`upload`) | File in MinIO; SQLite row; `instance` set | -| OCR / Translation / Chunking | System (worker activities) | Pages/chunks in SQLite | +| Upload → Registered | Contributor / Super Admin (`upload`) | File in MinIO; Postgres row; `instance` set | +| OCR / Translation / Chunking | System (worker activities) | Pages/chunks in Postgres | | OCR / Translation / Chunk / **Pre-ingestion** approve | State **review** (+ pipeline as needed) or Super Admin | Temporal signal continues workflow | | **Ingesting in Dev** | System after `approve_ingestion` | `ingest_document_from_db` → **Qdrant DEV** | | **Approval for Prod** (queue) | Document waits | Still **DEV-only** in search until promote | @@ -355,14 +355,14 @@ flowchart LR ```mermaid flowchart TB PDF[Source PDF in MinIO] - SQLITE[(SQLite: pages + chunks
+ instance + uploader)] + PGDB[(Postgres: pages + chunks
+ instance + uploader)] DEV[(Qdrant DEV collection)] PROD[(Qdrant PROD collection)] - PDF --> SQLITE - SQLITE -->|"ingest_document_from_db
after approve_ingestion"| DEV + PDF --> PGDB + PGDB -->|"ingest_document_from_db
after approve_ingestion"| DEV DEV -->|"promote_document_to_prod_qdrant
after Super Admin approve_prod"| PROD - SQLITE -.->|"stage mirror"| SQLITE + PGDB -.->|"stage mirror"| PGDB ``` - **DEV ingest** activity: `ingest_document_from_db` (worker). diff --git a/docs/openapi-search.yaml b/docs/openapi-search.yaml index d9c578e..5115ef8 100644 --- a/docs/openapi-search.yaml +++ b/docs/openapi-search.yaml @@ -9,7 +9,7 @@ info: `pipeline/models.py`, and `pipeline/vector_store/qdrant_store.py`. Not the full API surface — only `/search` (vector/hybrid semantic search) - and `/chunks/search` (SQLite keyword search across chunk records). + and `/chunks/search` (keyword search across chunk records). version: "1.0.0" contact: name: knowledge-provider (docs-pipeline) @@ -83,9 +83,9 @@ paths: /chunks/search: get: operationId: searchChunksAcrossDocuments - summary: Keyword search across chunk records (SQLite), for KB maintainer workflows + summary: Keyword search across chunk records (Postgres), for KB maintainer workflows description: | - Distinct from `/search`: this queries chunk rows directly in SQLite by + Distinct from `/search`: this queries chunk rows directly in Postgres by substring/keyword match, not the vector index. Results are scoped to the caller's instance(s) unless the caller is an unrestricted admin. tags: [search] @@ -468,7 +468,7 @@ components: type: array items: type: object - description: Raw chunk record from SQLite (db.search_chunks); shape follows the chunks table, not formally typed here. + description: Raw chunk record from Postgres (db.search_chunks); shape follows the chunks table, not formally typed here. additionalProperties: true total: type: integer From f285438ee0ffb316a022e081acff09686a727670 Mon Sep 17 00:00:00 2001 From: Naveen Date: Tue, 27 Oct 2026 16:45:19 +0530 Subject: [PATCH 33/53] feat: index advisory records into Qdrant for advisory search Port of docs-pipeline PR #171 onto the Postgres-backed knowledge-provider. - POST /webhook starts one AdvisoryIndexWorkflow per new or changed record; chunks (document_kind=advisory) go to the Qdrant collection - admin endpoints: /webhook/advisory/{reindex,promote,status} - AdvisoryPromoteWorkflow copies to the PROD collection (PROD_VECTOR_DB_*) - migration 0002 adds webhook_records.indexed_hash / promoted_hash - advisory payload fields on Qdrant points; embedding token counter - ADVISORY_BACKFILL_ON_START queues unindexed rows at worker startup - migration-runner tests use version 0099 as their probe --- ENV.md | 3 + .../0002_webhook_advisory_index_state.sql | 3 + pipeline/activities.py | 49 ++++ pipeline/advisory_chunks.py | 154 +++++++++++++ pipeline/advisory_index.py | 84 +++++++ pipeline/advisory_start.py | 58 +++++ pipeline/api/api.py | 84 ++++++- pipeline/db.py | 124 +++++++++- pipeline/vector_store/embeddings.py | 12 + pipeline/vector_store/qdrant_store.py | 16 ++ pipeline/worker.py | 13 ++ pipeline/workflows.py | 58 +++++ tests/test_activities.py | 29 +++ tests/test_advisory_activities.py | 126 ++++++++++ tests/test_advisory_api.py | 165 +++++++++++++ tests/test_advisory_chunks.py | 218 ++++++++++++++++++ tests/test_advisory_index.py | 152 ++++++++++++ tests/test_advisory_start.py | 98 ++++++++ tests/test_db.py | 10 +- tests/test_embeddings_count_tokens.py | 24 ++ tests/test_webhook_index_db.py | 107 +++++++++ 21 files changed, 1569 insertions(+), 18 deletions(-) create mode 100644 migrations/0002_webhook_advisory_index_state.sql create mode 100644 pipeline/advisory_chunks.py create mode 100644 pipeline/advisory_index.py create mode 100644 pipeline/advisory_start.py create mode 100644 tests/test_advisory_activities.py create mode 100644 tests/test_advisory_api.py create mode 100644 tests/test_advisory_chunks.py create mode 100644 tests/test_advisory_index.py create mode 100644 tests/test_advisory_start.py create mode 100644 tests/test_embeddings_count_tokens.py create mode 100644 tests/test_webhook_index_db.py diff --git a/ENV.md b/ENV.md index 0e6c601..c8ffacb 100644 --- a/ENV.md +++ b/ENV.md @@ -88,6 +88,9 @@ Root `.env` (see `.env.example`). Required by FastAPI (`pipeline/api/`), Tempora |----------|---------|---------| | `WEBHOOK_MAX_RECORDS` | `500` | Max records per `/webhook` request; more returns 413 | | `WEBHOOK_API_KEY` | *(empty — required)* | 32-char hex secret. Callers send it in the `x-api-key` header. Missing/invalid header → 401. Unset/empty → 503 (fails closed, ignores `AUTH_DISABLED`). Generate with `openssl rand -hex 16`. | +| `ADVISORY_BACKFILL_ON_START` | `false` | When `true`, the worker queues an index workflow for every unindexed `webhook_records` row once at startup (backfill of existing records). Leave off normally; `POST /webhook/advisory/reindex` does the same on demand. | + +Each saved record whose `payload_hash` differs from its `indexed_hash` also starts one Temporal workflow `advisory-` that writes advisory chunks (`document_kind=advisory`) to the Qdrant collection `VECTOR_DB_COLLECTION_NAME`. Admin endpoints (Keycloak admin): `POST /webhook/advisory/reindex`, `POST /webhook/advisory/promote` (copies to `PROD_VECTOR_DB_COLLECTION_NAME` (on `PROD_VECTOR_DB_URL`)), `GET /webhook/advisory/status`. Body is a JSON array of objects (max `WEBHOOK_MAX_RECORDS`); stored in the `webhook_records` table, upserted on `(source, external_id)` where `external_id` is `contentID` (else `id`, `slug`, payload hash). diff --git a/migrations/0002_webhook_advisory_index_state.sql b/migrations/0002_webhook_advisory_index_state.sql new file mode 100644 index 0000000..328fc93 --- /dev/null +++ b/migrations/0002_webhook_advisory_index_state.sql @@ -0,0 +1,3 @@ +-- Advisory indexing state: the payload_hash last written to the dev / PROD Qdrant collection. +ALTER TABLE webhook_records ADD COLUMN indexed_hash TEXT; +ALTER TABLE webhook_records ADD COLUMN promoted_hash TEXT; diff --git a/pipeline/activities.py b/pipeline/activities.py index d5a2cdb..0364884 100644 --- a/pipeline/activities.py +++ b/pipeline/activities.py @@ -705,6 +705,7 @@ def _prepare_records( "instance_name": instance_name, "provider_id": provider_id, "type": "scheme" if is_scheme else "document", + "document_kind": kind, "source": "docs-pipeline", "filename": external_slug, "name_gu": name_gu, @@ -1672,3 +1673,51 @@ async def detect_and_translate_pages_from_db( }, ) return {"page_count": len(translated), "translated_count": translated_count} + + +async def _run_advisory_activity(record_id: str, promote: bool) -> dict: + """Index (dev) or promote (PROD) one webhook advisory record; shared by both activities.""" + from . import advisory_index + from .vector_store import get_default_index_name, get_vector_store + from .vector_store.embeddings import count_tokens + + also_delete_from = None + if promote: + target = advisory_index.prod_target() + if target is None: + raise RuntimeError("PROD_VECTOR_DB_URL is required to promote advisory to prod") + store, collection = target + mark = "promoted" + else: + store, collection = get_vector_store(), get_default_index_name() + mark = "indexed" + try: + # An unpublished record must also disappear from PROD (deletion only). + also_delete_from = advisory_index.prod_target() + except Exception as exc: + activity.logger.warning("PROD target unavailable for advisory cleanup: %s", exc) + + # Embedding is CPU-bound and synchronous: keep it off the event loop. + result = await asyncio.to_thread( + advisory_index.index_advisory_record, + record_id, + store, + collection, + count_tokens, + mark, + also_delete_from, + ) + advisory_index.log_advisory_audit(record_id, mark, collection, result) + return result + + +@activity.defn +async def index_webhook_advisory(record_id: str) -> dict: + """Write one webhook advisory record's chunks to the dev collection (or delete if unpublished).""" + return await _run_advisory_activity(record_id, promote=False) + + +@activity.defn +async def promote_webhook_advisory(record_id: str) -> dict: + """Copy one webhook advisory record's chunks to the PROD collection (or delete if unpublished).""" + return await _run_advisory_activity(record_id, promote=True) diff --git a/pipeline/advisory_chunks.py b/pipeline/advisory_chunks.py new file mode 100644 index 0000000..2cd248d --- /dev/null +++ b/pipeline/advisory_chunks.py @@ -0,0 +1,154 @@ +"""Turn a published advisory ``webhook_records`` row into Qdrant chunk records. + +One advisory entry (``popList`` item) becomes one chunk when its whole text fits the +embedding model's token limit, otherwise up to three chunks that each carry the advice +(overview + advice, symptoms + advice, conditions + advice). The stored ``text`` is +always the complete chunk text; the embedding model reads only its first tokens. +""" + +from __future__ import annotations + +import hashlib +from typing import Callable + +MAX_EMBEDDING_TOKENS = 512 + +# (section, label) for the question-side fields, in display order. +_FIELD_SECTIONS = ( + ("overview", "Overview"), + ("symptoms", "Symptoms"), + ("conditions", "Conditions"), +) + + +def clean_value(value: object) -> str: + """Return a trimmed string with surrounding quote characters removed ('' for None).""" + if value is None: + return "" + return str(value).strip().strip('"').strip() + + +def _clean_list(value: object) -> str: + """Join the non-empty cleaned items of a list (or a single value) with spaces.""" + if value is None: + return "" + items = value if isinstance(value, (list, tuple)) else [value] + return " ".join(part for part in (clean_value(item) for item in items) if part) + + +def _is_published(row: dict) -> bool: + return row.get("is_published") in (1, True) and clean_value(row.get("status")) == "Published" + + +def _header(title: str, crop: str, problem_type: str, season: str) -> list[str]: + lines = [title] if title else [] + parts = [] + if crop: + parts.append(f"Crop: {crop}") + if problem_type: + parts.append(f"Problem type: {problem_type}") + if season: + parts.append(f"Season: {season}") + if parts: + lines.append(" | ".join(parts)) + return lines + + +def _chunk_id(doc_id: str, pop_id: object, section: str) -> str: + return hashlib.md5(f"{doc_id}:{pop_id}:{section}".encode("utf-8")).hexdigest() + + +def build_chunks( + row: dict, + count_tokens: Callable[[str], int], + max_tokens: int = MAX_EMBEDDING_TOKENS, +) -> list[dict]: + """Build Qdrant records for one ``webhook_records`` row (empty list if unpublished).""" + if not _is_published(row): + return [] + payload = row.get("payload") or {} + pop_list = payload.get("popList") + if not isinstance(pop_list, list): + return [] + + doc_id = str(row["id"]) + title = clean_value(row.get("title")) or clean_value(payload.get("title")) + crop = clean_value(payload.get("commodityName")) + base = { + "doc_id": doc_id, + "workflow_id": "", + "type": "document", + "document_kind": "advisory", + "source": "webhook", + "title_en": title, + "slug": clean_value(row.get("slug")), + "external_id": clean_value(row.get("external_id")), + "crop": crop, + "domain": clean_value(payload.get("domainName")), + "subdomain": clean_value(payload.get("subdomainName")), + "institute": clean_value(payload.get("instituteName")), + "creator": clean_value(payload.get("creatorName")), + "updated_at": clean_value(row.get("updated_at")), + } + + records: list[dict] = [] + + def add(pop: dict, index: int, section: str, text: str, meta: dict) -> None: + pop_id = pop.get("popID") + key = pop_id if pop_id is not None else f"idx{index}" + chunk_id = _chunk_id(doc_id, key, section) + records.append( + { + **base, + **meta, + "_id": chunk_id, + "chunk_id": chunk_id, + "pop_id": pop_id, + "section": section, + "chunk_num": len(records), + "text": text, + "text_for_embedding": text, + } + ) + + for index, pop in enumerate(pop_list): + if not isinstance(pop, dict) or clean_value(pop.get("status")) != "Published": + continue + problem_type = clean_value(pop.get("problemType")) + season = clean_value(pop.get("season")) + meta = { + "problem_type": problem_type, + "problem": clean_value(pop.get("problemDetails")), + "season": season, + "start_period": clean_value(pop.get("startPeriod")), + "end_period": clean_value(pop.get("endPeriod")), + } + header = _header(title, crop, problem_type, season) + fields = { + "overview": clean_value(pop.get("description")), + "symptoms": _clean_list(pop.get("applicableSymptoms")), + "conditions": _clean_list(pop.get("applicableCondition")), + } + advisory = clean_value(pop.get("advisory")) + labelled = [(label, fields[section]) for section, label in _FIELD_SECTIONS if fields[section]] + if advisory: + labelled.append(("Advisory", advisory)) + if not labelled: + continue + + full_text = "\n".join(header + [f"{label}: {body}" for label, body in labelled]) + if count_tokens(full_text) <= max_tokens: + add(pop, index, "full", full_text, meta) + continue + + others = [(section, label) for section, label in _FIELD_SECTIONS if fields[section]] + if not others: + add(pop, index, "advisory", full_text, meta) + continue + for section, label in others: + lines = header + [f"{label}: {fields[section]}"] + if advisory: + lines.append(f"Advisory: {advisory}") + add(pop, index, section, "\n".join(lines), meta) + + return records diff --git a/pipeline/advisory_index.py b/pipeline/advisory_index.py new file mode 100644 index 0000000..c91f2f3 --- /dev/null +++ b/pipeline/advisory_index.py @@ -0,0 +1,84 @@ +"""Index (or remove) one advisory ``webhook_records`` row in a vector store collection.""" + +from __future__ import annotations + +import logging +import os +from typing import Callable, Literal, Optional + +from . import db +from .advisory_chunks import build_chunks + +logger = logging.getLogger(__name__) + + +def _delete_points(store, collection: str, doc_id: str) -> None: + """Delete a record's points; a missing collection is fine, any other failure is not.""" + result = store.delete_by_doc_id(collection, doc_id) or {} + error = str(result.get("error") or "") + if error and "doesn't exist" not in error and "not found" not in error.lower(): + raise RuntimeError(f"Could not delete points for {doc_id} from {collection}: {error}") + + +def index_advisory_record( + record_id: str, + store, + collection: str, + count_tokens: Callable[[str], int], + mark: Literal["indexed", "promoted"] = "indexed", + also_delete_from: Optional[tuple[object, str]] = None, +) -> dict: + """Replace a record's points in ``collection`` with its current chunks. + + Order: read row, delete the record's old points, write the new chunks, then store the + row's payload_hash (``indexed_hash`` or ``promoted_hash``). An unpublished record (no + chunks) only gets its points deleted; ``also_delete_from=(store, collection)`` removes + them from a second collection too (deletion only, never a content push). + """ + row = db.get_webhook_record(record_id) + if row is None: + return {"record_id": record_id, "status": "missing", "chunks": 0} + + doc_id = str(row["id"]) + chunks = build_chunks(row, count_tokens) + _delete_points(store, collection, doc_id) + if chunks: + store.upsert(collection, chunks, batch_size=8) + elif also_delete_from is not None: + other_store, other_collection = also_delete_from + _delete_points(other_store, other_collection, doc_id) + + marker = db.mark_webhook_record_promoted if mark == "promoted" else db.mark_webhook_record_indexed + marker(record_id, row["payload_hash"]) + logger.info("advisory %s record=%s collection=%s chunks=%s", mark, record_id, collection, len(chunks)) + return {"record_id": record_id, "status": "indexed" if chunks else "deleted", "chunks": len(chunks)} + + +def prod_target() -> Optional[tuple[object, str]]: + """The PROD (store, collection) pair from the PROD_VECTOR_DB_* env, or None when not configured.""" + url = (os.environ.get("PROD_VECTOR_DB_URL") or "").strip() + if not url: + return None + from .vector_store.qdrant_store import QdrantVectorStore, get_qdrant_client + + api_key = (os.environ.get("PROD_VECTOR_DB_API_KEY") or "").strip() or None + collection = ( + os.environ.get("PROD_VECTOR_DB_COLLECTION_NAME") + or os.environ.get("VECTOR_DB_COLLECTION_NAME") + or "documents-index" + ).strip() + return QdrantVectorStore(client=get_qdrant_client(url=url, api_key=api_key)), collection + + +def log_advisory_audit(record_id: str, mark: str, collection: str, result: dict) -> None: + """Write one audit_logs row for an indexed / deleted / promoted advisory record.""" + action = "advisory_promoted" if mark == "promoted" else ( + "advisory_deleted" if result.get("status") == "deleted" else "advisory_indexed" + ) + db.log_audit( + workflow_id=f"advisory-{record_id}", + document_id=record_id, + action_type=action, + entity_type="webhook_record", + metadata={"collection": collection, "chunks": result.get("chunks", 0), "status": result.get("status")}, + ) diff --git a/pipeline/advisory_start.py b/pipeline/advisory_start.py new file mode 100644 index 0000000..cc3ef07 --- /dev/null +++ b/pipeline/advisory_start.py @@ -0,0 +1,58 @@ +"""Start advisory index / promote workflows: one Temporal workflow per webhook record.""" + +from __future__ import annotations + +import logging +import os + +from temporalio.exceptions import WorkflowAlreadyStartedError + +from .workflows import AdvisoryIndexWorkflow, AdvisoryPromoteWorkflow + +logger = logging.getLogger(__name__) + +TASK_QUEUE = "ocr-pipeline" + + +async def _start_each(client, workflow_run, id_prefix: str, record_ids: list[str]) -> int: + started = 0 + for record_id in record_ids: + try: + await client.start_workflow( + workflow_run, + args=[record_id], + id=f"{id_prefix}{record_id}", + task_queue=TASK_QUEUE, + ) + started += 1 + except WorkflowAlreadyStartedError: + logger.info("advisory workflow %s%s already running; skipped", id_prefix, record_id) + except Exception: + logger.exception("could not start advisory workflow %s%s", id_prefix, record_id) + return started + + +async def start_advisory_index(client, record_ids: list[str]) -> int: + """Start ``AdvisoryIndexWorkflow`` (id ``advisory-``) for each id; returns how many started.""" + return await _start_each(client, AdvisoryIndexWorkflow.run, "advisory-", record_ids) + + +async def start_advisory_promote(client, record_ids: list[str]) -> int: + """Start ``AdvisoryPromoteWorkflow`` (id ``advisory-promote-``) for each id.""" + return await _start_each(client, AdvisoryPromoteWorkflow.run, "advisory-promote-", record_ids) + + +def _backfill_enabled() -> bool: + return os.environ.get("ADVISORY_BACKFILL_ON_START", "false").strip().lower() in {"1", "true", "yes", "on"} + + +async def backfill_on_start(client) -> int: + """Worker startup hook: when ADVISORY_BACKFILL_ON_START is on (default off), queue every unindexed row.""" + if not _backfill_enabled(): + return 0 + from . import db + + record_ids = db.get_unindexed_webhook_record_ids() + started = await start_advisory_index(client, record_ids) + logger.info("advisory startup backfill: %s unindexed rows, %s workflows started", len(record_ids), started) + return started diff --git a/pipeline/api/api.py b/pipeline/api/api.py index 9507152..a471898 100644 --- a/pipeline/api/api.py +++ b/pipeline/api/api.py @@ -41,6 +41,7 @@ from .. import db, scheme_catalog from .. import display_name as display_name_rules +from ..advisory_start import start_advisory_index, start_advisory_promote from ..auth.config import load_auth_config, validate_auth_config from ..auth.deps import ( CurrentUser, @@ -121,6 +122,8 @@ datefmt='%H:%M:%S', ) +logger = logging.getLogger(__name__) + TASK_QUEUE = "ocr-pipeline" # Global clients @@ -249,6 +252,11 @@ async def lifespan(app: FastAPI): RATE_LIMIT_OTP = os.environ.get("RATE_LIMIT_OTP", "10/minute") _DOCUMENT_PATH_RE = re.compile(r"^/documents/([^/]+)(/.*)?$") +# Only identifier shapes with an indexed lookup are resolved before auth runs: +# the document UUID, the 32-hex fingerprint, or a ``doc-`` workflow id. +_RESOLVABLE_IDENTIFIER_RE = re.compile( + r"^(?:[0-9a-fA-F]{8}-(?:[0-9a-fA-F]{4}-){3}[0-9a-fA-F]{12}|[0-9a-f]{32}|doc-[0-9a-f]{32}.*)$" +) _static_document_segments: Optional[set[str]] = None @@ -271,10 +279,15 @@ async def canonicalize_document_identifier(request: Request, call_next): Handlers address a document by its Temporal workflow id (they signal and query Temporal with it), so the path is rewritten to that id before routing. - An identifier that matches no document is left alone and 404s in the handler. + An identifier of another shape, or one that matches no document, is left alone and + 404s in the handler; the lookup runs before auth, so it is limited to cheap shapes. """ match = _DOCUMENT_PATH_RE.match(request.scope["path"]) - if match and match.group(1) not in _static_document_path_segments(): + if ( + match + and _RESOLVABLE_IDENTIFIER_RE.match(match.group(1)) + and match.group(1) not in _static_document_path_segments() + ): document = await asyncio.to_thread(db.find_document_by_doc_identifier, match.group(1)) if document and document["workflow_id"] and document["workflow_id"] != match.group(1): new_path = f"/documents/{document['workflow_id']}{match.group(2) or ''}" @@ -3364,6 +3377,7 @@ async def webhook( count = db.upsert_webhook_records("webhook", records) except ValueError as e: raise HTTPException(422, str(e)) + await _start_advisory_index_for(records) return { "success": True, "message": "Webhook processed successfully.", @@ -3371,6 +3385,72 @@ async def webhook( } +async def _start_advisory_index_for(records: list) -> int: + """Start one advisory index workflow per record of this call that is not indexed yet. + + Best effort: the records are already saved, so a Temporal failure never fails the + webhook; the rows stay unindexed and the reindex endpoint (or worker startup backfill) + picks them up. + """ + if temporal_client is None: + logger.warning("Temporal client not ready; advisory indexing skipped for this webhook call") + return 0 + try: + external_ids = [db.webhook_external_id(r) for r in records if isinstance(r, dict)] + record_ids = db.get_unindexed_webhook_record_ids(external_ids=external_ids) + return await start_advisory_index(temporal_client, record_ids) + except Exception: + logger.exception("Could not start advisory indexing for this webhook call") + return 0 + + +class AdvisoryReindexBody(BaseModel): + record_ids: Optional[List[str]] = None + force: bool = False + + +class AdvisoryPromoteBody(BaseModel): + record_ids: Optional[List[str]] = None + + +def _require_temporal() -> Client: + if temporal_client is None: + raise HTTPException(503, "Temporal client is not ready") + return temporal_client + + +@app.post("/webhook/advisory/reindex") +async def advisory_reindex(body: AdvisoryReindexBody, user: RequireAdmin): + """Queue advisory indexing (dev collection): given record ids, else all unindexed (all with force).""" + client = _require_temporal() + if body.record_ids is not None: + record_ids = body.record_ids + elif body.force: + record_ids = db.get_webhook_record_ids() + else: + record_ids = db.get_unindexed_webhook_record_ids() + queued = await start_advisory_index(client, record_ids) + return {"success": True, "queued": queued} + + +@app.post("/webhook/advisory/promote") +async def advisory_promote(body: AdvisoryPromoteBody, user: RequireAdmin): + """Queue promotion to the PROD collection: given record ids, else all published records.""" + client = _require_temporal() + if body.record_ids is not None: + record_ids = body.record_ids + else: + record_ids = db.get_webhook_record_ids(published_only=True) + queued = await start_advisory_promote(client, record_ids) + return {"success": True, "queued": queued} + + +@app.get("/webhook/advisory/status") +async def advisory_status(user: RequireAdmin): + """Counts of webhook advisory records: total, published, indexed, unindexed, promoted.""" + return db.advisory_status_counts() + + # ============================================================================= # Vector index (passage schema) # ============================================================================= diff --git a/pipeline/db.py b/pipeline/db.py index 0799b44..6c5cfe9 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -110,8 +110,9 @@ def check_health() -> dict: with get_connection() as conn: conn.execute("SELECT 1") return {"reachable": True, "migrations_current": migrate.is_current(database_url())} - except Exception as exc: - return {"reachable": False, "migrations_current": False, "error": str(exc)} + except Exception: + logger.exception("database health check failed") + return {"reachable": False, "migrations_current": False} # ============================================================================= @@ -480,6 +481,21 @@ def _parse_timestamp(value: Any) -> Optional[datetime]: return parsed +def _webhook_payload_and_hash(rec: dict) -> tuple[str, str]: + payload = json.dumps(rec, sort_keys=True, ensure_ascii=False, default=str) + return payload, hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def webhook_external_id(rec: dict) -> str: + """The (source, external_id) key of a webhook record: contentID, else id, else slug, else payload hash.""" + _, payload_hash = _webhook_payload_and_hash(rec) + for key in ("contentID", "id", "slug"): + value = rec.get(key) + if value not in (None, ""): + return str(value) + return payload_hash + + def upsert_webhook_records(source: str, records: list) -> int: """Upsert records by (source, external_id) in one transaction; returns len(records). @@ -491,15 +507,8 @@ def upsert_webhook_records(source: str, records: list) -> int: for rec in records: if not isinstance(rec, dict): raise ValueError("Each record must be a JSON object") - payload_text = json.dumps(rec, sort_keys=True, ensure_ascii=False, default=str) - payload_hash = hashlib.sha256(payload_text.encode("utf-8")).hexdigest() - ext = rec.get("contentID") - if ext in (None, ""): - ext = rec.get("id") - if ext in (None, ""): - ext = rec.get("slug") - if ext in (None, ""): - ext = payload_hash + payload_text, payload_hash = _webhook_payload_and_hash(rec) + ext = webhook_external_id(rec) pub = rec.get("isPublished") rows.append(( source, str(ext), rec.get("title"), rec.get("slug"), rec.get("status"), @@ -533,6 +542,99 @@ def upsert_webhook_records(source: str, records: list) -> int: return len(rows) +_WEBHOOK_RECORD_COLUMNS = ( + "id, source, external_id, title, slug, status, is_published, category, payload, " + "payload_hash, created_at, updated_at, indexed_hash, promoted_hash" +) + + +def get_webhook_record(record_id: str) -> Optional[dict]: + """One webhook_records row as a dict (``payload`` already parsed from JSONB); None if missing.""" + if not _is_uuid(record_id): + return None + with get_connection() as conn: + row = conn.execute( + f"SELECT {_WEBHOOK_RECORD_COLUMNS} FROM webhook_records WHERE id = %s", (record_id,) + ).fetchone() + if row is None: + return None + record = dict(row) + record["id"] = str(record["id"]) + if not isinstance(record["payload"], dict): + record["payload"] = {} + return record + + +def get_unindexed_webhook_record_ids( + source: str = "webhook", external_ids: Optional[list] = None +) -> list[str]: + """Ids of rows whose payload changed since they were last indexed (or never indexed). + + ``external_ids`` limits the check to the records of one webhook call. + """ + sql = ( + "SELECT id FROM webhook_records WHERE source = %s " + "AND indexed_hash IS DISTINCT FROM payload_hash" + ) + params: list = [source] + if external_ids is not None: + if not external_ids: + return [] + sql += " AND external_id = ANY(%s)" + params.append([str(e) for e in external_ids]) + with get_connection() as conn: + return [str(r["id"]) for r in conn.execute(sql + " ORDER BY created_at, id", params)] + + +def get_webhook_record_ids(source: str = "webhook", published_only: bool = False) -> list[str]: + """All record ids for a source, optionally only published ones.""" + sql = "SELECT id FROM webhook_records WHERE source = %s" + if published_only: + sql += " AND is_published AND status = 'Published'" + with get_connection() as conn: + return [str(r["id"]) for r in conn.execute(sql + " ORDER BY created_at, id", (source,))] + + +def _mark_webhook_record(column: str, record_id: str, payload_hash: str) -> None: + with get_connection() as conn: + conn.execute(f"UPDATE webhook_records SET {column} = %s WHERE id = %s", (payload_hash, record_id)) + + +def mark_webhook_record_indexed(record_id: str, payload_hash: str) -> None: + """Record the payload_hash that was written to the dev collection.""" + _mark_webhook_record("indexed_hash", record_id, payload_hash) + + +def mark_webhook_record_promoted(record_id: str, payload_hash: str) -> None: + """Record the payload_hash that was written to the PROD collection.""" + _mark_webhook_record("promoted_hash", record_id, payload_hash) + + +def advisory_status_counts(source: str = "webhook") -> dict: + """Counts for the advisory status endpoint.""" + with get_connection() as conn: + row = conn.execute( + """ + SELECT + COUNT(*) AS total, + COUNT(*) FILTER (WHERE is_published AND status = 'Published') AS published, + COUNT(*) FILTER (WHERE indexed_hash = payload_hash) AS indexed, + COUNT(*) FILTER (WHERE promoted_hash = payload_hash) AS promoted + FROM webhook_records WHERE source = %s + """, + (source,), + ).fetchone() + total = int(row["total"]) + indexed = int(row["indexed"]) + return { + "total": total, + "published": int(row["published"]), + "indexed": indexed, + "unindexed": total - indexed, + "promoted": int(row["promoted"]), + } + + # ============================================================================= # Email OTPs # ============================================================================= diff --git a/pipeline/vector_store/embeddings.py b/pipeline/vector_store/embeddings.py index be9a30f..bc9feec 100644 --- a/pipeline/vector_store/embeddings.py +++ b/pipeline/vector_store/embeddings.py @@ -143,3 +143,15 @@ def embed_passages( batch = passages[i : i + batch_size] out.extend(embed_texts(batch, kind="passage", use_e5_prefix=use_e5_prefix)) return out + + +def count_tokens(text: str) -> int: + """Token count of ``text`` as the embedding model sees it (with the passage prefix). + + Uses the local model's tokenizer when embeddings run locally; for a remote + (OpenAI-compatible) provider there is no tokenizer here, so it estimates ~4 chars/token. + """ + prefixed = _prefix_passage(text) + if get_embedding_provider() == "openai_compatible": + return max(1, len(prefixed) // 4) + return len(_get_local_embedder().tokenizer.encode(prefixed)) diff --git a/pipeline/vector_store/qdrant_store.py b/pipeline/vector_store/qdrant_store.py index d2b7018..e89abe2 100644 --- a/pipeline/vector_store/qdrant_store.py +++ b/pipeline/vector_store/qdrant_store.py @@ -54,6 +54,22 @@ "scheme_aliases", "chunk_id", "chunk_index", + # Advisory payload (document_kind=advisory; webhook records and PDF advisory documents) + "document_kind", + "external_id", + "pop_id", + "slug", + "crop", + "domain", + "subdomain", + "institute", + "creator", + "problem_type", + "problem", + "season", + "start_period", + "end_period", + "updated_at", # Document validity - the period this chunk is searchable in. Absent on # points written before validity existed, which search treats as current. "start_date", diff --git a/pipeline/worker.py b/pipeline/worker.py index 6101442..23b5faa 100644 --- a/pipeline/worker.py +++ b/pipeline/worker.py @@ -16,18 +16,23 @@ create_chunks_from_db, detect_and_translate_pages, detect_and_translate_pages_from_db, + index_webhook_advisory, ingest_document_from_db, ingest_to_vector_db, persist_document_content, prepare_for_ingestion, promote_document_to_prod_qdrant, + promote_webhook_advisory, publish_catalog_to_network, run_ocr, run_ocr_and_store, update_document_state, ) +from .advisory_start import backfill_on_start from .network_group import validate_network_provider_config from .workflows import ( + AdvisoryIndexWorkflow, + AdvisoryPromoteWorkflow, ChunkingOnlyWorkflow, DocumentPipelineWorkflow, OcrOnlyWorkflow, @@ -84,6 +89,8 @@ async def main(): TranslationOnlyWorkflow, OcrOnlyWorkflow, ChunkingOnlyWorkflow, + AdvisoryIndexWorkflow, + AdvisoryPromoteWorkflow, ], activities=[ run_ocr, @@ -95,6 +102,8 @@ async def main(): ingest_to_vector_db, ingest_document_from_db, promote_document_to_prod_qdrant, + index_webhook_advisory, + promote_webhook_advisory, publish_catalog_to_network, update_document_state, detect_and_translate_pages, @@ -103,6 +112,10 @@ async def main(): ], ) + backfilled = await backfill_on_start(client) + if backfilled: + print(f"Advisory startup backfill queued {backfilled} workflows") + print(f"Worker started on queue: {TASK_QUEUE}") print("Waiting for workflows...") await worker.run() diff --git a/pipeline/workflows.py b/pipeline/workflows.py index d6ab900..4ce63fb 100644 --- a/pipeline/workflows.py +++ b/pipeline/workflows.py @@ -17,8 +17,10 @@ auto_tag_chunks_from_db, create_chunks_from_db, detect_and_translate_pages_from_db, + index_webhook_advisory, ingest_document_from_db, promote_document_to_prod_qdrant, + promote_webhook_advisory, publish_catalog_to_network, run_ocr_and_store, update_document_state, @@ -887,3 +889,59 @@ def get_state(self) -> dict: "chunk_count": self.state.chunk_count, "chunks_completed_at": self.state.chunks_completed_at, } + + +@workflow.defn +class AdvisoryIndexWorkflow: + """Index one webhook advisory record into the dev Qdrant collection.""" + + def __init__(self): + self.state = None + + @workflow.run + async def run(self, record_id: str) -> dict: + self.state = {"record_id": record_id, "stage": "indexing"} + try: + result = await workflow.execute_activity( + index_webhook_advisory, + record_id, + start_to_close_timeout=timedelta(minutes=30), + retry_policy=INGEST_RETRY, + ) + except Exception as e: + self.state.update(stage="failed", error_message=str(e)) + raise + self.state.update(stage="completed", status=result.get("status"), chunks=result.get("chunks")) + return result + + @workflow.query + def get_state(self) -> dict: + return self.state or {} + + +@workflow.defn +class AdvisoryPromoteWorkflow: + """Promote one webhook advisory record into the PROD Qdrant collection.""" + + def __init__(self): + self.state = None + + @workflow.run + async def run(self, record_id: str) -> dict: + self.state = {"record_id": record_id, "stage": "promoting"} + try: + result = await workflow.execute_activity( + promote_webhook_advisory, + record_id, + start_to_close_timeout=timedelta(minutes=30), + retry_policy=INGEST_RETRY, + ) + except Exception as e: + self.state.update(stage="failed", error_message=str(e)) + raise + self.state.update(stage="completed", status=result.get("status"), chunks=result.get("chunks")) + return result + + @workflow.query + def get_state(self) -> dict: + return self.state or {} diff --git a/tests/test_activities.py b/tests/test_activities.py index d14097c..73fb506 100644 --- a/tests/test_activities.py +++ b/tests/test_activities.py @@ -163,6 +163,35 @@ def test_prepare_records_carries_instance_name(self): assert records[0]["instance"] == code assert records[0]["instance_name"] == expected + @pytest.mark.unit + def test_prepare_records_carries_document_kind(self): + """Every record carries its document_kind; only scheme documents get type=scheme.""" + from pipeline.activities import _prepare_records + from pipeline.vector_store.qdrant_store import _record_payload + + chunks = [{"chunk_number": 1, "original_text": "Test chunk", "token_count": 5}] + + advisory = _prepare_records( + document_id="test-doc", filename="test.pdf", chunks=chunks, document_kind="advisory" + ) + assert advisory[0]["document_kind"] == "advisory" + assert advisory[0]["type"] == "document" + assert _record_payload(advisory[0])["document_kind"] == "advisory" + + default = _prepare_records(document_id="test-doc", filename="test.pdf", chunks=chunks) + assert default[0]["document_kind"] == "document" + + scheme = _prepare_records( + document_id="test-doc", + filename="test.pdf", + chunks=chunks, + document_kind="scheme", + scheme_code="pm-kisan", + scheme_name="PM-KISAN", + ) + assert scheme[0]["type"] == "scheme" + assert scheme[0]["document_kind"] == "scheme" + @pytest.mark.unit def test_prepare_records_fills_scheme_aliases(self): """A scheme must never index with scheme_aliases: [].""" diff --git a/tests/test_advisory_activities.py b/tests/test_advisory_activities.py new file mode 100644 index 0000000..0ac0dd2 --- /dev/null +++ b/tests/test_advisory_activities.py @@ -0,0 +1,126 @@ +"""Tests for the advisory index/promote activities, the PROD target and the workflow exports.""" + +import asyncio +import copy +from pathlib import Path + +import pytest + +from pipeline import advisory_index +from tests.test_advisory_chunks import SORGHUM_PAYLOAD +from tests.test_advisory_index import FakeStore + + +def _seed(db): + db.upsert_webhook_records("webhook", [copy.deepcopy(SORGHUM_PAYLOAD)]) + with db.get_connection() as conn: + return str(conn.execute("SELECT id FROM webhook_records").fetchone()["id"]) + + +def _audit_actions(db): + with db.get_connection() as conn: + return [r["action_type"] for r in conn.execute("SELECT action_type FROM audit_logs")] + + +@pytest.fixture +def qdrant_env(monkeypatch): + monkeypatch.setenv("VECTOR_DB_COLLECTION_NAME", "document-data") + monkeypatch.delenv("PROD_VECTOR_DB_URL", raising=False) + monkeypatch.setattr("pipeline.vector_store.embeddings.count_tokens", lambda text: 250) + + +@pytest.mark.unit +@pytest.mark.db +class TestActivities: + def test_index_activity_writes_to_the_dev_collection_and_audits(self, db_connection, qdrant_env, monkeypatch): + from pipeline.activities import index_webhook_advisory + + rid = _seed(db_connection) + store = FakeStore() + monkeypatch.setattr("pipeline.vector_store.get_vector_store", lambda: store) + result = asyncio.run(index_webhook_advisory(rid)) + assert result["status"] == "indexed" + assert store.calls[0][1] == "document-data" + assert db_connection.get_unindexed_webhook_record_ids() == [] + assert "advisory_indexed" in _audit_actions(db_connection) + + def test_promote_activity_writes_to_prod_and_marks_promoted(self, db_connection, qdrant_env, monkeypatch): + from pipeline.activities import promote_webhook_advisory + + rid = _seed(db_connection) + prod = FakeStore() + monkeypatch.setattr(advisory_index, "prod_target", lambda: (prod, "prod-collection")) + result = asyncio.run(promote_webhook_advisory(rid)) + assert result["status"] == "indexed" + assert prod.calls[0][1] == "prod-collection" + assert db_connection.advisory_status_counts()["promoted"] == 1 + assert "advisory_promoted" in _audit_actions(db_connection) + + def test_promote_activity_requires_a_prod_target(self, db_connection, qdrant_env, monkeypatch): + from pipeline.activities import promote_webhook_advisory + + monkeypatch.setattr(advisory_index, "prod_target", lambda: None) + with pytest.raises(RuntimeError, match="PROD_VECTOR_DB_URL"): + asyncio.run(promote_webhook_advisory("any")) + + def test_dev_index_of_unpublished_record_also_clears_prod(self, db_connection, qdrant_env, monkeypatch): + from pipeline.activities import index_webhook_advisory + + payload = copy.deepcopy(SORGHUM_PAYLOAD) + payload["isPublished"] = 0 + db_connection.upsert_webhook_records("webhook", [payload]) + with db_connection.get_connection() as conn: + rid = str(conn.execute("SELECT id FROM webhook_records").fetchone()["id"]) + dev, prod = FakeStore(), FakeStore() + monkeypatch.setattr("pipeline.vector_store.get_vector_store", lambda: dev) + monkeypatch.setattr(advisory_index, "prod_target", lambda: (prod, "prod-collection")) + result = asyncio.run(index_webhook_advisory(rid)) + assert result["status"] == "deleted" + assert prod.calls == [("delete", "prod-collection", rid)] + + def test_dev_index_survives_a_broken_prod_config(self, db_connection, qdrant_env, monkeypatch): + from pipeline.activities import index_webhook_advisory + + rid = _seed(db_connection) + monkeypatch.setattr("pipeline.vector_store.get_vector_store", lambda: FakeStore()) + + def broken(): + raise ValueError("QDRANT_API_KEY is required for remote Qdrant") + + monkeypatch.setattr(advisory_index, "prod_target", broken) + assert asyncio.run(index_webhook_advisory(rid))["status"] == "indexed" + + +class TestProdTarget: + def test_none_without_prod_url(self, monkeypatch): + monkeypatch.delenv("PROD_VECTOR_DB_URL", raising=False) + assert advisory_index.prod_target() is None + + def test_uses_prod_env(self, monkeypatch): + monkeypatch.setenv("PROD_VECTOR_DB_URL", "http://prod:6333") + monkeypatch.setenv("PROD_VECTOR_DB_API_KEY", "k") + monkeypatch.setenv("PROD_VECTOR_DB_COLLECTION_NAME", "prod-docs") + seen = {} + + def fake_client(url=None, api_key=None): + seen.update(url=url, api_key=api_key) + return object() + + monkeypatch.setattr("pipeline.vector_store.qdrant_store.get_qdrant_client", fake_client) + target = advisory_index.prod_target() + assert target is not None and target[1] == "prod-docs" + assert seen == {"url": "http://prod:6333", "api_key": "k"} + + +def test_workflows_are_defined_and_registered(): + from pipeline.workflows import AdvisoryIndexWorkflow, AdvisoryPromoteWorkflow + + assert callable(AdvisoryIndexWorkflow.run) and callable(AdvisoryPromoteWorkflow.run) + source = (Path(__file__).resolve().parent.parent / "pipeline" / "worker.py").read_text() + for name in ( + "AdvisoryIndexWorkflow", + "AdvisoryPromoteWorkflow", + "index_webhook_advisory", + "promote_webhook_advisory", + ): + assert source.count(name) >= 2, name # imported and registered diff --git a/tests/test_advisory_api.py b/tests/test_advisory_api.py new file mode 100644 index 0000000..46cc3ca --- /dev/null +++ b/tests/test_advisory_api.py @@ -0,0 +1,165 @@ +"""API tests: /webhook starts advisory workflows; admin backfill / promote / status endpoints.""" + +import os +from unittest.mock import AsyncMock, patch + +import pytest + +TEST_KEY = "0123456789abcdef0123456789abcdef" +H = {"x-api-key": TEST_KEY} + + +def _rec(content_id, published=1, status="Published"): + return { + "contentID": content_id, + "title": f"Advisory {content_id}", + "status": status, + "isPublished": published, + "popList": [{"popID": 1, "status": "Published"}], + } + + +@pytest.fixture +def client(db_connection, monkeypatch): + from fastapi.testclient import TestClient + + from pipeline import api + + monkeypatch.setattr(api.limiter, "enabled", False) # many posts per test process + with patch.dict(os.environ, {"WEBHOOK_API_KEY": TEST_KEY, "AUTH_DISABLED": "true"}): + yield TestClient(api.app) + + +@pytest.fixture +def temporal(monkeypatch): + from pipeline import api + + mock = AsyncMock() + monkeypatch.setattr(api, "temporal_client", mock) + return mock + + +def _row_ids(db, *content_ids): + with db.get_connection() as conn: + return { + c: str(conn.execute( + "SELECT id FROM webhook_records WHERE external_id = %s", (str(c),) + ).fetchone()["id"]) + for c in content_ids + } + + +def _started(mock): + return sorted(call.kwargs["id"] for call in mock.start_workflow.await_args_list) + + +@pytest.mark.api +@pytest.mark.unit +class TestWebhookStartsIndexing: + def test_one_workflow_per_record_in_the_call(self, client, db_connection, temporal): + r = client.post("/webhook", json=[_rec(1), _rec(2)], headers=H) + assert r.status_code == 200 + assert r.json() == { + "success": True, + "message": "Webhook processed successfully.", + "recordCount": 2, + } + ids = _row_ids(db_connection, 1, 2) + assert _started(temporal) == sorted(f"advisory-{i}" for i in ids.values()) + + def test_only_records_in_the_call_are_started(self, client, db_connection, temporal): + db_connection.upsert_webhook_records("webhook", [_rec(1)]) # existing, unindexed + client.post("/webhook", json=[_rec(2)], headers=H) + ids = _row_ids(db_connection, 1, 2) + assert _started(temporal) == [f"advisory-{ids[2]}"] + + def test_existing_never_indexed_row_is_started_when_resent(self, client, db_connection, temporal): + db_connection.upsert_webhook_records("webhook", [_rec(1)]) + client.post("/webhook", json=[_rec(1)], headers=H) + ids = _row_ids(db_connection, 1) + assert _started(temporal) == [f"advisory-{ids[1]}"] + + def test_nothing_started_when_already_indexed(self, client, db_connection, temporal): + client.post("/webhook", json=[_rec(1)], headers=H) + rid = _row_ids(db_connection, 1)[1] + db_connection.mark_webhook_record_indexed(rid, db_connection.get_webhook_record(rid)["payload_hash"]) + temporal.start_workflow.reset_mock() + r = client.post("/webhook", json=[_rec(1)], headers=H) + assert r.status_code == 200 + temporal.start_workflow.assert_not_called() + + def test_still_200_when_temporal_fails(self, client, db_connection, temporal): + temporal.start_workflow.side_effect = RuntimeError("temporal down") + r = client.post("/webhook", json=[_rec(1)], headers=H) + assert r.status_code == 200 + assert len(db_connection.get_unindexed_webhook_record_ids()) == 1 + + def test_still_200_when_temporal_client_is_none(self, client, db_connection, monkeypatch): + from pipeline import api + + monkeypatch.setattr(api, "temporal_client", None) + assert client.post("/webhook", json=[_rec(1)], headers=H).status_code == 200 + + +@pytest.mark.api +@pytest.mark.unit +class TestAdminEndpoints: + def _seed(self, client, temporal, *records): + client.post("/webhook", json=list(records), headers=H) + temporal.start_workflow.reset_mock() + + def test_reindex_queues_all_unindexed_by_default(self, client, db_connection, temporal): + self._seed(client, temporal, _rec(1), _rec(2)) + r = client.post("/webhook/advisory/reindex", json={}) + assert r.status_code == 200 and r.json() == {"success": True, "queued": 2} + assert len(_started(temporal)) == 2 + + def test_reindex_force_queues_indexed_rows_too(self, client, db_connection, temporal): + self._seed(client, temporal, _rec(1)) + rid = _row_ids(db_connection, 1)[1] + db_connection.mark_webhook_record_indexed(rid, db_connection.get_webhook_record(rid)["payload_hash"]) + assert client.post("/webhook/advisory/reindex", json={}).json()["queued"] == 0 + assert client.post("/webhook/advisory/reindex", json={"force": True}).json()["queued"] == 1 + + def test_reindex_explicit_ids(self, client, db_connection, temporal): + self._seed(client, temporal, _rec(1), _rec(2)) + rid = _row_ids(db_connection, 2)[2] + r = client.post("/webhook/advisory/reindex", json={"record_ids": [rid]}) + assert r.json()["queued"] == 1 + assert _started(temporal) == [f"advisory-{rid}"] + + def test_promote_defaults_to_published_records(self, client, db_connection, temporal): + self._seed(client, temporal, _rec(1), _rec(2, published=0, status="Draft")) + r = client.post("/webhook/advisory/promote", json={}) + assert r.json() == {"success": True, "queued": 1} + published = _row_ids(db_connection, 1)[1] + assert _started(temporal) == [f"advisory-promote-{published}"] + + def test_promote_explicit_ids(self, client, db_connection, temporal): + self._seed(client, temporal, _rec(1), _rec(2)) + rid = _row_ids(db_connection, 2)[2] + r = client.post("/webhook/advisory/promote", json={"record_ids": [rid]}) + assert r.json()["queued"] == 1 + assert _started(temporal) == [f"advisory-promote-{rid}"] + + def test_status_returns_counts(self, client, db_connection, temporal): + self._seed(client, temporal, _rec(1), _rec(2, published=0, status="Draft")) + r = client.get("/webhook/advisory/status") + assert r.status_code == 200 + assert r.json() == {"total": 2, "published": 1, "indexed": 0, "unindexed": 2, "promoted": 0} + + def test_endpoints_require_a_signed_in_admin(self, db_connection, temporal, monkeypatch): + from fastapi.testclient import TestClient + + from pipeline import api + + env = { + "AUTH_DISABLED": "false", + "KEYCLOAK_ISSUER": "https://example.com/realms/test", + "KEYCLOAK_JWKS_URL": "https://example.com/realms/test/protocol/openid-connect/certs", + } + with patch.dict(os.environ, env): + c = TestClient(api.app) + assert c.post("/webhook/advisory/reindex", json={}).status_code == 401 + assert c.post("/webhook/advisory/promote", json={}).status_code == 401 + assert c.get("/webhook/advisory/status").status_code == 401 diff --git a/tests/test_advisory_chunks.py b/tests/test_advisory_chunks.py new file mode 100644 index 0000000..913a805 --- /dev/null +++ b/tests/test_advisory_chunks.py @@ -0,0 +1,218 @@ +"""Tests for the webhook advisory chunk builder.""" + +import copy +import hashlib + +import pytest + +from pipeline.advisory_chunks import build_chunks, clean_value + +HEADER = ( + "Sorghum- Harvesting Management\n" + "Crop: Sorghum | Problem type: Harvesting Management | Season: Rabi" +) +OVERVIEW = ( + "Timely harvesting of AKJ-1 helps preserve grain appearance, processing quality and " + "suitability for flaking. Proper drying and handling after harvest are essential to " + "minimize grain damage and deterioration." +) +SYMPTOMS = ( + "Panicles become mature and dry, grains become hard and attain characteristic colour, " + "and most leaves begin to dry." +) +CONDITIONS = ( + "Crop has reached physiological maturity and panicles are fully developed; grains are " + "firm and suitable for flaking quality." +) +ADVISORY = ( + "Harvest AKJ-1 at physiological maturity when the grains are fully developed and firm. " + "Timely harvesting is important for maintaining grain quality required for flaking. " + "Avoid delayed harvesting, which may increase grain weathering, shattering and mould " + "risk. After harvesting, dry the panicles properly, thresh carefully, clean the grain " + "and reduce moisture to a safe level before storage or processing." +) + +SORGHUM_PAYLOAD = { + "contentID": 987, + "title": "Sorghum- Harvesting Management", + "commodityName": "Sorghum", + "creatorName": "jak", + "domainName": "Agriculture", + "subdomainName": "Cereals", + "instituteName": "ICAR–Directorate of Knowledge Management in Agriculture (DKMA)", + "isPublished": 1, + "status": "Published", + "popList": [ + { + "popID": 819, + "season": "Rabi", + "problemType": '"Harvesting Management"', + "problemDetails": '"Method Of Harvesting"', + "startPeriod": "105", + "endPeriod": "120", + "status": "Published", + "description": f'"{OVERVIEW}"', + "applicableSymptoms": [SYMPTOMS], + "applicableCondition": [CONDITIONS], + "advisory": f'"{ADVISORY}"', + } + ], +} + + +def make_row(payload=None, **overrides): + payload = copy.deepcopy(SORGHUM_PAYLOAD if payload is None else payload) + row = { + "id": "row-1", + "external_id": "987", + "title": payload.get("title", ""), + "slug": "987/sorghum--harvesting-management", + "status": payload.get("status"), + "is_published": payload.get("isPublished"), + "payload": payload, + "payload_hash": "hash-1", + "updated_at": "2026-10-16T09:15:10.000Z", + } + row.update(overrides) + return row + + +def long_when_combined(text: str) -> int: + """Over the limit only when the entry carries more than one question-side field.""" + return 600 if "Overview: " in text and "Symptoms: " in text else 300 + + +def pop_of(row): + return row["payload"]["popList"][0] + + +class TestCleanValue: + def test_strips_wrapping_quotes(self): + assert clean_value('"Aphids"') == "Aphids" + assert clean_value('""') == "" + + def test_handles_none_numbers_and_whitespace(self): + assert clean_value(None) == "" + assert clean_value(819) == "819" + assert clean_value(" x ") == "x" + + +class TestShortEntry: + def test_short_entry_is_one_full_chunk(self): + chunks = build_chunks(make_row(), lambda t: 250) + assert len(chunks) == 1 + assert chunks[0]["section"] == "full" + assert chunks[0]["text"] == "\n".join( + [ + HEADER, + f"Overview: {OVERVIEW}", + f"Symptoms: {SYMPTOMS}", + f"Conditions: {CONDITIONS}", + f"Advisory: {ADVISORY}", + ] + ) + assert chunks[0]["text_for_embedding"] == chunks[0]["text"] + + +class TestLongEntry: + def test_long_entry_splits_into_three_chunks_each_with_advice(self): + chunks = build_chunks(make_row(), long_when_combined) + assert [c["section"] for c in chunks] == ["overview", "symptoms", "conditions"] + for chunk in chunks: + assert chunk["text"].startswith(HEADER) + assert chunk["text"].endswith(f"Advisory: {ADVISORY}") + assert "Symptoms:" not in chunks[0]["text"] + assert f"Symptoms: {SYMPTOMS}" in chunks[1]["text"] + assert [c["chunk_num"] for c in chunks] == [0, 1, 2] + + def test_empty_conditions_gives_two_chunks(self): + row = make_row() + pop_of(row)["applicableCondition"] = [] + chunks = build_chunks(row, long_when_combined) + assert [c["section"] for c in chunks] == ["overview", "symptoms"] + + def test_no_advisory_gives_one_chunk_per_field(self): + row = make_row() + pop_of(row)["advisory"] = None + chunks = build_chunks(row, lambda t: 600) + assert [c["section"] for c in chunks] == ["overview", "symptoms", "conditions"] + for chunk in chunks: + assert "Advisory:" not in chunk["text"] + + def test_only_advisory_is_one_advisory_chunk(self): + row = make_row() + pop = pop_of(row) + pop["description"] = None + pop["applicableSymptoms"] = [] + pop["applicableCondition"] = [] + chunks = build_chunks(row, lambda t: 600) + assert [c["section"] for c in chunks] == ["advisory"] + assert chunks[0]["text"] == f"{HEADER}\nAdvisory: {ADVISORY}" + + +class TestPublishingAndShape: + @pytest.mark.parametrize( + "overrides", + [{"is_published": 0}, {"status": "Draft"}], + ) + def test_unpublished_record_returns_empty(self, overrides): + assert build_chunks(make_row(**overrides), lambda t: 250) == [] + + def test_unpublished_entry_is_skipped(self): + row = make_row() + row["payload"]["popList"].append( + {**copy.deepcopy(pop_of(row)), "popID": 820, "status": "Draft"} + ) + chunks = build_chunks(row, lambda t: 250) + assert [c["pop_id"] for c in chunks] == [819] + + @pytest.mark.parametrize("pop_list", [None, [], "nope"]) + def test_missing_or_empty_poplist_returns_empty(self, pop_list): + row = make_row() + row["payload"]["popList"] = pop_list + assert build_chunks(row, lambda t: 250) == [] + + def test_ids_are_stable_and_unique(self): + first = build_chunks(make_row(), long_when_combined) + second = build_chunks(make_row(), long_when_combined) + assert [c["_id"] for c in first] == [c["_id"] for c in second] + assert len({c["_id"] for c in first}) == 3 + full = build_chunks(make_row(), lambda t: 250)[0] + expected = hashlib.md5(b"row-1:819:full").hexdigest() + assert full["_id"] == expected + assert full["chunk_id"] == expected + + def test_metadata_fields(self): + chunk = build_chunks(make_row(), lambda t: 250)[0] + assert chunk["document_kind"] == "advisory" + assert chunk["type"] == "document" + assert chunk["source"] == "webhook" + assert chunk["workflow_id"] == "" + assert chunk["doc_id"] == "row-1" + assert chunk["external_id"] == "987" + assert chunk["pop_id"] == 819 + assert chunk["title_en"] == "Sorghum- Harvesting Management" + assert chunk["slug"] == "987/sorghum--harvesting-management" + assert chunk["crop"] == "Sorghum" + assert chunk["domain"] == "Agriculture" + assert chunk["subdomain"] == "Cereals" + assert chunk["institute"].startswith("ICAR") + assert chunk["creator"] == "jak" + assert chunk["problem_type"] == "Harvesting Management" + assert chunk["problem"] == "Method Of Harvesting" + assert chunk["season"] == "Rabi" + assert chunk["start_period"] == "105" + assert chunk["end_period"] == "120" + assert chunk["updated_at"] == "2026-10-16T09:15:10.000Z" + + def test_null_values_never_appear_as_text(self): + row = make_row() + pop = pop_of(row) + pop["description"] = None + pop["season"] = None + pop["applicableSymptoms"] = [None, "x"] + chunks = build_chunks(row, lambda t: 250) + assert len(chunks) == 1 + assert "None" not in chunks[0]["text"] + assert "Season:" not in chunks[0]["text"] + assert "Symptoms: x" in chunks[0]["text"] diff --git a/tests/test_advisory_index.py b/tests/test_advisory_index.py new file mode 100644 index 0000000..ab71be5 --- /dev/null +++ b/tests/test_advisory_index.py @@ -0,0 +1,152 @@ +"""Tests for indexing one webhook advisory record into a vector store.""" + +import copy + +import pytest + +from pipeline.advisory_chunks import build_chunks +from pipeline.advisory_index import index_advisory_record +from pipeline.vector_store.qdrant_store import _record_payload +from tests.test_advisory_chunks import SORGHUM_PAYLOAD, make_row + + +class FakeStore: + def __init__(self, delete_result=None, fail_upsert=False): + self.calls = [] + self.records = [] + self._delete_result = delete_result or {"deleted": -1} + self._fail_upsert = fail_upsert + + def delete_by_doc_id(self, name, doc_id): + self.calls.append(("delete", name, doc_id)) + return self._delete_result + + def upsert(self, name, records, batch_size=32): + self.calls.append(("upsert", name, len(records))) + if self._fail_upsert: + raise RuntimeError("qdrant down") + self.records = list(records) + return {"records_ingested": len(records)} + + +def _store_row(db, payload): + db.upsert_webhook_records("webhook", [payload]) + with db.get_connection() as conn: + return str(conn.execute( + "SELECT id FROM webhook_records WHERE external_id = %s", (str(payload["contentID"]),) + ).fetchone()["id"]) + + +def _count(_text): + return 250 + + +@pytest.mark.unit +@pytest.mark.db +class TestIndexAdvisoryRecord: + def test_published_record_is_deleted_then_upserted_then_marked(self, db_connection): + db = db_connection + rid = _store_row(db, copy.deepcopy(SORGHUM_PAYLOAD)) + store = FakeStore() + result = index_advisory_record(rid, store, "document-data", _count) + assert [c[0] for c in store.calls] == ["delete", "upsert"] + assert store.calls[0][1:] == ("document-data", rid) + assert result == {"record_id": rid, "status": "indexed", "chunks": 1} + assert db.get_unindexed_webhook_record_ids() == [] + + def test_unpublished_record_only_deletes(self, db_connection): + db = db_connection + payload = copy.deepcopy(SORGHUM_PAYLOAD) + payload["isPublished"] = 0 + rid = _store_row(db, payload) + store = FakeStore() + result = index_advisory_record(rid, store, "document-data", _count) + assert [c[0] for c in store.calls] == ["delete"] + assert result["status"] == "deleted" and result["chunks"] == 0 + assert db.get_unindexed_webhook_record_ids() == [] + + def test_unpublished_record_also_deletes_from_the_other_collection(self, db_connection): + db = db_connection + payload = copy.deepcopy(SORGHUM_PAYLOAD) + payload["isPublished"] = 0 + rid = _store_row(db, payload) + dev, prod = FakeStore(), FakeStore() + index_advisory_record( + rid, dev, "document-data", _count, also_delete_from=(prod, "prod-collection") + ) + assert prod.calls == [("delete", "prod-collection", rid)] + + def test_published_record_does_not_touch_the_other_collection(self, db_connection): + db = db_connection + rid = _store_row(db, copy.deepcopy(SORGHUM_PAYLOAD)) + dev, prod = FakeStore(), FakeStore() + index_advisory_record( + rid, dev, "document-data", _count, also_delete_from=(prod, "prod-collection") + ) + assert prod.calls == [] + + def test_missing_record_does_nothing(self, db_connection): + store = FakeStore() + result = index_advisory_record("nope", store, "document-data", _count) + assert store.calls == [] + assert result == {"record_id": "nope", "status": "missing", "chunks": 0} + + def test_record_without_poplist_is_marked_indexed(self, db_connection): + db = db_connection + payload = copy.deepcopy(SORGHUM_PAYLOAD) + payload["popList"] = [] + rid = _store_row(db, payload) + store = FakeStore() + result = index_advisory_record(rid, store, "document-data", _count) + assert [c[0] for c in store.calls] == ["delete"] + assert result["status"] == "deleted" + assert db.get_unindexed_webhook_record_ids() == [] + + def test_upsert_failure_leaves_record_unindexed(self, db_connection): + db = db_connection + rid = _store_row(db, copy.deepcopy(SORGHUM_PAYLOAD)) + with pytest.raises(RuntimeError, match="qdrant down"): + index_advisory_record(rid, FakeStore(fail_upsert=True), "document-data", _count) + assert db.get_unindexed_webhook_record_ids() == [rid] + + def test_delete_error_other_than_missing_collection_fails(self, db_connection): + db = db_connection + rid = _store_row(db, copy.deepcopy(SORGHUM_PAYLOAD)) + store = FakeStore(delete_result={"deleted": 0, "error": "connection refused"}) + with pytest.raises(RuntimeError, match="connection refused"): + index_advisory_record(rid, store, "document-data", _count) + assert db.get_unindexed_webhook_record_ids() == [rid] + + def test_missing_collection_on_delete_is_not_an_error(self, db_connection): + db = db_connection + rid = _store_row(db, copy.deepcopy(SORGHUM_PAYLOAD)) + store = FakeStore(delete_result={"deleted": 0, "error": "Collection `x` doesn't exist!"}) + result = index_advisory_record(rid, store, "x", _count) + assert result["status"] == "indexed" + + def test_promote_marks_promoted_not_indexed(self, db_connection): + db = db_connection + rid = _store_row(db, copy.deepcopy(SORGHUM_PAYLOAD)) + index_advisory_record(rid, FakeStore(), "prod", _count, mark="promoted") + assert db.get_unindexed_webhook_record_ids() == [rid] + assert db.advisory_status_counts()["promoted"] == 1 + + def test_reposting_unchanged_record_reuses_point_ids(self, db_connection): + db = db_connection + rid = _store_row(db, copy.deepcopy(SORGHUM_PAYLOAD)) + first, second = FakeStore(), FakeStore() + index_advisory_record(rid, first, "document-data", _count) + index_advisory_record(rid, second, "document-data", _count) + assert [r["_id"] for r in first.records] == [r["_id"] for r in second.records] + + +def test_payload_whitelist_keeps_advisory_fields(): + chunk = build_chunks(make_row(), _count)[0] + payload = _record_payload(chunk) + for key in ( + "document_kind", "external_id", "pop_id", "slug", "crop", "domain", "subdomain", + "institute", "creator", "problem_type", "problem", "season", "start_period", + "end_period", "updated_at", "section", "chunk_id", "text", "doc_id", "title_en", + ): + assert key in payload, key + assert payload["document_kind"] == "advisory" diff --git a/tests/test_advisory_start.py b/tests/test_advisory_start.py new file mode 100644 index 0000000..0c7056e --- /dev/null +++ b/tests/test_advisory_start.py @@ -0,0 +1,98 @@ +"""Tests for starting advisory workflows (one per record).""" + +import asyncio +from unittest.mock import AsyncMock + +import pytest +from temporalio.exceptions import WorkflowAlreadyStartedError + +from pipeline.advisory_start import start_advisory_index, start_advisory_promote +from pipeline.workflows import AdvisoryIndexWorkflow, AdvisoryPromoteWorkflow + + +def _started_ids(client): + return [call.kwargs["id"] for call in client.start_workflow.await_args_list] + + +def test_index_starts_one_workflow_per_record(): + client = AsyncMock() + started = asyncio.run(start_advisory_index(client, ["a", "b"])) + assert started == 2 + assert _started_ids(client) == ["advisory-a", "advisory-b"] + first = client.start_workflow.await_args_list[0] + assert first.args[0] is AdvisoryIndexWorkflow.run + assert first.kwargs["args"] == ["a"] + assert first.kwargs["task_queue"] == "ocr-pipeline" + + +def test_promote_uses_separate_ids(): + client = AsyncMock() + assert asyncio.run(start_advisory_promote(client, ["a"])) == 1 + assert _started_ids(client) == ["advisory-promote-a"] + assert client.start_workflow.await_args_list[0].args[0] is AdvisoryPromoteWorkflow.run + + +def test_already_running_workflow_is_skipped_and_others_still_start(): + client = AsyncMock() + client.start_workflow.side_effect = [ + WorkflowAlreadyStartedError("advisory-a", "AdvisoryIndexWorkflow"), + None, + ] + assert asyncio.run(start_advisory_index(client, ["a", "b"])) == 1 + + +def test_other_errors_are_logged_and_do_not_stop_the_loop(): + client = AsyncMock() + client.start_workflow.side_effect = [RuntimeError("temporal down"), None] + assert asyncio.run(start_advisory_index(client, ["a", "b"])) == 1 + + +def test_no_ids_starts_nothing(): + client = AsyncMock() + assert asyncio.run(start_advisory_index(client, [])) == 0 + client.start_workflow.assert_not_called() + + +@pytest.mark.db +class TestBackfillOnStart: + def _seed(self, db): + db.upsert_webhook_records( + "webhook", + [{"contentID": 1, "status": "Published", "isPublished": 1, "popList": []}], + ) + + def test_off_by_default(self, db_connection, monkeypatch): + from pipeline.advisory_start import backfill_on_start + + monkeypatch.delenv("ADVISORY_BACKFILL_ON_START", raising=False) + self._seed(db_connection) + client = AsyncMock() + assert asyncio.run(backfill_on_start(client)) == 0 + client.start_workflow.assert_not_called() + + @pytest.mark.parametrize("value", ["true", "1", "yes", "on", "TRUE"]) + def test_on_queues_all_unindexed_rows(self, db_connection, monkeypatch, value): + from pipeline.advisory_start import backfill_on_start + + monkeypatch.setenv("ADVISORY_BACKFILL_ON_START", value) + self._seed(db_connection) + client = AsyncMock() + assert asyncio.run(backfill_on_start(client)) == 1 + assert _started_ids(client)[0].startswith("advisory-") + + def test_on_with_nothing_unindexed_starts_nothing(self, db_connection, monkeypatch): + from pipeline.advisory_start import backfill_on_start + + monkeypatch.setenv("ADVISORY_BACKFILL_ON_START", "true") + client = AsyncMock() + assert asyncio.run(backfill_on_start(client)) == 0 + client.start_workflow.assert_not_called() + + def test_off_values(self, db_connection, monkeypatch): + from pipeline.advisory_start import backfill_on_start + + self._seed(db_connection) + for value in ("false", "0", "", "no"): + monkeypatch.setenv("ADVISORY_BACKFILL_ON_START", value) + client = AsyncMock() + assert asyncio.run(backfill_on_start(client)) == 0 diff --git a/tests/test_db.py b/tests/test_db.py index 87e00ea..496010d 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -1230,14 +1230,14 @@ def test_a_new_migration_runs_once(self, db_connection, postgres_url, tmp_path): from pipeline import migrate (tmp_path / "0001_initial_schema.sql").write_text("SELECT 1;") - (tmp_path / "0002_probe.sql").write_text("CREATE TABLE mig_probe (n int);") + (tmp_path / "0099_probe.sql").write_text("CREATE TABLE mig_probe (n int);") try: - assert migrate.apply_migrations(postgres_url, tmp_path) == ["0002"] + assert migrate.apply_migrations(postgres_url, tmp_path) == ["0099"] assert migrate.apply_migrations(postgres_url, tmp_path) == [] finally: with db_connection.get_connection() as conn: conn.execute("DROP TABLE IF EXISTS mig_probe") - conn.execute("DELETE FROM schema_migrations WHERE version = '0002'") + conn.execute("DELETE FROM schema_migrations WHERE version = '0099'") @pytest.mark.db @pytest.mark.unit @@ -1245,13 +1245,13 @@ def test_a_failing_migration_applies_nothing_of_itself(self, db_connection, post from pipeline import migrate (tmp_path / "0001_initial_schema.sql").write_text("SELECT 1;") - (tmp_path / "0002_bad.sql").write_text("CREATE TABLE mig_half (n int); SELECT 1/0;") + (tmp_path / "0099_bad.sql").write_text("CREATE TABLE mig_half (n int); SELECT 1/0;") with pytest.raises(psycopg.errors.DivisionByZero): migrate.apply_migrations(postgres_url, tmp_path) with db_connection.get_connection() as conn: assert conn.execute("SELECT to_regclass('mig_half') AS t").fetchone()["t"] is None assert conn.execute( - "SELECT count(*) AS n FROM schema_migrations WHERE version = '0002'" + "SELECT count(*) AS n FROM schema_migrations WHERE version = '0099'" ).fetchone()["n"] == 0 @pytest.mark.db diff --git a/tests/test_embeddings_count_tokens.py b/tests/test_embeddings_count_tokens.py new file mode 100644 index 0000000..c469f9e --- /dev/null +++ b/tests/test_embeddings_count_tokens.py @@ -0,0 +1,24 @@ +"""Tests for the embedding token counter used to size advisory chunks.""" + +from types import SimpleNamespace + +from pipeline.vector_store import embeddings + + +def test_count_tokens_local_uses_tokenizer(monkeypatch): + monkeypatch.setenv("EMBEDDING_PROVIDER", "sentence_transformers") + fake = SimpleNamespace(tokenizer=SimpleNamespace(encode=lambda text: text.split())) + monkeypatch.setattr(embeddings, "_get_local_embedder", lambda: fake) + # "passage: a b c" -> passage:, a, b, c + assert embeddings.count_tokens("a b c") == 4 + + +def test_count_tokens_remote_estimates_from_length(monkeypatch): + monkeypatch.setenv("EMBEDDING_PROVIDER", "openai_compatible") + text = "x" * 400 + assert embeddings.count_tokens(text) == len("passage: " + text) // 4 + + +def test_count_tokens_remote_is_at_least_one(monkeypatch): + monkeypatch.setenv("EMBEDDING_PROVIDER", "openai_compatible") + assert embeddings.count_tokens("") >= 1 diff --git a/tests/test_webhook_index_db.py b/tests/test_webhook_index_db.py new file mode 100644 index 0000000..0c09b1a --- /dev/null +++ b/tests/test_webhook_index_db.py @@ -0,0 +1,107 @@ +"""Tests for the webhook_records indexing state (indexed_hash / promoted_hash) helpers.""" + + +import pytest + + +def _rec(content_id, title="Wheat Insect Management", published=1, status="Published"): + return { + "contentID": content_id, + "title": title, + "status": status, + "isPublished": published, + "domainName": "Agriculture", + "slug": f"{content_id}/x", + "popList": [{"popID": 1, "status": "Published"}], + } + + +def _row_id(db, content_id): + with db.get_connection() as conn: + return str(conn.execute( + "SELECT id FROM webhook_records WHERE external_id = %s", (str(content_id),) + ).fetchone()["id"]) + + +@pytest.mark.unit +@pytest.mark.db +class TestIndexState: + def test_new_rows_are_unindexed_until_marked(self, db_connection): + db = db_connection + db.upsert_webhook_records("webhook", [_rec(1)]) + rid = _row_id(db, 1) + assert db.get_unindexed_webhook_record_ids() == [rid] + row = db.get_webhook_record(rid) + db.mark_webhook_record_indexed(rid, row["payload_hash"]) + assert db.get_unindexed_webhook_record_ids() == [] + + def test_changed_payload_becomes_unindexed_again(self, db_connection): + db = db_connection + db.upsert_webhook_records("webhook", [_rec(1)]) + rid = _row_id(db, 1) + db.mark_webhook_record_indexed(rid, db.get_webhook_record(rid)["payload_hash"]) + + db.upsert_webhook_records("webhook", [_rec(1)]) # identical + assert db.get_unindexed_webhook_record_ids() == [] + + db.upsert_webhook_records("webhook", [_rec(1, title="Changed")]) + assert db.get_unindexed_webhook_record_ids() == [rid] + + def test_marking_a_stale_hash_leaves_the_row_unindexed(self, db_connection): + db = db_connection + db.upsert_webhook_records("webhook", [_rec(1)]) + rid = _row_id(db, 1) + stale = db.get_webhook_record(rid)["payload_hash"] + db.upsert_webhook_records("webhook", [_rec(1, title="Changed")]) + db.mark_webhook_record_indexed(rid, stale) + assert db.get_unindexed_webhook_record_ids() == [rid] + + def test_unindexed_ids_can_be_scoped_to_external_ids(self, db_connection): + db = db_connection + db.upsert_webhook_records("webhook", [_rec(1), _rec(2), _rec(3)]) + ids = db.get_unindexed_webhook_record_ids(external_ids=["1", "3"]) + assert sorted(ids) == sorted([_row_id(db, 1), _row_id(db, 3)]) + assert db.get_unindexed_webhook_record_ids(external_ids=[]) == [] + + def test_get_webhook_record_parses_payload(self, db_connection): + db = db_connection + db.upsert_webhook_records("webhook", [_rec(101)]) + row = db.get_webhook_record(_row_id(db, 101)) + assert row["payload"]["contentID"] == 101 + assert row["external_id"] == "101" + assert row["is_published"] == 1 + assert row["status"] == "Published" + assert db.get_webhook_record("missing") is None + + def test_webhook_external_id_matches_upsert_rule(self, db_connection): + db = db_connection + assert db.webhook_external_id({"contentID": 7}) == "7" + assert db.webhook_external_id({"id": "abc"}) == "abc" + assert db.webhook_external_id({"slug": "s"}) == "s" + assert len(db.webhook_external_id({"title": "x"})) == 64 # payload hash + + def test_record_id_lists(self, db_connection): + db = db_connection + db.upsert_webhook_records( + "webhook", [_rec(1), _rec(2, published=0, status="Draft")] + ) + assert len(db.get_webhook_record_ids()) == 2 + assert db.get_webhook_record_ids(published_only=True) == [_row_id(db, 1)] + + def test_status_counts(self, db_connection): + db = db_connection + db.upsert_webhook_records( + "webhook", [_rec(1), _rec(2), _rec(3, published=0, status="Draft")] + ) + a, b = _row_id(db, 1), _row_id(db, 2) + db.mark_webhook_record_indexed(a, db.get_webhook_record(a)["payload_hash"]) + db.mark_webhook_record_promoted(a, db.get_webhook_record(a)["payload_hash"]) + counts = db.advisory_status_counts() + assert counts == { + "total": 3, + "published": 2, + "indexed": 1, + "unindexed": 2, + "promoted": 1, + } + assert b # silence unused From 6d6bda30f051faa23715798d81c87aeda9f0a909 Mon Sep 17 00:00:00 2001 From: Naveen Date: Tue, 27 Oct 2026 16:47:14 +0530 Subject: [PATCH 34/53] fix: bulk advisory promote only pushes records indexed in dev POST /webhook/advisory/promote without record_ids promoted every published record, building chunks from the current payload. A record edited after its last dev index could reach PROD without being indexed in dev. The default now selects only rows where indexed_hash = payload_hash. --- pipeline/api/api.py | 4 ++-- pipeline/db.py | 13 +++++++++++-- tests/test_advisory_api.py | 16 +++++++++++++--- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/pipeline/api/api.py b/pipeline/api/api.py index a471898..136e816 100644 --- a/pipeline/api/api.py +++ b/pipeline/api/api.py @@ -3435,12 +3435,12 @@ async def advisory_reindex(body: AdvisoryReindexBody, user: RequireAdmin): @app.post("/webhook/advisory/promote") async def advisory_promote(body: AdvisoryPromoteBody, user: RequireAdmin): - """Queue promotion to the PROD collection: given record ids, else all published records.""" + """Queue promotion to the PROD collection: given record ids, else all published records already indexed in dev.""" client = _require_temporal() if body.record_ids is not None: record_ids = body.record_ids else: - record_ids = db.get_webhook_record_ids(published_only=True) + record_ids = db.get_webhook_record_ids(published_only=True, indexed_only=True) queued = await start_advisory_promote(client, record_ids) return {"success": True, "queued": queued} diff --git a/pipeline/db.py b/pipeline/db.py index 6c5cfe9..203522f 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -586,11 +586,20 @@ def get_unindexed_webhook_record_ids( return [str(r["id"]) for r in conn.execute(sql + " ORDER BY created_at, id", params)] -def get_webhook_record_ids(source: str = "webhook", published_only: bool = False) -> list[str]: - """All record ids for a source, optionally only published ones.""" +def get_webhook_record_ids( + source: str = "webhook", published_only: bool = False, indexed_only: bool = False +) -> list[str]: + """All record ids for a source, optionally only published and/or dev-indexed ones. + + ``indexed_only`` keeps rows whose current payload is what was last written to the dev + collection (``indexed_hash = payload_hash``), so a bulk promote never pushes a payload + that has not been indexed in dev. + """ sql = "SELECT id FROM webhook_records WHERE source = %s" if published_only: sql += " AND is_published AND status = 'Published'" + if indexed_only: + sql += " AND indexed_hash = payload_hash" with get_connection() as conn: return [str(r["id"]) for r in conn.execute(sql + " ORDER BY created_at, id", (source,))] diff --git a/tests/test_advisory_api.py b/tests/test_advisory_api.py index 46cc3ca..b0b7f8c 100644 --- a/tests/test_advisory_api.py +++ b/tests/test_advisory_api.py @@ -128,12 +128,22 @@ def test_reindex_explicit_ids(self, client, db_connection, temporal): assert r.json()["queued"] == 1 assert _started(temporal) == [f"advisory-{rid}"] - def test_promote_defaults_to_published_records(self, client, db_connection, temporal): + def test_promote_defaults_to_published_records_indexed_in_dev(self, client, db_connection, temporal): self._seed(client, temporal, _rec(1), _rec(2, published=0, status="Draft")) + ids = _row_ids(db_connection, 1, 2) + for rid in ids.values(): + db_connection.mark_webhook_record_indexed(rid, db_connection.get_webhook_record(rid)["payload_hash"]) + temporal.start_workflow.reset_mock() r = client.post("/webhook/advisory/promote", json={}) assert r.json() == {"success": True, "queued": 1} - published = _row_ids(db_connection, 1)[1] - assert _started(temporal) == [f"advisory-promote-{published}"] + assert _started(temporal) == [f"advisory-promote-{ids[1]}"] + + def test_default_promote_skips_records_not_indexed_in_dev(self, client, db_connection, temporal): + self._seed(client, temporal, _rec(1)) + temporal.start_workflow.reset_mock() + r = client.post("/webhook/advisory/promote", json={}) + assert r.json() == {"success": True, "queued": 0} + assert _started(temporal) == [] def test_promote_explicit_ids(self, client, db_connection, temporal): self._seed(client, temporal, _rec(1), _rec(2)) From ef2647f1067fdb5cab3e02a36413619ae6244452 Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 11:07:16 +0530 Subject: [PATCH 35/53] feat(db): Postgres schema v2 with groups, catalogs and code-level documents Rewrite migration 0001 in place (no Postgres holds v1 yet): - add groups (tenant node, keycloak id filled on login sync) and users.group_id - replace catalog_entries with catalogs, one row per (group, schema type) - documents carry catalog_id, code, name and aliases; period, uploader, index_status and flags live in metadata; counts are derived from pages/chunks - audit_logs snapshot group_id instead of instance - enums become plain TEXT, validated in pipeline/models.py - catalog view becomes catalog_status (one row per document) db.py gets group/catalog get-or-create, find_document_by_code, reset_document_for_reupload and DuplicateDocumentCode. The flat document dict and db params are renamed to group/kind/code/name/aliases. instances.py becomes group_names.py. Scheme/advisory map to schema types in network_constants.py (placeholder values). Bootstrap catalog seeding is removed. --- migrations/0001_initial_schema.sql | 186 +++--- pipeline/catalog_builder.py | 16 +- pipeline/db.py | 747 ++++++++++++---------- pipeline/document_repository.py | 4 +- pipeline/{instances.py => group_names.py} | 26 +- pipeline/models.py | 96 ++- pipeline/network_constants.py | 30 + pipeline/scheme_catalog.py | 160 ++--- 8 files changed, 688 insertions(+), 577 deletions(-) rename pipeline/{instances.py => group_names.py} (77%) diff --git a/migrations/0001_initial_schema.sql b/migrations/0001_initial_schema.sql index 3fc75d0..6ea17de 100644 --- a/migrations/0001_initial_schema.sql +++ b/migrations/0001_initial_schema.sql @@ -6,26 +6,26 @@ BEGIN RETURN NEW; END $$ LANGUAGE plpgsql; -CREATE TYPE workflow_stage AS ENUM ( - 'registered', 'ocr_processing', 'ocr_review', 'translation_processing', - 'translation_review', 'chunking', 'chunk_review', 'ready_for_ingestion', - 'ingesting', 'publishing_to_network', 'approval_for_prod', 'ingesting_prod', - 'completed', 'failed' -); -CREATE TYPE workflow_type_enum AS ENUM ( - 'pipeline', 'ocr_only', 'ocr_retry', 'translation_retry', 'chunking_retry', - 'reingest', 'promote_prod' +-- 1. groups: one row per tenant node (provider), mirrored from Keycloak +CREATE TABLE groups ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + keycloak_group_id TEXT UNIQUE, + name TEXT NOT NULL, + path TEXT NOT NULL UNIQUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() ); -CREATE TYPE workflow_status_enum AS ENUM ('running', 'waiting_review', 'completed', 'failed'); -CREATE TYPE audit_entity_type AS ENUM ('document', 'page', 'chunk', 'setting', 'catalog'); +CREATE TRIGGER trg_groups_updated_at BEFORE UPDATE ON groups + FOR EACH ROW EXECUTE FUNCTION set_updated_at(); --- 1. users +-- 2. users CREATE TABLE users ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), keycloak_user_id TEXT NOT NULL UNIQUE, username TEXT, email TEXT, roles TEXT[] NOT NULL DEFAULT '{}', + group_id UUID REFERENCES groups(id) ON DELETE SET NULL, first_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(), last_seen_at TIMESTAMPTZ NOT NULL DEFAULT now() ); @@ -34,61 +34,51 @@ CREATE UNIQUE INDEX users_email_lower_key ON users (lower(email)) WHERE email IS INSERT INTO users (keycloak_user_id, username, roles) VALUES ('local-admin', 'local-admin', '{superadmin}'); --- 2. catalog_entries -CREATE TABLE catalog_entries ( +-- 3. catalogs: one row per provider group and resource type +CREATE TABLE catalogs ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - type TEXT NOT NULL CHECK (type ~ '^[a-z0-9]+([_-][a-z0-9]+)*$'), - code TEXT NOT NULL CHECK (code ~ '^[a-z0-9]+(-[a-z0-9]+)*$'), + group_id UUID NOT NULL REFERENCES groups(id) ON DELETE RESTRICT, + type TEXT NOT NULL CHECK (type ~ '^[a-z0-9]+:[A-Za-z0-9]+$'), name TEXT NOT NULL, - aliases TEXT[] NOT NULL DEFAULT '{}' CHECK (array_position(aliases, '') IS NULL), - metadata JSONB NOT NULL DEFAULT '{"source": "pipeline"}' CHECK (jsonb_typeof(metadata) = 'object'), + metadata JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(metadata) = 'object'), created_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), - UNIQUE (type, code), - UNIQUE (id, type) + UNIQUE (group_id, type) ); -CREATE TRIGGER trg_catalog_entries_updated_at BEFORE UPDATE ON catalog_entries +CREATE TRIGGER trg_catalogs_updated_at BEFORE UPDATE ON catalogs FOR EACH ROW EXECUTE FUNCTION set_updated_at(); --- 3. documents +-- 4. documents CREATE TABLE documents ( - id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - instance TEXT NOT NULL, - source JSONB NOT NULL - CHECK (jsonb_typeof(source) = 'object') - CHECK (source->>'fingerprint' ~ '^[0-9a-f]{32}$'), - document_kind TEXT NOT NULL DEFAULT 'document' CHECK (document_kind ~ '^[a-z0-9]+([_-][a-z0-9]+)*$'), - catalog_entry_id UUID, - valid_from DATE, - valid_to DATE CHECK (valid_from <= valid_to), - is_disabled BOOLEAN NOT NULL DEFAULT FALSE, - index_status JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(index_status) = 'object'), - metadata JSONB NOT NULL - DEFAULT '{"catalog_visible": true, "network_visible": true, "is_demo": false, "chunk_count": 0, "reindex_required": false}' - CHECK (jsonb_typeof(metadata) = 'object') - CHECK ((metadata->>'chunk_count')::int >= 0), - uploaded_by_user_id UUID REFERENCES users(id), - created_at TIMESTAMPTZ NOT NULL DEFAULT now(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), - FOREIGN KEY (catalog_entry_id, document_kind) REFERENCES catalog_entries (id, type) - ON UPDATE CASCADE ON DELETE RESTRICT + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + catalog_id UUID NOT NULL REFERENCES catalogs(id) ON DELETE RESTRICT, + code TEXT NOT NULL CHECK (code ~ '^[a-z0-9]+(-[a-z0-9]+)*$'), + name TEXT NOT NULL, + aliases TEXT[] NOT NULL DEFAULT '{}' CHECK (array_position(aliases, '') IS NULL), + source JSONB NOT NULL + CHECK (jsonb_typeof(source) = 'object') + CHECK (source->>'fingerprint' ~ '^[0-9a-f]{32}$'), + is_disabled BOOLEAN NOT NULL DEFAULT FALSE, + metadata JSONB NOT NULL + DEFAULT '{"index_status": {}, "catalog_visible": true, "network_visible": true, "reindex_required": false}' + CHECK (jsonb_typeof(metadata) = 'object'), + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (catalog_id, code) ); -CREATE INDEX documents_instance_idx ON documents (instance); CREATE INDEX documents_created_at_idx ON documents (created_at DESC); -CREATE INDEX documents_catalog_entry_idx ON documents (catalog_entry_id) WHERE catalog_entry_id IS NOT NULL; -CREATE INDEX documents_fingerprint_idx ON documents (instance, (source->>'fingerprint')); -CREATE INDEX documents_is_demo_idx ON documents (((metadata->>'is_demo')::boolean)); +CREATE INDEX documents_fingerprint_idx ON documents (catalog_id, (source->>'fingerprint')); CREATE TRIGGER trg_documents_updated_at BEFORE UPDATE ON documents FOR EACH ROW EXECUTE FUNCTION set_updated_at(); --- 4. workflows +-- 5. workflows CREATE TABLE workflows ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE, - workflow_type workflow_type_enum NOT NULL, + workflow_type TEXT NOT NULL, temporal_workflow_id TEXT NOT NULL, - status workflow_status_enum NOT NULL DEFAULT 'running', - current_stage workflow_stage NOT NULL DEFAULT 'registered', + status TEXT NOT NULL DEFAULT 'running', + current_stage TEXT NOT NULL DEFAULT 'registered', stage_timestamps JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(stage_timestamps) = 'object' AND (stage_timestamps - ARRAY['ocr', 'translation', 'chunks']) = '{}'::jsonb), @@ -111,12 +101,12 @@ SELECT DISTINCT ON (document_id) * FROM workflows ORDER BY document_id, started_at DESC, id DESC; --- 5. workflow_artifacts +-- 6. workflow_artifacts CREATE TABLE workflow_artifacts ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), workflow_id UUID NOT NULL REFERENCES workflows(id) ON DELETE CASCADE, artifact_type TEXT NOT NULL CHECK (btrim(artifact_type) <> ''), - stage workflow_stage, + stage TEXT, storage_uri TEXT NOT NULL, mime_type TEXT, filename TEXT, @@ -127,7 +117,7 @@ CREATE TABLE workflow_artifacts ( CREATE INDEX workflow_artifacts_workflow_idx ON workflow_artifacts (workflow_id, created_at DESC); CREATE INDEX workflow_artifacts_type_idx ON workflow_artifacts (workflow_id, artifact_type, created_at DESC); --- 6. pages +-- 7. pages CREATE TABLE pages ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE, @@ -143,7 +133,7 @@ CREATE TABLE pages ( CREATE TRIGGER trg_pages_updated_at BEFORE UPDATE ON pages FOR EACH ROW EXECUTE FUNCTION set_updated_at(); --- 7. chunks +-- 8. chunks CREATE TABLE chunks ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE, @@ -170,14 +160,14 @@ CREATE INDEX chunks_text_trgm_idx ON chunks CREATE TRIGGER trg_chunks_updated_at BEFORE UPDATE ON chunks FOR EACH ROW EXECUTE FUNCTION set_updated_at(); --- 8. audit_logs +-- 9. audit_logs CREATE TABLE audit_logs ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), document_id UUID REFERENCES documents(id) ON DELETE SET NULL, workflow_id TEXT, - instance TEXT, + group_id UUID REFERENCES groups(id) ON DELETE SET NULL, action_type TEXT NOT NULL CHECK (btrim(action_type) <> ''), - entity_type audit_entity_type, + entity_type TEXT, entity_number INTEGER, field_name TEXT, old_value TEXT, @@ -189,11 +179,12 @@ CREATE TABLE audit_logs ( CHECK (entity_type IN ('setting', 'catalog') OR workflow_id IS NOT NULL) ); CREATE INDEX audit_logs_document_idx ON audit_logs (document_id, created_at DESC); +CREATE INDEX audit_logs_group_idx ON audit_logs (group_id, created_at DESC); CREATE INDEX audit_logs_created_idx ON audit_logs (created_at DESC); CREATE INDEX audit_logs_action_idx ON audit_logs (action_type, created_at DESC); CREATE INDEX audit_logs_settings_idx ON audit_logs (created_at DESC) WHERE entity_type = 'setting'; --- 9. search_settings +-- 10. search_settings CREATE TABLE search_settings ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), singleton BOOLEAN NOT NULL DEFAULT TRUE UNIQUE CHECK (singleton), @@ -208,7 +199,7 @@ CREATE TRIGGER trg_search_settings_updated_at BEFORE UPDATE ON search_settings FOR EACH ROW EXECUTE FUNCTION set_updated_at(); INSERT INTO search_settings DEFAULT VALUES; --- 10. email_otps +-- 11. email_otps CREATE TABLE email_otps ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), email TEXT NOT NULL UNIQUE CHECK (email = lower(email)), @@ -237,56 +228,35 @@ CREATE TABLE webhook_records ( ); CREATE INDEX webhook_records_source_status_idx ON webhook_records (source, status); --- catalog view -CREATE VIEW catalog AS +-- catalog_status view: one row per document, computed from catalogs, documents and workflows +CREATE VIEW catalog_status AS SELECT - e.id AS entry_id, - e.type, - e.code, - e.name, - e.aliases, - e.metadata->>'source' AS source, + d.id AS document_id, + c.id AS catalog_id, + c.group_id, + g.name AS group_name, + c.type, + d.code, + d.name, + d.aliases, CASE - WHEN COALESCE(a.has_pending_reindex, FALSE) THEN 'pending_reindex' - WHEN COALESCE(a.live_count, 0) > 0 THEN 'live' - WHEN COALESCE(a.pending_prod_count, 0) > 0 THEN 'pending_prod' - WHEN COALESCE(a.doc_count, 0) = 0 AND e.metadata->>'source' = 'bootstrap' THEN 'live' + WHEN x.is_live AND COALESCE((d.metadata->>'reindex_required')::boolean, FALSE) THEN 'pending_reindex' + WHEN x.is_live THEN 'live' + WHEN cw.current_stage = 'approval_for_prod' AND NOT d.is_disabled THEN 'pending_prod' ELSE 'disabled' END AS status, - CASE WHEN COALESCE(cardinality(a.instances), 0) = 0 THEN ARRAY['default']::TEXT[] ELSE a.instances END AS instances, - COALESCE(a.workflow_ids, '{}'::TEXT[]) AS workflow_ids, - COALESCE(a.chunk_count, 0)::INTEGER AS chunk_count, - CASE - WHEN COALESCE(a.doc_count, 0) = 0 THEN COALESCE(e.metadata->>'source' = 'bootstrap', FALSE) - ELSE COALESCE(a.network_visible, FALSE) - END AS network_visible, - a.promoted_at -FROM catalog_entries e -LEFT JOIN LATERAL ( - SELECT - COUNT(*) AS doc_count, - COUNT(*) FILTER (WHERE x.is_live) AS live_count, - BOOL_OR(x.is_live AND COALESCE((x.metadata->>'reindex_required')::boolean, FALSE)) AS has_pending_reindex, - COUNT(*) FILTER (WHERE x.current_stage = 'approval_for_prod' AND NOT x.is_disabled) AS pending_prod_count, - ARRAY(SELECT DISTINCT d2.instance FROM documents d2 - WHERE d2.catalog_entry_id = e.id ORDER BY d2.instance) AS instances, - ARRAY(SELECT w.temporal_workflow_id FROM documents d3 - JOIN LATERAL ( - SELECT w2.temporal_workflow_id FROM workflows w2 - WHERE w2.document_id = d3.id AND w2.workflow_type = 'pipeline' - ORDER BY w2.started_at ASC, w2.id ASC LIMIT 1 - ) w ON TRUE - WHERE d3.catalog_entry_id = e.id - ORDER BY d3.created_at, d3.id) AS workflow_ids, - COALESCE(SUM((x.metadata->>'chunk_count')::int), 0) AS chunk_count, - BOOL_OR(COALESCE((x.metadata->>'network_visible')::boolean, TRUE)) AS network_visible, - MAX((x.index_status->'prod'->>'last_indexed_at')::timestamptz) FILTER (WHERE x.is_live) AS promoted_at - FROM ( - SELECT d.*, cw.current_stage, - (cw.current_stage = 'completed' AND NOT d.is_disabled - AND COALESCE((d.metadata->>'catalog_visible')::boolean, TRUE)) AS is_live - FROM documents d - LEFT JOIN document_current_workflow cw ON cw.document_id = d.id - WHERE d.catalog_entry_id = e.id - ) x -) a ON TRUE; + (SELECT w.temporal_workflow_id FROM workflows w + WHERE w.document_id = d.id AND w.workflow_type = 'pipeline' + ORDER BY w.started_at ASC, w.id ASC LIMIT 1) AS workflow_id, + (SELECT count(*) FROM chunks ch WHERE ch.document_id = d.id)::INTEGER AS chunk_count, + COALESCE((d.metadata->>'network_visible')::boolean, TRUE) AS network_visible, + CASE WHEN x.is_live + THEN (d.metadata->'index_status'->'prod'->>'last_indexed_at')::timestamptz END AS promoted_at +FROM documents d +JOIN catalogs c ON c.id = d.catalog_id +JOIN groups g ON g.id = c.group_id +LEFT JOIN document_current_workflow cw ON cw.document_id = d.id +CROSS JOIN LATERAL ( + SELECT (cw.current_stage = 'completed' AND NOT d.is_disabled + AND COALESCE((d.metadata->>'catalog_visible')::boolean, TRUE)) AS is_live +) x; diff --git a/pipeline/catalog_builder.py b/pipeline/catalog_builder.py index 4b74978..fd8d3d4 100644 --- a/pipeline/catalog_builder.py +++ b/pipeline/catalog_builder.py @@ -72,28 +72,28 @@ PUBLISHABLE_DOCUMENT_KINDS = tuple(_KNOWLEDGE_KIND_CATALOGS) -def normalize_document_kind(document_kind: Optional[str]) -> str: - """Coerce a stored `documents.document_kind` to its lookup form.""" - return (document_kind or DEFAULT_DOCUMENT_KIND).strip().lower() +def normalize_document_kind(kind: Optional[str]) -> str: + """Coerce a document's knowledge kind (from its catalog type) to its lookup form.""" + return (kind or DEFAULT_DOCUMENT_KIND).strip().lower() -def is_publishable_kind(document_kind: Optional[str]) -> bool: - return normalize_document_kind(document_kind) in _KNOWLEDGE_KIND_CATALOGS +def is_publishable_kind(kind: Optional[str]) -> bool: + return normalize_document_kind(kind) in _KNOWLEDGE_KIND_CATALOGS def build_catalog( - document_kind: Optional[str], + kind: Optional[str], provider: dict, catalog_name: str, group: Optional[Group] = None, ) -> Optional[dict]: - """Return the catalog to announce for `document_kind`, or None. + """Return the catalog to announce for `kind`, or None. None means this kind has nothing to announce - `document`, `video` and any operator-entered custom slug. Callers must skip the publish rather than send an empty `catalogs` array, which the spec rejects (`minItems: 1`). """ - spec = _KNOWLEDGE_KIND_CATALOGS.get(normalize_document_kind(document_kind)) + spec = _KNOWLEDGE_KIND_CATALOGS.get(normalize_document_kind(kind)) if spec is None: return None diff --git a/pipeline/db.py b/pipeline/db.py index 203522f..c90ed21 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -41,6 +41,7 @@ ResultSettings, RetrievalSettings, ) +from .network_constants import catalog_type_for_kind, kind_for_catalog_type logger = logging.getLogger(__name__) @@ -136,13 +137,13 @@ def check_health() -> dict: DEV_INDEX_KEY = "dev" -def _default_instance() -> str: +def _default_group() -> str: return (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" -def _normalize_instances(instances: list[str]) -> list[str]: - default = _default_instance() - return sorted({(i or "").strip().lower() or default for i in instances}) +def _normalize_groups(groups: list[str]) -> list[str]: + default = _default_group() + return sorted({(i or "").strip().lower() or default for i in groups}) def _iso_now() -> str: @@ -245,7 +246,8 @@ def _resolve_document_uuid(conn, identifier: Optional[str]) -> Optional[str]: """ SELECT id FROM documents d WHERE EXISTS ( - SELECT 1 FROM jsonb_each(d.index_status) e WHERE e.value->>'doc_id' = %s + SELECT 1 FROM jsonb_each(COALESCE(d.metadata->'index_status', '{}'::jsonb)) e + WHERE e.value->>'doc_id' = %s ) OR md5(d.source->>'fingerprint') = %s ORDER BY created_at DESC LIMIT 1 """, @@ -267,15 +269,16 @@ def _require_document_uuid(conn, identifier: str) -> str: _DOC_SELECT = """ SELECT - d.id, d.instance, d.source, d.document_kind, d.catalog_entry_id, - d.valid_from, d.valid_to, d.is_disabled, d.index_status, d.metadata, + d.id, d.catalog_id, d.code, d.name, d.aliases, d.source, d.is_disabled, d.metadata, d.created_at, d.updated_at, + c.type AS catalog_type, g.name AS group_name, + (SELECT count(*) FROM pages p WHERE p.document_id = d.id) AS page_total, + (SELECT count(*) FROM chunks ch WHERE ch.document_id = d.id) AS chunk_total, root.temporal_workflow_id AS root_workflow_id, root.stop_after_ocr AS root_stop_after_ocr, cw.id AS current_workflow_pk, cw.current_stage AS current_stage, cw.error_message AS current_error_message, - ce.code AS entry_code, ce.name AS entry_name, ce.aliases AS entry_aliases, u.keycloak_user_id AS up_user_id, u.username AS up_username, u.email AS up_email, u.roles AS up_roles, ts.ocr_completed_at, ts.translation_completed_at, ts.chunks_completed_at, @@ -286,6 +289,8 @@ def _require_document_uuid(conn, identifier: str) -> str: oa.id AS original_artifact_id, na.id AS normalized_artifact_id FROM documents d + JOIN catalogs c ON c.id = d.catalog_id + JOIN groups g ON g.id = c.group_id LEFT JOIN document_current_workflow cw ON cw.document_id = d.id LEFT JOIN LATERAL ( SELECT w.temporal_workflow_id, w.stop_after_ocr FROM workflows w @@ -293,8 +298,7 @@ def _require_document_uuid(conn, identifier: str) -> str: ORDER BY (w.workflow_type IN ('pipeline', 'ocr_only')) DESC, w.started_at ASC, w.id ASC LIMIT 1 ) root ON TRUE - LEFT JOIN catalog_entries ce ON ce.id = d.catalog_entry_id - LEFT JOIN users u ON u.id = d.uploaded_by_user_id + LEFT JOIN users u ON u.id = NULLIF(d.metadata->>'uploaded_by_user_id', '')::uuid LEFT JOIN LATERAL ( SELECT min(w.stage_timestamps->>'ocr') AS ocr_completed_at, min(w.stage_timestamps->>'translation') AS translation_completed_at, @@ -324,6 +328,16 @@ def _require_document_uuid(conn, identifier: str) -> str: """ +def _kind_of(catalog_type: Optional[str]) -> Optional[str]: + """Knowledge kind a catalog type stands for; an unmapped type is returned as is.""" + if not catalog_type: + return None + try: + return kind_for_catalog_type(catalog_type) + except ValueError: + return catalog_type + + def _document_from_row(row: Optional[dict]) -> Optional[dict]: if row is None: return None @@ -331,39 +345,37 @@ def _document_from_row(row: Optional[dict]) -> Optional[dict]: meta = row["metadata"] or {} fingerprint = source.get("fingerprint") roles = row["up_roles"] or [] - entry_aliases = row["entry_aliases"] + entry_aliases = row["aliases"] doc = { "id": row["id"], "workflow_id": row["root_workflow_id"], "document_id": fingerprint, "canonical_document_id": fingerprint, "filename": source.get("original_filename") or "", - "display_name": meta.get("display_name"), + "name": row["name"], "source_filename": source.get("original_filename"), "source_manifest_name": None, "source_file_fingerprint": fingerprint, "filepath": source.get("storage_uri") or "", - "instance": row["instance"], + "group": (row["group_name"] or "").lower(), "stage": row["current_stage"] or "registered", - "page_count": source.get("page_count") or 0, - "chunk_count": meta.get("chunk_count") or 0, + "page_count": int(row["page_total"] or 0), + "chunk_count": int(row["chunk_total"] or 0), "error_message": row["current_error_message"], - "is_demo": bool(meta.get("is_demo")), "is_disabled": bool(row["is_disabled"]), "source_type": source.get("source_type"), "canonical_input_type": source.get("canonical_input_type"), "stop_after_ocr": bool(row["root_stop_after_ocr"]), "reindex_required": bool(meta.get("reindex_required")), "reindex_reason": meta.get("reindex_reason"), - "document_kind": row["document_kind"], - "scheme_code": row["entry_code"], - "scheme_name": row["entry_name"], - "scheme_aliases_json": json.dumps(list(entry_aliases)) if entry_aliases is not None else None, - "tool_routing": "qdrant" if row["entry_code"] else None, + "kind": _kind_of(row["catalog_type"]), + "code": row["code"], + "aliases": list(entry_aliases or []), + "tool_routing": "qdrant", "catalog_visible": bool(meta.get("catalog_visible", True)), "network_visible": bool(meta.get("network_visible", True)), - "valid_from": row["valid_from"], - "valid_to": row["valid_to"], + "valid_from": meta.get("valid_from"), + "valid_to": meta.get("valid_to"), "uploaded_by_user_id": row["up_user_id"], "uploaded_by_username": row["up_username"], "uploaded_by_email": row["up_email"], @@ -378,8 +390,8 @@ def _document_from_row(row: Optional[dict]) -> Optional[dict]: "original_artifact_id": row["original_artifact_id"], "normalized_artifact_id": row["normalized_artifact_id"], "latest_job_id": row["current_workflow_pk"], - "catalog_entry_id": row["catalog_entry_id"], - "index_status": row["index_status"] or {}, + "catalog_id": row["catalog_id"], + "index_status": meta.get("index_status") or {}, "created_at": row["created_at"], "updated_at": row["updated_at"], } @@ -394,23 +406,80 @@ def _fetch_document(conn, doc_uuid: Optional[str]) -> Optional[dict]: def _document_filters( - include_demo: bool, include_disabled: bool, - instances: Optional[list[str]], + groups: Optional[list[str]], alias: str = "d", + group_alias: str = "g", ) -> tuple[list[str], list]: + """WHERE clauses for document lists. The query must join ``groups`` as ``group_alias``.""" clauses: list[str] = [] params: list = [] - if not include_demo: - clauses.append(f"NOT COALESCE(({alias}.metadata->>'is_demo')::boolean, FALSE)") if not include_disabled: clauses.append(f"NOT {alias}.is_disabled") - if instances is not None: - clauses.append(f"lower({alias}.instance) = ANY(%s)") - params.append(_normalize_instances(instances)) + if groups is not None: + clauses.append(f"lower({group_alias}.name) = ANY(%s)") + params.append(_normalize_groups(groups)) return clauses, params +# ============================================================================= +# Groups +# ============================================================================= + +def _tenant_path(group: str) -> str: + """Keycloak group path of a tenant code: ``bv`` is ``/global``, others ``/states/``.""" + return "/global" if group == "bv" else f"/states/{group.upper()}" + + +def _tenant_code_from_path(path: str) -> str: + return "bv" if path == "/global" else path.rstrip("/").rsplit("/", 1)[-1].lower() + + +def upsert_group(path: str, keycloak_group_id: Optional[str] = None, conn=None) -> str: + """Create or refresh the ``groups`` row for a Keycloak group path; returns its UUID. + + Rows made by the SQLite move have no Keycloak id yet; the first sync that + sees the path fills it in. + """ + if conn is None: + with get_connection() as own: + return upsert_group(path, keycloak_group_id, conn=own) + row = conn.execute( + """ + INSERT INTO groups (keycloak_group_id, name, path) VALUES (%s, %s, %s) + ON CONFLICT (path) DO UPDATE SET + keycloak_group_id = COALESCE(EXCLUDED.keycloak_group_id, groups.keycloak_group_id) + RETURNING id + """, + (keycloak_group_id, _tenant_code_from_path(path), path), + ).fetchone() + return str(row["id"]) + + +def get_or_create_group(group: Optional[str], conn=None) -> str: + """UUID of the group for a tenant code (``bv``, ``mh``, ...), created when missing.""" + code = (group or _default_group()).strip().lower() or _default_group() + return upsert_group(_tenant_path(code), conn=conn) + + +def get_or_create_catalog(group_id: str, kind: str, conn=None) -> dict: + """The ``(group, kind)`` catalog row, created with a default name when missing.""" + if conn is None: + with get_connection() as own: + return get_or_create_catalog(group_id, kind, conn=own) + catalog_type = catalog_type_for_kind(kind) + row = conn.execute( + """ + INSERT INTO catalogs (group_id, type, name) + SELECT g.id, %s, g.name || ' ' || %s FROM groups g WHERE g.id = %s::uuid + ON CONFLICT (group_id, type) DO UPDATE SET type = EXCLUDED.type + RETURNING id, group_id, type, name, metadata + """, + (catalog_type, kind, group_id), + ).fetchone() + return _clean_row(row) + + # ============================================================================= # Users # ============================================================================= @@ -421,16 +490,29 @@ def upsert_user( email: Optional[str] = None, roles: Any = None, conn=None, + tenants: Optional[list[str]] = None, ) -> Optional[str]: - """Create or refresh the ``users`` row for a Keycloak subject; returns its UUID.""" + """Create or refresh the ``users`` row for a Keycloak subject; returns its UUID. + + ``tenants`` are the tenant codes in the user's token. A user has one + ``group_id``, so the first code (sorted) wins and a warning is logged when + there are several. Authorization still reads the token, not this column. + """ keycloak_user_id = (keycloak_user_id or "").strip() if not keycloak_user_id: return None if conn is None: with get_connection() as own: - return upsert_user(keycloak_user_id, username, email, roles, conn=own) + return upsert_user(keycloak_user_id, username, email, roles, conn=own, tenants=tenants) role_list = _csv_roles(roles) + tenant_codes = sorted({(t or "").strip().lower() for t in (tenants or []) if (t or "").strip()}) + if len(tenant_codes) > 1: + logger.warning( + "user %s has several tenants %s; users.group_id uses %s", + keycloak_user_id, tenant_codes, tenant_codes[0], + ) + group_uuid = get_or_create_group(tenant_codes[0], conn=conn) if tenant_codes else None email = (email or "").strip() or None username = (username or "").strip() or None for candidate_email in ([email, None] if email else [None]): @@ -438,17 +520,18 @@ def upsert_user( with conn.transaction(): row = conn.execute( """ - INSERT INTO users (keycloak_user_id, username, email, roles) - VALUES (%s, %s, %s, %s) + INSERT INTO users (keycloak_user_id, username, email, roles, group_id) + VALUES (%s, %s, %s, %s, %s::uuid) ON CONFLICT (keycloak_user_id) DO UPDATE SET username = COALESCE(EXCLUDED.username, users.username), email = COALESCE(EXCLUDED.email, users.email), roles = CASE WHEN cardinality(EXCLUDED.roles) > 0 THEN EXCLUDED.roles ELSE users.roles END, + group_id = COALESCE(EXCLUDED.group_id, users.group_id), last_seen_at = now() RETURNING id """, - (keycloak_user_id, username, candidate_email, role_list), + (keycloak_user_id, username, candidate_email, role_list, group_uuid), ).fetchone() return str(row["id"]) except psycopg.errors.UniqueViolation: @@ -742,100 +825,65 @@ def bump_catalog_version(reason: str = "") -> int: return int(row["version"]) -def upsert_catalog_entry( - *, - type: str, - code: str, - name: Optional[str] = None, - aliases: Optional[list[str]] = None, - source: Optional[str] = None, - conn=None, -) -> dict: - """Insert or update one ``catalog_entries`` row. ``None`` fields keep stored values.""" - if conn is None: - with get_connection() as own: - return upsert_catalog_entry( - type=type, code=code, name=name, aliases=aliases, source=source, conn=own - ) - alias_list = [a for a in (aliases or []) if a] if aliases is not None else None - row = conn.execute( - """ - INSERT INTO catalog_entries (type, code, name, aliases, metadata) - VALUES (%s, %s, COALESCE(%s, %s), COALESCE(%s, '{}'::text[]), %s) - ON CONFLICT (type, code) DO UPDATE SET - name = COALESCE(%s, catalog_entries.name), - aliases = COALESCE(%s, catalog_entries.aliases), - metadata = CASE WHEN %s::text IS NULL THEN catalog_entries.metadata - ELSE catalog_entries.metadata || jsonb_build_object('source', %s::text) END - RETURNING id, type, code, name, aliases, metadata - """, - ( - type, code, name, code, alias_list, Jsonb({"source": source or "pipeline"}), - name, alias_list, source, source, - ), - ).fetchone() - return _clean_row(row) - - -def list_catalog_entry_codes(type: str) -> set[str]: +def list_document_codes(kind: str) -> set[str]: + """Every document code in catalogs of one knowledge kind.""" with get_connection() as conn: - rows = conn.execute("SELECT code FROM catalog_entries WHERE type = %s", (type,)).fetchall() + rows = conn.execute( + "SELECT d.code FROM documents d JOIN catalogs c ON c.id = d.catalog_id WHERE c.type = %s", + (catalog_type_for_kind(kind),), + ).fetchall() return {r["code"] for r in rows} def _catalog_row(row: dict) -> dict: cleaned = _clean_row(row) + cleaned.pop("group_name", None) cleaned["aliases"] = list(row["aliases"] or []) - cleaned["instances"] = list(row["instances"] or []) - cleaned["workflow_ids"] = list(row["workflow_ids"] or []) + cleaned["groups"] = [(row["group_name"] or "").lower()] + cleaned["workflow_ids"] = [row["workflow_id"]] if row["workflow_id"] else [] + cleaned["source"] = "pipeline" return cleaned -def list_catalog_entries(status: Optional[str] = None, type: str = "scheme") -> list[dict]: - """Rows of the ``catalog`` view for one entry type.""" +def list_catalog_entries(status: Optional[str] = None, kind: str = "scheme") -> list[dict]: + """Rows of the ``catalog_status`` view (one per document) for one knowledge kind.""" + catalog_type = catalog_type_for_kind(kind) with get_connection() as conn: if status: rows = conn.execute( - "SELECT * FROM catalog WHERE type = %s AND status = %s ORDER BY code", - (type, status), + "SELECT * FROM catalog_status WHERE type = %s AND status = %s ORDER BY code", + (catalog_type, status), ).fetchall() else: rows = conn.execute( - "SELECT * FROM catalog WHERE type = %s ORDER BY code", (type,) + "SELECT * FROM catalog_status WHERE type = %s ORDER BY code", (catalog_type,) ).fetchall() return [_catalog_row(r) for r in rows] -def get_catalog_entry(code: str, type: str = "scheme") -> Optional[dict]: +def get_catalog_entry(code: str, kind: str = "scheme") -> Optional[dict]: with get_connection() as conn: row = conn.execute( - "SELECT * FROM catalog WHERE type = %s AND code = %s", (type, code) + "SELECT * FROM catalog_status WHERE type = %s AND code = %s LIMIT 1", + (catalog_type_for_kind(kind), code), ).fetchone() return _catalog_row(row) if row else None -def list_used_scheme_codes(exclude_workflow_id: Optional[str] = None) -> set[str]: - """Every scheme code already taken - the collision domain for auto-generated codes. +def list_used_scheme_codes(exclude_workflow_id: Optional[str] = None, kind: str = "scheme") -> set[str]: + """Every document code already taken in a kind - the collision domain for auto-generated codes. - The entry the excluded document alone holds is not counted, so re-deriving a - document's own code does not collide with itself. + The excluded document's own code is not counted, so re-deriving a + document's code does not collide with itself. """ with get_connection() as conn: doc_uuid = _resolve_document_uuid(conn, exclude_workflow_id) if exclude_workflow_id else None rows = conn.execute( """ - SELECT e.code FROM catalog_entries e - WHERE e.type = 'scheme' - AND NOT ( - %s::uuid IS NOT NULL - AND e.metadata->>'source' <> 'bootstrap' - AND EXISTS (SELECT 1 FROM documents d - WHERE d.catalog_entry_id = e.id AND d.id = %s::uuid) - AND NOT EXISTS (SELECT 1 FROM documents d - WHERE d.catalog_entry_id = e.id AND d.id <> %s::uuid) - ) + SELECT d.code FROM documents d JOIN catalogs c ON c.id = d.catalog_id + WHERE c.type = %s AND (%s::uuid IS NULL OR d.id <> %s::uuid) """, - (doc_uuid, doc_uuid, doc_uuid), + (catalog_type_for_kind(kind), doc_uuid, doc_uuid), ).fetchall() return {r["code"] for r in rows} @@ -844,59 +892,30 @@ def list_scheme_catalog_entries_for_workflow(workflow_id: str) -> list[dict]: """Catalog rows whose documents still include this workflow.""" with get_connection() as conn: rows = conn.execute( - "SELECT code, workflow_ids FROM catalog WHERE type = 'scheme' AND %s = ANY(workflow_ids)", - (workflow_id,), + "SELECT code, workflow_id FROM catalog_status WHERE type = %s AND workflow_id = %s", + (catalog_type_for_kind("scheme"), workflow_id), ).fetchall() - return [{"scheme_code": r["code"], "workflow_ids": list(r["workflow_ids"])} for r in rows] + return [{"code": r["code"], "workflow_ids": [r["workflow_id"]]} for r in rows] -def _link_catalog_entry( - conn, - doc_uuid: str, - updates: dict, -) -> dict[str, Any]: - """Resolve document_kind / scheme_* updates into documents column changes. +def find_document_by_code(group: str, kind: str, code: str) -> Optional[dict]: + """The document holding ``code`` in the group's ``kind`` catalog, if any.""" + tenant = (group or _default_group()).strip().lower() or _default_group() + with get_connection() as conn: + row = conn.execute( + """ + SELECT d.id FROM documents d + JOIN catalogs c ON c.id = d.catalog_id + JOIN groups g ON g.id = c.group_id + WHERE lower(g.name) = %s AND c.type = %s AND d.code = %s + """, + (tenant, catalog_type_for_kind(kind), code), + ).fetchone() + return _fetch_document(conn, str(row["id"])) if row else None - Returns the extra ``documents`` assignments (``document_kind``, - ``catalog_entry_id``) so the kind and its entry change in one statement, - which the composite foreign key requires. - """ - current = conn.execute( - """ - SELECT d.document_kind, d.catalog_entry_id, ce.type AS entry_type, - ce.code AS entry_code, ce.name AS entry_name, ce.aliases AS entry_aliases - FROM documents d LEFT JOIN catalog_entries ce ON ce.id = d.catalog_entry_id - WHERE d.id = %s::uuid - """, - (doc_uuid,), - ).fetchone() - kind = updates.get("document_kind") or current["document_kind"] - code = updates["scheme_code"] if "scheme_code" in updates else current["entry_code"] - name = updates["scheme_name"] if "scheme_name" in updates else None - aliases = None - if "scheme_aliases_json" in updates: - raw = updates["scheme_aliases_json"] - aliases = json.loads(raw) if isinstance(raw, str) else list(raw or []) - - changes: dict[str, Any] = {} - if "document_kind" in updates: - changes["document_kind"] = kind - - current_entry_id = str(current["catalog_entry_id"]) if current["catalog_entry_id"] else None - new_entry_id = None - if kind != "document" and code: - moved = current["entry_type"] != kind or current["entry_code"] != code - if moved: - if name is None: - name = current["entry_name"] - if aliases is None and current["entry_aliases"] is not None: - aliases = list(current["entry_aliases"]) - new_entry_id = upsert_catalog_entry( - type=kind, code=code, name=name, aliases=aliases, source="pipeline", conn=conn - )["id"] - if new_entry_id != current_entry_id: - changes["catalog_entry_id"] = new_entry_id - return changes + +class DuplicateDocumentCode(ValueError): + """The ``(catalog, code)`` pair is already held by another document.""" # ============================================================================= @@ -909,7 +928,7 @@ def upsert_document( filename: str, filepath: str, canonical_document_id: Optional[str] = None, - display_name: Optional[str] = None, + name: Optional[str] = None, source_filename: Optional[str] = None, source_manifest_name: Optional[str] = None, source_file_fingerprint: Optional[str] = None, @@ -927,7 +946,7 @@ def upsert_document( original_artifact_id: Optional[str] = None, normalized_artifact_id: Optional[str] = None, latest_job_id: Optional[str] = None, - instance: Optional[str] = None, + group: Optional[str] = None, uploaded_by_user_id: Optional[str] = None, uploaded_by_username: Optional[str] = None, uploaded_by_email: Optional[str] = None, @@ -935,11 +954,20 @@ def upsert_document( valid_from: Optional[str] = None, valid_to: Optional[str] = None, with_workflow: bool = True, + kind: Optional[str] = None, + code: Optional[str] = None, + aliases: Optional[list[str]] = None, ) -> str: """Insert or update a document record; returns the document UUID. - ``instance`` is applied on INSERT only. Updates leave the existing tenant - stamp alone so a re-upload / restart cannot silently reassign tenants. + A new document needs ``kind`` (``scheme`` / ``advisory``) and ``code``: + ``group`` (a tenant code) and the kind pick its ``(group, kind)`` catalog, + which is created when missing. ``name`` becomes ``documents.name`` (the + code when absent). ``DuplicateDocumentCode`` is raised when another + document already holds ``code`` in that catalog. + + ``group`` and the catalog are applied on INSERT only. Updates leave them + alone so a re-upload / restart cannot silently reassign tenants. Uploader identity is set on INSERT, and filled on UPDATE only when currently null. ``valid_from``/``valid_to`` are the document's validity period, stored as @@ -948,12 +976,15 @@ def upsert_document( to is the caller's call (see ``pipeline/document_validity.py``); omitting both stores NULL, which search reads as "no stated period". + ``page_count`` and ``chunk_count`` are accepted for call-site compatibility + and ignored: both are counted from ``pages`` and ``chunks``. + A new document also gets its first workflow row (``temporal_workflow_id`` = ``workflow_id``) unless ``with_workflow`` is False, for callers that create it themselves with ``create_document_job`` (addressing the document by the returned UUID, since no workflow id resolves to it yet). """ - instance_value = (instance or _default_instance()).strip().lower() or _default_instance() + group_value = (group or _default_group()).strip().lower() or _default_group() fingerprint = _fingerprint_for( source_file_fingerprint or canonical_document_id, document_id, workflow_id ) @@ -963,30 +994,29 @@ def upsert_document( "storage_uri": filepath, "source_type": source_type, "canonical_input_type": canonical_input_type, - "page_count": page_count, } with get_connection() as conn: doc_uuid = _resolve_document_uuid(conn, workflow_id) if workflow_id else None if doc_uuid: patch = {k: v for k, v in source_patch.items() if v is not None} DocumentSource.model_validate({**_load(conn, doc_uuid, "source"), **patch}) - metadata_patch: dict[str, Any] = {"chunk_count": chunk_count} - if display_name: - metadata_patch["display_name"] = display_name uploader_uuid = ( upsert_user(uploaded_by_user_id, uploaded_by_username, uploaded_by_email, - uploaded_by_roles, conn=conn) + uploaded_by_roles, conn=conn, tenants=[group_value]) if uploaded_by_user_id else None ) conn.execute( """ UPDATE documents SET source = source || %s::jsonb, - metadata = metadata || %s::jsonb, - uploaded_by_user_id = COALESCE(uploaded_by_user_id, %s::uuid) + name = COALESCE(%s, name), + metadata = CASE + WHEN %s::text IS NULL OR metadata ? 'uploaded_by_user_id' THEN metadata + ELSE metadata || jsonb_build_object('uploaded_by_user_id', %s::text) + END WHERE id = %s::uuid """, - (Jsonb(patch), Jsonb(metadata_patch), uploader_uuid, doc_uuid), + (Jsonb(patch), name or None, uploader_uuid, uploader_uuid, doc_uuid), ) cw = conn.execute( "SELECT id FROM document_current_workflow WHERE document_id = %s::uuid", (doc_uuid,) @@ -994,7 +1024,7 @@ def upsert_document( if cw: conn.execute( """ - UPDATE workflows SET current_stage = %s::workflow_stage, error_message = %s, + UPDATE workflows SET current_stage = %s, error_message = %s, stop_after_ocr = %s WHERE id = %s::uuid """, @@ -1002,32 +1032,46 @@ def upsert_document( ) return doc_uuid + if not kind or not code: + raise ValueError("A new document needs kind and code") source = DocumentSource.model_validate( {k: v for k, v in source_patch.items() if v is not None} ).model_dump(exclude_none=True) - metadata = DocumentMetadata( - display_name=display_name, chunk_count=chunk_count - ).model_dump(exclude_none=True) user_uuid = ( upsert_user(uploaded_by_user_id, uploaded_by_username, uploaded_by_email, - uploaded_by_roles, conn=conn) + uploaded_by_roles, conn=conn, tenants=[group_value]) if uploaded_by_user_id else None ) - row = conn.execute( - """ - INSERT INTO documents (instance, source, metadata, valid_from, valid_to, uploaded_by_user_id) - VALUES (%s, %s, %s, %s::date, %s::date, %s::uuid) - RETURNING id - """, - (instance_value, Jsonb(source), Jsonb(metadata), valid_from, valid_to, user_uuid), - ).fetchone() + metadata = DocumentMetadata( + valid_from=valid_from, valid_to=valid_to, uploaded_by_user_id=user_uuid + ).model_dump(exclude_none=True) + catalog = get_or_create_catalog(get_or_create_group(group_value, conn=conn), kind, conn=conn) + try: + with conn.transaction(): + row = conn.execute( + """ + INSERT INTO documents (catalog_id, code, name, aliases, source, metadata) + VALUES (%s::uuid, %s, %s, %s, %s, %s) + RETURNING id + """, + ( + catalog["id"], code, name or code, + [a for a in (aliases or []) if a], Jsonb(source), Jsonb(metadata), + ), + ).fetchone() + except psycopg.errors.UniqueViolation as exc: + if exc.diag.constraint_name == "documents_catalog_id_code_key": + raise DuplicateDocumentCode( + f"Code '{code}' is already used in the {kind} catalog of {group_value}" + ) from exc + raise if with_workflow: conn.execute( """ INSERT INTO workflows (document_id, workflow_type, temporal_workflow_id, current_stage, error_message, stop_after_ocr, started_by_user_id) - VALUES (%s::uuid, %s::workflow_type_enum, %s, %s::workflow_stage, %s, %s, %s::uuid) + VALUES (%s::uuid, %s, %s, %s, %s, %s, %s::uuid) """, ( str(row["id"]), @@ -1038,6 +1082,43 @@ def upsert_document( return str(row["id"]) +def reset_document_for_reupload( + doc_uuid: str, + *, + filename: str, + filepath: str, + fingerprint: str, + source_type: Optional[str] = None, + canonical_input_type: Optional[str] = None, +) -> None: + """Point an existing document at a re-uploaded file. + + ``source`` is replaced, pages and chunks are cleared for the new run, and + the document is flagged ``reindex_required`` because its vector index is now + stale. Earlier workflows and artifacts stay as history. + """ + source = DocumentSource.model_validate({ + "fingerprint": fingerprint, + "original_filename": filename, + "storage_uri": filepath, + "source_type": source_type, + "canonical_input_type": canonical_input_type, + }).model_dump(exclude_none=True) + with get_connection() as conn: + conn.execute("DELETE FROM pages WHERE document_id = %s::uuid", (doc_uuid,)) + conn.execute("DELETE FROM chunks WHERE document_id = %s::uuid", (doc_uuid,)) + conn.execute( + """ + UPDATE documents SET + source = %s::jsonb, + metadata = (metadata - 'chunking') + || jsonb_build_object('reindex_required', TRUE, 'reindex_reason', 'source_replaced') + WHERE id = %s::uuid + """, + (Jsonb(source), doc_uuid), + ) + + def _load(conn, doc_uuid: str, column: str) -> dict: row = conn.execute(f"SELECT {column} FROM documents WHERE id = %s::uuid", (doc_uuid,)).fetchone() return dict(row[column]) if row else {} @@ -1050,7 +1131,11 @@ def update_document_stage( chunk_count: Optional[int] = None, error_message: Optional[str] = None ): - """Update the stage (of the document's current workflow) and counts.""" + """Update the stage (of the document's current workflow). + + ``page_count`` and ``chunk_count`` are accepted for call-site compatibility; + both are counted from ``pages`` and ``chunks``, so only the audit entry keeps them. + """ old_stage = None audit_workflow_id = workflow_id with get_connection() as conn: @@ -1065,7 +1150,7 @@ def update_document_stage( return old_stage = cw["current_stage"] - assignments = ["current_stage = %s::workflow_stage"] + assignments = ["current_stage = %s"] params: list = [stage] if error_message is not None: assignments.append("error_message = %s") @@ -1087,18 +1172,7 @@ def update_document_stage( f"UPDATE workflows SET {', '.join(assignments)} WHERE id = %s::uuid", params ) - doc_assignments = ["updated_at = now()"] - doc_params: list = [] - if page_count is not None: - doc_assignments.append("source = source || jsonb_build_object('page_count', %s::int)") - doc_params.append(page_count) - if chunk_count is not None: - doc_assignments.append("metadata = metadata || jsonb_build_object('chunk_count', %s::int)") - doc_params.append(chunk_count) - doc_params.append(doc_uuid) - conn.execute( - f"UPDATE documents SET {', '.join(doc_assignments)} WHERE id = %s::uuid", doc_params - ) + conn.execute("UPDATE documents SET updated_at = now() WHERE id = %s::uuid", (doc_uuid,)) audit_workflow_id = _root_workflow_id(conn, doc_uuid) or workflow_id if old_stage and old_stage != stage: @@ -1250,25 +1324,23 @@ def list_documents( stage: Optional[str] = None, limit: int = 100, offset: int = 0, - include_demo: bool = False, include_disabled: bool = False, - instances: Optional[list[str]] = None, + groups: Optional[list[str]] = None, ) -> list[dict]: - """List documents with optional stage / instance filters. + """List documents with optional stage / group filters. Args: stage: Filter by document stage limit: Max documents to return offset: Pagination offset - include_demo: If False (default), excludes demo documents from results include_disabled: If False (default), excludes soft-deleted documents from results - instances: If provided, only return docs whose instance is in this list + groups: If provided, only return docs whose group is in this list """ - if instances is not None and not _normalize_instances(instances): + if groups is not None and not _normalize_groups(groups): return [] - clauses, params = _document_filters(include_demo, include_disabled, instances) + clauses, params = _document_filters(include_disabled, groups) if stage: - clauses.append("cw.current_stage = %s::workflow_stage") + clauses.append("cw.current_stage = %s") params.append(stage) where = f"WHERE {' AND '.join(clauses)}" if clauses else "" with get_connection() as conn: @@ -1300,14 +1372,13 @@ def list_documents( def get_document_summary_counts( - include_demo: bool = False, include_disabled: bool = False, - instances: Optional[list[str]] = None, + groups: Optional[list[str]] = None, ) -> dict: """Return aggregate document counts for dashboard and migration planning.""" - if instances is not None and not _normalize_instances(instances): + if groups is not None and not _normalize_groups(groups): return {key: 0 for key in _SUMMARY_KEYS} - clauses, params = _document_filters(include_demo, include_disabled, instances) + clauses, params = _document_filters(include_disabled, groups) where = f"WHERE {' AND '.join(clauses)}" if clauses else "" with get_connection() as conn: row = conn.execute( @@ -1330,6 +1401,8 @@ def get_document_summary_counts( COALESCE(SUM((SELECT COUNT(*) FROM workflows j WHERE j.document_id = d.id AND j.status = 'running')), 0) AS running_jobs FROM documents d + JOIN catalogs c ON c.id = d.catalog_id + JOIN groups g ON g.id = c.group_id LEFT JOIN document_current_workflow cw ON cw.document_id = d.id {where} """, @@ -1338,37 +1411,38 @@ def get_document_summary_counts( return {key: int(row[key] or 0) for key in _SUMMARY_KEYS} -def get_document_counts_by_instance( - include_demo: bool = False, +def get_document_counts_by_group( include_disabled: bool = False, - instances: Optional[list[str]] = None, + groups: Optional[list[str]] = None, ) -> list[dict]: - """Return document counts grouped by instance (state), for dashboard breakdowns.""" - if instances is not None and not _normalize_instances(instances): + """Return document counts grouped by group (state), for dashboard breakdowns.""" + if groups is not None and not _normalize_groups(groups): return [] - clauses, params = _document_filters(include_demo, include_disabled, instances) + clauses, params = _document_filters(include_disabled, groups) where = f"WHERE {' AND '.join(clauses)}" if clauses else "" with get_connection() as conn: rows = conn.execute( f""" SELECT - lower(d.instance) AS instance, + lower(g.name) AS "group", COUNT(*) AS count, COUNT(*) FILTER (WHERE cw.current_stage = 'completed') AS success, COUNT(*) FILTER (WHERE cw.current_stage = 'failed') AS failed, COUNT(*) FILTER (WHERE cw.current_stage IN ('ocr_review', 'translation_review', 'chunk_review', 'approval_for_prod')) AS dev_approval FROM documents d + JOIN catalogs c ON c.id = d.catalog_id + JOIN groups g ON g.id = c.group_id LEFT JOIN document_current_workflow cw ON cw.document_id = d.id {where} - GROUP BY lower(d.instance) - ORDER BY count DESC, instance ASC + GROUP BY lower(g.name) + ORDER BY count DESC, "group" ASC """, params, ).fetchall() return [ { - "instance": row["instance"], + "group": row["group"], "count": int(row["count"] or 0), "success": int(row["success"] or 0), "failed": int(row["failed"] or 0), @@ -1378,18 +1452,6 @@ def get_document_counts_by_instance( ] -def set_document_demo(workflow_id: str, is_demo: bool = True): - """Mark a document as demo (filtered from UI by default).""" - with get_connection() as conn: - doc_uuid = _resolve_document_uuid(conn, workflow_id) - if doc_uuid: - conn.execute( - "UPDATE documents SET metadata = metadata || jsonb_build_object('is_demo', %s::boolean) " - "WHERE id = %s::uuid", - (bool(is_demo), doc_uuid), - ) - - def set_document_disabled(workflow_id: str, is_disabled: bool = True): """Soft delete a document (filtered from all queries by default). @@ -1421,7 +1483,7 @@ def purge_document(workflow_id: str, keep_audit: bool = True) -> dict: Audit rows are kept by default: they are the record that the deletion happened, not part of the document itself. Their ``document_id`` becomes - NULL; the ``workflow_id`` and ``instance`` snapshots stay. + NULL; the ``workflow_id`` and ``group`` snapshots stay. Returns a per-table count of deleted rows. """ @@ -1467,7 +1529,7 @@ def get_document_count(stage: Optional[str] = None) -> int: """ SELECT COUNT(*) AS cnt FROM documents d JOIN document_current_workflow cw ON cw.document_id = d.id - WHERE cw.current_stage = %s::workflow_stage + WHERE cw.current_stage = %s """, (stage,), ).fetchone() @@ -1479,9 +1541,10 @@ def get_document_count(stage: Optional[str] = None) -> int: def update_document_fields(workflow_id: str, **updates: object) -> Optional[dict]: """Update selected document fields and return the document. - Legacy flat names are mapped onto the tables: ``stage``/``error_message`` act - on the current workflow, ``scheme_*`` on the catalog entry, counts and - flags on the JSONB columns. Names derived from other rows (artifact and job + Flat names are mapped onto the tables: ``stage``/``error_message`` act on the + current workflow, ``code``/``name``/``aliases`` on the document columns, + ``kind`` moves the document to the group's catalog of that kind, and flags, + period and uploader go into ``metadata``. Names derived from other rows (counts, artifact and job pointers, completion timestamps) are accepted and ignored. """ with get_connection() as conn: @@ -1501,15 +1564,11 @@ def update_document_fields(workflow_id: str, **updates: object) -> Optional[dict for key, value in updates.items(): if key == "stage": - wf_sets.append("current_stage = %s::workflow_stage") + wf_sets.append("current_stage = %s") wf_params.append(value) elif key == "error_message": wf_sets.append("error_message = %s") wf_params.append(value) - elif key == "page_count": - source_patch["page_count"] = value - elif key == "chunk_count": - meta_patch["chunk_count"] = value elif key == "source_type": source_patch["source_type"] = value elif key == "canonical_input_type": @@ -1522,11 +1581,17 @@ def update_document_fields(workflow_id: str, **updates: object) -> Optional[dict elif key in ("document_id", "canonical_document_id", "source_file_fingerprint"): if isinstance(value, str) and _FINGERPRINT_RE.match(value): source_patch["fingerprint"] = value - elif key == "display_name": + elif key == "name": if value: - meta_patch["display_name"] = value - else: - meta_remove.append("display_name") + doc_sets.append("name = %s") + doc_params.append(value) + elif key == "code": + if value: + doc_sets.append("code = %s") + doc_params.append(value) + elif key == "aliases": + doc_sets.append("aliases = %s") + doc_params.append([a for a in (value or []) if a]) elif key in ("reindex_required", "catalog_visible", "network_visible"): meta_patch[key] = bool(value) elif key == "reindex_reason": @@ -1542,24 +1607,33 @@ def update_document_fields(workflow_id: str, **updates: object) -> Optional[dict (bool(value), doc_uuid), ) elif key in ("valid_from", "valid_to"): - doc_sets.append(f"{key} = %s::date") - doc_params.append(value) + meta_patch[key] = value.isoformat() if isinstance(value, date) else value elif key == "uploaded_by_user_id" and value: - doc_sets.append("uploaded_by_user_id = COALESCE(uploaded_by_user_id, %s::uuid)") - doc_params.append( - upsert_user( + has_uploader = conn.execute( + "SELECT metadata ? 'uploaded_by_user_id' AS has FROM documents WHERE id = %s::uuid", + (doc_uuid,), + ).fetchone()["has"] + if not has_uploader: + meta_patch["uploaded_by_user_id"] = upsert_user( str(value), updates.get("uploaded_by_username"), updates.get("uploaded_by_email"), updates.get("uploaded_by_roles"), conn=conn, ) - ) - if any(k in updates for k in ("document_kind", "scheme_code", "scheme_name", "scheme_aliases_json")): - for column, new_value in _link_catalog_entry(conn, doc_uuid, updates).items(): - doc_sets.append(f"{column} = %s" + ("::uuid" if column == "catalog_entry_id" else "")) - doc_params.append(new_value) + if updates.get("kind"): + current = conn.execute( + "SELECT c.group_id, c.type FROM documents d JOIN catalogs c ON c.id = d.catalog_id " + "WHERE d.id = %s::uuid", + (doc_uuid,), + ).fetchone() + if current["type"] != catalog_type_for_kind(str(updates["kind"])): + target = get_or_create_catalog( + str(current["group_id"]), str(updates["kind"]), conn=conn + ) + doc_sets.append("catalog_id = %s::uuid") + doc_params.append(target["id"]) if source_patch: doc_sets.append("source = source || %s::jsonb") @@ -1580,10 +1654,18 @@ def update_document_fields(workflow_id: str, **updates: object) -> Optional[dict (*wf_params, doc_uuid), ) if doc_sets: - conn.execute( - f"UPDATE documents SET {', '.join(doc_sets)} WHERE id = %s::uuid", - (*doc_params, doc_uuid), - ) + try: + with conn.transaction(): + conn.execute( + f"UPDATE documents SET {', '.join(doc_sets)} WHERE id = %s::uuid", + (*doc_params, doc_uuid), + ) + except psycopg.errors.UniqueViolation as exc: + if exc.diag.constraint_name == "documents_catalog_id_code_key": + raise DuplicateDocumentCode( + "Another document in the target catalog already uses this code" + ) from exc + raise elif wf_sets: conn.execute("UPDATE documents SET updated_at = now() WHERE id = %s::uuid", (doc_uuid,)) return _fetch_document(conn, doc_uuid) @@ -1732,8 +1814,8 @@ def create_document_job( if job_type == "promote_prod": adopted = conn.execute( """ - UPDATE workflows SET temporal_workflow_id = %s, status = %s::workflow_status_enum, - current_stage = COALESCE(%s::workflow_stage, current_stage), + UPDATE workflows SET temporal_workflow_id = %s, status = %s, + current_stage = COALESCE(%s, current_stage), config = config || %s::jsonb WHERE id = ( SELECT id FROM workflows @@ -1751,8 +1833,8 @@ def create_document_job( """ INSERT INTO workflows (document_id, workflow_type, temporal_workflow_id, status, current_stage, stop_after_ocr, config, started_by_user_id) - VALUES (%s::uuid, %s::workflow_type_enum, %s, %s::workflow_status_enum, - COALESCE(%s, 'registered')::workflow_stage, %s, %s, %s::uuid) + VALUES (%s::uuid, %s, %s, %s, + COALESCE(%s, 'registered'), %s, %s, %s::uuid) RETURNING id """, ( @@ -1778,9 +1860,9 @@ def update_document_job(job_id: str, **updates: object) -> Optional[dict]: if key not in allowed_fields: continue if key == "status": - set_clauses.append("status = %s::workflow_status_enum") + set_clauses.append("status = %s") elif key == "current_stage": - set_clauses.append("current_stage = %s::workflow_stage") + set_clauses.append("current_stage = %s") elif key == "completed_at": set_clauses.append("completed_at = %s::timestamptz") elif key == "temporal_run_id": @@ -1838,20 +1920,19 @@ def list_document_jobs(workflow_id: str, limit: int = 50) -> list[dict]: def list_operations_queue( limit: int = 100, offset: int = 0, - include_demo: bool = False, include_disabled: bool = False, - instances: Optional[list[str]] = None, + groups: Optional[list[str]] = None, ) -> tuple[list[dict], int]: - """List documents needing action, optionally scoped to tenant instances. + """List documents needing action, optionally scoped to tenant groups. - ``instances``: + ``groups``: - ``None`` - unrestricted (super admin / bypass) - ``[]`` - restricted user with no states -> empty result - - non-empty - only those document.instance values + - non-empty - only those document.group values """ - if instances is not None and not _normalize_instances(instances): + if groups is not None and not _normalize_groups(groups): return [], 0 - clauses, params = _document_filters(include_demo, include_disabled, instances) + clauses, params = _document_filters(include_disabled, groups) clauses.append( "(cw.current_stage IN ('ocr_review', 'translation_review', 'chunk_review', " "'ready_for_ingestion', 'approval_for_prod', 'failed') " @@ -1861,6 +1942,8 @@ def list_operations_queue( where = f"WHERE {' AND '.join(clauses)}" from_sql = """ FROM documents d + JOIN catalogs c ON c.id = d.catalog_id + JOIN groups g ON g.id = c.group_id JOIN document_current_workflow cw ON cw.document_id = d.id """ with get_connection() as conn: @@ -1874,7 +1957,7 @@ def list_operations_queue( LIMIT 1) AS workflow_id, d.source->>'original_filename' AS filename, cw.current_stage AS stage, - d.instance, + lower(g.name) AS "group", cw.error_message, COALESCE((d.metadata->>'reindex_required')::boolean, FALSE) AS reindex_required, d.metadata->>'reindex_reason' AS reindex_reason, @@ -1896,24 +1979,24 @@ def list_runs( limit: int = 100, offset: int = 0, status: Optional[str] = None, - instances: Optional[list[str]] = None, + groups: Optional[list[str]] = None, ) -> list[dict]: """List workflows with document title fields for the Runs UI. Args: - instances: If provided, only return runs whose document instance is in + groups: If provided, only return runs whose document group is in this list. Empty list returns no rows. None = unrestricted. """ where_parts: list[str] = [] params: list = [] if status: - where_parts.append("w.status = %s::workflow_status_enum") + where_parts.append("w.status = %s") params.append(status) - if instances is not None: - normalized = _normalize_instances(instances) + if groups is not None: + normalized = _normalize_groups(groups) if not normalized: return [] - where_parts.append("lower(d.instance) = ANY(%s)") + where_parts.append("lower(g.name) = ANY(%s)") params.append(normalized) where_sql = f"WHERE {' AND '.join(where_parts)}" if where_parts else "" with get_connection() as conn: @@ -1925,12 +2008,14 @@ def list_runs( ORDER BY (r.workflow_type IN ('pipeline', 'ocr_only')) DESC, r.started_at ASC, r.id ASC LIMIT 1) AS doc_workflow_id, d.source->>'original_filename' AS filename, - d.metadata->>'display_name' AS display_name, + d.name AS name, d.source->>'original_filename' AS source_filename, cw.current_stage AS document_stage, - d.instance AS instance + lower(g.name) AS "group" FROM workflows w JOIN documents d ON d.id = w.document_id + JOIN catalogs c ON c.id = d.catalog_id + JOIN groups g ON g.id = c.group_id LEFT JOIN document_current_workflow cw ON cw.document_id = d.id {where_sql} ORDER BY w.started_at DESC, w.id DESC @@ -1942,7 +2027,7 @@ def list_runs( for row in rows: job = _job_from_row(row) job["document_uuid"] = str(row["document_id"]) - for extra in ("filename", "display_name", "source_filename", "document_stage", "instance"): + for extra in ("filename", "name", "source_filename", "document_stage", "group"): job[extra] = row[extra] result.append(job) return result @@ -2024,7 +2109,7 @@ def add_document_artifact( """ INSERT INTO workflow_artifacts (workflow_id, artifact_type, stage, storage_uri, mime_type, filename, size_bytes, metadata) - VALUES (%s::uuid, %s, %s::workflow_stage, %s, %s, %s, %s, %s) + VALUES (%s::uuid, %s, %s, %s, %s, %s, %s, %s) RETURNING id """, ( @@ -2063,7 +2148,7 @@ def get_document_artifact(workflow_id: str, artifact_id: str) -> Optional[dict]: # ============================================================================= -# Vector index state (documents.index_status) +# Vector index state (documents.metadata.index_status) # ============================================================================= def _index_entry_to_legacy(workflow_id: str, key: str, entry: dict) -> dict: @@ -2092,7 +2177,7 @@ def upsert_document_index_status( status: str = "unknown", details: Optional[dict] = None, ) -> dict: - """Merge one index's state into ``documents.index_status``. + """Merge one index's state into ``documents.metadata.index_status``. The key is ``prod`` for the PROD collection names and ``dev`` otherwise; the collection name is kept inside the entry. ``None`` arguments leave the @@ -2102,7 +2187,7 @@ def upsert_document_index_status( with get_connection() as conn: doc_uuid = _require_document_uuid(conn, workflow_id) row = conn.execute( - "SELECT index_status->%s::text AS entry FROM documents WHERE id = %s::uuid FOR UPDATE", + "SELECT metadata->'index_status'->%s::text AS entry FROM documents WHERE id = %s::uuid FOR UPDATE", (key, doc_uuid), ).fetchone() existing = dict(row["entry"] or {}) @@ -2118,7 +2203,9 @@ def upsert_document_index_status( merged = {**existing, **{k: v for k, v in incoming.items() if v is not None}, "status": status} validated = IndexState.model_validate(merged).model_dump() conn.execute( - "UPDATE documents SET index_status = index_status || jsonb_build_object(%s::text, %s::jsonb) " + "UPDATE documents SET metadata = jsonb_set(" + "metadata || jsonb_build_object('index_status', COALESCE(metadata->'index_status', '{}'::jsonb)), " + "ARRAY['index_status', %s::text], %s::jsonb) " "WHERE id = %s::uuid", (key, Jsonb(validated), doc_uuid), ) @@ -2147,8 +2234,8 @@ def get_document_index_status(workflow_id: str, index_name: str) -> Optional[dic return None -def list_index_summaries(include_demo: bool = False, include_disabled: bool = False) -> list[dict]: - clauses, params = _document_filters(include_demo, include_disabled, None) +def list_index_summaries(include_disabled: bool = False) -> list[dict]: + clauses, params = _document_filters(include_disabled, None) where = f"AND {' AND '.join(clauses)}" if clauses else "" with get_connection() as conn: rows = conn.execute( @@ -2161,7 +2248,7 @@ def list_index_summaries(include_demo: bool = False, include_disabled: bool = Fa THEN 1 ELSE 0 END) AS stale_documents, MAX(e.value->>'last_indexed_at') AS last_indexed_at, array_agg(DISTINCT COALESCE(e.value->>'status', 'unknown')) AS statuses - FROM documents d, jsonb_each(d.index_status) e + FROM documents d, jsonb_each(COALESCE(d.metadata->'index_status', '{{}}'::jsonb)) e WHERE TRUE {where} GROUP BY COALESCE(e.value->>'index_name', e.key) ORDER BY index_name @@ -2212,13 +2299,13 @@ def resolve_chunk_provenance( excerpt = text[:320] + ("..." if len(text) > 320 else "") page_start = int(chunk.get("page_start") or 1) page_end = int(chunk.get("page_end") or page_start) - display_name = doc.get("display_name") or doc.get("filename") or workflow_id + doc_name = doc.get("name") or doc.get("filename") or workflow_id return { "workflow_id": workflow_id, "doc_id": doc.get("document_id"), "filename": workflow_id, - "doc_name": display_name, + "doc_name": doc_name, "chunk_num": int(chunk_num), "page_start": page_start, "page_end": page_end, @@ -2275,13 +2362,15 @@ def log_audit( roles = _csv_roles(actor_roles) with get_connection() as conn: doc_uuid = None if system_entry else _resolve_document_uuid(conn, workflow_id) - instance = None + group_uuid = None stored_workflow_id = None if system_entry else workflow_id if doc_uuid: doc_row = conn.execute( - "SELECT instance FROM documents WHERE id = %s::uuid", (doc_uuid,) + "SELECT c.group_id FROM documents d JOIN catalogs c ON c.id = d.catalog_id " + "WHERE d.id = %s::uuid", + (doc_uuid,), ).fetchone() - instance = doc_row["instance"] + group_uuid = str(doc_row["group_id"]) if _is_uuid(workflow_id): stored_workflow_id = _root_workflow_id(conn, doc_uuid) or workflow_id if stored_workflow_id is None and entity not in ("setting", "catalog"): @@ -2292,14 +2381,14 @@ def log_audit( ) row = conn.execute( """ - INSERT INTO audit_logs (document_id, workflow_id, instance, action_type, entity_type, + INSERT INTO audit_logs (document_id, workflow_id, group_id, action_type, entity_type, entity_number, field_name, old_value, new_value, metadata, actor_user_id, actor) - VALUES (%s::uuid, %s, %s, %s, %s::audit_entity_type, %s, %s, %s, %s, %s, %s::uuid, %s) + VALUES (%s::uuid, %s, %s::uuid, %s, %s, %s, %s, %s, %s, %s, %s::uuid, %s) RETURNING id """, ( - doc_uuid, stored_workflow_id, instance, action_type, entity, entity_id, + doc_uuid, stored_workflow_id, group_uuid, action_type, entity, entity_id, field_name, old_value, new_value, Jsonb(metadata or {}), actor_uuid, Jsonb({"username": actor_username or None, "email": actor_email or None, "roles": roles}) if (actor_user_id or actor_username or actor_email or roles) else Jsonb({}), @@ -2371,17 +2460,20 @@ def _normalize_audit_row(row) -> dict: _AUDIT_SELECT = """ SELECT a.id, a.workflow_id, a.action_type, a.entity_type, a.entity_number, a.field_name, - a.old_value, a.new_value, a.metadata, a.created_at, a.actor, a.instance AS audit_instance, + a.old_value, a.new_value, a.metadata, a.created_at, a.actor, ag.name AS audit_group, au.keycloak_user_id AS actor_keycloak_id, d.source->>'fingerprint' AS doc_fingerprint, d.source->>'original_filename' AS filename, - d.metadata->>'display_name' AS display_name, - d.instance AS doc_instance, + d.name AS document_name, + dg.name AS doc_group, uu.username AS uploaded_by_username, uu.email AS uploaded_by_email FROM audit_logs a LEFT JOIN documents d ON d.id = a.document_id + LEFT JOIN groups ag ON ag.id = a.group_id + LEFT JOIN catalogs dc ON dc.id = d.catalog_id + LEFT JOIN groups dg ON dg.id = dc.group_id LEFT JOIN users au ON au.id = a.actor_user_id - LEFT JOIN users uu ON uu.id = d.uploaded_by_user_id + LEFT JOIN users uu ON uu.id = NULLIF(d.metadata->>'uploaded_by_user_id', '')::uuid """ @@ -2414,8 +2506,8 @@ def _audit_row_to_legacy(row: dict) -> dict: "actor_email": actor.get("email"), "actor_roles": ",".join(roles) if roles else None, "filename": row["filename"], - "display_name": row["display_name"], - "instance": row["doc_instance"] or row["audit_instance"], + "name": row["document_name"], + "group": row["doc_group"] or row["audit_group"], "uploaded_by_username": row["uploaded_by_username"], "uploaded_by_email": row["uploaded_by_email"], } @@ -2472,16 +2564,16 @@ def get_audit_log_count(workflow_id: str, action_type: Optional[str] = None) -> def _global_audit_filters( - action_type: Optional[str], instances: Optional[list[str]] + action_type: Optional[str], groups: Optional[list[str]] ) -> tuple[list[str], list]: where_parts: list[str] = ["a.action_type <> %s"] params: list = [CATALOG_VERSION_ACTION] if action_type: where_parts.append("a.action_type = %s") params.append(action_type) - if instances is not None: - where_parts.append("lower(COALESCE(a.instance, d.instance, %s)) = ANY(%s)") - params.extend([_default_instance(), _normalize_instances(instances)]) + if groups is not None: + where_parts.append("lower(COALESCE(ag.name, dg.name, %s)) = ANY(%s)") + params.extend([_default_group(), _normalize_groups(groups)]) return where_parts, params @@ -2489,7 +2581,7 @@ def get_all_audit_logs( action_type: Optional[str] = None, limit: int = 100, offset: int = 0, - instances: Optional[list[str]] = None, + groups: Optional[list[str]] = None, ) -> list[dict]: """ Get all audit logs across all documents. @@ -2498,14 +2590,14 @@ def get_all_audit_logs( action_type: Optional filter by action type limit: Maximum number of entries to return offset: Offset for pagination - instances: Tenant scope (None = all, [] = none) + groups: Tenant scope (None = all, [] = none) Returns: List of audit log entries as dicts, with document filename included """ - if instances is not None and not _normalize_instances(instances): + if groups is not None and not _normalize_groups(groups): return [] - where_parts, params = _global_audit_filters(action_type, instances) + where_parts, params = _global_audit_filters(action_type, groups) with get_connection() as conn: rows = conn.execute( f"{_AUDIT_SELECT} WHERE {' AND '.join(where_parts)} " @@ -2517,15 +2609,19 @@ def get_all_audit_logs( def get_all_audit_log_count( action_type: Optional[str] = None, - instances: Optional[list[str]] = None, + groups: Optional[list[str]] = None, ) -> int: """Get total count of all audit logs (optionally tenant-scoped).""" - if instances is not None and not _normalize_instances(instances): + if groups is not None and not _normalize_groups(groups): return 0 - where_parts, params = _global_audit_filters(action_type, instances) + where_parts, params = _global_audit_filters(action_type, groups) with get_connection() as conn: row = conn.execute( - f"SELECT COUNT(*) AS cnt FROM audit_logs a LEFT JOIN documents d ON d.id = a.document_id " + "SELECT COUNT(*) AS cnt FROM audit_logs a " + "LEFT JOIN documents d ON d.id = a.document_id " + "LEFT JOIN groups ag ON ag.id = a.group_id " + "LEFT JOIN catalogs dc ON dc.id = d.catalog_id " + "LEFT JOIN groups dg ON dg.id = dc.group_id " f"WHERE {' AND '.join(where_parts)}", params, ).fetchone() @@ -2876,8 +2972,8 @@ def save_chunks(workflow_id: str, chunks: list[dict]): Save all chunks for a document. Re-chunking runs in one transaction: the document's chunks are deleted, the - new ones inserted, and ``metadata.chunk_count`` and ``metadata.chunking`` - (with ``run_number`` incremented) updated together. + new ones inserted, and ``metadata.chunking`` (with ``run_number`` + incremented) updated together. """ with get_connection() as conn: doc_uuid = _require_document_uuid(conn, workflow_id) @@ -2921,7 +3017,7 @@ def save_chunks(workflow_id: str, chunks: list[dict]): chunk.get("reviewer_notes"), ), ) - meta_patch: dict[str, Any] = {"chunk_count": len(chunks)} + meta_patch: dict[str, Any] = {} if chunks: first = chunks[0] meta_patch["chunking"] = ChunkingRun( @@ -2931,10 +3027,13 @@ def save_chunks(workflow_id: str, chunks: list[dict]): config=_load_json_value(first.get("chunking_config_json"), {}), created_at=_iso_now(), ).model_dump() - conn.execute( - "UPDATE documents SET metadata = metadata || %s::jsonb WHERE id = %s::uuid", - (Jsonb(meta_patch), doc_uuid), - ) + if meta_patch: + conn.execute( + "UPDATE documents SET metadata = metadata || %s::jsonb WHERE id = %s::uuid", + (Jsonb(meta_patch), doc_uuid), + ) + else: + conn.execute("UPDATE documents SET updated_at = now() WHERE id = %s::uuid", (doc_uuid,)) def get_chunks(workflow_id: str, include_excluded: bool = False) -> list[dict]: @@ -2959,12 +3058,12 @@ def search_chunks( offset: int = 0, include_excluded: bool = False, stage: Optional[str] = None, - instances: Optional[list[str]] = None, + groups: Optional[list[str]] = None, ) -> tuple[list[dict], int]: """Search chunks across documents for maintainer workflows. - instances: If provided, only return chunks whose owning document's instance - is in this list (tenant scoping). ``None`` means no instance restriction. + groups: If provided, only return chunks whose owning document's group + is in this list (tenant scoping). ``None`` means no group restriction. """ where_clauses = ["TRUE"] params: list = [] @@ -2972,14 +3071,14 @@ def search_chunks( if not include_excluded: where_clauses.append("NOT c.is_excluded") if stage: - where_clauses.append("cw.current_stage = %s::workflow_stage") + where_clauses.append("cw.current_stage = %s") params.append(stage) - if instances is not None: - normalized = _normalize_instances(instances) + if groups is not None: + normalized = _normalize_groups(groups) if not normalized: where_clauses.append("FALSE") else: - where_clauses.append("lower(d.instance) = ANY(%s)") + where_clauses.append("lower(g.name) = ANY(%s)") params.append(normalized) if query and query.strip(): where_clauses.append("lower(coalesce(c.edited_text, c.original_text)) LIKE %s") @@ -2988,6 +3087,8 @@ def search_chunks( base_from_sql = f""" FROM chunks c JOIN documents d ON d.id = c.document_id + JOIN catalogs cat ON cat.id = d.catalog_id + JOIN groups g ON g.id = cat.group_id LEFT JOIN document_current_workflow cw ON cw.document_id = d.id WHERE {' AND '.join(where_clauses)} """ @@ -3003,7 +3104,7 @@ def search_chunks( d.metadata->'chunking' AS chunking, d.source->>'fingerprint' AS document_id, d.source->>'original_filename' AS filename, - d.metadata->>'display_name' AS display_name, + d.name AS name, cw.current_stage AS stage, d.updated_at AS document_updated_at, (SELECT r.temporal_workflow_id FROM workflows r WHERE r.document_id = d.id @@ -3020,7 +3121,7 @@ def search_chunks( for row in rows: row = dict(row) chunk = _chunk_from_row(row, row["root_workflow_id"]) - for extra in ("document_id", "filename", "display_name", "stage", "document_updated_at"): + for extra in ("document_id", "filename", "name", "stage", "document_updated_at"): chunk[extra] = _clean(row[extra]) chunks.append(chunk) return chunks, total diff --git a/pipeline/document_repository.py b/pipeline/document_repository.py index 83ff1ce..654570e 100644 --- a/pipeline/document_repository.py +++ b/pipeline/document_repository.py @@ -28,10 +28,10 @@ def get_document_kind(self, workflow_id: str) -> Optional[str]: doc = self._db.get_document(workflow_id) if not doc: return None - return doc.get("document_kind") + return doc.get("kind") def get_document_group(self, workflow_id: str) -> Optional[str]: doc = self._db.get_document(workflow_id) if not doc: return None - return group_for_instance(doc.get("instance")) + return group_for_instance(doc.get("group")) diff --git a/pipeline/instances.py b/pipeline/group_names.py similarity index 77% rename from pipeline/instances.py rename to pipeline/group_names.py index 74fa154..2dee0bf 100644 --- a/pipeline/instances.py +++ b/pipeline/group_names.py @@ -1,11 +1,13 @@ -"""Instance (state / tenant) code → human-readable name. +"""Group (state / tenant) code → human-readable name. -``documents.instance`` stores a short lowercase code — ``bv`` for the Bharat +A group code is a short lowercase ``groups.name`` — ``bv`` for the Bharat Vistaar portal, otherwise the state code matching the Keycloak group path -(``/states/MH/...`` → ``mh``). That code is what the pipeline filters on, but -it is not what a reader wants to see: consumers of the vector payload and the -master catalog get ``instance_name`` alongside it so they can show "Maharashtra" -without shipping their own lookup table. +(``/states/MH/...`` → ``mh``). A document's group is derived through its +catalog (``documents.catalog_id`` → ``catalogs.group_id``). That code is what +the pipeline filters on, but it is not what a reader wants to see: consumers of +the vector payload and the master catalog get ``instance_name`` (their field +name, unchanged) alongside the code so they can show "Maharashtra" without +shipping their own lookup table. Codes follow the standard two-letter Indian state/UT abbreviations. Unknown codes fall back to a title-cased version of the code itself rather than raising @@ -17,7 +19,7 @@ import os # Portal / platform-wide documents (not a state). -PORTAL_INSTANCE = "bv" +PORTAL_GROUP = "bv" def publish_to_network_enabled() -> bool: @@ -40,7 +42,7 @@ def prod_stage_disabled() -> bool: return (os.environ.get("DISABLE_PROD_SETTING", "false").strip().lower() in {"true", "1", "yes"}) -INSTANCE_NAMES: dict[str, str] = { +GROUP_NAMES: dict[str, str] = { # Platform "bv": "Bharat Vistaar", "default": "Default", @@ -88,15 +90,15 @@ def prod_stage_disabled() -> bool: } -def instance_display_name(code: str | None) -> str: - """Human-readable name for an instance code (``mh`` → ``Maharashtra``). +def group_display_name(code: str | None) -> str: + """Human-readable name for a group code (``mh`` → ``Maharashtra``). Unknown codes are title-cased so the field is never empty. """ normalized = (code or "").strip().lower() if not normalized: - return INSTANCE_NAMES["default"] - known = INSTANCE_NAMES.get(normalized) + return GROUP_NAMES["default"] + known = GROUP_NAMES.get(normalized) if known: return known # Unknown code: make it presentable rather than failing the ingest. diff --git a/pipeline/models.py b/pipeline/models.py index 0746ef0..07edd53 100644 --- a/pipeline/models.py +++ b/pipeline/models.py @@ -7,7 +7,7 @@ from enum import Enum from typing import Any, Optional -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator class DocumentStage(str, Enum): @@ -28,6 +28,34 @@ class DocumentStage(str, Enum): FAILED = "failed" +class WorkflowType(str, Enum): + """``workflows.workflow_type``: kind of processing run. Stored as TEXT.""" + PIPELINE = "pipeline" + OCR_ONLY = "ocr_only" + OCR_RETRY = "ocr_retry" + TRANSLATION_RETRY = "translation_retry" + CHUNKING_RETRY = "chunking_retry" + REINGEST = "reingest" + PROMOTE_PROD = "promote_prod" + + +class WorkflowStatus(str, Enum): + """``workflows.status``. Stored as TEXT.""" + RUNNING = "running" + WAITING_REVIEW = "waiting_review" + COMPLETED = "completed" + FAILED = "failed" + + +class AuditEntityType(str, Enum): + """``audit_logs.entity_type``. Stored as TEXT.""" + DOCUMENT = "document" + PAGE = "page" + CHUNK = "chunk" + SETTING = "setting" + CATALOG = "catalog" + + # Pipeline stage order for stepper UI PIPELINE_STAGES = [ ("registered", "Registered", "Document uploaded"), @@ -185,12 +213,12 @@ class DocumentSummary(BaseModel): canonical_document_id: Optional[str] = None workflow_id: str # The Temporal workflow ID (use this for API calls) filename: str - display_name: Optional[str] = None + name: Optional[str] = None # documents.name source_filename: Optional[str] = None source_manifest_name: Optional[str] = None source_file_fingerprint: Optional[str] = None authoritative: bool = False - instance: str = "default" + group: str = "default" # tenant code (groups.name) of the document's catalog stage: DocumentStage page_count: int chunk_count: int @@ -204,11 +232,11 @@ class DocumentSummary(BaseModel): uploaded_by_username: Optional[str] = None uploaded_by_email: Optional[str] = None uploaded_by_roles: list[str] = [] - # Scheme catalog metadata (document_kind scheme → Master Catalog) - document_kind: str = "document" - scheme_code: Optional[str] = None - scheme_name: Optional[str] = None - scheme_aliases: list[str] = [] + # Catalog placement: kind comes from the catalog type, code/name/aliases are + # the document's own (a scheme's code feeds the Master Catalog). + kind: Optional[str] = None # scheme | advisory + code: Optional[str] = None + aliases: list[str] = [] tool_routing: Optional[str] = None catalog_visible: bool = True network_visible: bool = True @@ -231,10 +259,10 @@ class SchemeMetadataUpdate(BaseModel): `document_validity.parse_period` so a bad period comes back as the same 400 a bad kind does, not a Pydantic 422. """ - document_kind: Optional[str] = None # document | scheme - scheme_code: Optional[str] = None - scheme_name: Optional[str] = None - scheme_aliases: Optional[list[str]] = None + kind: Optional[str] = None # scheme | advisory + code: Optional[str] = None + name: Optional[str] = None + aliases: Optional[list[str]] = None tool_routing: Optional[str] = None # qdrant | legacy | both catalog_visible: Optional[bool] = None network_visible: Optional[bool] = None @@ -325,8 +353,8 @@ class DocumentGraph(BaseModel): runtime: dict[str, Any] -class InstanceDocumentCount(BaseModel): - instance: str +class GroupDocumentCount(BaseModel): + group: str count: int success: int = 0 failed: int = 0 @@ -341,7 +369,7 @@ class DocumentCohortsResponse(BaseModel): review_queue: int failed_documents: int by_stage: dict[str, int] - by_instance: list[InstanceDocumentCount] = [] + by_group: list[GroupDocumentCount] = [] needs_reindex: int running_jobs: int = 0 @@ -417,8 +445,8 @@ class AuditLogEntry(BaseModel): # Document context — joined from documents. Without these declared, FastAPI # strips them from the response and the UI can only show a raw doc_id hash. filename: Optional[str] = None - display_name: Optional[str] = None - instance: Optional[str] = None + name: Optional[str] = None + group: Optional[str] = None uploaded_by_username: Optional[str] = None uploaded_by_email: Optional[str] = None @@ -506,7 +534,6 @@ class DocumentSource(BaseModel): storage_uri: Optional[str] = None source_type: Optional[str] = None # spreadsheet | document | unknown canonical_input_type: Optional[str] = None # spreadsheet | pdf - page_count: Optional[int] = Field(default=None, ge=0) class IndexState(BaseModel): @@ -534,18 +561,43 @@ class ChunkingRun(BaseModel): class DocumentMetadata(BaseModel): - """``documents.metadata``: display and flag fields.""" + """``documents.metadata``: period, uploader, index state and flag fields.""" model_config = ConfigDict(extra="forbid") - display_name: Optional[str] = None + valid_from: Optional[str] = None # YYYY-MM-DD + valid_to: Optional[str] = None # YYYY-MM-DD, None = no end + uploaded_by_user_id: Optional[str] = None # users.id (UUID string) + index_status: dict[str, IndexState] = Field(default_factory=dict) # keys dev / prod catalog_visible: bool = True network_visible: bool = True - is_demo: bool = False - chunk_count: int = Field(default=0, ge=0) reindex_required: bool = False reindex_reason: Optional[str] = None chunking: Optional[ChunkingRun] = None + @model_validator(mode="after") + def _period_is_ordered(self) -> "DocumentMetadata": + if self.valid_from and self.valid_to and self.valid_from > self.valid_to: + raise ValueError("valid_from must not be after valid_to") + return self + + +class CatalogNetworkState(BaseModel): + """``catalogs.metadata.network``: set once the catalog is published.""" + model_config = ConfigDict(extra="forbid") + + catalog_id: Optional[str] = None + last_published_at: Optional[str] = None + + +class CatalogMetadata(BaseModel): + """``catalogs.metadata``: Beckn publish information.""" + model_config = ConfigDict(extra="forbid") + + publish_directives: dict[str, Any] = Field(default_factory=dict) + visible_networks: list[str] = Field(default_factory=list) + criteria: dict[str, Any] = Field(default_factory=dict) + network: Optional[CatalogNetworkState] = None + class PageMetadata(BaseModel): """``pages.metadata``: OCR review and language fields.""" diff --git a/pipeline/network_constants.py b/pipeline/network_constants.py index eb6a494..fdc4575 100644 --- a/pipeline/network_constants.py +++ b/pipeline/network_constants.py @@ -35,3 +35,33 @@ PROVIDER_ID_TEMPLATE = "{instance}.{network_id}" GROUP_CATALOG_NAME_TEMPLATE = "{group_name} {catalog_name}" + +# Knowledge kinds the pipeline serves, and the network schema each maps to. +# ``catalogs.type`` stores the schema name; code asks this map what a type means +# instead of comparing ``openagrinet:*`` strings. The scheme mapping is a +# placeholder (ADR 0003 types schemes as KnowledgeAdvisory) - change it here only. +KIND_SCHEME = "scheme" +KIND_ADVISORY = "advisory" +KIND_TO_CATALOG_TYPE = { + KIND_SCHEME: "openagrinet:AgricultureResource", + KIND_ADVISORY: "openagrinet:KnowledgeAdvisory", +} +CATALOG_TYPE_TO_KIND = {schema: kind for kind, schema in KIND_TO_CATALOG_TYPE.items()} + + +def catalog_type_for_kind(kind: str) -> str: + """``catalogs.type`` for a knowledge kind; ValueError for an unknown kind.""" + try: + return KIND_TO_CATALOG_TYPE[(kind or "").strip().lower()] + except KeyError: + raise ValueError( + f"Unknown document kind {kind!r}. Use one of: {', '.join(KIND_TO_CATALOG_TYPE)}" + ) from None + + +def kind_for_catalog_type(catalog_type: str) -> str: + """Knowledge kind a ``catalogs.type`` stands for; ValueError for an unknown type.""" + try: + return CATALOG_TYPE_TO_KIND[catalog_type] + except KeyError: + raise ValueError(f"Unknown catalog type {catalog_type!r}") from None diff --git a/pipeline/scheme_catalog.py b/pipeline/scheme_catalog.py index 28c6c45..5657199 100644 --- a/pipeline/scheme_catalog.py +++ b/pipeline/scheme_catalog.py @@ -16,14 +16,10 @@ from . import db, document_validity from . import display_name as display_name_rules +from .network_constants import catalog_type_for_kind SCHEME_CODE_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") -# document_kind is a free-form slug (scheme, advisory, video, or a custom -# type entered in the UI) — same character rules as scheme_code but allows -# underscores too since kind names read more naturally that way (e.g. "how_to"). -DOCUMENT_KIND_RE = re.compile(r"^[a-z0-9]+(?:[_-][a-z0-9]+)*$") -SUGGESTED_DOCUMENT_KINDS = ("document", "scheme", "advisory", "video") _SCHEME_CODE_STOPWORDS = { "a", "an", "the", "of", "on", "for", "and", "or", "to", "in", "at", "by", @@ -192,14 +188,14 @@ def utc_now_iso() -> str: return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") -def central_instances() -> set[str]: +def central_groups() -> set[str]: raw = (os.environ.get("CENTRAL_INSTANCES") or "default").strip() return {p.strip().lower() for p in raw.split(",") if p.strip()} or {"default"} -def default_network_visible(instance: str | None) -> bool: - inst = (instance or "default").strip().lower() or "default" - return inst in central_instances() +def default_network_visible(group: str | None) -> bool: + code = (group or "default").strip().lower() or "default" + return code in central_groups() def scheme_collection_name() -> str: @@ -234,19 +230,21 @@ def _acronym_from_title(title: str) -> str: return letters -def derive_scheme_code(title: str, exclude_workflow_id: Optional[str] = None) -> str: +def derive_scheme_code( + title: str, exclude_workflow_id: Optional[str] = None, kind: str = "scheme" +) -> str: """Auto-generate a scheme_code from a document title: acronym of significant words, falling back to a truncated slug if the title has too few words to produce a useful acronym (e.g. a single-word title). Appends -2, -3, ... on - collision against every scheme_code already in use (documents table + - scheme_catalog_entries registry) so two schemes never share a code. + collision against every code already in use for the kind (``documents.code``) + so two documents of a kind never share a code across tenants. """ base = _acronym_from_title(title) if len(base) < 2: slug = re.sub(r"[^a-z0-9]+", "", (title or "").strip().lower()) base = slug[:8] or "scheme" - used = db.list_used_scheme_codes(exclude_workflow_id=exclude_workflow_id) + used = db.list_used_scheme_codes(exclude_workflow_id=exclude_workflow_id, kind=kind) if base not in used: return base n = 2 @@ -345,8 +343,8 @@ def _dedupe_aliases(aliases: list[str]) -> list[str]: def resolve_scheme_aliases( - scheme_code: str | None, - scheme_name: str | None, + code: str | None, + name: str | None, explicit: Any = None, ) -> list[str]: """Aliases to index for a scheme, in precedence order. @@ -358,60 +356,26 @@ def resolve_scheme_aliases( All three are merged rather than short-circuited, so a curator adding one alias does not silently discard the bootstrap and derived ones. """ - code = (scheme_code or "").strip().lower() + code = (code or "").strip().lower() merged: list[str] = list(parse_aliases(explicit)) bootstrap = BOOTSTRAP_VECTOR_SCHEMES.get(code) if bootstrap: merged.extend(bootstrap.get("scheme_aliases") or []) - if not (scheme_name or "").strip(): - scheme_name = bootstrap.get("scheme_name") + if not (name or "").strip(): + name = bootstrap.get("scheme_name") - merged.extend(_derive_aliases(code, scheme_name or "")) + merged.extend(_derive_aliases(code, name or "")) return _dedupe_aliases(merged) -def _seed_bootstrap_entry(code: str, definition: dict[str, Any]) -> None: - db.upsert_catalog_entry( - type="scheme", - code=code, - name=definition["scheme_name"], - aliases=list(definition.get("scheme_aliases") or []), - source="bootstrap", - ) - - -def bootstrap_catalog_if_empty(*, force: bool = False) -> dict[str, Any]: - """Seed bootstrap vector schemes that have no catalog entry yet (all of them if force).""" - existing = db.list_catalog_entry_codes("scheme") - missing = {c: d for c, d in BOOTSTRAP_VECTOR_SCHEMES.items() if force or c not in existing} - if not missing: - meta = db.get_catalog_meta() - return {"bootstrapped": False, "version": meta.get("version", 0), "entry_count": len(existing)} - - for code, definition in missing.items(): - _seed_bootstrap_entry(code, definition) - version = db.bump_catalog_version("bootstrap_vector_schemes") - return { - "bootstrapped": True, - "version": version, - "vector_count": len(BOOTSTRAP_VECTOR_SCHEMES), - "legacy_count": len(BOOTSTRAP_LEGACY_SCHEMES), - } - - def rebuild_catalog_from_documents(*, reason: str = "rebuild") -> int: """ - Re-seed bootstrap codes that have no entry, then bump the catalog version. + Bump the catalog version. - Status, instances, chunk counts and the rest are computed by the ``catalog`` - view, so there is nothing to recompute here. Bootstrap-only codes with no - documents stay live. + Status, groups, chunk counts and the rest are computed by the + ``catalog_status`` view, so there is nothing to recompute here. """ - existing = db.list_catalog_entry_codes("scheme") - for code, definition in BOOTSTRAP_VECTOR_SCHEMES.items(): - if code not in existing: - _seed_bootstrap_entry(code, definition) return db.bump_catalog_version(reason) @@ -420,7 +384,7 @@ def on_scheme_document_promoted(workflow_id: str) -> Optional[int]: doc = db.get_document(workflow_id) if not doc: return None - if (doc.get("document_kind") or "document") != "scheme": + if doc.get("kind") != "scheme": return None # Clear reindex flag on successful promote (new code is live) if int(doc.get("reindex_required") or 0): @@ -432,7 +396,7 @@ def on_scheme_document_promoted(workflow_id: str) -> Optional[int]: return rebuild_catalog_from_documents(reason=f"promote_scheme:{workflow_id}") -def on_scheme_document_disabled(workflow_id: str, scheme_code: Optional[str] = None) -> Optional[int]: +def on_scheme_document_disabled(workflow_id: str, code: Optional[str] = None) -> Optional[int]: return rebuild_catalog_from_documents(reason=f"disable_scheme:{workflow_id}") @@ -491,10 +455,10 @@ def build_snapshot( # Live snapshot for warmers: only status=live (pending excluded unless include_pending) if not include_pending and st != "live": continue - instances = e["instances"] + groups = e["groups"] if instance: inst = instance.strip().lower() - if inst not in {i.lower() for i in instances} and instances: + if inst not in {g.lower() for g in groups} and groups: continue nv = bool(e.get("network_visible")) if network_visible is not None and nv != network_visible: @@ -506,12 +470,12 @@ def build_snapshot( "scheme_aliases": e["aliases"], "tool": "search_schemes", "tool_routing": "qdrant", - "instances": instances or ["default"], + "instances": groups or ["default"], "network_visible": nv, "chunk_count": int(e.get("chunk_count") or 0), "workflow_ids": e["workflow_ids"], "promoted_at": e.get("promoted_at"), - "source": e.get("source") or "bootstrap", + "source": e.get("source") or "pipeline", "status": st, } ) @@ -551,10 +515,10 @@ def build_snapshot( def apply_scheme_metadata( workflow_id: str, *, - document_kind: Optional[str] = None, - scheme_code: Optional[str] = None, - scheme_name: Optional[str] = None, - scheme_aliases: Optional[list[str]] = None, + kind: Optional[str] = None, + code: Optional[str] = None, + name: Optional[str] = None, + aliases: Optional[list[str]] = None, tool_routing: Optional[str] = None, catalog_visible: Optional[bool] = None, network_visible: Optional[bool] = None, @@ -562,7 +526,7 @@ def apply_scheme_metadata( valid_to: Optional[str] = None, display_name: Optional[str] = None, ) -> dict[str, Any]: - """Update document scheme fields and rebuild catalog when needed. + """Update document catalog fields and rebuild catalog when needed. `valid_from`/`valid_to` set the document's validity period (see `pipeline/document_validity.py`). Either end alone is enough - the other is @@ -575,65 +539,61 @@ def apply_scheme_metadata( raise LookupError(f"Document not found: {workflow_id}") updates: dict[str, Any] = {} - old_code = (doc.get("scheme_code") or "").strip().lower() or None - old_kind = (doc.get("document_kind") or "document").strip().lower() + old_code = (doc.get("code") or "").strip().lower() or None + old_kind = (doc.get("kind") or "").strip().lower() code_renamed = False pending_reindex = False - stored_display_name = doc.get("display_name") + stored_display_name = doc.get("name") # v2: the display name is documents.name if display_name is not None and (stored_display_name is None or display_name.strip() != stored_display_name): display_name_rules.ensure_editable(doc) - updates["display_name"] = display_name_rules.normalize(display_name) - - if document_kind is not None: - kind = document_kind.strip().lower().replace(" ", "-") - if not kind or not DOCUMENT_KIND_RE.match(kind): - raise ValueError( - f"Invalid document_kind '{document_kind}'. Use a lowercase slug, " - f"e.g. one of {SUGGESTED_DOCUMENT_KINDS} or a custom type." - ) - updates["document_kind"] = kind + updates["name"] = display_name_rules.normalize(display_name) + + if kind is not None: + new_kind = kind.strip().lower() + catalog_type_for_kind(new_kind) # ValueError for a kind with no catalog type + updates["kind"] = new_kind else: - kind = old_kind + new_kind = old_kind new_code = old_code - if scheme_code is not None: - new_code = validate_scheme_code(scheme_code) if scheme_code else None - updates["scheme_code"] = new_code + if code is not None: + new_code = validate_scheme_code(code) if code else None + updates["code"] = new_code if old_code and new_code and old_code != new_code: code_renamed = True - if scheme_name is not None: - updates["scheme_name"] = scheme_name.strip() if scheme_name else None + if name is not None: + updates["name"] = name.strip() if name else None - # Auto-derive scheme_code from the title when the caller set a name but no + # Auto-derive the code from the title when the caller set a name but no # explicit code (the UI form only collects a name and previews the code; # this is the authoritative, collision-checked source of truth for it). - final_name = (updates["scheme_name"] if "scheme_name" in updates else doc.get("scheme_name")) or "" - if scheme_code is None and not new_code and kind == "scheme" and final_name.strip(): - new_code = derive_scheme_code(final_name, exclude_workflow_id=workflow_id) - updates["scheme_code"] = new_code + final_name = (updates["name"] if "name" in updates else doc.get("name")) or "" + if code is None and not new_code and new_kind == "scheme" and final_name.strip(): + new_code = derive_scheme_code(final_name, exclude_workflow_id=workflow_id, kind=new_kind) + updates["code"] = new_code - if scheme_aliases is not None: - updates["scheme_aliases_json"] = json.dumps(parse_aliases(scheme_aliases)) + if aliases is not None: + updates["aliases"] = parse_aliases(aliases) if tool_routing is not None: tr = tool_routing.strip().lower() if tr not in ("qdrant", "legacy", "both"): raise ValueError("tool_routing must be qdrant|legacy|both") updates["tool_routing"] = tr - elif kind == "scheme" and not doc.get("tool_routing"): + elif new_kind == "scheme" and not doc.get("tool_routing"): updates["tool_routing"] = "qdrant" if catalog_visible is not None: updates["catalog_visible"] = 1 if catalog_visible else 0 - elif kind == "scheme" and doc.get("catalog_visible") is None: + elif new_kind == "scheme" and doc.get("catalog_visible") is None: updates["catalog_visible"] = 1 if network_visible is not None: updates["network_visible"] = 1 if network_visible else 0 - elif kind == "scheme" and doc.get("network_visible") is None: - updates["network_visible"] = 1 if default_network_visible(doc.get("instance")) else 0 + elif new_kind == "scheme" and doc.get("network_visible") is None: + updates["network_visible"] = 1 if default_network_visible(doc.get("group")) else 0 if valid_from is not None or valid_to is not None: # Validated here rather than at the SQL layer, and against the stored @@ -646,10 +606,6 @@ def apply_scheme_metadata( updates["valid_from"] = period.start_date updates["valid_to"] = period.end_date - if kind == "scheme" and not (new_code or updates.get("scheme_code") or old_code): - # Allow setting kind first without code only if not completing as scheme - pass - if code_renamed and (doc.get("stage") == "completed") and not int(doc.get("is_disabled") or 0): # Rule (b): hide new code until reindex updates["reindex_required"] = 1 @@ -661,7 +617,7 @@ def apply_scheme_metadata( # Rebuild catalog if scheme-related final = db.get_document(workflow_id) or doc - final_kind = (final.get("document_kind") or "document").strip().lower() + final_kind = (final.get("kind") or "").strip().lower() catalog_version = None if final_kind == "scheme" or old_kind == "scheme" or code_renamed: reason = "scheme_metadata_patch" From f3d30b26c694051fe2df0cbbb33579b117cba0ac Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 11:07:17 +0530 Subject: [PATCH 36/53] feat(api): require document kind at upload and rename tenancy to group - /upload, /documents and /documents/batch take kind (scheme|advisory) and an optional code; an existing code updates that document with the new file - group, kind, code, name and aliases replace instance, document_kind, scheme_code, scheme_name and scheme_aliases in requests and responses - tenancy helpers renamed to group; users.group_id set from the token tenants - demo flag, X-Include-Demo, POST /catalog/v1/bootstrap and the never-classified reingest guard are removed - /catalog/v1 keys, Qdrant payload keys and Temporal arguments keep their names --- pipeline/activities.py | 87 +++++------ pipeline/api/api.py | 216 ++++++++++++-------------- pipeline/api/documents.py | 122 ++++++++------- pipeline/api/upload.py | 81 ++++++---- pipeline/auth/deps.py | 5 +- pipeline/auth/groups.py | 10 +- pipeline/auth/keycloak_admin.py | 10 +- pipeline/auth/models.py | 28 ++-- pipeline/auth/tenancy.py | 79 +++++----- pipeline/discovery_publish_service.py | 16 +- pipeline/network_group.py | 12 +- 11 files changed, 337 insertions(+), 329 deletions(-) diff --git a/pipeline/activities.py b/pipeline/activities.py index 0364884..ff5adfe 100644 --- a/pipeline/activities.py +++ b/pipeline/activities.py @@ -29,7 +29,7 @@ from .chunking import chunk_pages, load_chunking_config from .discovery_publish_service import DiscoveryPublishService, PublishConfigError from .document_repository import DocumentRepository -from .instances import instance_display_name, publish_to_network_enabled +from .group_names import group_display_name, publish_to_network_enabled from .models import PUBLISH_SKIPPED_DISABLED, PUBLISH_SKIPPED_UNMAPPED_KIND from .network_constants import RESULT_REJECTED from .network_group import load_provider_naming, named_group, provider_for @@ -623,8 +623,8 @@ def clean_text_for_ingestion(text: str) -> str: return result.strip() -def _normalize_instance(value: str | None) -> str: - """Ingest-side instance normalizer (mirrors auth.tenancy without importing FastAPI).""" +def _normalize_group(value: str | None) -> str: + """Ingest-side group normalizer (mirrors auth.tenancy without importing FastAPI).""" text = (value or "").strip().lower() if text: return text @@ -640,18 +640,18 @@ def _prepare_records( name_en: str | None = None, description: str | None = None, include_e5_prefix_field: bool = True, - instance: str | None = None, - document_kind: str = "document", - scheme_code: str | None = None, - scheme_name: str | None = None, - scheme_aliases: list[str] | None = None, + group: str | None = None, + kind: str = "document", + code: str | None = None, + name: str | None = None, + aliases: list[str] | None = None, valid_from: str | None = None, valid_to: str | None = None, provider_id: str | None = None, display_name: str | None = None, ) -> list[dict]: metadata = _get_doc_metadata(filename) - resolved_instance = _normalize_instance(instance) + resolved_group = _normalize_group(group) llm_doc_description = _get_doc_description(filename) doc_hash = hashlib.md5(document_id.encode()).hexdigest() @@ -667,19 +667,17 @@ def _prepare_records( short_description = metadata.get("doc_short_description", "") effective_description = description or llm_doc_description or short_description - kind = (document_kind or "document").strip().lower() - is_scheme = kind == "scheme" and bool((scheme_code or "").strip()) - resolved_scheme_name = scheme_name or name_en or default_name + kind = (kind or "document").strip().lower() + is_scheme = kind == "scheme" and bool((code or "").strip()) + resolved_name = name or name_en or default_name # Merge curator-supplied, bootstrap and derived aliases so a scheme is never # indexed with an empty alias list (it would then only match its exact name). - aliases = ( - scheme_catalog.resolve_scheme_aliases( - scheme_code, resolved_scheme_name, scheme_aliases - ) + resolved_aliases = ( + scheme_catalog.resolve_scheme_aliases(code, resolved_name, aliases) if is_scheme else [] ) - instance_name = instance_display_name(resolved_instance) + group_name = group_display_name(resolved_group) # One period for the whole document, resolved before the loop: every chunk # of a document expires together, and re-deriving it per chunk would let a # midnight boundary split one document across two periods. @@ -701,8 +699,8 @@ def _prepare_records( "_id": point_id, "doc_id": document_id, "workflow_id": workflow_id or "", - "instance": resolved_instance, - "instance_name": instance_name, + "instance": resolved_group, # vector payload key, kept for consumers + "instance_name": group_name, "provider_id": provider_id, "type": "scheme" if is_scheme else "document", "document_kind": kind, @@ -738,9 +736,9 @@ def _prepare_records( if validity.end_date is not None: record["end_date"] = validity.end_date if is_scheme: - record["scheme_code"] = (scheme_code or "").strip().lower() - record["scheme_name"] = resolved_scheme_name - record["scheme_aliases"] = aliases + record["scheme_code"] = (code or "").strip().lower() + record["scheme_name"] = resolved_name + record["scheme_aliases"] = resolved_aliases if include_e5_prefix_field: record["text_for_embedding"] = f"passage: {text}" if text else "passage:" records.append(record) @@ -749,34 +747,21 @@ def _prepare_records( def _scheme_fields_from_doc(doc: dict | None) -> dict: - """Extract scheme kwargs for _prepare_records from a documents row.""" - import json as _json - + """Extract kind/code/name/aliases kwargs for _prepare_records from a documents row.""" doc = doc or {} - aliases_raw = doc.get("scheme_aliases_json") - aliases: list[str] = [] - if isinstance(aliases_raw, list): - aliases = [str(a) for a in aliases_raw] - elif isinstance(aliases_raw, str) and aliases_raw.strip(): - try: - parsed = _json.loads(aliases_raw) - if isinstance(parsed, list): - aliases = [str(a) for a in parsed] - except Exception: - aliases = [] return { - "document_kind": (doc.get("document_kind") or "document"), - "scheme_code": doc.get("scheme_code"), - "scheme_name": doc.get("scheme_name"), - "scheme_aliases": aliases, + "kind": (doc.get("kind") or "document"), + "code": doc.get("code"), + "name": doc.get("name"), + "aliases": [str(a) for a in (doc.get("aliases") or [])], } def _provider_id_from_doc(doc: dict | None) -> str | None: doc = doc or {} - group = named_group(group_for_instance(doc.get("instance"))) + group = named_group(group_for_instance(doc.get("group"))) try: - provider = provider_for(load_provider_naming(), doc.get("document_kind"), group) + provider = provider_for(load_provider_naming(), doc.get("kind"), group) except InvalidProviderIdError as exc: raise ApplicationError(str(exc), non_retryable=True) from exc return provider["id"] if provider else None @@ -807,7 +792,7 @@ def prepare_ingestion_records( name_gu: str | None = None, name_en: str | None = None, description: str | None = None, - instance: str | None = None, + group: str | None = None, ) -> list[dict]: """Public helper used by tests and scripts when preparing vector-index payloads.""" return _prepare_records( @@ -818,7 +803,7 @@ def prepare_ingestion_records( name_gu=name_gu, name_en=name_en, description=description, - instance=instance, + group=group, ) @@ -1248,7 +1233,7 @@ async def prepare_for_ingestion( name_gu=name_gu, name_en=name_en, description=description, - instance=(doc or {}).get("instance"), + group=(doc or {}).get("group"), **_validity_fields_from_doc(doc), provider_id=_provider_id_from_doc(doc), display_name=(doc or {}).get("display_name"), @@ -1324,8 +1309,8 @@ async def promote_document_to_prod_qdrant( chunks = db.get_chunks(workflow_id, include_excluded=True) doc = db.get_document(workflow_id) or {} scheme_kwargs = _scheme_fields_from_doc(doc) - kind = (scheme_kwargs.get("document_kind") or "document").strip().lower() - is_scheme = kind == "scheme" and bool((scheme_kwargs.get("scheme_code") or "").strip()) + kind = (scheme_kwargs.get("kind") or "document").strip().lower() + is_scheme = kind == "scheme" and bool((scheme_kwargs.get("code") or "").strip()) if is_scheme: prod_url = ( @@ -1359,7 +1344,7 @@ async def promote_document_to_prod_qdrant( filename, chunks, workflow_id=workflow_id, - instance=doc.get("instance"), + group=doc.get("group"), **scheme_kwargs, **_validity_fields_from_doc(doc), provider_id=_provider_id_from_doc(doc), @@ -1385,8 +1370,8 @@ async def promote_document_to_prod_qdrant( "collection": prod_collection, "records_ingested": result.get("records_ingested", len(records)), "prod_qdrant_url": prod_url, - "document_kind": kind, - "scheme_code": scheme_kwargs.get("scheme_code"), + "kind": kind, + "code": scheme_kwargs.get("code"), }, ) catalog_version = None @@ -1424,7 +1409,7 @@ async def ingest_document_from_db( filename, chunks, workflow_id=workflow_id, - instance=doc.get("instance"), + group=doc.get("group"), **_scheme_fields_from_doc(doc), **_validity_fields_from_doc(doc), provider_id=_provider_id_from_doc(doc), diff --git a/pipeline/api/api.py b/pipeline/api/api.py index 136e816..c2b1603 100644 --- a/pipeline/api/api.py +++ b/pipeline/api/api.py @@ -68,15 +68,15 @@ from ..auth.models import AuthUser from ..auth.permissions import Permission from ..auth.tenancy import ( - PORTAL_INSTANCE, - allowed_instances, - assert_document_instance_access, - normalize_instance, - resolve_create_instance, + PORTAL_GROUP, + allowed_groups, + assert_document_group_access, + normalize_group, + resolve_create_group, unrestricted, - user_can_access_instance, + user_can_access_group, ) -from ..instances import prod_stage_disabled +from ..group_names import prod_stage_disabled from ..models import ( PIPELINE_STAGES, PROD_ONLY_STAGES, @@ -100,7 +100,7 @@ SearchSettingsUpdate, SettingsAuditResponse, ) -from ..network_constants import RESULT_REJECTED +from ..network_constants import RESULT_REJECTED, catalog_type_for_kind from ..network_group import validate_network_provider_config from ..workflows import ( ChunkingOnlyWorkflow, @@ -144,7 +144,7 @@ async def lifespan(app: FastAPI): if auth_cfg.disabled: logging.warning( "WARNING: AUTH_DISABLED=true — every caller is treated as synthetic " - "master_admin with unrestricted instance access. Do not expose this " + "master_admin with unrestricted group access. Do not expose this " "API beyond the internal network or set AUTH_DISABLED=false until " "the maintainer UI sends Bearer tokens." ) @@ -385,15 +385,15 @@ def _compute_file_fingerprint(filepath: Path) -> str: return md5.hexdigest() -def _instance_scope_for_user(user: AuthUser) -> Optional[list[str]]: - """None = unrestricted; otherwise only these instance ids.""" - allowed = allowed_instances(user) +def _group_scope_for_user(user: AuthUser) -> Optional[list[str]]: + """None = unrestricted; otherwise only these group codes.""" + allowed = allowed_groups(user) if allowed is None: return None return sorted(allowed) -def _document_cohorts_payload(summary: dict, by_instance: Optional[list[dict]] = None) -> dict: +def _document_cohorts_payload(summary: dict, by_group: Optional[list[dict]] = None) -> dict: """Shape Postgres aggregate counts for DocumentCohortsResponse (all ints).""" def _i(key: str, default: int = 0) -> int: val = summary.get(key, default) @@ -417,24 +417,61 @@ def _i(key: str, default: int = 0) -> int: "ready_for_ingestion": _i("ready_for_ingestion_documents"), "failed": _i("failed_documents"), }, - "by_instance": by_instance or [], + "by_group": by_group or [], } -def _resolve_create_instance(user: AuthUser, requested: Optional[str] = None) -> str: - """Plan 2: stamp document.instance from Keycloak-allowed states / portal.""" - return resolve_create_instance(user, requested) +def _resolve_create_group(user: AuthUser, requested: Optional[str] = None) -> str: + """Pick the document's group from Keycloak-allowed states / portal.""" + return resolve_create_group(user, requested) + + +def _document_title(filename: str) -> str: + """Human title for a new document: the file name without extension, separators as spaces.""" + stem = Path(filename or "").stem + return re.sub(r"[_\-\s]+", " ", stem).strip() or "Untitled document" + + +def _resolve_document_identity( + user: AuthUser, + group: str, + kind: str, + code: str, + filename: str, +) -> tuple[str, str, Optional[dict]]: + """Kind, code and (when ``code`` already exists) the document an upload replaces. + + An explicit ``code`` that is already held in the tenant's catalog of that + kind means "update this document": the caller reuses it instead of inserting. + Without a code one is derived from the file name. + """ + kind = (kind or "").strip().lower() + try: + catalog_type_for_kind(kind) + except ValueError as exc: + raise HTTPException(400, str(exc)) from exc + code = (code or "").strip() + if code: + try: + code = scheme_catalog.validate_scheme_code(code) + except ValueError as exc: + raise HTTPException(400, str(exc)) from exc + existing = db.find_document_by_code(group, kind, code) + if existing: + existing = assert_document_group_access(user, existing) + return kind, code, existing + return kind, scheme_catalog.derive_scheme_code(_document_title(filename), kind=kind), None def _require_document_for_user(workflow_id: str, user: AuthUser) -> dict: - """Load a document or 404 if missing / outside the caller's instance scope.""" - return assert_document_instance_access(user, db.get_document(workflow_id)) + """Load a document or 404 if missing / outside the caller's group scope.""" + return assert_document_group_access(user, db.get_document(workflow_id)) def _document_for_user_or_none(workflow_id: str, user: AuthUser) -> Optional[dict]: """Like _require_document_for_user but returns None instead of raising (bulk paths).""" doc = db.get_document(workflow_id) - if not doc or not user_can_access_instance(user, doc.get("instance")): + if not doc or not user_can_access_group(user, doc.get("group")): return None return doc @@ -484,12 +521,12 @@ def _document_summary_from_row(doc: dict, current_job: Optional[dict] = None) -> canonical_document_id=doc.get("canonical_document_id"), workflow_id=doc["workflow_id"], filename=doc["filename"], - display_name=doc.get("display_name"), + name=doc.get("name"), source_filename=doc.get("source_filename"), source_manifest_name=doc.get("source_manifest_name"), source_file_fingerprint=doc.get("source_file_fingerprint"), authoritative=bool(doc.get("source_manifest_name")), - instance=normalize_instance(doc.get("instance")), + group=normalize_group(doc.get("group")), stage=DocumentStage(doc["stage"]), page_count=doc.get("page_count") or 0, chunk_count=doc.get("chunk_count") or 0, @@ -506,10 +543,9 @@ def _document_summary_from_row(doc: dict, current_job: Optional[dict] = None) -> uploaded_by_username=doc.get("uploaded_by_username"), uploaded_by_email=doc.get("uploaded_by_email"), uploaded_by_roles=_parse_roles_field(doc.get("uploaded_by_roles")), - document_kind=(doc.get("document_kind") or "document"), - scheme_code=doc.get("scheme_code"), - scheme_name=doc.get("scheme_name"), - scheme_aliases=_parse_scheme_aliases(doc), + kind=doc.get("kind"), + code=doc.get("code"), + aliases=list(doc.get("aliases") or []), tool_routing=doc.get("tool_routing"), catalog_visible=bool(int(doc["catalog_visible"])) if doc.get("catalog_visible") is not None else True, network_visible=bool(int(doc["network_visible"])) if doc.get("network_visible") is not None else True, @@ -735,8 +771,8 @@ async def auth_me(user: CurrentUser): "groups": list(user.groups or []), "state_roles": dict(user.state_roles or {}), "is_super_admin": bool(user.is_superadmin), - # Plan 2: portal instance code for BV / platform docs (not a state). - "portal_instance": PORTAL_INSTANCE, + # Portal group code for BV / platform docs (not a state). + "portal_group": PORTAL_GROUP, "auth_disabled": user.token_disabled_mode, } @@ -834,22 +870,19 @@ async def get_operations_queue( user: RequireSearch, limit: int = Query(100, le=500), offset: int = Query(0, ge=0), - x_include_demo: Optional[str] = Header(None, alias="X-Include-Demo"), x_include_disabled: Optional[str] = Header(None, alias="X-Include-Disabled"), ): """Return documents that currently need operator or agent action. - Results are limited to instances the caller can access (state users never + Results are limited to groups the caller can access (state users never see portal ``bv`` or other states' documents). """ - include_demo = x_include_demo and x_include_demo.lower() == "true" include_disabled = x_include_disabled and x_include_disabled.lower() == "true" rows, total = db.list_operations_queue( limit=limit, offset=offset, - include_demo=include_demo, include_disabled=include_disabled, - instances=_instance_scope_for_user(user), + groups=_group_scope_for_user(user), ) items = [ OperationQueueEntry( @@ -877,14 +910,14 @@ async def list_runs( ): """List recent document jobs visible to the caller (tenant/role scoped). - Restricted users only see jobs whose document ``instance`` is in their + Restricted users only see jobs whose document ``group`` is in their allowed states. Super-admin / bypass users see all tenants. """ return db.list_runs( limit=limit, offset=offset, status=status, - instances=_instance_scope_for_user(user), + groups=_group_scope_for_user(user), ) @@ -894,7 +927,7 @@ async def get_run(job_id: str, user: RequireSearch): run = db.get_document_job(job_id) if not run: raise HTTPException(404, f"Run not found: {job_id}") - # Enforce same instance scope as list: hide other tenants' runs as 404. + # Enforce same group scope as list: hide other tenants' runs as 404. workflow_id = run.get("workflow_id") if workflow_id: _require_document_for_user(str(workflow_id), user) @@ -903,10 +936,6 @@ async def get_run(job_id: str, user: RequireSearch): return run -def _parse_scheme_aliases(doc: dict) -> list[str]: - return scheme_catalog.parse_aliases(doc.get("scheme_aliases_json")) - - NETWORK_PUBLISH_ARTIFACT_TYPES = {"network_publish_payload", "network_publish_skipped"} @@ -944,12 +973,12 @@ def _build_document_detail(doc: dict) -> DocumentDetail: canonical_document_id=doc.get("canonical_document_id"), workflow_id=workflow_id, filename=doc["filename"], - display_name=doc.get("display_name"), + name=doc.get("name"), source_filename=doc.get("source_filename"), source_manifest_name=doc.get("source_manifest_name"), source_file_fingerprint=doc.get("source_file_fingerprint"), authoritative=bool(doc.get("source_manifest_name")), - instance=normalize_instance(doc.get("instance")), + group=normalize_group(doc.get("group")), filepath=doc["filepath"], stage=DocumentStage(doc["stage"]), page_count=doc.get("page_count", 0), @@ -979,10 +1008,9 @@ def _build_document_detail(doc: dict) -> DocumentDetail: uploaded_by_username=doc.get("uploaded_by_username"), uploaded_by_email=doc.get("uploaded_by_email"), uploaded_by_roles=_parse_roles_field(doc.get("uploaded_by_roles")), - document_kind=(doc.get("document_kind") or "document"), - scheme_code=doc.get("scheme_code"), - scheme_name=doc.get("scheme_name"), - scheme_aliases=_parse_scheme_aliases(doc), + kind=doc.get("kind"), + code=doc.get("code"), + aliases=list(doc.get("aliases") or []), tool_routing=doc.get("tool_routing"), catalog_visible=bool(int(doc["catalog_visible"])) if doc.get("catalog_visible") is not None else True, network_visible=bool(int(doc["network_visible"])) if doc.get("network_visible") is not None else True, @@ -1446,7 +1474,7 @@ async def disable_document( doc = _require_document_for_user(workflow_id, user) if not user.has_permission(Permission.ADMIN): - if not user.has_permission_for_instance(Permission.DELETE_OWN, doc.get("instance")): + if not user.has_permission_for_group(Permission.DELETE_OWN, doc.get("group")): raise HTTPException(403, "Missing permission: delete_own") if purge: raise HTTPException( @@ -1503,13 +1531,13 @@ async def disable_document( result["vector_error"] = vector_result["error"] # PROD Qdrant cleanup, if this document was ever promoted - kind = (doc.get("document_kind") or "document").strip().lower() - if kind == "scheme" or doc.get("scheme_code"): + kind = (doc.get("kind") or "").strip().lower() + if kind == "scheme": q_result = _delete_scheme_points_from_prod(doc) result["scheme_qdrant_deleted"] = q_result try: result["catalog_version"] = scheme_catalog.on_scheme_document_disabled( - workflow_id, scheme_code=doc.get("scheme_code") + workflow_id, code=doc.get("code") ) except Exception as exc: result["catalog_error"] = str(exc) @@ -1531,7 +1559,7 @@ async def disable_document( entity_type="document", metadata={ "filename": doc.get("filename"), - "scheme_code": doc.get("scheme_code"), + "code": doc.get("code"), "stage": doc.get("stage"), "minio_artifacts_deleted": result["minio_artifacts_deleted"], "keep_audit": keep_audit, @@ -1552,7 +1580,7 @@ async def disable_document( "vector_deleted": result["vector_deleted"], "scheme_qdrant_deleted": result.get("scheme_qdrant_deleted"), "prod_qdrant_deleted": result.get("prod_qdrant_deleted"), - "scheme_code": doc.get("scheme_code"), + "code": doc.get("code"), "catalog_version": result.get("catalog_version"), }, user=user, @@ -1568,19 +1596,19 @@ async def patch_scheme_metadata( user: RequireReview, ): """ - Set document_kind / scheme fields used by Master Catalog and Qdrant scheme promote. + Set kind / code / name / aliases used by Master Catalog and Qdrant scheme promote. - On completed docs, renaming scheme_code sets pending_reindex (excluded from live + On completed docs, renaming the code sets pending_reindex (excluded from live vector_schemes until re-promote). Other metadata edits bump catalog version only. """ _require_document_for_user(workflow_id, user) try: result = scheme_catalog.apply_scheme_metadata( workflow_id, - document_kind=body.document_kind, - scheme_code=body.scheme_code, - scheme_name=body.scheme_name, - scheme_aliases=body.scheme_aliases, + kind=body.kind, + code=body.code, + name=body.name, + aliases=body.aliases, tool_routing=body.tool_routing, catalog_visible=body.catalog_visible, network_visible=body.network_visible, @@ -1609,10 +1637,10 @@ async def patch_scheme_metadata( doc = result["document"] return { "workflow_id": workflow_id, - "document_kind": doc.get("document_kind"), - "scheme_code": doc.get("scheme_code"), - "scheme_name": doc.get("scheme_name"), - "scheme_aliases": scheme_catalog.parse_aliases(doc.get("scheme_aliases_json")), + "kind": doc.get("kind"), + "code": doc.get("code"), + "name": doc.get("name"), + "aliases": list(doc.get("aliases") or []), "tool_routing": doc.get("tool_routing"), "catalog_visible": bool(int(doc["catalog_visible"])) if doc.get("catalog_visible") is not None else True, "network_visible": bool(int(doc["network_visible"])) if doc.get("network_visible") is not None else True, @@ -1635,7 +1663,6 @@ async def patch_scheme_metadata( async def catalog_version(request: Request, user: CurrentUser): """Cheap poll for cache warmers: { version, updated_at }.""" _require_catalog_read(request, user) - scheme_catalog.bootstrap_catalog_if_empty() meta = db.get_catalog_meta() return { "version": int(meta.get("version") or 0), @@ -1655,7 +1682,6 @@ async def catalog_schemes( ): """List schemes from the master catalog (vector-indexed + filters).""" _require_catalog_read(request, user) - scheme_catalog.bootstrap_catalog_if_empty() snap = scheme_catalog.build_snapshot( instance=instance, status=status, @@ -1687,7 +1713,6 @@ async def catalog_snapshot( collections metadata, and monotonic catalog_version. """ _require_catalog_read(request, user) - scheme_catalog.bootstrap_catalog_if_empty() return scheme_catalog.build_snapshot( instance=instance, status=status, @@ -1700,7 +1725,6 @@ async def catalog_snapshot( async def catalog_tool_prompt(request: Request, user: CurrentUser): """Prompt fragments only (for AI system prompt assembly).""" _require_catalog_read(request, user) - scheme_catalog.bootstrap_catalog_if_empty() snap = scheme_catalog.build_snapshot() return { "catalog_version": snap["catalog_version"], @@ -1714,14 +1738,12 @@ async def catalog_tool_prompt(request: Request, user: CurrentUser): @app.post("/catalog/v1/rebuild") async def catalog_rebuild(request: Request, user: RequireAdmin): - """Admin: recompute catalog entries from documents + preserve bootstrap codes.""" + """Admin: bump the catalog version (entries are computed from documents).""" from ..auth.permissions import Permission # Admin JWT always OK; service key also accepted for automation if not _service_key_ok(request) and not user.has_permission(Permission.ADMIN): raise HTTPException(403, "Rebuild requires admin") - # Ensure bootstrap seeds exist before rebuild merge - scheme_catalog.bootstrap_catalog_if_empty() version = scheme_catalog.rebuild_catalog_from_documents(reason="admin_rebuild") snap = scheme_catalog.build_snapshot() _log_audit( @@ -1739,13 +1761,6 @@ async def catalog_rebuild(request: Request, user: RequireAdmin): } -@app.post("/catalog/v1/bootstrap") -async def catalog_bootstrap(request: Request, user: RequireAdmin, force: bool = Query(False)): - """Admin: seed 13 vector schemes if empty (or force=true).""" - result = scheme_catalog.bootstrap_catalog_if_empty(force=force) - return result - - @app.post("/documents/{workflow_id}/restore") async def restore_document(workflow_id: str, user: RequireAdmin): """ @@ -1789,24 +1804,6 @@ async def reingest_document( # Get document from Postgres doc = _require_document_for_user(workflow_id, user) - # Reingest skips the entire review pipeline (including ready_for_ingestion, - # where document type / scheme name get set) and jumps straight to pushing - # chunks to Qdrant. That's fine for re-pushing already-classified content - # (e.g. after an embedding/index change), but a trap if it's used as a - # substitute for the real approval flow on a never-classified document — - # it would silently ingest with document_kind='document' and no scheme - # metadata, the exact raw-hash/filename-fallback catalog rows we've - # had to clean up by hand. 'document' is never offered as a choice in the - # classification panel, so seeing it here means classification was never done. - if (doc.get("document_kind") or "document") == "document" and not doc.get("scheme_code") and not doc.get("scheme_name"): - raise HTTPException( - 400, - "This document has never been classified (document type / scheme name not set). " - "Set it via PATCH /documents/{workflow_id}/scheme-metadata (or the classification " - "panel at the ready_for_ingestion stage) before reingesting — reingest skips that " - "review stage entirely and would otherwise ingest with no scheme metadata.", - ) - # Get chunks from Postgres chunks = db.get_chunks(workflow_id, include_excluded=False) if not chunks: @@ -2033,19 +2030,6 @@ async def clear_reindex_required(workflow_id: str, user: RequirePipeline): } -@app.post("/documents/{workflow_id}/demo") -async def set_document_demo(workflow_id: str, user: RequireAdmin, is_demo: bool = Query(True)): - """ - Mark a document as demo. - - Demo documents are excluded from the UI by default but always available - for API testing via include_demo=true parameter. - """ - _require_document_for_user(workflow_id, user) - db.set_document_demo(workflow_id, is_demo) - return {"workflow_id": workflow_id, "is_demo": is_demo} - - async def _reconcile_single_document(doc: dict) -> dict: workflow_id = doc.get("workflow_id") current_stage = doc.get("stage") @@ -2622,14 +2606,14 @@ async def get_all_audit_logs( Each entry includes the document filename for context. """ - scope = _instance_scope_for_user(user) + scope = _group_scope_for_user(user) logs = db.get_all_audit_logs( action_type=action_type, limit=limit, offset=offset, - instances=scope, + groups=scope, ) - total = db.get_all_audit_log_count(action_type, instances=scope) + total = db.get_all_audit_log_count(action_type, groups=scope) return AuditLogResponse( logs=logs, @@ -2863,14 +2847,14 @@ async def search_chunks_across_documents( stage: Optional[DocumentStage] = Query(None, description="Optional document stage filter"), ): """Search chunks across all documents for KB maintainer workflows.""" - # Tenant-scope via the owning document's instance (None = unrestricted admin). + # Tenant-scope via the owning document's group (None = unrestricted admin). chunks, total = db.search_chunks( query=q, limit=limit, offset=offset, include_excluded=include_excluded, stage=stage.value if stage else None, - instances=_instance_scope_for_user(user), + groups=_group_scope_for_user(user), ) return { "items": chunks, @@ -3293,16 +3277,13 @@ async def get_vector_index_stats(index_name: str, user: RequireSearch): @app.get("/indexes/summary") async def get_vector_indexes_summary( user: RequireSearch, - x_include_demo: Optional[str] = Header(None, alias="X-Include-Demo"), x_include_disabled: Optional[str] = Header(None, alias="X-Include-Disabled"), ): """Summarize index coverage from Postgres-backed index status plus live vector-store stats.""" from ..vector_store import get_vector_store - include_demo = x_include_demo and x_include_demo.lower() == "true" include_disabled = x_include_disabled and x_include_disabled.lower() == "true" summaries = db.list_index_summaries( - include_demo=include_demo, include_disabled=include_disabled, ) if not summaries: @@ -3565,12 +3546,11 @@ async def reconcile_document_states(user: RequirePipeline): 'ready_for_ingestion', 'ingesting' ] - # Scope to caller's instances (None = unrestricted bypass / all tenants). + # Scope to caller's groups (None = unrestricted bypass / all tenants). docs = db.list_documents( limit=1000, - include_demo=True, include_disabled=True, - instances=_instance_scope_for_user(user), + groups=_group_scope_for_user(user), ) active_docs = [d for d in docs if d.get('stage') in active_stages] diff --git a/pipeline/api/documents.py b/pipeline/api/documents.py index 2c65771..a35607b 100644 --- a/pipeline/api/documents.py +++ b/pipeline/api/documents.py @@ -9,8 +9,8 @@ from .. import db, document_validity from ..auth.deps import CurrentUser, RequireUpload -from ..auth.tenancy import assert_document_instance_access, normalize_instance -from ..instances import prod_stage_disabled +from ..auth.tenancy import assert_document_group_access, normalize_group +from ..group_names import prod_stage_disabled from ..models import ( DocumentCohortsResponse, DocumentStage, @@ -36,7 +36,9 @@ async def start_document_workflow( min_tokens: int = 100, index_name: str = "documents-index", stop_after_ocr: bool = False, - instance: str = "", + group: str = "", + kind: str = Query(..., description="scheme or advisory"), + code: str = "", ): """ Start a new document processing workflow. @@ -52,7 +54,7 @@ async def start_document_workflow( Rate limited to 10 requests/minute per IP. Requires permission: upload (no-op while AUTH_DISABLED=true). """ - create_instance = _api._resolve_create_instance(user, instance) + create_group = _api._resolve_create_group(user, group) # Validate file path to prevent path traversal attacks filepath = _api.validate_file_path(data.filepath) source_filename = _api.get_filename_from_path(filepath) @@ -67,7 +69,7 @@ async def start_document_workflow( existing_doc = db.get_document(workflow_id) if existing_doc: # Same fingerprint/path must not leak or restart another tenant's doc. - existing_doc = assert_document_instance_access(user, existing_doc) + existing_doc = assert_document_group_access(user, existing_doc) try: handle = _api.temporal_client.get_workflow_handle(workflow_id) state = await handle.query("get_state") @@ -80,7 +82,7 @@ async def start_document_workflow( source_filename=source_filename, source_file_fingerprint=source_file_fingerprint, authoritative=bool(existing_doc.get("source_manifest_name")) if existing_doc else False, - instance=normalize_instance(existing_doc.get("instance")), + group=normalize_group(existing_doc.get("group")), stage=DocumentStage(state.get("stage", "registered")), page_count=state.get("page_count", 0), chunk_count=state.get("chunk_count", 0), @@ -118,25 +120,40 @@ async def start_document_workflow( # that knows the upload day. A reviewer confirms or moves it later, on the # form that sets the document type. validity = document_validity.default_period() - document_uuid = db.upsert_document( - workflow_id=workflow_id, - document_id=document_id, - canonical_document_id=canonical_document_id, - filename=source_filename, - source_filename=source_filename, - source_file_fingerprint=source_file_fingerprint, - filepath=str(filepath), - stage="registered", - stop_after_ocr=stop_after_ocr, - instance=create_instance, - uploaded_by_user_id=actor.get("user_id") or None, - uploaded_by_username=actor.get("username") or None, - uploaded_by_email=actor.get("email") or None, - uploaded_by_roles=actor.get("roles_csv") or None, - valid_from=validity.start_date, - valid_to=validity.end_date, - with_workflow=False, - ) + if replace_doc: + document_uuid = replace_doc["id"] + db.reset_document_for_reupload( + document_uuid, + filename=source_filename, + filepath=str(filepath), + fingerprint=source_file_fingerprint, + ) + else: + try: + document_uuid = db.upsert_document( + workflow_id=workflow_id, + document_id=document_id, + canonical_document_id=canonical_document_id, + filename=source_filename, + source_filename=source_filename, + source_file_fingerprint=source_file_fingerprint, + filepath=str(filepath), + stage="registered", + stop_after_ocr=stop_after_ocr, + group=create_group, + uploaded_by_user_id=actor.get("user_id") or None, + uploaded_by_username=actor.get("username") or None, + uploaded_by_email=actor.get("email") or None, + uploaded_by_roles=actor.get("roles_csv") or None, + valid_from=validity.start_date, + valid_to=validity.end_date, + with_workflow=False, + kind=doc_kind, + code=doc_code, + name=_api._document_title(source_filename), + ) + except db.DuplicateDocumentCode as exc: + raise HTTPException(409, str(exc)) from exc db.create_document_job( workflow_id=document_uuid, job_type="ocr_only" if stop_after_ocr else "pipeline", @@ -160,7 +177,13 @@ async def start_document_workflow( entity_type="document", field_name="registered", new_value=source_filename, - metadata={"source": "register_endpoint", "instance": create_instance}, + metadata={ + "source": "register_endpoint", + "group": create_group, + "kind": doc_kind, + "code": doc_code, + "replaced_existing": bool(replace_doc), + }, user=user, ) @@ -172,7 +195,7 @@ async def start_document_workflow( source_filename=source_filename, source_file_fingerprint=source_file_fingerprint, authoritative=False, - instance=create_instance, + group=create_group, stage=DocumentStage.REGISTERED, page_count=0, chunk_count=0, @@ -194,10 +217,11 @@ async def start_batch_workflows( chunk_overlap: int = 128, min_tokens: int = 100, stop_after_ocr: bool = False, - instance: str = "", + group: str = "", + kind: str = Query(..., description="scheme or advisory"), ): """Start workflows for all supported documents in a directory.""" - create_instance = _api._resolve_create_instance(user, instance) + create_group = _api._resolve_create_group(user, group) directory = Path(data.directory) if not directory.exists(): raise HTTPException(404, f"Directory not found: {data.directory}") @@ -218,7 +242,9 @@ async def start_batch_workflows( chunk_overlap=chunk_overlap, min_tokens=min_tokens, stop_after_ocr=stop_after_ocr, - instance=create_instance, + group=create_group, + kind=kind, + code="", ) results.append(result) except Exception as e: @@ -244,23 +270,20 @@ async def list_documents( stage: Optional[DocumentStage] = None, limit: int = Query(100, le=500), offset: int = Query(0, ge=0), - x_include_demo: Optional[str] = Header(None, alias="X-Include-Demo"), x_include_disabled: Optional[str] = Header(None, alias="X-Include-Disabled") ): """ List all document workflows. Uses Postgres for fast listing (no Temporal queries for performance). - Demo documents are excluded by default - use X-Include-Demo: true header to show them. Disabled (soft-deleted) documents are excluded by default - use X-Include-Disabled: true to show them. - Results are limited to instances the caller can access. + Results are limited to groups the caller can access. Pagination: - limit: Max documents to return (default 100, max 500) - offset: Skip first N documents (default 0) """ stage_filter = stage.value if stage else None - include_demo = x_include_demo and x_include_demo.lower() == "true" include_disabled = x_include_disabled and x_include_disabled.lower() == "true" # Use Postgres only for fast listing - no Temporal queries @@ -268,9 +291,8 @@ async def list_documents( stage=stage_filter, limit=limit, offset=offset, - include_demo=include_demo, include_disabled=include_disabled, - instances=_api._instance_scope_for_user(user), + groups=_api._group_scope_for_user(user), ) return [_api._document_summary_from_row(doc) for doc in docs] @@ -279,44 +301,36 @@ async def list_documents( @router.get("/summary", response_model=DocumentCohortsResponse) async def get_documents_summary( user: CurrentUser, - x_include_demo: Optional[str] = Header(None, alias="X-Include-Demo"), x_include_disabled: Optional[str] = Header(None, alias="X-Include-Disabled") ): """Return aggregate Postgres counts for dashboard totals and migration planning.""" - include_demo = x_include_demo and x_include_demo.lower() == "true" include_disabled = x_include_disabled and x_include_disabled.lower() == "true" - instance_scope = _api._instance_scope_for_user(user) + group_scope = _api._group_scope_for_user(user) summary = db.get_document_summary_counts( - include_demo=include_demo, include_disabled=include_disabled, - instances=instance_scope, + groups=group_scope, ) - by_instance = db.get_document_counts_by_instance( - include_demo=include_demo, + by_group = db.get_document_counts_by_group( include_disabled=include_disabled, - instances=instance_scope, + groups=group_scope, ) - return _api._document_cohorts_payload(summary, by_instance) + return _api._document_cohorts_payload(summary, by_group) @router.get("/cohorts", response_model=DocumentCohortsResponse) async def get_document_cohorts( user: CurrentUser, - x_include_demo: Optional[str] = Header(None, alias="X-Include-Demo"), x_include_disabled: Optional[str] = Header(None, alias="X-Include-Disabled") ): """Return machine-friendly cohort counts for queueing and orchestration.""" - include_demo = x_include_demo and x_include_demo.lower() == "true" include_disabled = x_include_disabled and x_include_disabled.lower() == "true" - instance_scope = _api._instance_scope_for_user(user) + group_scope = _api._group_scope_for_user(user) summary = db.get_document_summary_counts( - include_demo=include_demo, include_disabled=include_disabled, - instances=instance_scope, + groups=group_scope, ) - by_instance = db.get_document_counts_by_instance( - include_demo=include_demo, + by_group = db.get_document_counts_by_group( include_disabled=include_disabled, - instances=instance_scope, + groups=group_scope, ) - return _api._document_cohorts_payload(summary, by_instance) + return _api._document_cohorts_payload(summary, by_group) diff --git a/pipeline/api/upload.py b/pipeline/api/upload.py index 320e08d..f4fb5c7 100644 --- a/pipeline/api/upload.py +++ b/pipeline/api/upload.py @@ -5,13 +5,13 @@ from pathlib import Path from typing import Optional -from fastapi import APIRouter, File, Form, HTTPException, Request, UploadFile +from fastapi import APIRouter, File, Form, HTTPException, Query, Request, UploadFile from .. import db, document_validity from .. import display_name as display_name_rules from ..auth.deps import RequireUpload -from ..auth.tenancy import assert_document_instance_access, normalize_instance -from ..instances import prod_stage_disabled +from ..auth.tenancy import assert_document_group_access, normalize_group +from ..group_names import prod_stage_disabled from ..models import DocumentStage, DocumentSummary from ..workflows import DocumentPipelineWorkflow from . import api as _api @@ -31,7 +31,9 @@ async def upload_and_process( min_tokens: int = 100, index_name: str = "documents-index", stop_after_ocr: bool = False, - instance: str = "", + group: str = "", + kind: str = Query(..., description="scheme or advisory"), + code: str = "", display_name: Optional[str] = Form(None), ): """ @@ -42,7 +44,10 @@ async def upload_and_process( Rate limited to 10 requests/minute per IP. Requires permission: upload (no-op while AUTH_DISABLED=true). """ - create_instance = _api._resolve_create_instance(user, instance) + create_group = _api._resolve_create_group(user, group) + doc_kind, doc_code, replace_doc = _api._resolve_document_identity( + user, create_group, kind, code, file.filename or "" + ) suffix = Path(file.filename or "").suffix.lower() if suffix not in _api.ALLOWED_EXTENSIONS: raise HTTPException(400, f"Unsupported file type: {suffix}") @@ -86,7 +91,7 @@ async def upload_and_process( # If Postgres was purged, avoid returning stale Temporal state and create a fresh run ID. existing_doc = db.get_document(workflow_id) if existing_doc: - existing_doc = assert_document_instance_access(user, existing_doc) + existing_doc = assert_document_group_access(user, existing_doc) try: handle = _api.temporal_client.get_workflow_handle(workflow_id) state = await handle.query("get_state") @@ -99,7 +104,7 @@ async def upload_and_process( source_filename=file.filename, source_file_fingerprint=file_hash, authoritative=bool(existing_doc.get("source_manifest_name")) if existing_doc else False, - instance=normalize_instance(existing_doc.get("instance")), + group=normalize_group(existing_doc.get("group")), stage=DocumentStage(state.get("stage", "registered")), page_count=state.get("page_count", 0), chunk_count=state.get("chunk_count", 0), @@ -135,26 +140,41 @@ async def upload_and_process( actor = _api._actor_from_user(user) # See the manifest upload path above: the period starts at the upload. validity = document_validity.default_period() - document_uuid = db.upsert_document( - workflow_id=workflow_id, - document_id=document_id, - canonical_document_id=canonical_document_id, - filename=file.filename, - source_filename=file.filename, - source_file_fingerprint=file_hash, - filepath=minio_path, - display_name=display_name, - stage="registered", - stop_after_ocr=stop_after_ocr, - instance=create_instance, - uploaded_by_user_id=actor.get("user_id") or None, - uploaded_by_username=actor.get("username") or None, - uploaded_by_email=actor.get("email") or None, - uploaded_by_roles=actor.get("roles_csv") or None, - valid_from=validity.start_date, - valid_to=validity.end_date, - with_workflow=False, - ) + if replace_doc: + document_uuid = replace_doc["id"] + db.reset_document_for_reupload( + document_uuid, + filename=file.filename, + filepath=minio_path, + fingerprint=file_hash, + ) + else: + try: + document_uuid = db.upsert_document( + workflow_id=workflow_id, + document_id=document_id, + canonical_document_id=canonical_document_id, + filename=file.filename, + source_filename=file.filename, + source_file_fingerprint=file_hash, + filepath=minio_path, + stage="registered", + stop_after_ocr=stop_after_ocr, + group=create_group, + uploaded_by_user_id=actor.get("user_id") or None, + uploaded_by_username=actor.get("username") or None, + uploaded_by_email=actor.get("email") or None, + uploaded_by_roles=actor.get("roles_csv") or None, + valid_from=validity.start_date, + valid_to=validity.end_date, + with_workflow=False, + kind=doc_kind, + code=doc_code, + # v2: the contributor's Display Name is documents.name. + name=display_name or _api._document_title(file.filename or ""), + ) + except db.DuplicateDocumentCode as exc: + raise HTTPException(409, str(exc)) from exc job_id = db.create_document_job( workflow_id=document_uuid, job_type="ocr_only" if stop_after_ocr else "pipeline", @@ -191,7 +211,10 @@ async def upload_and_process( new_value=file.filename, metadata={ "source": "upload_endpoint", - "instance": create_instance, + "group": create_group, + "kind": doc_kind, + "code": doc_code, + "replaced_existing": bool(replace_doc), "size_bytes": file_size, "auto_approve": auto_approve, }, @@ -214,7 +237,7 @@ async def upload_and_process( source_filename=file.filename, source_file_fingerprint=file_hash, authoritative=False, - instance=create_instance, + group=create_group, stage=DocumentStage.REGISTERED, page_count=0, chunk_count=0, diff --git a/pipeline/auth/deps.py b/pipeline/auth/deps.py index 4b192f3..5a3af81 100644 --- a/pipeline/auth/deps.py +++ b/pipeline/auth/deps.py @@ -87,7 +87,10 @@ def _sync_user_row(user: AuthUser) -> None: if last is not None and now - last < _USER_SYNC_INTERVAL_SECONDS: return try: - db.upsert_user(user.user_id, user.username, user.email, user.roles) + db.upsert_user( + user.user_id, user.username, user.email, user.roles, + tenants=list(user.instances or []), + ) except Exception: logging.warning("users upsert failed for %s", user.user_id, exc_info=True) return diff --git a/pipeline/auth/groups.py b/pipeline/auth/groups.py index f907c02..4104b6e 100644 --- a/pipeline/auth/groups.py +++ b/pipeline/auth/groups.py @@ -217,17 +217,17 @@ def extract_groups_claim(claims: dict[str, Any]) -> list[str]: return [] -def role_for_instance(access: GroupAccess, instance: str | None) -> str | None: - """Return the user's role for a state instance, or SUPER_ADMIN when global. +def role_for_group(access: GroupAccess, group: str | None) -> str | None: + """Return the user's role for a state group, or SUPER_ADMIN when global. A ``bh_viewer`` has no per-state group, but reads every state — so any - instance resolves to STATE_VIEW unless a stronger state role is held. + group resolves to STATE_VIEW unless a stronger state role is held. """ if access.is_super_admin: return ROLE_SUPER_ADMIN - if not instance: + if not group: return None - role = access.state_roles.get(normalize_state_code(instance)) + role = access.state_roles.get(normalize_state_code(group)) if role: return role return ROLE_STATE_VIEW if access.is_bh_viewer else None diff --git a/pipeline/auth/keycloak_admin.py b/pipeline/auth/keycloak_admin.py index 0b67380..f4b1a2c 100644 --- a/pipeline/auth/keycloak_admin.py +++ b/pipeline/auth/keycloak_admin.py @@ -28,7 +28,7 @@ group_leaf_for_role, normalize_role, ) -from .tenancy import PORTAL_INSTANCE +from .tenancy import PORTAL_GROUP # Realm-role descriptions written when a role is auto-created. STATE_ROLE_DESCRIPTIONS: dict[str, str] = { @@ -506,7 +506,7 @@ def provision_user( "roles": [r.get("name") for r in (roles or []) if isinstance(r, dict)], "app_url": ui_url, "login_url": login_url, - "portal_instance": PORTAL_INSTANCE, + "portal_group": PORTAL_GROUP, "share_message": "\n".join(share_lines), } @@ -517,7 +517,7 @@ def list_access_options() -> dict[str, Any]: return { "keycloak_admin_configured": cfg.configured, "realm": cfg.realm, - "portal_instance": PORTAL_INSTANCE, + "portal_group": PORTAL_GROUP, "access_types": [ { "id": "super_admin", @@ -579,9 +579,9 @@ def list_access_options() -> dict[str, Any]: "notes": [ "Users sign in with Google SSO using the same email you enter.", "No app password is required for normal SSO login.", - "State codes must match document instance tags (mh, up, …).", + "State codes must match document group codes (mh, up, …).", "A centre is modelled as just another state code.", - "Super admin (BV) documents can use portal instance 'bv'.", + "Super admin (BV) documents can use portal group 'bv'.", "Access comes from group membership only — no direct role assignment.", "Keycloak groups: /global/super-admin, /global/bh-viewer, " "/states/{STATE}/{admin|approver|contributor|view}", diff --git a/pipeline/auth/models.py b/pipeline/auth/models.py index dd7fcdd..6a3ab40 100644 --- a/pipeline/auth/models.py +++ b/pipeline/auth/models.py @@ -29,7 +29,7 @@ # Roles not limited by the JWT ``instances`` claim. # State-level roles are restricted to their claimed instances (tenants/states). -INSTANCE_UNRESTRICTED_ROLES = SUPERADMIN_ROLES | GLOBAL_READ_ROLES +GROUP_UNRESTRICTED_ROLES = SUPERADMIN_ROLES | GLOBAL_READ_ROLES @dataclass @@ -73,44 +73,44 @@ def is_admin(self) -> bool: """ return self.is_superadmin - def is_instance_unrestricted(self) -> bool: - """True when the caller may access every instance (all tenants/states). + def is_group_unrestricted(self) -> bool: + """True when the caller may access every group (all tenants/states). Two cases: local bypass mode with no scoped claim, or platform superadmin even when the token carries a narrow ``instances`` claim. """ if self.token_disabled_mode and not self.instances: return True - return bool(INSTANCE_UNRESTRICTED_ROLES & self._role_keys) + return bool(GROUP_UNRESTRICTED_ROLES & self._role_keys) - def has_instance(self, instance: str) -> bool: + def has_group(self, group: str) -> bool: if not self.instances: - # Empty instance list means "no tenant restriction yet" only in disabled mode. + # Empty list means "no tenant restriction yet" only in disabled mode. return self.token_disabled_mode - return instance.strip().lower() in {i.lower() for i in self.instances} + return group.strip().lower() in {i.lower() for i in self.instances} def has_env(self, env: str) -> bool: if not self.envs: return self.token_disabled_mode return env.strip().lower() in {e.lower() for e in self.envs} - def role_for_instance(self, instance: str | None) -> str | None: + def role_for_group(self, group: str | None) -> str | None: """Role in a given state; super_admin always returns super_admin. - A ``bh_viewer`` reads every state, so any instance falls back to + A ``bh_viewer`` reads every state, so any group falls back to STATE_VIEW when no stronger per-state role is held. """ if self.is_superadmin: return ROLE_SUPER_ADMIN - if not instance: + if not group: return None - role = self.state_roles.get(normalize_state_code(instance)) + role = self.state_roles.get(normalize_state_code(group)) if role: return role return ROLE_STATE_VIEW if self.is_bh_viewer else None - def has_permission_for_instance( - self, permission: Permission | str, instance: str | None + def has_permission_for_group( + self, permission: Permission | str, group: str | None ) -> bool: """Permission check scoped to a state when per-state roles are present. @@ -126,7 +126,7 @@ def has_permission_for_instance( if not self.state_roles: return needed in self.permissions - role = self.role_for_instance(instance) + role = self.role_for_group(group) if not role: return False from .permissions import permissions_for_roles diff --git a/pipeline/auth/tenancy.py b/pipeline/auth/tenancy.py index e7813f6..ad20b66 100644 --- a/pipeline/auth/tenancy.py +++ b/pipeline/auth/tenancy.py @@ -1,12 +1,14 @@ -"""Instance (tenant / state) access helpers — Plan 2 (Keycloak + document.instance). +"""Group (tenant / state) access helpers — Keycloak groups + document groups. -Access who-sees-what-state comes from the JWT (Keycloak groups). The database -only stores which state a *document* belongs to (``documents.instance``). +Who-sees-what-state comes from the JWT (Keycloak groups). The database only +stores which group a *document* belongs to (``groups`` via +``documents.catalog_id -> catalogs.group_id``); the document dict carries its +tenant code as ``group``. -Plan 2 conventions: +Conventions: - State codes: lowercase (``mh``, ``up``, …) matching Keycloak ``/states/MH/...`` - Portal / BV platform docs: ``bv`` (Bharat Vistaar) - - Super admin (``/global/super-admin``): unrestricted instances + - Super admin (``/global/super-admin``): unrestricted groups """ from __future__ import annotations @@ -18,63 +20,64 @@ from .models import AuthUser # Bharat Vistaar portal / platform-wide documents (not a state). -PORTAL_INSTANCE = "bv" +PORTAL_GROUP = "bv" -def default_instance() -> str: - """Fallback instance id for new docs and legacy rows without a value.""" +def default_group() -> str: + """Fallback group code for new docs and legacy rows without a value.""" return (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" -def portal_instance() -> str: - return (os.environ.get("PORTAL_INSTANCE") or PORTAL_INSTANCE).strip().lower() or PORTAL_INSTANCE +def portal_group() -> str: + return (os.environ.get("PORTAL_INSTANCE") or PORTAL_GROUP).strip().lower() or PORTAL_GROUP -def group_for_instance(instance: str | None) -> str | None: - code = (instance or "").strip().lower() - if not code or code in {portal_instance(), default_instance()}: +def group_for_instance(group: str | None) -> str | None: + """The named network group of a tenant code, or None for the portal / default tenant.""" + code = (group or "").strip().lower() + if not code or code in {portal_group(), default_group()}: return None return code -def normalize_instance(value: str | None) -> str: +def normalize_group(value: str | None) -> str: text = (value or "").strip().lower() - return text or default_instance() + return text or default_group() def unrestricted(user: AuthUser) -> bool: - """True when the caller may see all instances (super admin / local bypass).""" - return user.is_instance_unrestricted() + """True when the caller may see all groups (super admin / local bypass).""" + return user.is_group_unrestricted() -def allowed_instances(user: AuthUser) -> set[str] | None: +def allowed_groups(user: AuthUser) -> set[str] | None: """ - Return the set of instance ids the user may access, or None if unrestricted. + Return the set of group codes the user may access, or None if unrestricted. """ if unrestricted(user): return None - return {normalize_instance(i) for i in user.instances if str(i).strip()} + return {normalize_group(i) for i in user.instances if str(i).strip()} -def user_can_access_instance(user: AuthUser, instance: str | None) -> bool: - allowed = allowed_instances(user) +def user_can_access_group(user: AuthUser, group: str | None) -> bool: + allowed = allowed_groups(user) if allowed is None: return True if not allowed: return False - return normalize_instance(instance) in allowed + return normalize_group(group) in allowed -def assert_instance_access(user: AuthUser, instance: str | None) -> str: - """Raise 403 if user cannot access instance; return normalized instance id.""" - normalized = normalize_instance(instance) - if not user_can_access_instance(user, normalized): - raise HTTPException(403, f"No access to instance: {normalized}") +def assert_group_access(user: AuthUser, group: str | None) -> str: + """Raise 403 if user cannot access group; return normalized group code.""" + normalized = normalize_group(group) + if not user_can_access_group(user, normalized): + raise HTTPException(403, f"No access to group: {normalized}") return normalized -def resolve_create_instance(user: AuthUser, requested: str | None = None) -> str: - """Pick document ``instance`` at create/upload time (Plan 2). +def resolve_create_group(user: AuthUser, requested: str | None = None) -> str: + """Pick the document's ``group`` at create/upload time. - Super admin / bypass: use requested, else portal ``bv``, else DEFAULT_INSTANCE. - Single-state user: that state (requested must match if provided). @@ -85,11 +88,11 @@ def resolve_create_instance(user: AuthUser, requested: str | None = None) -> str if unrestricted(user): if requested_norm: - return normalize_instance(requested_norm) + return normalize_group(requested_norm) # Prefer portal tag for platform operators when nothing chosen. - return normalize_instance(portal_instance()) + return normalize_group(portal_group()) - allowed = allowed_instances(user) or set() + allowed = allowed_groups(user) or set() if not allowed: raise HTTPException( 403, @@ -98,7 +101,7 @@ def resolve_create_instance(user: AuthUser, requested: str | None = None) -> str if requested_norm: if requested_norm not in allowed: - raise HTTPException(403, f"No access to instance: {requested_norm}") + raise HTTPException(403, f"No access to group: {requested_norm}") return requested_norm if len(allowed) == 1: @@ -106,18 +109,18 @@ def resolve_create_instance(user: AuthUser, requested: str | None = None) -> str raise HTTPException( 400, - "instance is required when you have multiple states; " + "group is required when you have multiple states; " f"choose one of: {', '.join(sorted(allowed))}", ) -def assert_document_instance_access(user: AuthUser, doc: dict | None) -> dict: +def assert_document_group_access(user: AuthUser, doc: dict | None) -> dict: """ - Ensure the document exists and the user may access its instance. + Ensure the document exists and the user may access its group. Missing / forbidden both return 404 to avoid leaking other tenants' ids. """ if not doc: raise HTTPException(404, "Document not found") - if not user_can_access_instance(user, doc.get("instance")): + if not user_can_access_group(user, doc.get("group")): raise HTTPException(404, "Document not found") return doc diff --git a/pipeline/discovery_publish_service.py b/pipeline/discovery_publish_service.py index 175fdb7..88b2b43 100644 --- a/pipeline/discovery_publish_service.py +++ b/pipeline/discovery_publish_service.py @@ -50,12 +50,12 @@ def __init__( def publish( self, transaction_id: str, - document_kind: Optional[str], + kind: Optional[str], workflow_id: Optional[str] = None, group: Optional[str] = None, ) -> dict: """ - POST a catalog/publish envelope for `document_kind`. + POST a catalog/publish envelope for `kind`. `transaction_id` is supplied by the caller (generated once per publish, reused across retries) - only `messageId` is generated fresh here, per @@ -65,18 +65,18 @@ def publish( with `skipped=True`: the spec declares `message.catalogs` as `minItems: 1`, so there is no valid "publish nothing" request to send. """ - if not is_publishable_kind(document_kind): + if not is_publishable_kind(kind): logger.info( - "workflow_id=%s document_kind=%s network_publish_skipped=True", + "workflow_id=%s kind=%s network_publish_skipped=True", workflow_id, - document_kind, + kind, ) return self._skipped() catalog_group = named_group(group) - provider = provider_for(self.provider_naming, document_kind, catalog_group) - catalog_name = self.provider_naming.catalog_name(normalize_document_kind(document_kind), catalog_group) - catalog = build_catalog(document_kind, provider, catalog_name, catalog_group) + provider = provider_for(self.provider_naming, kind, catalog_group) + catalog_name = self.provider_naming.catalog_name(normalize_document_kind(kind), catalog_group) + catalog = build_catalog(kind, provider, catalog_name, catalog_group) envelope = { "context": { diff --git a/pipeline/network_group.py b/pipeline/network_group.py index ce2aad0..edd71f2 100644 --- a/pipeline/network_group.py +++ b/pipeline/network_group.py @@ -6,9 +6,9 @@ from pathlib import Path from typing import Optional -from .auth.tenancy import portal_instance +from .auth.tenancy import portal_group from .catalog_builder import normalize_document_kind -from .instances import INSTANCE_NAMES, instance_display_name +from .group_names import GROUP_NAMES, group_display_name from .network_constants import ( ADVISORY_CATALOG_NAME, GROUP_CATALOG_NAME_TEMPLATE, @@ -117,7 +117,7 @@ def _checked_id_template(shared_instance: str, network_id: Optional[str]) -> str instance=shared_instance, network_id=sample_network_id, ) - for instance in dict.fromkeys((shared_instance, *INSTANCE_NAMES)): + for instance in dict.fromkeys((shared_instance, *GROUP_NAMES)): _check_provider_id( template.format(instance=instance, network_id=sample_network_id), f"{PROVIDER_ID_TEMPLATE_ENV}={template!r} with {NETWORK_ID_ENV}={sample_network_id!r}", @@ -126,11 +126,11 @@ def _checked_id_template(shared_instance: str, network_id: Optional[str]) -> str def load_provider_naming(network_id: Optional[str] = None) -> ProviderNaming: - shared_instance = portal_instance() + shared_instance = portal_group() network_id = (network_id or os.environ.get(NETWORK_ID_ENV) or "").strip() or None return ProviderNaming( shared_instance=shared_instance, - shared_name=instance_display_name(shared_instance), + shared_name=group_display_name(shared_instance), network_id=network_id, id_template=_checked_id_template(shared_instance, network_id), group_catalog_name_template=_checked_template( @@ -155,7 +155,7 @@ def validate_network_provider_config() -> None: def named_group(code: Optional[str]) -> Optional[Group]: if code is None: return None - return Group(code=code, name=instance_display_name(code)) + return Group(code=code, name=group_display_name(code)) def provider_for(naming: ProviderNaming, document_kind: Optional[str], group: Optional[Group]) -> Optional[dict]: From 1b7e9beb4d963c4462083a9d76c46f45a545f2f5 Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 11:07:18 +0530 Subject: [PATCH 37/53] feat(scripts): move SQLite data into the v2 schema Build groups and catalogs from legacy instance codes, fold period, uploader and index state into documents.metadata, skip kind=document and demo documents with counts in the report. Deploy migration plan is still to be written. --- scripts/migrate_sqlite_to_postgres.py | 232 +++++++++++++++++--------- 1 file changed, 155 insertions(+), 77 deletions(-) diff --git a/scripts/migrate_sqlite_to_postgres.py b/scripts/migrate_sqlite_to_postgres.py index dc583c0..a37214b 100644 --- a/scripts/migrate_sqlite_to_postgres.py +++ b/scripts/migrate_sqlite_to_postgres.py @@ -12,10 +12,18 @@ second run finds the rows already there and changes nothing. A document is migrated in one transaction. ``--dry-run`` runs everything and rolls it back. -Mapping: one legacy ``documents`` row becomes one document plus its main -workflow (``temporal_workflow_id`` = the legacy ``workflow_id``); each -``document_jobs`` row becomes a workflow; ``scheme_catalog_*`` becomes -``catalog_entries``; ``document_manifest_entries`` is dropped. +Mapping (Schema v2): one legacy ``documents`` row becomes one document plus +its main workflow (``temporal_workflow_id`` = the legacy ``workflow_id``); each +``document_jobs`` row becomes a workflow. The legacy ``instance`` becomes a +``groups`` row (``bv`` is ``/global``, others ``/states/``, Keycloak id +filled in by the first login sync) and the group's ``catalogs`` row for the +document's kind. ``scheme_code`` / ``scheme_name`` / ``scheme_aliases_json`` +(or the ``scheme_catalog_entries`` registry entry for the code) become the +document's ``code`` / ``name`` / ``aliases``; the validity period, uploader and +vector index state move into ``documents.metadata``. Not migrated, and listed in +the report: documents of kind ``document`` (no catalog), demo documents +(``is_demo``), registry entries no document uses (bootstrap-only), and +``document_manifest_entries``. """ import argparse @@ -45,11 +53,11 @@ PageMetadata, PageTranslation, ) +from pipeline.network_constants import KIND_TO_CATALOG_TYPE, catalog_type_for_kind # noqa: E402 _NAMESPACE = uuid.UUID("3f2b6c0e-5d1a-4a53-9a77-0c6a5c1d7e10") _FINGERPRINT_RE = re.compile(r"^[0-9a-f]{32}$") _CODE_RE = re.compile(r"^[a-z0-9]+(-[a-z0-9]+)*$") -_KIND_RE = re.compile(r"^[a-z0-9]+([_-][a-z0-9]+)*$") _STAGES = { "registered", "ocr_processing", "ocr_review", "translation_processing", "translation_review", "chunking", "chunk_review", "ready_for_ingestion", @@ -221,7 +229,10 @@ def _get(row: sqlite3.Row, key: str, default: Any = None) -> Any: # Catalog # ----------------------------------------------------------------------------- -def migrate_catalog_registry(sq, conn, report: Report) -> None: +def load_catalog_registry(sq, report: Report) -> dict[str, dict[str, Any]]: + """Legacy ``scheme_catalog_entries`` by code. Their name and aliases are merged + into the documents holding the code; see ``settle_catalog_registry``.""" + registry: dict[str, dict[str, Any]] = {} for row in _rows(sq, "scheme_catalog_entries"): report.count("scheme_catalog_entries", "read") code = (row["scheme_code"] or "").strip().lower() @@ -229,17 +240,24 @@ def migrate_catalog_registry(sq, conn, report: Report) -> None: report.skip("scheme_catalog_entries", str(row["scheme_code"]), "code is not a valid slug") continue aliases = [a for a in load_json(_get(row, "scheme_aliases_json"), []) if isinstance(a, str) and a] - source = _get(row, "source") if _get(row, "source") in ("bootstrap", "pipeline", "import") else "pipeline" - inserted = conn.execute( - """ - INSERT INTO catalog_entries (id, type, code, name, aliases, metadata) - VALUES (%s::uuid, 'scheme', %s, %s, %s, %s) - ON CONFLICT (type, code) DO NOTHING RETURNING id - """, - (det_id("entry", f"scheme:{code}"), code, row["scheme_name"] or code, aliases, - Jsonb({"source": source})), - ).fetchone() - report.count("scheme_catalog_entries", "inserted" if inserted else "existing") + registry[code] = {"name": row["scheme_name"] or code, "aliases": aliases} + return registry + + +def settle_catalog_registry(registry: dict[str, dict[str, Any]], consumed: set[str], report: Report) -> None: + """Registry entries a migrated document used count as inserted (merged into the + document); the rest have no document and are dropped.""" + unused = [] + for code in sorted(registry): + if code in consumed: + report.count("scheme_catalog_entries", "inserted") + else: + report.count("scheme_catalog_entries", "dropped") + unused.append(code) + if unused: + report.notes.append( + f"scheme_catalog_entries without a document were not migrated: {', '.join(unused)}" + ) def migrate_catalog_version(sq, conn, report: Report) -> None: @@ -267,20 +285,49 @@ def migrate_catalog_version(sq, conn, report: Report) -> None: ) -def _ensure_doc_catalog_entry(conn, kind: str, code: str, name: str, aliases: list[str]) -> str: - conn.execute( +def _instance_of(doc: sqlite3.Row) -> str: + return (_get(doc, "instance") or _default_instance()).strip().lower() or _default_instance() + + +def _group_for(conn, instance: str) -> str: + """The ``groups`` row of a tenant code, created with a deterministic id.""" + path = db._tenant_path(instance) + row = conn.execute( """ - INSERT INTO catalog_entries (id, type, code, name, aliases, metadata) - VALUES (%s::uuid, %s, %s, %s, %s, %s) ON CONFLICT (type, code) DO NOTHING + INSERT INTO groups (id, name, path) VALUES (%s::uuid, %s, %s) + ON CONFLICT (path) DO UPDATE SET name = groups.name RETURNING id """, - (det_id("entry", f"{kind}:{code}"), kind, code, name or code, aliases, - Jsonb({"source": "pipeline"})), - ) - return str( - conn.execute( - "SELECT id FROM catalog_entries WHERE type = %s AND code = %s", (kind, code) - ).fetchone()["id"] - ) + (det_id("group", path), db._tenant_code_from_path(path), path), + ).fetchone() + return str(row["id"]) + + +def _catalog_for(conn, group_id: str, kind: str) -> str: + """The ``(group, kind)`` catalog, created with a deterministic id.""" + catalog_type = catalog_type_for_kind(kind) + row = conn.execute( + """ + INSERT INTO catalogs (id, group_id, type, name) + SELECT %s::uuid, g.id, %s, g.name || ' ' || %s FROM groups g WHERE g.id = %s::uuid + ON CONFLICT (group_id, type) DO UPDATE SET type = EXCLUDED.type RETURNING id + """, + (det_id("catalog", f"{group_id}:{catalog_type}"), catalog_type, kind, group_id), + ).fetchone() + return str(row["id"]) + + +def _slug(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", (text or "").lower()).strip("-")[:40].strip("-") + + +def _unique_code(conn, catalog_id: str, base: str) -> str: + code, n = base, 1 + while conn.execute( + "SELECT 1 FROM documents WHERE catalog_id = %s::uuid AND code = %s", (catalog_id, code) + ).fetchone(): + n += 1 + code = f"{base}-{n}" + return code # ----------------------------------------------------------------------------- @@ -393,7 +440,23 @@ def _build_workflows( return workflows, job_ids -def migrate_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workflow: dict[str, list]) -> bool: +def _skip_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workflow: dict[str, list], reason: str) -> None: + """Leave a document behind. Its history rows are counted as skipped; its audit + entries are kept, tied to the tenant's group but to no document.""" + workflow_id = doc["workflow_id"] + report.skip("documents", workflow_id, reason) + for table in ("document_jobs", "document_artifacts", "pages", "chunks"): + report.count(table, "skipped", len(_rows(sq, table, "WHERE workflow_id = ?", (workflow_id,)))) + for audit in audit_by_workflow.pop(workflow_id, []): + insert_audit(conn, audit, None, _instance_of(doc), report) + + +def migrate_document( + sq, conn, doc: sqlite3.Row, report: Report, audit_by_workflow: dict[str, list], + registry: Optional[dict[str, dict[str, Any]]] = None, consumed: Optional[set[str]] = None, +) -> bool: + registry = registry if registry is not None else {} + consumed = consumed if consumed is not None else set() workflow_id = doc["workflow_id"] doc_uuid = det_id("document", workflow_id) exists = conn.execute("SELECT 1 FROM documents WHERE id = %s::uuid", (doc_uuid,)).fetchone() @@ -406,6 +469,14 @@ def migrate_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workfl report.count(label, "existing", len(_rows(sq, table, "WHERE workflow_id = ?", (workflow_id,)))) return False + kind = (_get(doc, "document_kind") or "document").strip().lower() + if truthy(_get(doc, "is_demo")): + _skip_document(sq, conn, doc, report, audit_by_workflow, "demo document (is_demo)") + return False + if kind not in KIND_TO_CATALOG_TYPE: + _skip_document(sq, conn, doc, report, audit_by_workflow, f"document_kind {kind!r} has no catalog") + return False + fingerprint = next( ( c for c in ( @@ -420,12 +491,7 @@ def migrate_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workfl fingerprint = hashlib.md5((doc["document_id"] or workflow_id).encode()).hexdigest() report.notes.append(f"{workflow_id}: no hex fingerprint, derived md5 of document_id") - page_count = max(int(doc["page_count"] or 0), 0) chunk_rows = _rows(sq, "chunks", "WHERE workflow_id = ? ORDER BY chunk_number", (workflow_id,)) - kind = (_get(doc, "document_kind") or "document").strip().lower() - if not _KIND_RE.match(kind): - report.notes.append(f"{workflow_id}: invalid document_kind {kind!r}, stored as 'document'") - kind = "document" chunking = None if chunk_rows: @@ -436,12 +502,24 @@ def migrate_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workfl model=_get(first, "chunking_model"), config=load_json(_get(first, "chunking_config_json"), {}), ) + + valid_from, valid_to = parse_date(_get(doc, "valid_from")), parse_date(_get(doc, "valid_to")) + if valid_from and valid_to and valid_from > valid_to: + report.notes.append(f"{workflow_id}: valid_from after valid_to, valid_to cleared") + valid_to = None + + uploader = _user_for( + conn, _get(doc, "uploaded_by_user_id"), _get(doc, "uploaded_by_username"), + _get(doc, "uploaded_by_email"), _get(doc, "uploaded_by_roles"), + ) + metadata = DocumentMetadata( - display_name=_get(doc, "display_name") or None, + valid_from=valid_from.isoformat() if valid_from else None, + valid_to=valid_to.isoformat() if valid_to else None, + uploaded_by_user_id=uploader, + index_status=_build_index_status(sq, workflow_id), catalog_visible=truthy(_get(doc, "catalog_visible"), True), network_visible=truthy(_get(doc, "network_visible"), True), - is_demo=truthy(_get(doc, "is_demo")), - chunk_count=len(chunk_rows) if chunk_rows else max(int(doc["chunk_count"] or 0), 0), reindex_required=truthy(_get(doc, "reindex_required")), reindex_reason=_get(doc, "reindex_reason") or None, chunking=chunking, @@ -452,46 +530,41 @@ def migrate_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workfl storage_uri=doc["filepath"], source_type=_get(doc, "source_type"), canonical_input_type=_get(doc, "canonical_input_type"), - page_count=page_count, ).model_dump(exclude_none=True) - valid_from, valid_to = parse_date(_get(doc, "valid_from")), parse_date(_get(doc, "valid_to")) - if valid_from and valid_to and valid_from > valid_to: - report.notes.append(f"{workflow_id}: valid_from after valid_to, valid_to cleared") - valid_to = None - - uploader = _user_for( - conn, _get(doc, "uploaded_by_user_id"), _get(doc, "uploaded_by_username"), - _get(doc, "uploaded_by_email"), _get(doc, "uploaded_by_roles"), - ) - - entry_id = None + catalog_id = _catalog_for(conn, _group_for(conn, _instance_of(doc)), kind) code = (_get(doc, "scheme_code") or "").strip().lower() - if code and kind != "document": - if _CODE_RE.match(code): - aliases = [a for a in load_json(_get(doc, "scheme_aliases_json"), []) if isinstance(a, str) and a] - entry_id = _ensure_doc_catalog_entry( - conn, kind, code, _get(doc, "scheme_name") or code, aliases - ) - else: - report.notes.append(f"{workflow_id}: scheme_code {code!r} is not a valid slug, not linked") - elif code: - report.notes.append(f"{workflow_id}: scheme_code {code!r} on a plain document, not linked") + entry = registry.get(code) if kind == "scheme" and code else None + if entry is not None: + consumed.add(code) + name = (entry or {}).get("name") or _get(doc, "scheme_name") or _get(doc, "display_name") or None + aliases = (entry or {}).get("aliases") or [ + a for a in load_json(_get(doc, "scheme_aliases_json"), []) if isinstance(a, str) and a + ] + if code and not _CODE_RE.match(code): + report.notes.append(f"{workflow_id}: scheme_code {code!r} is not a valid slug, code derived") + code = "" + if not code: + code = _slug(name or "") or _slug(Path(doc["filename"] or "").stem) + if not _CODE_RE.match(code): + code = f"doc-{fingerprint[:8]}" + report.notes.append(f"{workflow_id}: no usable scheme_code, derived {code!r}") + unique = _unique_code(conn, catalog_id, code) + if unique != code: + report.notes.append(f"{workflow_id}: code {code!r} already used in its catalog, stored as {unique!r}") + code = unique created_at = parse_ts(doc["created_at"]) or datetime.now(timezone.utc) updated_at = parse_ts(doc["updated_at"]) or created_at conn.execute( """ - INSERT INTO documents (id, instance, source, document_kind, catalog_entry_id, valid_from, - valid_to, is_disabled, index_status, metadata, uploaded_by_user_id, + INSERT INTO documents (id, catalog_id, code, name, aliases, source, is_disabled, metadata, created_at, updated_at) - VALUES (%s::uuid, %s, %s, %s, %s::uuid, %s, %s, %s, %s, %s, %s::uuid, %s, %s) + VALUES (%s::uuid, %s::uuid, %s, %s, %s, %s, %s, %s, %s, %s) """, ( - doc_uuid, (_get(doc, "instance") or _default_instance()).strip().lower() or _default_instance(), - Jsonb(source), kind, entry_id, valid_from, valid_to, truthy(_get(doc, "is_disabled")), - Jsonb(_build_index_status(sq, workflow_id)), Jsonb(metadata), uploader, created_at, - updated_at, + doc_uuid, catalog_id, code, name or code, aliases, Jsonb(source), + truthy(_get(doc, "is_disabled")), Jsonb(metadata), created_at, updated_at, ), ) report.count("documents", "inserted") @@ -507,8 +580,8 @@ def migrate_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workfl INSERT INTO workflows (id, document_id, workflow_type, temporal_workflow_id, status, current_stage, stage_timestamps, stop_after_ocr, error_message, config, started_by_user_id, started_at, completed_at, updated_at) - VALUES (%s::uuid, %s::uuid, %s::workflow_type_enum, %s, %s::workflow_status_enum, - %s::workflow_stage, %s, %s, %s, %s, %s::uuid, %s, %s, %s) + VALUES (%s::uuid, %s::uuid, %s, %s, %s, + %s, %s, %s, %s, %s, %s::uuid, %s, %s, %s) """, ( w["id"], doc_uuid, w["type"], w["temporal_id"], w["status"], w["stage"], @@ -526,7 +599,7 @@ def migrate_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workfl """ INSERT INTO workflow_artifacts (id, workflow_id, artifact_type, stage, storage_uri, mime_type, filename, size_bytes, metadata, created_at) - VALUES (%s::uuid, %s::uuid, %s, %s::workflow_stage, %s, %s, %s, %s, %s, %s) + VALUES (%s::uuid, %s::uuid, %s, %s, %s, %s, %s, %s, %s, %s) """, ( det_id("artifact", f"{workflow_id}:{art['id']}"), @@ -604,7 +677,7 @@ def migrate_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workfl report.count("chunks", "inserted") for audit in audit_by_workflow.pop(workflow_id, []): - insert_audit(conn, audit, doc_uuid, source_instance=(_get(doc, "instance") or _default_instance()), report=report) + insert_audit(conn, audit, doc_uuid, source_instance=_instance_of(doc), report=report) return True @@ -636,16 +709,16 @@ def insert_audit(conn, audit: sqlite3.Row, doc_uuid: Optional[str], source_insta ) inserted = conn.execute( """ - INSERT INTO audit_logs (id, document_id, workflow_id, instance, action_type, entity_type, + INSERT INTO audit_logs (id, document_id, workflow_id, group_id, action_type, entity_type, entity_number, field_name, old_value, new_value, metadata, actor_user_id, actor, created_at) - VALUES (%s::uuid, %s::uuid, %s, %s, %s, %s::audit_entity_type, %s, %s, %s, %s, %s, + VALUES (%s::uuid, %s::uuid, %s, %s::uuid, %s, %s, %s, %s, %s, %s, %s, %s::uuid, %s, %s) ON CONFLICT (id) DO NOTHING RETURNING id """, ( det_id("audit", str(audit["id"])), doc_uuid, workflow_id, - (source_instance or "").strip().lower() or None if doc_uuid else None, + _group_for(conn, source_instance.strip().lower()) if (source_instance or "").strip() else None, audit["action_type"], entity, audit["entity_id"], _legacy_audit_field(audit["field_name"], entity), audit["old_value"], audit["new_value"], Jsonb(metadata), actor_uuid, @@ -794,13 +867,18 @@ def run_migration(sqlite_path: str, database_url: str, dry_run: bool = False) -> else: orphan_audit.append(audit) - with conn.transaction(): - migrate_catalog_registry(sq, conn, report) + registry = load_catalog_registry(sq, report) + consumed: set[str] = set() for doc in documents: report.count("documents", "read") with conn.transaction(): - migrate_document(sq, conn, doc, report, audit_by_workflow) + migrate_document(sq, conn, doc, report, audit_by_workflow, registry, consumed) + settle_catalog_registry(registry, consumed, report) + + skipped = Counter(i["reason"] for i in report.skip_reasons if i["table"] == "documents") + for reason, n in sorted(skipped.items()): + report.notes.append(f"documents not migrated: {n} x {reason}") with conn.transaction(): for audits in audit_by_workflow.values(): From 37e777a8e141c41a81e198306d5187c9d4bdbff8 Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 11:07:18 +0530 Subject: [PATCH 38/53] test: update for schema v2 and add constraint tests Rename to group/kind/code/name/aliases, cover upload with kind, re-upload of an existing code, catalog_status, group snapshots in audit_logs and the new unique/foreign-key constraints in tests/test_schema_v2.py. --- tests/conftest.py | 6 +- tests/test_activities.py | 69 +- tests/test_api.py | 255 +++++++- tests/test_auth.py | 2 +- tests/test_db.py | 192 +++--- tests/test_discovery_publish_service.py | 32 +- tests/test_document_repository.py | 42 +- ...{test_instances.py => test_group_names.py} | 30 +- tests/test_groups.py | 46 +- tests/test_migrate_sqlite_to_postgres.py | 79 ++- tests/test_provenance.py | 16 +- tests/test_schema_v2.py | 598 ++++++++++++++++++ tests/test_scheme_catalog.py | 301 ++++----- tests/test_tenancy.py | 343 ++++++---- 14 files changed, 1508 insertions(+), 503 deletions(-) rename tests/{test_instances.py => test_group_names.py} (80%) create mode 100644 tests/test_schema_v2.py diff --git a/tests/conftest.py b/tests/conftest.py index e023105..fa276dd 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,7 +23,7 @@ _TRUNCATE_SQL = """ - TRUNCATE documents, users, catalog_entries, audit_logs, email_otps, + TRUNCATE documents, users, groups, catalogs, audit_logs, email_otps, webhook_records, search_settings CASCADE """ @@ -234,7 +234,9 @@ def sample_document(db_connection): document_id=document_id, filename="test.pdf", filepath="/app/books/test.pdf", - stage="registered" + stage="registered", + kind="scheme", + code="test-scheme", ) return { diff --git a/tests/test_activities.py b/tests/test_activities.py index 73fb506..e10e205 100644 --- a/tests/test_activities.py +++ b/tests/test_activities.py @@ -10,6 +10,7 @@ """ import asyncio +import itertools import os from unittest.mock import MagicMock @@ -21,6 +22,14 @@ os.environ["TRANSLATION_VLLM_BASE_URL"] = "http://localhost:8000/v1" +_code_seq = itertools.count(1) + + +def _next_code() -> str: + """A unique document code for fixtures (documents need one in v2).""" + return f"doc-{next(_code_seq)}" + + class TestChunkingActivity: """Tests for the chunking activity.""" @@ -158,7 +167,7 @@ def test_prepare_records_carries_instance_name(self): document_id="test-doc", filename="test.pdf", chunks=chunks, - instance=code, + group=code, ) assert records[0]["instance"] == code assert records[0]["instance_name"] == expected @@ -172,7 +181,7 @@ def test_prepare_records_carries_document_kind(self): chunks = [{"chunk_number": 1, "original_text": "Test chunk", "token_count": 5}] advisory = _prepare_records( - document_id="test-doc", filename="test.pdf", chunks=chunks, document_kind="advisory" + document_id="test-doc", filename="test.pdf", chunks=chunks, kind="advisory" ) assert advisory[0]["document_kind"] == "advisory" assert advisory[0]["type"] == "document" @@ -185,9 +194,9 @@ def test_prepare_records_carries_document_kind(self): document_id="test-doc", filename="test.pdf", chunks=chunks, - document_kind="scheme", - scheme_code="pm-kisan", - scheme_name="PM-KISAN", + kind="scheme", + code="pm-kisan", + name="PM-KISAN", ) assert scheme[0]["type"] == "scheme" assert scheme[0]["document_kind"] == "scheme" @@ -203,11 +212,11 @@ def test_prepare_records_fills_scheme_aliases(self): document_id="test-doc", filename="NMEO-OPGUIEDELINES.pdf", chunks=chunks, - instance="bv", - document_kind="scheme", - scheme_code="nmeoop", - scheme_name="National Mission on Edible Oils - Oil Palm", - scheme_aliases=None, + group="bv", + kind="scheme", + code="nmeoop", + name="National Mission on Edible Oils - Oil Palm", + aliases=None, ) record = records[0] @@ -229,8 +238,8 @@ def test_prepare_records_non_scheme_has_no_aliases(self): document_id="test-doc", filename="test.pdf", chunks=chunks, - instance="mh", - document_kind="document", + group="mh", + kind="document", ) assert "scheme_aliases" not in records[0] @@ -429,6 +438,8 @@ def test_update_state(self, db_connection): workflow_id = "state-update-test" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-state", filename="test.pdf", @@ -436,6 +447,12 @@ def test_update_state(self, db_connection): stage="registered" ) + # Page and chunk counts are derived from the pages table, not stored. + db_connection.save_pages( + workflow_id, + [{"page_number": n, "original_markdown": f"page {n}"} for n in range(1, 6)], + ) + asyncio.run(update_document_state( workflow_id=workflow_id, stage="ocr_processing", @@ -559,9 +576,9 @@ class FakeService: def __init__(self): pass - def publish(self, transaction_id, document_kind, workflow_id=None, group=None): + def publish(self, transaction_id, kind, workflow_id=None, group=None): assert transaction_id == "txn-1" - assert document_kind == "advisory" + assert kind == "advisory" assert workflow_id == "wf-1" return fake_result @@ -571,7 +588,7 @@ def publish(self, transaction_id, document_kind, workflow_id=None, group=None): "_upload_file_to_minio", lambda *a, **k: ("minio://documents/network_publish.json", 123, "application/json"), ) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "advisory"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory"}) monkeypatch.setattr(db, "get_latest_document_job", lambda workflow_id: {"id": 42}) recorded = {} @@ -608,11 +625,11 @@ async def test_publish_failure_propagates_and_records_nothing(self, monkeypatch) import pipeline.db as db class FakeService: - def publish(self, transaction_id, document_kind, workflow_id=None, group=None): + def publish(self, transaction_id, kind, workflow_id=None, group=None): raise RuntimeError("discovery service unreachable") monkeypatch.setattr(activities, "DiscoveryPublishService", FakeService) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "advisory"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory"}) add_artifact_calls = [] monkeypatch.setattr(db, "add_document_artifact", lambda **kwargs: add_artifact_calls.append(kwargs)) @@ -635,7 +652,7 @@ async def test_missing_publish_config_fails_without_retry(self, monkeypatch, uns monkeypatch.setenv("DISCOVERY_SERVICE_ENDPOINT", "https://discovery.example.com") monkeypatch.setenv("NETWORK_SENDER_ID", "provider.oan.dev") monkeypatch.delenv(unset) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "advisory"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory"}) with pytest.raises(ApplicationError, match=unset) as excinfo: await activities.publish_catalog_to_network("wf-1", "txn-1") @@ -649,7 +666,7 @@ async def test_unmapped_kind_records_a_skipped_artifact(self, monkeypatch): import pipeline.db as db class FakeService: - def publish(self, transaction_id, document_kind, workflow_id=None, group=None): + def publish(self, transaction_id, kind, workflow_id=None, group=None): return { "skipped": True, "envelope": None, @@ -665,7 +682,7 @@ def publish(self, transaction_id, document_kind, workflow_id=None, group=None): "_upload_file_to_minio", lambda *a, **k: ("minio://documents/network_publish_skipped.json", 45, "application/json"), ) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "document"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "document"}) monkeypatch.setattr(db, "get_latest_document_job", lambda workflow_id: None) recorded = {} @@ -708,7 +725,7 @@ async def test_disabled_publish_records_a_skip_without_calling_the_network(self, "_upload_file_to_minio", lambda *a, **k: ("minio://documents/network_publish_skipped.json", 45, "application/json"), ) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "advisory"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory"}) monkeypatch.setattr(db, "get_latest_document_job", lambda workflow_id: None) recorded = {} monkeypatch.setattr(db, "add_document_artifact", lambda **kwargs: recorded.update(kwargs)) @@ -729,7 +746,7 @@ async def test_non_rejected_results_complete_the_stage(self, monkeypatch, result import pipeline.db as db class FakeService: - def publish(self, transaction_id, document_kind, workflow_id=None, group=None): + def publish(self, transaction_id, kind, workflow_id=None, group=None): return { "skipped": False, "envelope": {"context": {"messageId": "msg-3"}}, @@ -745,7 +762,7 @@ def publish(self, transaction_id, document_kind, workflow_id=None, group=None): "_upload_file_to_minio", lambda *a, **k: ("minio://documents/network_publish.json", 12, "application/json"), ) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "advisory"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory"}) monkeypatch.setattr(db, "get_latest_document_job", lambda workflow_id: None) monkeypatch.setattr(db, "add_document_artifact", lambda **kwargs: 1) @@ -763,7 +780,7 @@ async def test_rejected_result_records_the_errors_then_fails(self, monkeypatch): errors = [{"code": "SCH_SCHEMA_VALIDATION_FAILED", "message": "topics is required"}] class FakeService: - def publish(self, transaction_id, document_kind, workflow_id=None, group=None): + def publish(self, transaction_id, kind, workflow_id=None, group=None): return { "skipped": False, "envelope": {"context": {"messageId": "msg-4"}}, @@ -779,7 +796,7 @@ def publish(self, transaction_id, document_kind, workflow_id=None, group=None): "_upload_file_to_minio", lambda *a, **k: ("minio://documents/network_publish.json", 12, "application/json"), ) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "advisory"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory"}) monkeypatch.setattr(db, "get_latest_document_job", lambda workflow_id: None) recorded = {} @@ -890,7 +907,7 @@ def publish(self, transaction_id, document_kind, workflow_id=None, group=None): raise InvalidProviderIdError("Provider id 'advisory.ladakh_ut.provider.oan.dev' is invalid") monkeypatch.setattr(activities, "DiscoveryPublishService", FakeService) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "advisory", "instance": "ladakh_ut"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory", "instance": "ladakh_ut"}) recorded = {} monkeypatch.setattr(db, "add_document_artifact", lambda **kwargs: recorded.update(kwargs)) diff --git a/tests/test_api.py b/tests/test_api.py index cba7a64..7fbd9c8 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -11,6 +11,7 @@ """ import hashlib +import itertools import os import pytest @@ -21,6 +22,14 @@ os.environ["ALLOWED_FILE_PATHS"] = "/app/books,/tmp" +_code_seq = itertools.count(1) + + +def _next_code() -> str: + """A unique document code for fixtures (documents need one in v2).""" + return f"doc-{next(_code_seq)}" + + class TestHealthEndpoints: """Tests for health check endpoints.""" @@ -53,6 +62,8 @@ def test_list_documents(self, test_client, db_connection): """Test listing documents.""" # Create a test document db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id="api-test-001", document_id="doc-001", filename="test.pdf", @@ -70,6 +81,8 @@ def test_list_documents(self, test_client, db_connection): def test_list_documents_with_stage_filter(self, test_client, db_connection): """Test filtering documents by stage.""" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id="filter-api-001", document_id="doc-filter-001", filename="completed.pdf", @@ -89,6 +102,8 @@ def test_get_document_from_db(self, test_client, db_connection): """Test getting document details from the database.""" workflow_id = "get-test-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-get-001", filename="test.pdf", @@ -117,6 +132,8 @@ class TestDocumentIdentifiers: def _document(self, db_connection): db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=self.WORKFLOW_ID, document_id=self.FINGERPRINT, filename="identifiers.pdf", @@ -170,6 +187,8 @@ def test_the_uuid_is_rewritten_to_the_temporal_id_before_handlers_run( @pytest.mark.unit def test_static_document_routes_are_not_mistaken_for_identifiers(self, test_client, db_connection): db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id="summary", document_id=hashlib.md5(b"summary").hexdigest(), filename="summary", filepath="/tmp/s.pdf", ) @@ -188,6 +207,8 @@ class TestArtifactAndJobIdentifiers: def _document(self, db_connection): db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id="wf-art-api", document_id=hashlib.md5(b"art").hexdigest(), filename="a.pdf", filepath="minio://documents/a.pdf", stage="ocr_review", ) @@ -256,6 +277,8 @@ class TestProdApproval: def _document_at_the_gate(self, db_connection, workflow_id): db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id=f"doc-{workflow_id}", filename="test.pdf", @@ -319,6 +342,8 @@ def test_the_document_carries_no_validity_window(self, test_client, db_connectio def test_the_wrong_stage_is_still_refused(self, test_client, db_connection): workflow_id = "prod-approval-005" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id=f"doc-{workflow_id}", filename="test.pdf", @@ -341,6 +366,8 @@ class TestDocumentValidityEndpoints: def _uploaded_document(self, db_connection, workflow_id): db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id=f"doc-{workflow_id}", filename="test.pdf", @@ -360,6 +387,7 @@ def test_upload_starts_today_and_never_expires(self, test_client, sample_pdf_con response = test_client.post( "/upload", + params={"kind": "scheme"}, files={"file": ("validity-upload.pdf", sample_pdf_content, "application/pdf")}, ) @@ -378,13 +406,13 @@ def test_an_end_date_can_be_set_and_then_cleared(self, test_client, db_connectio set_response = test_client.patch( f"/documents/{workflow_id}/scheme-metadata", - json={"document_kind": "advisory", "valid_from": "2026-01-01", "valid_to": "2026-06-30"}, + json={"kind": "advisory", "valid_from": "2026-01-01", "valid_to": "2026-06-30"}, ) assert set_response.json()["valid_to"] == "2026-06-30" cleared = test_client.patch( f"/documents/{workflow_id}/scheme-metadata", - json={"document_kind": "advisory", "valid_to": ""}, + json={"kind": "advisory", "valid_to": ""}, ) assert cleared.status_code == 200 @@ -414,7 +442,7 @@ def test_stores_the_reviewers_edit(self, test_client, db_connection): response = test_client.patch( f"/documents/{workflow_id}/scheme-metadata", json={ - "document_kind": "advisory", + "kind": "advisory", "valid_from": "2026-10-01", "valid_to": "2026-12-31", }, @@ -436,7 +464,7 @@ def test_moving_only_the_end_keeps_the_stored_start(self, test_client, db_connec response = test_client.patch( f"/documents/{workflow_id}/scheme-metadata", - json={"document_kind": "advisory", "valid_to": "2027-06-30"}, + json={"kind": "advisory", "valid_to": "2027-06-30"}, ) assert response.status_code == 200 @@ -452,7 +480,7 @@ def test_classifying_without_dates_leaves_the_period_alone(self, test_client, db test_client.patch( f"/documents/{workflow_id}/scheme-metadata", - json={"document_kind": "advisory"}, + json={"kind": "advisory"}, ) stored = db_connection.get_document(workflow_id) @@ -468,7 +496,7 @@ def test_rejects_an_end_before_the_start(self, test_client, db_connection): response = test_client.patch( f"/documents/{workflow_id}/scheme-metadata", json={ - "document_kind": "advisory", + "kind": "advisory", "valid_from": "2026-12-31", "valid_to": "2026-01-01", }, @@ -485,7 +513,7 @@ def test_rejects_a_malformed_date(self, test_client, db_connection): response = test_client.patch( f"/documents/{workflow_id}/scheme-metadata", - json={"document_kind": "advisory", "valid_to": "31-12-2026"}, + json={"kind": "advisory", "valid_to": "31-12-2026"}, ) assert response.status_code == 400 @@ -502,13 +530,13 @@ def test_a_rejected_period_stores_nothing(self, test_client, db_connection): test_client.patch( f"/documents/{workflow_id}/scheme-metadata", - json={"document_kind": "advisory", "valid_to": "not-a-date"}, + json={"kind": "advisory", "valid_to": "not-a-date"}, ) stored = db_connection.get_document(workflow_id) assert stored["valid_from"] == "2026-01-01" assert stored["valid_to"] == "2026-06-30" - assert (stored["document_kind"] or "document") == "document" + assert stored["kind"] == "scheme" @pytest.mark.api @pytest.mark.unit @@ -718,6 +746,8 @@ def test_list_pages(self, test_client, db_connection): """Test listing pages for a document.""" workflow_id = "pages-test-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-pages", filename="test.pdf", @@ -744,6 +774,8 @@ def test_get_page(self, test_client, db_connection): """Test getting a specific page.""" workflow_id = "page-get-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-page", filename="test.pdf", @@ -767,6 +799,8 @@ def test_get_page_invalid_number(self, test_client, db_connection): """Test validation rejects invalid page numbers.""" workflow_id = "page-invalid-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-invalid", filename="test.pdf", @@ -788,6 +822,8 @@ def test_update_page(self, test_client, db_connection): """Test updating a page.""" workflow_id = "page-update-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-update", filename="test.pdf", @@ -818,6 +854,8 @@ def test_list_chunks(self, test_client, db_connection): """Test listing chunks for a document.""" workflow_id = "chunks-test-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-chunks", filename="test.pdf", @@ -844,6 +882,8 @@ def test_list_chunks_include_excluded(self, test_client, db_connection): """Test including excluded chunks in listing.""" workflow_id = "chunks-excluded-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-excluded", filename="test.pdf", @@ -874,6 +914,8 @@ def test_get_chunk_invalid_number(self, test_client, db_connection): """Test validation rejects invalid chunk numbers.""" workflow_id = "chunk-invalid-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-chunk-invalid", filename="test.pdf", @@ -894,6 +936,8 @@ def test_get_document_audit(self, test_client, db_connection): """Test getting audit logs for a document.""" workflow_id = "audit-test-001" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-audit", filename="test.pdf", @@ -988,6 +1032,8 @@ def test_page_not_found(self, test_client, db_connection): """Test 404 for missing page.""" workflow_id = "page-404-test" db_connection.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id="doc-404", filename="test.pdf", @@ -1021,3 +1067,194 @@ def test_inline_content_disposition_strips_crlf(self): assert header.count(":") >= 1 assert "\r\nX-Injected" not in header assert header.startswith('inline; filename="') + + +class TestUploadCatalogIdentity: + """/upload picks the tenant's catalog from ``kind`` and names the document by ``code``.""" + + @pytest.fixture(autouse=True) + def _fresh_rate_limit(self): + # /upload is limited per IP; these tests post more often than one minute allows. + import pipeline.api as api + + api.limiter.reset() + yield + api.limiter.reset() + + @staticmethod + def _pdf(sample_pdf_content: bytes, tag: str) -> bytes: + # A different trailer gives a different fingerprint, hence a new workflow id. + return sample_pdf_content + f"\n% {tag}".encode() + + @pytest.mark.api + @pytest.mark.unit + def test_kind_is_required(self, test_client, sample_pdf_content): + response = test_client.post( + "/upload", files={"file": ("a.pdf", sample_pdf_content, "application/pdf")} + ) + + assert response.status_code == 422 + + @pytest.mark.api + @pytest.mark.unit + @pytest.mark.parametrize("kind", ["document", "video", "", "Scheme-X"]) + def test_an_unknown_kind_is_rejected(self, test_client, sample_pdf_content, kind): + response = test_client.post( + "/upload", + params={"kind": kind}, + files={"file": ("a.pdf", sample_pdf_content, "application/pdf")}, + ) + + assert response.status_code == 400 + assert "scheme" in response.json()["detail"] + + @pytest.mark.api + @pytest.mark.unit + def test_an_invalid_code_is_rejected(self, test_client, sample_pdf_content): + response = test_client.post( + "/upload", + params={"kind": "scheme", "code": "Not A Slug"}, + files={"file": ("a.pdf", sample_pdf_content, "application/pdf")}, + ) + + assert response.status_code == 400 + + @pytest.mark.api + @pytest.mark.unit + def test_upload_creates_the_catalog_and_derives_the_code_from_the_filename( + self, test_client, db_connection, sample_pdf_content + ): + response = test_client.post( + "/upload", + params={"kind": "scheme", "group": "mh"}, + files={"file": ("National Mission on Natural Farming.pdf", sample_pdf_content, "application/pdf")}, + ) + + assert response.status_code == 200 + doc = test_client.get(f"/documents/{response.json()['workflow_id']}").json() + assert doc["kind"] == "scheme" + assert doc["group"] == "mh" + assert doc["code"] == "nmnf" + assert doc["name"] == "National Mission on Natural Farming" + with db_connection.get_connection() as conn: + catalog = conn.execute( + "SELECT c.type, g.name, g.path FROM catalogs c JOIN groups g ON g.id = c.group_id" + ).fetchall() + assert [(r["type"], r["name"], r["path"]) for r in catalog] == [ + ("openagrinet:AgricultureResource", "mh", "/states/MH") + ] + + @pytest.mark.api + @pytest.mark.unit + def test_a_second_upload_reuses_the_catalog_and_gets_a_fresh_code( + self, test_client, db_connection, sample_pdf_content + ): + first = test_client.post( + "/upload", + params={"kind": "advisory", "group": "mh"}, + files={"file": ("Crop Advisory.pdf", self._pdf(sample_pdf_content, "one"), "application/pdf")}, + ) + second = test_client.post( + "/upload", + params={"kind": "advisory", "group": "mh"}, + files={"file": ("Crop Advisory.pdf", self._pdf(sample_pdf_content, "two"), "application/pdf")}, + ) + + assert first.status_code == second.status_code == 200 + codes = { + test_client.get(f"/documents/{r.json()['workflow_id']}").json()["code"] + for r in (first, second) + } + assert codes == {"ca", "ca-2"} + with db_connection.get_connection() as conn: + assert conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] == 1 + + @pytest.mark.api + @pytest.mark.unit + def test_an_existing_code_updates_the_same_document( + self, test_client, db_connection, sample_pdf_content + ): + first = test_client.post( + "/upload", + params={"kind": "scheme", "group": "mh", "code": "pm-kisan"}, + files={"file": ("pmkisan.pdf", self._pdf(sample_pdf_content, "v1"), "application/pdf")}, + ) + assert first.status_code == 200 + before = test_client.get(f"/documents/{first.json()['workflow_id']}").json() + db_connection.save_pages( + before["id"], [{"page_number": 1, "original_markdown": "old text"}] + ) + db_connection.save_chunks(before["id"], [{"chunk_number": 1, "original_text": "old"}]) + + second = test_client.post( + "/upload", + params={"kind": "scheme", "group": "mh", "code": "pm-kisan"}, + files={"file": ("pmkisan-2026.pdf", self._pdf(sample_pdf_content, "v2"), "application/pdf")}, + ) + + assert second.status_code == 200 + assert second.json()["workflow_id"] != first.json()["workflow_id"] + after = test_client.get(f"/documents/{before['id']}").json() + assert after["id"] == before["id"] + assert after["filename"] == "pmkisan-2026.pdf" + assert after["source_file_fingerprint"] != before["source_file_fingerprint"] + assert after["reindex_required"] is True + assert after["reindex_reason"] == "source_replaced" + assert (after["page_count"], after["chunk_count"]) == (0, 0) + with db_connection.get_connection() as conn: + assert conn.execute("SELECT count(*) AS n FROM documents").fetchone()["n"] == 1 + assert conn.execute("SELECT count(*) AS n FROM workflows").fetchone()["n"] == 2 + + @pytest.mark.api + @pytest.mark.unit + def test_the_same_code_in_another_tenant_is_a_different_document( + self, test_client, db_connection, sample_pdf_content + ): + for group, tag in (("mh", "a"), ("gj", "b")): + response = test_client.post( + "/upload", + params={"kind": "scheme", "group": group, "code": "pm-kisan"}, + files={"file": ("p.pdf", self._pdf(sample_pdf_content, tag), "application/pdf")}, + ) + assert response.status_code == 200 + + with db_connection.get_connection() as conn: + assert conn.execute("SELECT count(*) AS n FROM documents").fetchone()["n"] == 2 + assert conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] == 2 + + @pytest.mark.api + @pytest.mark.unit + def test_replacing_a_document_needs_access_to_its_tenant(self, db_connection): + from fastapi import HTTPException + + import pipeline.api as api + from pipeline.auth.jwt import claims_to_user + + db_connection.upsert_document( + workflow_id="wf-mh-doc", + document_id="d-mh", + filename="a.pdf", + filepath="/tmp/a.pdf", + group="mh", + kind="scheme", + code="pm-kisan", + ) + outsider = claims_to_user( + {"sub": "u1", "realm_access": {"roles": ["content_curator"]}, "instances": ["gj"]} + ) + insider = claims_to_user( + {"sub": "u2", "realm_access": {"roles": ["content_curator"]}, "instances": ["mh"]} + ) + + with pytest.raises(HTTPException) as exc: + api._resolve_document_identity(outsider, "mh", "scheme", "pm-kisan", "a.pdf") + assert exc.value.status_code == 404 + + kind, code, existing = api._resolve_document_identity(insider, "mh", "scheme", "pm-kisan", "a.pdf") + assert (kind, code, existing["workflow_id"]) == ("scheme", "pm-kisan", "wf-mh-doc") + + @pytest.mark.api + @pytest.mark.unit + def test_the_demo_and_bootstrap_routes_are_gone(self, test_client): + assert test_client.post("/documents/anything/demo").status_code in (404, 405) + assert test_client.post("/catalog/v1/bootstrap").status_code in (404, 405) diff --git a/tests/test_auth.py b/tests/test_auth.py index 8623866..5f00285 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -85,7 +85,7 @@ def test_claims_to_user_maps_keycloak_shape(): assert Permission.UPLOAD in user.permissions assert user.instances == ["tenant-a", "tenant-b"] assert user.envs == ["dev"] - assert user.has_instance("Tenant-A") + assert user.has_group("Tenant-A") assert not user.has_env("prod") diff --git a/tests/test_db.py b/tests/test_db.py index 496010d..2d2d5b6 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -12,6 +12,7 @@ """ import hashlib +import re import psycopg import pytest @@ -21,6 +22,30 @@ def _md5(text): return hashlib.md5(text.encode()).hexdigest() +def _code_for(workflow_id): + return re.sub(r"[^a-z0-9]+", "-", (workflow_id or "doc").lower()).strip("-") or "doc" + + +@pytest.fixture(autouse=True) +def _new_documents_default_to_a_scheme(monkeypatch): + """A new v2 document needs a kind and a code; these tests are about everything else. + + The requirement itself is pinned in test_schema_v2.py. Here a new document is a + scheme whose code is its workflow id, unless a test says otherwise. + """ + from pipeline import db + + real = db.upsert_document + + def with_defaults(*args, **kwargs): + kwargs.setdefault("kind", "scheme") + if not kwargs.get("code"): + kwargs["code"] = _code_for(kwargs.get("workflow_id")) + return real(*args, **kwargs) + + monkeypatch.setattr(db, "upsert_document", with_defaults) + + class TestDocumentOperations: """Tests for document CRUD operations.""" @@ -133,34 +158,17 @@ def test_update_document_stage(self, db_connection): stage="registered" ) - db_connection.update_document_stage( - workflow_id=workflow_id, - stage="ocr_review", - page_count=5, - chunk_count=10 - ) + db_connection.update_document_stage(workflow_id=workflow_id, stage="ocr_review") doc = db_connection.get_document(workflow_id) assert doc["stage"] == "ocr_review" - assert doc["page_count"] == 5 - assert doc["chunk_count"] == 10 - - @pytest.mark.db - @pytest.mark.unit - def test_set_document_demo(self, db_connection): - """Test marking document as demo.""" - workflow_id = "demo-test" - db_connection.upsert_document( - workflow_id=workflow_id, - document_id="doc-demo", - filename="test.pdf", - filepath="/app/books/test.pdf", - stage="registered" - ) - - db_connection.set_document_demo(workflow_id, True) + # Counts are derived from pages and chunks, not stored. + assert (doc["page_count"], doc["chunk_count"]) == (0, 0) + db_connection.save_pages(workflow_id, [{"page_number": 1, "original_markdown": "a"}]) + db_connection.save_chunks(workflow_id, [{"chunk_number": 1, "original_text": "x"}, + {"chunk_number": 2, "original_text": "y"}]) doc = db_connection.get_document(workflow_id) - assert doc["is_demo"] == 1 + assert (doc["page_count"], doc["chunk_count"]) == (1, 2) @pytest.mark.db @pytest.mark.unit @@ -606,26 +614,31 @@ def _new_doc(db, workflow_id, **kwargs): class TestSchemaConstraints: """Rules the database itself enforces.""" + @staticmethod + def _catalog_id(db): + with db.get_connection() as conn: + return db.get_or_create_catalog(db.get_or_create_group("mh", conn=conn), "scheme", conn=conn)["id"] + @pytest.mark.db @pytest.mark.unit - def test_document_kind_must_match_its_catalog_entry_type(self, db_connection): + def test_document_code_must_be_a_slug(self, db_connection): + catalog_id = self._catalog_id(db_connection) with db_connection.get_connection() as conn: - entry = db_connection.upsert_catalog_entry(type="scheme", code="x-scheme", name="X", conn=conn) - with pytest.raises(psycopg.errors.ForeignKeyViolation): + with pytest.raises(psycopg.errors.CheckViolation): conn.execute( - "INSERT INTO documents (instance, source, document_kind, catalog_entry_id) " - "VALUES ('default', %s, 'advisory', %s::uuid)", - (psycopg.types.json.Jsonb({"fingerprint": _md5("k")}), entry["id"]), + "INSERT INTO documents (catalog_id, code, name, source) VALUES (%s::uuid, 'Bad Code', 'N', %s)", + (catalog_id, psycopg.types.json.Jsonb({"fingerprint": _md5("k")})), ) @pytest.mark.db @pytest.mark.unit def test_source_requires_a_hex_fingerprint(self, db_connection): + catalog_id = self._catalog_id(db_connection) with db_connection.get_connection() as conn: with pytest.raises(psycopg.errors.CheckViolation): conn.execute( - "INSERT INTO documents (instance, source) VALUES ('default', %s)", - (psycopg.types.json.Jsonb({"fingerprint": "not-hex"}),), + "INSERT INTO documents (catalog_id, code, name, source) VALUES (%s::uuid, 'c', 'C', %s)", + (catalog_id, psycopg.types.json.Jsonb({"fingerprint": "not-hex"})), ) @pytest.mark.db @@ -642,7 +655,8 @@ def test_stage_timestamps_only_allow_known_keys(self, db_connection): @pytest.mark.db @pytest.mark.unit def test_validity_period_must_not_end_before_it_starts(self, db_connection): - with pytest.raises(psycopg.errors.CheckViolation): + # The period lives in documents.metadata, so Pydantic enforces the order. + with pytest.raises(ValueError): _new_doc(db_connection, "wf-bad-period", valid_from="2026-02-01", valid_to="2026-01-01") @pytest.mark.db @@ -889,14 +903,14 @@ def test_source_spans_round_trip_and_derive_page_numbers(self, db_connection): @pytest.mark.db @pytest.mark.unit def test_search_chunks_matches_edited_text_first(self, db_connection): - _new_doc(db_connection, "wf-search", instance="mh") + _new_doc(db_connection, "wf-search", group="mh") db_connection.save_chunks("wf-search", [ {"chunk_number": 1, "original_text": "alpha beta", "edited_text": "gamma delta"}, ]) hits, total = db_connection.search_chunks(query="gamma") assert total == 1 and hits[0]["workflow_id"] == "wf-search" assert db_connection.search_chunks(query="alpha")[1] == 0 - assert db_connection.search_chunks(query="gamma", instances=["up"])[1] == 0 + assert db_connection.search_chunks(query="gamma", groups=["up"])[1] == 0 class TestIndexStatus: @@ -952,7 +966,7 @@ class TestAuditTrail: @pytest.mark.db @pytest.mark.unit def test_audit_outlives_the_document_and_keeps_tenant_scope(self, db_connection): - _new_doc(db_connection, "wf-aud", instance="mh") + _new_doc(db_connection, "wf-aud", group="mh") db_connection.log_audit( workflow_id="wf-aud", document_id="x", action_type="purge_document", entity_type="document", actor_user_id="kc-9", actor_username="boss", @@ -964,8 +978,8 @@ def test_audit_outlives_the_document_and_keeps_tenant_scope(self, db_connection) assert [entry["action_type"] for entry in logs] == ["purge_document"] assert logs[0]["actor"] == "b@x.in (superadmin)" assert logs[0]["actor_user_id"] == "kc-9" - assert db_connection.get_all_audit_log_count(instances=["mh"]) == 1 - assert db_connection.get_all_audit_log_count(instances=["up"]) == 0 + assert db_connection.get_all_audit_log_count(groups=["mh"]) == 1 + assert db_connection.get_all_audit_log_count(groups=["up"]) == 0 @pytest.mark.db @pytest.mark.unit @@ -1017,10 +1031,10 @@ def test_a_partial_update_keeps_the_rest_of_its_group(self, db_connection): class TestCatalogView: @staticmethod - def _scheme_doc(db, workflow_id, code, stage, **kwargs): - _new_doc(db, workflow_id, stage=stage, **kwargs) - db.update_document_fields( - workflow_id, document_kind="scheme", scheme_code=code, scheme_name=code.upper(), + def _scheme_doc(db, workflow_id, code, stage, kind="scheme", **kwargs): + _new_doc( + db, workflow_id, stage=stage, kind=kind, code=code, + name=code.upper(), **kwargs, ) @pytest.mark.db @@ -1034,7 +1048,6 @@ def test_status_rules(self, db_connection): self._scheme_doc(db, "wf-wip", "wip-one", "chunk_review") self._scheme_doc(db, "wf-off", "off-one", "completed") db.set_document_disabled("wf-off", True) - db.upsert_catalog_entry(type="scheme", code="boot-one", name="Boot", source="bootstrap") status = {e["code"]: e["status"] for e in db.list_catalog_entries()} assert status == { @@ -1043,62 +1056,71 @@ def test_status_rules(self, db_connection): "gate-one": "pending_prod", "wip-one": "disabled", "off-one": "disabled", - "boot-one": "live", } @pytest.mark.db @pytest.mark.unit - def test_computed_columns(self, db_connection): + def test_computed_columns(self, db_connection, monkeypatch): + monkeypatch.setenv("PROD_VECTOR_DB_COLLECTION_NAME", "prod-docs") db = db_connection - self._scheme_doc(db, "wf-a", "multi", "completed", instance="mh") - self._scheme_doc(db, "wf-b", "multi", "completed", instance="up") + self._scheme_doc(db, "wf-a", "scheme-a", "completed", group="mh") + self._scheme_doc(db, "wf-b", "scheme-b", "completed", group="up") db.save_chunks("wf-a", [{"chunk_number": 1, "original_text": "x"}]) db.save_chunks("wf-b", [{"chunk_number": 1, "original_text": "y"}, {"chunk_number": 2, "original_text": "z"}]) db.update_document_fields("wf-b", network_visible=False) - db.upsert_document_index_status("wf-a", "documents-index", last_indexed_at="2026-05-01T00:00:00") + db.upsert_document_index_status("wf-a", "documents-index", last_indexed_at="2026-04-01T00:00:00") + db.upsert_document_index_status("wf-a", "prod-docs", last_indexed_at="2026-05-01T00:00:00") - entry = db.get_catalog_entry("multi") - assert entry["instances"] == ["mh", "up"] - assert sorted(entry["workflow_ids"]) == ["wf-a", "wf-b"] - assert entry["chunk_count"] == 3 - assert entry["network_visible"] is True - assert entry["promoted_at"] is None + a = db.get_catalog_entry("scheme-a") + b = db.get_catalog_entry("scheme-b") + assert (a["groups"], a["workflow_ids"], a["chunk_count"]) == (["mh"], ["wf-a"], 1) + assert (b["groups"], b["workflow_ids"], b["chunk_count"]) == (["up"], ["wf-b"], 2) + assert a["network_visible"] is True and b["network_visible"] is False + assert a["promoted_at"].startswith("2026-05-01T00:00:00") + assert b["promoted_at"] is None @pytest.mark.db @pytest.mark.unit - def test_entries_without_documents_default_to_the_default_instance(self, db_connection): - db_connection.upsert_catalog_entry(type="scheme", code="lonely", name="Lonely", source="import") - entry = db_connection.get_catalog_entry("lonely") - assert entry["instances"] == ["default"] - assert entry["status"] == "disabled" - assert entry["network_visible"] is False + def test_catalogs_without_documents_are_not_listed(self, db_connection): + db = db_connection + with db.get_connection() as conn: + db.get_or_create_catalog(db.get_or_create_group("mh", conn=conn), "scheme", conn=conn) + assert db.list_catalog_entries() == [] + assert db.get_catalog_entry("anything") is None @pytest.mark.db @pytest.mark.unit - def test_other_entry_types_stay_out_of_the_scheme_listing(self, db_connection): - _new_doc(db_connection, "wf-adv", stage="completed") - db_connection.update_document_fields( - "wf-adv", document_kind="advisory", scheme_code="kharif", scheme_name="Kharif", - ) + def test_other_kinds_stay_out_of_the_scheme_listing(self, db_connection): + self._scheme_doc(db_connection, "wf-adv", "kharif", "completed", kind="advisory") assert db_connection.list_catalog_entries() == [] - assert [e["code"] for e in db_connection.list_catalog_entries(type="advisory")] == ["kharif"] + assert [e["code"] for e in db_connection.list_catalog_entries(kind="advisory")] == ["kharif"] + assert db_connection.list_document_codes("advisory") == {"kharif"} + assert db_connection.list_document_codes("scheme") == set() @pytest.mark.db @pytest.mark.unit - def test_changing_the_kind_moves_or_unlinks_the_entry(self, db_connection): - _new_doc(db_connection, "wf-move") - db_connection.update_document_fields( - "wf-move", document_kind="scheme", scheme_code="movable", scheme_name="Movable", - ) - db_connection.update_document_fields("wf-move", document_kind="advisory") + def test_changing_the_kind_moves_the_document_and_rejects_unknown_kinds(self, db_connection): + _new_doc(db_connection, "wf-move", kind="scheme", code="movable", name="Movable") + before = db_connection.get_document("wf-move")["catalog_id"] + db_connection.update_document_fields("wf-move", kind="advisory") doc = db_connection.get_document("wf-move") - assert (doc["document_kind"], doc["scheme_code"], doc["scheme_name"]) == ( + assert (doc["kind"], doc["code"], doc["name"]) == ( "advisory", "movable", "Movable", ) - db_connection.update_document_fields("wf-move", document_kind="document") - doc = db_connection.get_document("wf-move") - assert doc["scheme_code"] is None and doc["catalog_entry_id"] is None + assert doc["catalog_id"] != before + with pytest.raises(ValueError): + db_connection.update_document_fields("wf-move", kind="document") + assert db_connection.get_document("wf-move")["kind"] == "advisory" + + @pytest.mark.db + @pytest.mark.unit + def test_used_codes_exclude_the_document_itself(self, db_connection): + self._scheme_doc(db_connection, "wf-u1", "taken", "completed") + self._scheme_doc(db_connection, "wf-u2", "other", "completed") + assert db_connection.list_used_scheme_codes() == {"taken", "other"} + assert db_connection.list_used_scheme_codes(exclude_workflow_id="wf-u1") == {"other"} + assert db_connection.list_used_scheme_codes(kind="advisory") == set() @pytest.mark.db @pytest.mark.unit @@ -1157,34 +1179,32 @@ class TestOperationsViews: @pytest.mark.db @pytest.mark.unit def test_summary_counts_and_queue_use_the_current_workflow(self, db_connection): - _new_doc(db_connection, "wf-q1", stage="ocr_review", instance="mh") - _new_doc(db_connection, "wf-q2", stage="completed", instance="up") + _new_doc(db_connection, "wf-q1", stage="ocr_review", group="mh") + _new_doc(db_connection, "wf-q2", stage="completed", group="up") db_connection.create_document_job("wf-q2", "reingest", current_stage="ingesting") counts = db_connection.get_document_summary_counts() assert counts["total_documents"] == 2 assert counts["ocr_review_documents"] == 1 assert counts["completed_documents"] == 0 assert counts["running_jobs"] == 3 - assert db_connection.get_document_summary_counts(instances=[])["total_documents"] == 0 - by_instance = {r["instance"]: r for r in db_connection.get_document_counts_by_instance()} - assert by_instance["mh"]["dev_approval"] == 1 + assert db_connection.get_document_summary_counts(groups=[])["total_documents"] == 0 + by_group = {r["group"]: r for r in db_connection.get_document_counts_by_group()} + assert by_group["mh"]["dev_approval"] == 1 rows, total = db_connection.list_operations_queue() assert total == 2 assert {r["workflow_id"]: r["job_type"] for r in rows} == { "wf-q1": "pipeline", "wf-q2": "reingest", } - assert db_connection.list_operations_queue(instances=["up"])[1] == 1 + assert db_connection.list_operations_queue(groups=["up"])[1] == 1 @pytest.mark.db @pytest.mark.unit - def test_demo_and_disabled_documents_are_hidden_by_default(self, db_connection): - _new_doc(db_connection, "wf-demo") + def test_disabled_documents_are_hidden_by_default(self, db_connection): _new_doc(db_connection, "wf-off") - db_connection.set_document_demo("wf-demo", True) db_connection.set_document_disabled("wf-off", True) assert db_connection.list_documents() == [] - assert len(db_connection.list_documents(include_demo=True, include_disabled=True)) == 2 + assert len(db_connection.list_documents(include_disabled=True)) == 1 @pytest.mark.db @pytest.mark.unit diff --git a/tests/test_discovery_publish_service.py b/tests/test_discovery_publish_service.py index 3539abf..834f437 100644 --- a/tests/test_discovery_publish_service.py +++ b/tests/test_discovery_publish_service.py @@ -98,7 +98,7 @@ def test_publish_success_envelope_shape(self, monkeypatch): mock_client.post.return_value = mock_response with patch("pipeline.discovery_publish_service.httpx.Client", return_value=mock_client): - result = service.publish(transaction_id="txn-123", document_kind="advisory") + result = service.publish(transaction_id="txn-123", kind="advisory") mock_client.post.assert_called_once() call_args = mock_client.post.call_args @@ -153,8 +153,8 @@ def test_publish_generates_fresh_message_id_per_call(self, monkeypatch): mock_client.post.return_value = mock_response with patch("pipeline.discovery_publish_service.httpx.Client", return_value=mock_client): - first = service.publish(transaction_id="txn-shared", document_kind="advisory") - second = service.publish(transaction_id="txn-shared", document_kind="advisory") + first = service.publish(transaction_id="txn-shared", kind="advisory") + second = service.publish(transaction_id="txn-shared", kind="advisory") assert first["envelope"]["context"]["transactionId"] == "txn-shared" assert second["envelope"]["context"]["transactionId"] == "txn-shared" @@ -180,7 +180,7 @@ def test_publish_raises_on_http_error(self, monkeypatch): with patch("pipeline.discovery_publish_service.httpx.Client", return_value=mock_client): with pytest.raises(httpx.HTTPStatusError): - service.publish(transaction_id="txn-123", document_kind="advisory") + service.publish(transaction_id="txn-123", kind="advisory") @pytest.mark.unit def test_publish_raises_on_connection_error(self, monkeypatch): @@ -197,7 +197,7 @@ def test_publish_raises_on_connection_error(self, monkeypatch): with patch("pipeline.discovery_publish_service.httpx.Client", return_value=mock_client): with pytest.raises(httpx.ConnectError): - service.publish(transaction_id="txn-123", document_kind="advisory") + service.publish(transaction_id="txn-123", kind="advisory") class TestDiscoveryPublishServiceSchemeCatalog: @@ -220,7 +220,7 @@ def test_scheme_kind_publishes_the_schemes_catalog(self, monkeypatch): mock_client.post.return_value = mock_response with patch("pipeline.discovery_publish_service.httpx.Client", return_value=mock_client): - result = service.publish(transaction_id="txn-9", document_kind="scheme") + result = service.publish(transaction_id="txn-9", kind="scheme") envelope = mock_client.post.call_args.kwargs["json"] catalog = envelope["message"]["catalogs"][0] @@ -244,7 +244,7 @@ def test_no_http_call_is_made(self, monkeypatch, kind): mock_client = MagicMock() with patch("pipeline.discovery_publish_service.httpx.Client", return_value=mock_client): - result = service.publish(transaction_id="txn-skip", document_kind=kind) + result = service.publish(transaction_id="txn-skip", kind=kind) mock_client.post.assert_not_called() assert result["skipped"] is True @@ -265,7 +265,7 @@ def test_reports_the_result_status_for_our_catalog(self, monkeypatch, status): "pipeline.discovery_publish_service.httpx.Client", return_value=_client_returning(response), ): - result = service.publish(transaction_id="txn-v", document_kind="advisory") + result = service.publish(transaction_id="txn-v", kind="advisory") assert result["result_status"] == status @@ -281,7 +281,7 @@ def test_carries_the_errors_array_through(self, monkeypatch): "pipeline.discovery_publish_service.httpx.Client", return_value=_client_returning(response), ): - result = service.publish(transaction_id="txn-v", document_kind="advisory") + result = service.publish(transaction_id="txn-v", kind="advisory") assert result["errors"] == errors @@ -306,7 +306,7 @@ def test_picks_our_catalog_out_of_several_results(self, monkeypatch): "pipeline.discovery_publish_service.httpx.Client", return_value=_client_returning(response), ): - result = service.publish(transaction_id="txn-v", document_kind="advisory") + result = service.publish(transaction_id="txn-v", kind="advisory") assert result["result_status"] == "ACCEPTED" @@ -333,7 +333,7 @@ def test_unreadable_result_is_unknown_not_a_failure(self, monkeypatch, body): "pipeline.discovery_publish_service.httpx.Client", return_value=_client_returning(response), ): - result = service.publish(transaction_id="txn-v", document_kind="advisory") + result = service.publish(transaction_id="txn-v", kind="advisory") assert result["result_status"] is None assert result["errors"] == [] @@ -351,7 +351,7 @@ def test_non_json_body_is_unknown_not_a_failure(self, monkeypatch): "pipeline.discovery_publish_service.httpx.Client", return_value=_client_returning(response), ): - result = service.publish(transaction_id="txn-v", document_kind="advisory") + result = service.publish(transaction_id="txn-v", kind="advisory") assert result["result_status"] is None assert result["errors"] == [] @@ -361,7 +361,7 @@ def test_skipped_publish_has_no_result_status(self, monkeypatch): service = _service(monkeypatch) with patch("pipeline.discovery_publish_service.httpx.Client", return_value=MagicMock()): - result = service.publish(transaction_id="txn-v", document_kind="video") + result = service.publish(transaction_id="txn-v", kind="video") assert result["skipped"] is True assert result["result_status"] is None @@ -384,7 +384,7 @@ def test_logs_the_request_and_the_response_status(self, monkeypatch, caplog): ) with patch("pipeline.discovery_publish_service.httpx.Client", return_value=ok): service.publish( - transaction_id="txn-log", document_kind="advisory", workflow_id="wf-log" + transaction_id="txn-log", kind="advisory", workflow_id="wf-log" ) messages = [r.getMessage() for r in caplog.records] @@ -410,7 +410,7 @@ def test_logs_an_error_when_the_call_fails(self, monkeypatch, caplog): with patch("pipeline.discovery_publish_service.httpx.Client", return_value=client): with pytest.raises(httpx.ConnectError): service.publish( - transaction_id="txn-log", document_kind="advisory", workflow_id="wf-log" + transaction_id="txn-log", kind="advisory", workflow_id="wf-log" ) errors = [r.getMessage() for r in caplog.records if r.levelno == logging.ERROR] @@ -430,7 +430,7 @@ def test_skipped_publish_logs_no_request(self, monkeypatch, caplog): "pipeline.discovery_publish_service.httpx.Client", return_value=MagicMock() ): service.publish( - transaction_id="txn-log", document_kind="video", workflow_id="wf-log" + transaction_id="txn-log", kind="video", workflow_id="wf-log" ) messages = [r.getMessage() for r in caplog.records] diff --git a/tests/test_document_repository.py b/tests/test_document_repository.py index 1b9673e..7888493 100644 --- a/tests/test_document_repository.py +++ b/tests/test_document_repository.py @@ -24,7 +24,7 @@ def get_document(self, workflow_id): class TestGetDocumentKind: @pytest.mark.unit def test_returns_the_stored_kind(self): - repo = DocumentRepository(FakeDb(document={"document_kind": "advisory"})) + repo = DocumentRepository(FakeDb(document={"kind": "advisory"})) assert repo.get_document_kind("wf-1") == "advisory" @@ -35,42 +35,56 @@ def test_returns_none_for_a_missing_document(self): assert repo.get_document_kind("wf-nope") is None @pytest.mark.unit - def test_returns_none_when_the_column_is_null(self): - # Old rows predate the column default, so it can be NULL. - repo = DocumentRepository(FakeDb(document={"document_kind": None})) + def test_returns_none_when_the_kind_is_missing(self): + # A catalog type the pipeline has no kind for reads as None. + repo = DocumentRepository(FakeDb(document={"kind": None})) assert repo.get_document_kind("wf-1") is None @pytest.mark.unit def test_does_not_normalise_the_value(self): - repo = DocumentRepository(FakeDb(document={"document_kind": " Advisory "})) + repo = DocumentRepository(FakeDb(document={"kind": " Advisory "})) assert repo.get_document_kind("wf-1") == " Advisory " @pytest.mark.unit def test_passes_the_workflow_id_through(self): - fake = FakeDb(document={"document_kind": "scheme"}) + fake = FakeDb(document={"kind": "scheme"}) DocumentRepository(fake).get_document_kind("wf-42") assert fake.get_document_called_with == "wf-42" -class TestAgainstRealSqlite: +@pytest.fixture +def scheme_document(db_connection): + """A document registered the v2 way (it needs a kind and a code).""" + db_connection.upsert_document( + workflow_id="repo-wf-1", + document_id="repo-doc-1", + filename="repo.pdf", + filepath="/app/books/repo.pdf", + stage="registered", + kind="scheme", + code="repo-scheme", + ) + return "repo-wf-1" + + +class TestAgainstRealPostgres: """Guards against db.py drifting out from under the repository.""" @pytest.mark.unit @pytest.mark.db - def test_reads_a_kind_written_through_the_db_layer(self, db_connection, sample_document): - workflow_id = sample_document["workflow_id"] - db_connection.update_document_fields(workflow_id, document_kind="advisory") - - assert DocumentRepository().get_document_kind(workflow_id) == "advisory" + def test_reads_the_kind_of_the_documents_catalog(self, db_connection, scheme_document): + assert DocumentRepository().get_document_kind(scheme_document) == "scheme" @pytest.mark.unit @pytest.mark.db - def test_unclassified_document_reads_as_the_column_default(self, db_connection, sample_document): - assert DocumentRepository().get_document_kind(sample_document["workflow_id"]) == "document" + def test_reads_a_kind_written_through_the_db_layer(self, db_connection, scheme_document): + db_connection.update_document_fields(scheme_document, kind="advisory") + + assert DocumentRepository().get_document_kind(scheme_document) == "advisory" @pytest.mark.unit @pytest.mark.db diff --git a/tests/test_instances.py b/tests/test_group_names.py similarity index 80% rename from tests/test_instances.py rename to tests/test_group_names.py index 560a375..857ca7a 100644 --- a/tests/test_instances.py +++ b/tests/test_group_names.py @@ -1,9 +1,9 @@ -"""Tests for instance code → state name resolution.""" +"""Tests for group (tenant) code → state name resolution.""" import pytest -class TestInstanceDisplayName: +class TestGroupDisplayName: @pytest.mark.unit @pytest.mark.parametrize( "code,expected", @@ -22,30 +22,30 @@ class TestInstanceDisplayName: ], ) def test_known_codes(self, code, expected): - from pipeline.instances import instance_display_name + from pipeline.group_names import group_display_name - assert instance_display_name(code) == expected + assert group_display_name(code) == expected @pytest.mark.unit @pytest.mark.parametrize("code,expected", [("or", "Odisha"), ("od", "Odisha"), ("ts", "Telangana"), ("tg", "Telangana")]) def test_legacy_code_aliases(self, code, expected): - from pipeline.instances import instance_display_name + from pipeline.group_names import group_display_name - assert instance_display_name(code) == expected + assert group_display_name(code) == expected @pytest.mark.unit def test_blank_falls_back_to_default(self): - from pipeline.instances import instance_display_name + from pipeline.group_names import group_display_name - assert instance_display_name("") == "Default" - assert instance_display_name(None) == "Default" + assert group_display_name("") == "Default" + assert group_display_name(None) == "Default" @pytest.mark.unit def test_unknown_code_is_made_presentable(self): """A new state must degrade to a readable label, never break ingestion.""" - from pipeline.instances import instance_display_name + from pipeline.group_names import group_display_name - assert instance_display_name("new-state") == "New State" + assert group_display_name("new-state") == "New State" class TestProdStageDisabled: @@ -54,7 +54,7 @@ class TestProdStageDisabled: @pytest.mark.unit @pytest.mark.parametrize("value", ["true", "TRUE", "True", "1", "yes", "YES"]) def test_truthy_values_disable_prod(self, monkeypatch, value): - from pipeline.instances import prod_stage_disabled + from pipeline.group_names import prod_stage_disabled monkeypatch.setenv("DISABLE_PROD_SETTING", value) assert prod_stage_disabled() is True @@ -62,7 +62,7 @@ def test_truthy_values_disable_prod(self, monkeypatch, value): @pytest.mark.unit @pytest.mark.parametrize("value", ["false", "FALSE", "0", "no", "", "anything-else"]) def test_other_values_keep_prod_enabled(self, monkeypatch, value): - from pipeline.instances import prod_stage_disabled + from pipeline.group_names import prod_stage_disabled monkeypatch.setenv("DISABLE_PROD_SETTING", value) assert prod_stage_disabled() is False @@ -70,14 +70,14 @@ def test_other_values_keep_prod_enabled(self, monkeypatch, value): @pytest.mark.unit def test_defaults_to_enabled_when_unset(self, monkeypatch): """Absent flag must keep the full pipeline — never silently skip PROD.""" - from pipeline.instances import prod_stage_disabled + from pipeline.group_names import prod_stage_disabled monkeypatch.delenv("DISABLE_PROD_SETTING", raising=False) assert prod_stage_disabled() is False @pytest.mark.unit def test_whitespace_is_tolerated(self, monkeypatch): - from pipeline.instances import prod_stage_disabled + from pipeline.group_names import prod_stage_disabled monkeypatch.setenv("DISABLE_PROD_SETTING", " true ") assert prod_stage_disabled() is True diff --git a/tests/test_groups.py b/tests/test_groups.py index 96bcb23..f90d1f7 100644 --- a/tests/test_groups.py +++ b/tests/test_groups.py @@ -9,11 +9,11 @@ ROLE_STATE_VIEW, ROLE_SUPER_ADMIN, parse_group_paths, - role_for_instance, + role_for_group, ) from pipeline.auth.jwt import claims_to_user from pipeline.auth.permissions import Permission, UserRole, permissions_for_roles -from pipeline.auth.tenancy import allowed_instances, user_can_access_instance +from pipeline.auth.tenancy import allowed_groups, user_can_access_group def test_parse_global_super_admin(): @@ -33,9 +33,9 @@ def test_parse_multi_state_roles(): assert access.is_super_admin is False assert access.state_roles == {"mh": ROLE_STATE_ADMIN, "up": ROLE_STATE_VIEW} assert set(access.instances) == {"mh", "up"} - assert role_for_instance(access, "MH") == ROLE_STATE_ADMIN - assert role_for_instance(access, "up") == ROLE_STATE_VIEW - assert role_for_instance(access, "bh") is None + assert role_for_group(access, "MH") == ROLE_STATE_ADMIN + assert role_for_group(access, "up") == ROLE_STATE_VIEW + assert role_for_group(access, "bh") is None def test_approver_and_contributor_paths(): @@ -61,17 +61,17 @@ def test_contributor_approves_stages_but_cannot_publish_to_dev(): # OCR / translation / chunk approvals all ride on REVIEW. assert user.has_permission(Permission.REVIEW) - assert user.has_permission_for_instance(Permission.REVIEW, "up") + assert user.has_permission_for_group(Permission.REVIEW, "up") # …but publishing to DEV is its own permission. assert not user.has_permission(Permission.APPROVE_INGESTION) - assert not user.has_permission_for_instance(Permission.APPROVE_INGESTION, "up") + assert not user.has_permission_for_group(Permission.APPROVE_INGESTION, "up") assert not user.has_permission(Permission.DELETE_OWN) # Approver and admin both clear the DEV publish gate. for group, role in (("approver", "state_approver"), ("admin", "state_admin")): stronger = claims_to_user({"sub": role, "groups": [f"/states/UP/{group}"]}) assert stronger.has_permission(Permission.APPROVE_INGESTION), role - assert stronger.has_permission_for_instance(Permission.APPROVE_INGESTION, "up"), role + assert stronger.has_permission_for_group(Permission.APPROVE_INGESTION, "up"), role # Only the admin may delete. approver = claims_to_user({"sub": "a", "groups": ["/states/UP/approver"]}) @@ -109,14 +109,14 @@ def test_bh_viewer_reads_every_state_without_write(): assert access.is_super_admin is False assert access.state_roles == {} # No per-state group, but any state resolves to view. - assert role_for_instance(access, "mh") == ROLE_STATE_VIEW - assert role_for_instance(access, "wb") == ROLE_STATE_VIEW + assert role_for_group(access, "mh") == ROLE_STATE_VIEW + assert role_for_group(access, "wb") == ROLE_STATE_VIEW def test_bh_viewer_keeps_stronger_state_role(): access = parse_group_paths(["/global/bh-viewer", "/states/MH/admin"]) - assert role_for_instance(access, "mh") == ROLE_STATE_ADMIN - assert role_for_instance(access, "up") == ROLE_STATE_VIEW + assert role_for_group(access, "mh") == ROLE_STATE_ADMIN + assert role_for_group(access, "up") == ROLE_STATE_VIEW def test_higher_role_wins_same_state(): @@ -172,17 +172,17 @@ def test_claims_to_user_from_groups_only(): assert user.state_roles["mh"] == "state_admin" assert user.state_roles["up"] == "state_view" assert user.is_superadmin is False - assert allowed_instances(user) == {"mh", "up"} - assert user_can_access_instance(user, "mh") - assert not user_can_access_instance(user, "bh") + assert allowed_groups(user) == {"mh", "up"} + assert user_can_access_group(user, "mh") + assert not user_can_access_group(user, "bh") # Union of state_admin + state_view permissions assert Permission.UPLOAD in user.permissions assert Permission.SEARCH in user.permissions assert Permission.MANAGE_USERS not in user.permissions - assert user.has_permission_for_instance(Permission.UPLOAD, "mh") is True - assert user.has_permission_for_instance(Permission.UPLOAD, "up") is False - assert user.has_permission_for_instance(Permission.SEARCH, "up") is True - assert user.has_permission_for_instance(Permission.REVIEW, "up") is False + assert user.has_permission_for_group(Permission.UPLOAD, "mh") is True + assert user.has_permission_for_group(Permission.UPLOAD, "up") is False + assert user.has_permission_for_group(Permission.SEARCH, "up") is True + assert user.has_permission_for_group(Permission.REVIEW, "up") is False def test_claims_super_admin_unrestricted(): @@ -194,8 +194,8 @@ def test_claims_super_admin_unrestricted(): } ) assert user.is_superadmin is True - assert allowed_instances(user) is None - assert user_can_access_instance(user, "mh") + assert allowed_groups(user) is None + assert user_can_access_group(user, "mh") assert Permission.MANAGE_USERS in user.permissions @@ -210,5 +210,5 @@ def test_legacy_instances_claim_still_works(): ) assert user.instances == ["tenant-a"] assert Permission.UPLOAD in user.permissions - assert user_can_access_instance(user, "tenant-a") - assert not user_can_access_instance(user, "mh") + assert user_can_access_group(user, "tenant-a") + assert not user_can_access_group(user, "mh") diff --git a/tests/test_migrate_sqlite_to_postgres.py b/tests/test_migrate_sqlite_to_postgres.py index e963846..99b2884 100644 --- a/tests/test_migrate_sqlite_to_postgres.py +++ b/tests/test_migrate_sqlite_to_postgres.py @@ -100,6 +100,8 @@ WF1 = f"doc-{FP1}" WF2 = f"doc-{FP2}" WF3 = "doc-broken" +WF4 = "doc-plain" +WF5 = "doc-demo" FUTURE = "2099-01-01T00:00:00" @@ -139,7 +141,7 @@ def legacy_db(tmp_path): conn, "documents", workflow_id=WF2, document_id=FP2, filename="inflight.pdf", filepath="/data/documents/inflight.pdf", stage="approval_for_prod", page_count=1, created_at="2026-04-01T10:00:00", updated_at="2026-04-01T12:00:00", instance="up", - is_demo=1, prod_ready_requested_at="2026-04-02T09:00:00", + document_kind="advisory", prod_ready_requested_at="2026-04-02T09:00:00", prod_ready_requested_by_user_id="kc-req", prod_ready_requested_by_username="rita", ocr_completed_at="2026-04-01T11:00:00", ) @@ -147,7 +149,18 @@ def legacy_db(tmp_path): conn, "documents", workflow_id=WF3, document_id="not-a-hash", filename="broken.pdf", filepath="/tmp/broken.pdf", stage="failed", error_message="OCR blew up", created_at="2026-05-01T10:00:00", updated_at="2026-05-01T10:30:00", instance=" ", - document_kind="document", scheme_code="Bad Code", valid_from="2026-06-01", valid_to="2026-01-01", + document_kind="advisory", scheme_code="Bad Code", valid_from="2026-06-01", valid_to="2026-01-01", + ) + _insert( + conn, "documents", workflow_id=WF4, document_id="plain", filename="plain.pdf", + filepath="/tmp/plain.pdf", stage="registered", created_at="2026-05-02T10:00:00", + updated_at="2026-05-02T10:00:00", instance="mh", document_kind="document", + ) + _insert( + conn, "documents", workflow_id=WF5, document_id="demo", filename="demo.pdf", + filepath="/tmp/demo.pdf", stage="completed", created_at="2026-05-03T10:00:00", + updated_at="2026-05-03T10:00:00", instance="mh", document_kind="scheme", scheme_code="demo-s", + is_demo=1, ) _insert(conn, "document_jobs", id=1, workflow_id=WF1, job_type="pipeline", temporal_workflow_id=WF1, @@ -175,6 +188,7 @@ def legacy_db(tmp_path): _insert(conn, "pages", workflow_id=WF1, page_number=2, original_markdown="english page", detected_language="en", reviewer_notes="fine") _insert(conn, "pages", workflow_id="doc-purged", page_number=1, original_markdown="stray") + _insert(conn, "pages", workflow_id=WF5, page_number=1, original_markdown="demo page") _insert(conn, "chunks", workflow_id=WF1, chunk_number=1, original_text="chunk one", token_count=5, page_start=1, page_end=2, source_spans_json=json.dumps([{"page_number": 1, "start_char": 0, "end_char": 5}]), @@ -195,6 +209,8 @@ def legacy_db(tmp_path): _insert(conn, "audit_logs", workflow_id=WF1, document_id=FP1, action_type="page_edit", entity_type="page", entity_id=1, metadata="not json", timestamp="2026-03-02T12:00:00", actor_user_id="kc-rev", actor_username="ravi", actor_email="ravi@x.in", actor_roles="reviewer") + _insert(conn, "audit_logs", workflow_id=WF5, document_id="demo", action_type="stage_change", + entity_type="document", timestamp="2026-05-03T11:00:00") _insert(conn, "audit_logs", workflow_id="doc-purged", document_id="gone", action_type="purge_document", entity_type="document", timestamp="2026-02-01T00:00:00") _insert(conn, "audit_logs", workflow_id="__system__", document_id="__settings__", @@ -238,7 +254,7 @@ def _counts(db): table: conn.execute(f"SELECT count(*) AS n FROM {table}").fetchone()["n"] for table in ( "documents", "workflows", "workflow_artifacts", "pages", "chunks", "audit_logs", - "users", "catalog_entries", "email_otps", "webhook_records", + "users", "groups", "catalogs", "email_otps", "webhook_records", ) } @@ -252,6 +268,7 @@ def test_dry_run_writes_nothing_but_reports_everything(self, db_connection, post report = script.run_migration(legacy_db, postgres_url, dry_run=True) assert _counts(db_connection) == before assert report.tables["documents"]["inserted"] == 3 + assert report.tables["documents"]["skipped"] == 2 assert report.mismatches() == [] def test_a_full_run_maps_every_table(self, db_connection, postgres_url, legacy_db): @@ -260,21 +277,28 @@ def test_a_full_run_maps_every_table(self, db_connection, postgres_url, legacy_d assert report.mismatches() == [] tables = report.as_dict()["tables"] - assert tables["documents"]["inserted"] == 3 + assert tables["documents"] == {"read": 5, "inserted": 3, "skipped": 2} assert tables["document_jobs"] == {"read": 3, "inserted": 2, "skipped": 1} - assert tables["pages"] == {"read": 3, "inserted": 2, "skipped": 1} + assert tables["pages"] == {"read": 4, "inserted": 2, "skipped": 2} assert tables["chunks"]["inserted"] == 2 assert tables["document_manifest_entries"] == {"read": 2, "dropped": 2} assert tables["email_otps"] == {"read": 2, "inserted": 1, "dropped": 1} assert tables["settings"]["skipped"] == 2 + assert tables["scheme_catalog_entries"] == {"read": 2, "inserted": 1, "dropped": 1} + skipped = {i["key"]: i["reason"] for i in report.skip_reasons if i["table"] == "documents"} + assert skipped == { + WF4: "document_kind 'document' has no catalog", + WF5: "demo document (is_demo)", + } + assert db.get_document(WF4) is None and db.get_document(WF5) is None doc = db.get_document(WF1) assert doc["stage"] == "completed" - assert doc["instance"] == "mh" + assert doc["group"] == "mh" assert doc["document_id"] == FP1 - assert doc["display_name"] == "Guideline" + assert doc["name"] == "Registry name" # one name per document in v2 assert (doc["valid_from"], doc["valid_to"]) == ("2026-03-01", "2027-03-01") - assert (doc["scheme_code"], doc["document_kind"]) == ("pm-kmy", "scheme") + assert (doc["code"], doc["kind"]) == ("pm-kmy", "scheme") assert doc["network_visible"] is False and doc["reindex_required"] is True assert doc["uploaded_by_username"] == "uma" assert doc["uploaded_by_roles"] == "contributor,reviewer" @@ -282,6 +306,8 @@ def test_a_full_run_maps_every_table(self, db_connection, postgres_url, legacy_d assert doc["ocr_completed_at"] == "2026-03-01T11:00:00" assert doc["chunk_count"] == 2 assert doc["page_count"] == 2 + assert doc["catalog_id"] + assert (doc["name"], doc["aliases"]) == ("Registry name", ["registry"]) assert set(doc["index_status"]) == {"dev"} assert doc["index_status"]["dev"]["index_name"] == "schemes-index" @@ -311,12 +337,32 @@ def test_a_full_run_maps_every_table(self, db_connection, postgres_url, legacy_d assert chunks[1]["page_end"] == 2 and chunks[1]["is_excluded"] is True entries = {e["code"]: e for e in db.list_catalog_entries()} - assert entries["cdp"]["source"] == "bootstrap" and entries["cdp"]["status"] == "live" + assert set(entries) == {"pm-kmy"} # the bootstrap-only "cdp" entry has no document assert entries["pm-kmy"]["name"] == "Registry name" assert entries["pm-kmy"]["status"] == "pending_reindex" assert entries["pm-kmy"]["workflow_ids"] == [WF1] assert db.get_catalog_meta()["version"] == 6 + with db.get_connection() as conn: + groups = {g["path"]: g for g in conn.execute("SELECT * FROM groups").fetchall()} + catalogs = conn.execute( + "SELECT g.path, c.type FROM catalogs c JOIN groups g ON g.id = c.group_id ORDER BY 1, 2" + ).fetchall() + meta = conn.execute( + "SELECT metadata FROM documents WHERE id = %s::uuid", (doc["id"],) + ).fetchone()["metadata"] + assert set(groups) == {"/states/MH", "/states/UP", "/states/DEFAULT"} + assert all(g["keycloak_group_id"] is None for g in groups.values()) + assert [(c["path"], c["type"]) for c in catalogs] == [ + ("/states/DEFAULT", "openagrinet:KnowledgeAdvisory"), + ("/states/MH", "openagrinet:AgricultureResource"), + ("/states/UP", "openagrinet:KnowledgeAdvisory"), + ] + assert (meta["valid_from"], meta["valid_to"]) == ("2026-03-01", "2027-03-01") + assert meta["uploaded_by_user_id"] == db.get_user_by_keycloak_id("kc-up")["id"] + assert meta["index_status"]["dev"]["index_name"] == "schemes-index" + assert not {"is_demo", "chunk_count", "display_name"} & set(meta) + users = {u: db.get_user_by_keycloak_id(u) for u in ("kc-up", "kc-rev", "kc-req")} assert all(users.values()) @@ -327,6 +373,9 @@ def test_a_full_run_maps_every_table(self, db_connection, postgres_url, legacy_d assert edit["metadata"] == json.dumps({"raw": "not json"}) purged = db.get_audit_logs("doc-purged") assert [e["action_type"] for e in purged] == ["purge_document"] + demo_trail = db.get_audit_logs(WF5) # kept for the skipped demo document, tied to its group + assert [e["action_type"] for e in demo_trail] == ["stage_change"] + assert demo_trail[0]["group"] == "mh" settings_log = db.get_settings_audit_logs() assert settings_log[0]["field_name"] == "retrieval.alpha" @@ -348,10 +397,10 @@ def test_in_flight_documents_keep_their_stage_and_prod_request(self, db_connecti doc = db.get_document(WF2) assert doc["stage"] == "approval_for_prod" - assert doc["is_demo"] is True + assert doc["kind"] == "advisory" and doc["code"] == "inflight" assert doc["prod_ready_requested_at"] == "2026-04-02T09:00:00" assert doc["prod_ready_requested_by_username"] == "rita" - assert doc["instance"] == "up" + assert doc["group"] == "up" assert doc["ocr_completed_at"] == "2026-04-01T11:00:00" types = sorted(j["job_type"] for j in db.list_document_jobs(WF2)) assert types == ["pipeline", "promote_prod"] @@ -366,11 +415,11 @@ def test_unusable_values_are_repaired_and_noted(self, db_connection, postgres_ur doc = db.get_document(WF3) assert doc["stage"] == "failed" and doc["error_message"] == "OCR blew up" assert doc["document_id"] == hashlib.md5(b"not-a-hash").hexdigest() - assert doc["instance"] == "default" - assert doc["scheme_code"] is None + assert doc["group"] == "default" + assert doc["kind"] == "advisory" and doc["code"] == "broken" assert doc["valid_from"] == "2026-06-01" and doc["valid_to"] is None notes = " ".join(report.notes) - assert "valid_to cleared" in notes and "not a valid slug" in notes or "plain document" in notes + assert "valid_to cleared" in notes and "not a valid slug, code derived" in notes def test_running_it_twice_changes_nothing(self, db_connection, postgres_url, legacy_db): script = _load_script() @@ -380,7 +429,7 @@ def test_running_it_twice_changes_nothing(self, db_connection, postgres_url, leg report = script.run_migration(legacy_db, postgres_url) assert _counts(db_connection) == first - assert report.tables["documents"] == {"read": 3, "existing": 3} + assert report.tables["documents"] == {"read": 5, "existing": 3, "skipped": 2} assert report.tables["audit_logs"]["inserted"] == 0 assert report.tables["webhook_records"]["existing"] == 1 assert report.mismatches() == [] diff --git a/tests/test_provenance.py b/tests/test_provenance.py index f7a1794..5b52532 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -1,6 +1,7 @@ """Tests for vector-store provenance ingest fields and /provenance/chunk resolver.""" import hashlib +import itertools import pytest from fastapi.testclient import TestClient @@ -8,6 +9,13 @@ from pipeline.activities import prepare_ingestion_records from pipeline.api import app +_code_seq = itertools.count(1) + + +def _next_code() -> str: + """A unique document code for fixtures (documents need one in v2).""" + return f"doc-{next(_code_seq)}" + @pytest.mark.unit def test_prepare_records_includes_provenance_fields(): @@ -52,12 +60,14 @@ def test_find_document_by_legacy_doc_id_hash(db_connection): document_id = "legacy-doc-id-001" workflow_id = "doc-legacy000001" db.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id=document_id, filename="legacy.pdf", filepath="/tmp/legacy.pdf", stage="completed", - display_name="Legacy PDF", + name="Legacy PDF", ) db.save_chunks( workflow_id, @@ -97,12 +107,14 @@ def test_provenance_chunk_endpoint(db_connection): workflow_id = "doc-provenance01" document_id = hashlib.md5(b"prov-doc-id-xyz").hexdigest() db.upsert_document( + kind="scheme", + code=_next_code(), workflow_id=workflow_id, document_id=document_id, filename="prov.pdf", filepath="/tmp/prov.pdf", stage="completed", - display_name="Provenance PDF", + name="Provenance PDF", ) db.save_chunks( workflow_id, diff --git a/tests/test_schema_v2.py b/tests/test_schema_v2.py new file mode 100644 index 0000000..df13c54 --- /dev/null +++ b/tests/test_schema_v2.py @@ -0,0 +1,598 @@ +""" +Schema v2 rules: groups, catalogs, documents keyed by (catalog, code), the +group snapshot on audit_logs and the catalog_status view. + +Everything runs against the real Postgres from conftest.py. +""" + +import hashlib +import logging + +import psycopg +import pytest +from psycopg.types.json import Jsonb + + +def _md5(text): + return hashlib.md5(text.encode()).hexdigest() + + +def _doc(db, workflow_id, **kwargs): + kwargs.setdefault("document_id", _md5(workflow_id)) + kwargs.setdefault("filename", f"{workflow_id}.pdf") + kwargs.setdefault("filepath", f"minio://documents/{workflow_id}.pdf") + kwargs.setdefault("kind", "scheme") + kwargs.setdefault("code", workflow_id) + return db.upsert_document(workflow_id=workflow_id, **kwargs) + + +def _group(db, group="mh"): + return db.get_or_create_group(group) + + +class TestGroups: + @pytest.mark.db + @pytest.mark.unit + def test_get_or_create_group_is_idempotent_and_maps_tenant_codes_to_paths(self, db_connection): + db = db_connection + mh = _group(db, "mh") + assert _group(db, "MH") == mh + assert _group(db, "up") != mh + bv = _group(db, "bv") + with db.get_connection() as conn: + rows = {r["name"]: r["path"] for r in conn.execute("SELECT name, path FROM groups").fetchall()} + assert rows == {"mh": "/states/MH", "up": "/states/UP", "bv": "/global"} + assert _group(db, "bv") == bv + + @pytest.mark.db + @pytest.mark.unit + def test_upsert_group_fills_the_keycloak_id_of_an_existing_path(self, db_connection): + db = db_connection + first = db.upsert_group("/states/MH") # as created by the SQLite move: no Keycloak id yet + again = db.upsert_group("/states/MH", keycloak_group_id="kc-group-1") + assert again == first + with db.get_connection() as conn: + row = conn.execute("SELECT keycloak_group_id, name FROM groups WHERE id = %s::uuid", (first,)).fetchone() + assert (row["keycloak_group_id"], row["name"]) == ("kc-group-1", "mh") + # A later sync that does not know the id must not erase it. + db.upsert_group("/states/MH") + with db.get_connection() as conn: + row = conn.execute("SELECT keycloak_group_id FROM groups WHERE id = %s::uuid", (first,)).fetchone() + assert row["keycloak_group_id"] == "kc-group-1" + + @pytest.mark.db + @pytest.mark.unit + def test_keycloak_group_id_is_unique_but_nullable(self, db_connection): + with db_connection.get_connection() as conn: + conn.execute("INSERT INTO groups (name, path) VALUES ('a', '/states/A')") + conn.execute("INSERT INTO groups (name, path) VALUES ('b', '/states/B')") # two NULLs are fine + conn.execute("INSERT INTO groups (keycloak_group_id, name, path) VALUES ('k', 'c', '/states/C')") + with pytest.raises(psycopg.errors.UniqueViolation): + conn.execute("INSERT INTO groups (keycloak_group_id, name, path) VALUES ('k', 'd', '/states/D')") + + @pytest.mark.db + @pytest.mark.unit + def test_a_group_with_catalogs_cannot_be_deleted(self, db_connection): + db = db_connection + group_id = _group(db) + db.get_or_create_catalog(group_id, "scheme") + with db.get_connection() as conn: + with pytest.raises(psycopg.errors.ForeignKeyViolation): + conn.execute("DELETE FROM groups WHERE id = %s::uuid", (group_id,)) + + @pytest.mark.db + @pytest.mark.unit + def test_deleting_a_group_clears_users_and_audit_snapshots(self, db_connection): + db = db_connection + group_id = _group(db, "zz") + db.upsert_user("kc-zz", "zed", None, [], tenants=["zz"]) + with db.get_connection() as conn: + conn.execute( + "INSERT INTO audit_logs (action_type, entity_type, group_id) VALUES ('x', 'setting', %s::uuid)", + (group_id,), + ) + conn.execute("DELETE FROM groups WHERE id = %s::uuid", (group_id,)) + assert conn.execute("SELECT group_id FROM audit_logs WHERE action_type = 'x'").fetchone()["group_id"] is None + assert db.get_user_by_keycloak_id("kc-zz")["group_id"] is None + + +class TestCatalogs: + @pytest.mark.db + @pytest.mark.unit + def test_get_or_create_catalog_is_idempotent_per_group_and_kind(self, db_connection): + db = db_connection + mh = _group(db, "mh") + scheme = db.get_or_create_catalog(mh, "scheme") + assert db.get_or_create_catalog(mh, "scheme")["id"] == scheme["id"] + assert scheme["type"] == "openagrinet:AgricultureResource" + assert scheme["name"] == "mh scheme" + advisory = db.get_or_create_catalog(mh, "advisory") + assert advisory["id"] != scheme["id"] + assert advisory["type"] == "openagrinet:KnowledgeAdvisory" + assert db.get_or_create_catalog(_group(db, "up"), "scheme")["id"] != scheme["id"] + + @pytest.mark.db + @pytest.mark.unit + def test_a_kind_without_a_catalog_type_is_rejected(self, db_connection): + with pytest.raises(ValueError): + db_connection.get_or_create_catalog(_group(db_connection), "video") + + @pytest.mark.db + @pytest.mark.unit + def test_one_catalog_per_group_and_type(self, db_connection): + db = db_connection + group_id = _group(db) + with db.get_connection() as conn: + conn.execute( + "INSERT INTO catalogs (group_id, type, name) VALUES (%s::uuid, 'openagrinet:MandiPrice', 'A')", + (group_id,), + ) + with pytest.raises(psycopg.errors.UniqueViolation): + conn.execute( + "INSERT INTO catalogs (group_id, type, name) VALUES (%s::uuid, 'openagrinet:MandiPrice', 'B')", + (group_id,), + ) + + @pytest.mark.db + @pytest.mark.unit + @pytest.mark.parametrize("bad_type", ["scheme", "openagrinet:", ":Thing", "open agrinet:X", "a:b:c", ""]) + def test_type_must_look_like_a_schema_name(self, db_connection, bad_type): + group_id = _group(db_connection) + with db_connection.get_connection() as conn: + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute( + "INSERT INTO catalogs (group_id, type, name) VALUES (%s::uuid, %s, 'N')", + (group_id, bad_type), + ) + + @pytest.mark.db + @pytest.mark.unit + def test_catalog_metadata_must_be_an_object(self, db_connection): + group_id = _group(db_connection) + with db_connection.get_connection() as conn: + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute( + "INSERT INTO catalogs (group_id, type, name, metadata) " + "VALUES (%s::uuid, 'openagrinet:MandiPrice', 'N', '[]'::jsonb)", + (group_id,), + ) + + @pytest.mark.db + @pytest.mark.unit + def test_a_catalog_with_documents_cannot_be_deleted(self, db_connection): + db = db_connection + _doc(db, "wf-held") + with db.get_connection() as conn: + with pytest.raises(psycopg.errors.ForeignKeyViolation): + conn.execute("DELETE FROM catalogs") + + +class TestDocumentIdentity: + @pytest.mark.db + @pytest.mark.unit + def test_a_new_document_needs_a_kind_and_a_code(self, db_connection): + db = db_connection + common = dict(workflow_id="wf-x", document_id=_md5("x"), filename="x.pdf", filepath="/x.pdf") + with pytest.raises(ValueError): + db.upsert_document(**common, code="x") + with pytest.raises(ValueError): + db.upsert_document(**common, kind="scheme") + assert db.get_document("wf-x") is None + + @pytest.mark.db + @pytest.mark.unit + def test_a_new_document_lands_in_its_tenants_catalog_of_that_kind(self, db_connection): + db = db_connection + _doc(db, "wf-land", group="mh", kind="advisory", code="kharif", name="Kharif advice") + doc = db.get_document("wf-land") + assert (doc["group"], doc["kind"], doc["code"], doc["name"]) == ( + "mh", "advisory", "kharif", "Kharif advice", + ) + expected = db.get_or_create_catalog(_group(db, "mh"), "advisory")["id"] + assert doc["catalog_id"] == expected + + @pytest.mark.db + @pytest.mark.unit + def test_the_name_defaults_to_the_code(self, db_connection): + _doc(db_connection, "wf-name", code="pm-kisan") + assert db_connection.get_document("wf-name")["name"] == "pm-kisan" + + @pytest.mark.db + @pytest.mark.unit + def test_a_code_is_unique_within_its_catalog(self, db_connection): + db = db_connection + _doc(db, "wf-one", group="mh", code="pm-kisan") + with pytest.raises(db.DuplicateDocumentCode): + _doc(db, "wf-two", group="mh", code="pm-kisan") + assert db.get_document("wf-two") is None + + @pytest.mark.db + @pytest.mark.unit + def test_the_same_code_is_fine_in_another_group_or_kind(self, db_connection): + db = db_connection + _doc(db, "wf-mh", group="mh", code="pm-kisan") + _doc(db, "wf-up", group="up", code="pm-kisan") + _doc(db, "wf-adv", group="mh", code="pm-kisan", kind="advisory") + assert {d["workflow_id"] for d in db.list_documents()} == {"wf-mh", "wf-up", "wf-adv"} + + @pytest.mark.db + @pytest.mark.unit + def test_a_duplicate_code_leaves_no_half_written_rows(self, db_connection): + db = db_connection + _doc(db, "wf-first", code="dup") + with pytest.raises(db.DuplicateDocumentCode): + _doc(db, "wf-second", code="dup") + with db.get_connection() as conn: + assert conn.execute("SELECT count(*) AS n FROM documents").fetchone()["n"] == 1 + assert conn.execute("SELECT count(*) AS n FROM workflows").fetchone()["n"] == 1 + + @pytest.mark.db + @pytest.mark.unit + def test_find_document_by_code(self, db_connection): + db = db_connection + _doc(db, "wf-find", group="mh", code="pm-kisan") + found = db.find_document_by_code("mh", "scheme", "pm-kisan") + assert found and found["workflow_id"] == "wf-find" + assert db.find_document_by_code("MH", "scheme", "pm-kisan")["id"] == found["id"] + assert db.find_document_by_code("up", "scheme", "pm-kisan") is None + assert db.find_document_by_code("mh", "advisory", "pm-kisan") is None + assert db.find_document_by_code("mh", "scheme", "other") is None + + @pytest.mark.db + @pytest.mark.unit + def test_changing_the_kind_moves_the_document_within_its_group(self, db_connection): + db = db_connection + _doc(db, "wf-kind", group="mh", code="movable") + db.update_document_fields("wf-kind", kind="advisory") + doc = db.get_document("wf-kind") + assert doc["kind"] == "advisory" + with db.get_connection() as conn: + row = conn.execute( + "SELECT c.group_id, c.type FROM catalogs c WHERE c.id = %s::uuid", (doc["catalog_id"],) + ).fetchone() + assert str(row["group_id"]) == _group(db, "mh") + assert row["type"] == "openagrinet:KnowledgeAdvisory" + + @pytest.mark.db + @pytest.mark.unit + def test_moving_to_a_kind_that_already_holds_the_code_is_a_duplicate(self, db_connection): + db = db_connection + _doc(db, "wf-s", group="mh", code="same") + _doc(db, "wf-a", group="mh", code="same", kind="advisory") + with pytest.raises(db.DuplicateDocumentCode): + db.update_document_fields("wf-s", kind="advisory") + assert db.get_document("wf-s")["kind"] == "scheme" + + +class TestReupload: + @pytest.mark.db + @pytest.mark.unit + def test_reset_for_reupload_replaces_the_source_and_clears_the_content(self, db_connection): + db = db_connection + doc_uuid = _doc(db, "wf-re", filename="old.pdf", valid_from="2026-01-01", valid_to="2026-12-31") + db.save_pages("wf-re", [{"page_number": 1, "original_markdown": "old"}]) + db.save_chunks("wf-re", [{"chunk_number": 1, "original_text": "old", "chunking_provider": "p"}]) + db.upsert_document_index_status("wf-re", "documents-index", status="indexed") + artifact_id = db.add_document_artifact("wf-re", "original_upload", "minio://documents/old.pdf") + new_fingerprint = _md5("new content") + + db.reset_document_for_reupload( + doc_uuid, filename="new.pdf", filepath="minio://documents/new.pdf", fingerprint=new_fingerprint, + ) + + doc = db.get_document("wf-re") + assert doc["id"] == doc_uuid + assert doc["source_file_fingerprint"] == new_fingerprint + assert (doc["filename"], doc["filepath"]) == ("new.pdf", "minio://documents/new.pdf") + assert db.get_pages("wf-re") == [] + assert db.get_chunks("wf-re") == [] + assert (doc["page_count"], doc["chunk_count"]) == (0, 0) + assert doc["reindex_required"] is True + assert doc["reindex_reason"] == "source_replaced" + + # What is not content survives: workflows, artifacts, period, index state, code. + assert [j["job_type"] for j in db.list_document_jobs("wf-re")] == ["pipeline"] + assert db.get_document_artifact("wf-re", artifact_id) is not None + assert (doc["valid_from"], doc["valid_to"]) == ("2026-01-01", "2026-12-31") + assert "dev" in doc["index_status"] + assert doc["code"] == "wf-re" + with db.get_connection() as conn: + assert "chunking" not in conn.execute( + "SELECT metadata FROM documents WHERE id = %s::uuid", (doc_uuid,) + ).fetchone()["metadata"] + + @pytest.mark.db + @pytest.mark.unit + def test_a_rechunk_after_reupload_starts_a_fresh_run_number(self, db_connection): + db = db_connection + doc_uuid = _doc(db, "wf-run") + db.save_chunks("wf-run", [{"chunk_number": 1, "original_text": "a", "chunking_provider": "p"}]) + db.reset_document_for_reupload( + doc_uuid, filename="n.pdf", filepath="/n.pdf", fingerprint=_md5("n"), + ) + db.save_chunks("wf-run", [{"chunk_number": 1, "original_text": "b", "chunking_provider": "p"}]) + assert db.get_chunk("wf-run", 1)["chunk_version"] == 1 + + @pytest.mark.db + @pytest.mark.unit + def test_a_fingerprint_must_be_hex(self, db_connection): + doc_uuid = _doc(db_connection, "wf-bad") + with pytest.raises(ValueError): + db_connection.reset_document_for_reupload( + doc_uuid, filename="n.pdf", filepath="/n.pdf", fingerprint="not-hex", + ) + + +class TestUserGroup: + @pytest.mark.db + @pytest.mark.unit + def test_a_single_tenant_sets_the_group_without_a_warning(self, db_connection, caplog): + db = db_connection + with caplog.at_level(logging.WARNING, logger="pipeline.db"): + db.upsert_user("kc-one", "one", None, [], tenants=["MH"]) + assert db.get_user_by_keycloak_id("kc-one")["group_id"] == _group(db, "mh") + assert not [r for r in caplog.records if "tenants" in r.getMessage()] + + @pytest.mark.db + @pytest.mark.unit + def test_several_tenants_use_the_first_sorted_one_and_warn(self, db_connection, caplog): + db = db_connection + with caplog.at_level(logging.WARNING, logger="pipeline.db"): + db.upsert_user("kc-many", "many", None, [], tenants=["up", "mh", "up"]) + assert db.get_user_by_keycloak_id("kc-many")["group_id"] == _group(db, "mh") + warnings = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] + assert any("kc-many" in w and "mh" in w for w in warnings) + + @pytest.mark.db + @pytest.mark.unit + def test_a_login_without_tenants_keeps_the_stored_group(self, db_connection): + db = db_connection + db.upsert_user("kc-keep", "keep", None, [], tenants=["up"]) + db.upsert_user("kc-keep", "keep", None, []) + db.upsert_user("kc-keep", "keep", None, [], tenants=[]) + assert db.get_user_by_keycloak_id("kc-keep")["group_id"] == _group(db, "up") + + @pytest.mark.db + @pytest.mark.unit + def test_a_new_login_moves_the_user_to_the_new_first_tenant(self, db_connection): + db = db_connection + db.upsert_user("kc-move", "move", None, [], tenants=["up"]) + db.upsert_user("kc-move", "move", None, [], tenants=["mh"]) + assert db.get_user_by_keycloak_id("kc-move")["group_id"] == _group(db, "mh") + + @pytest.mark.db + @pytest.mark.unit + def test_users_without_tenants_have_no_group(self, db_connection): + db_connection.upsert_user("kc-none", "none", None, []) + assert db_connection.get_user_by_keycloak_id("kc-none")["group_id"] is None + + @pytest.mark.db + @pytest.mark.unit + def test_the_uploader_lands_in_the_documents_metadata_and_gets_the_tenant_group(self, db_connection): + db = db_connection + _doc(db, "wf-up", group="mh", uploaded_by_user_id="kc-up", uploaded_by_username="uma") + doc = db.get_document("wf-up") + user = db.get_user_by_keycloak_id("kc-up") + assert doc["uploaded_by_user_id"] == "kc-up" + assert user["group_id"] == _group(db, "mh") + with db.get_connection() as conn: + meta = conn.execute("SELECT metadata FROM documents").fetchone()["metadata"] + assert meta["uploaded_by_user_id"] == user["id"] + + +class TestAuditGroupSnapshot: + @pytest.mark.db + @pytest.mark.unit + def test_an_entry_snapshots_the_documents_group(self, db_connection): + db = db_connection + _doc(db, "wf-aud", group="mh") + db.log_audit(workflow_id="wf-aud", document_id="x", action_type="page_edit", entity_type="page") + with db.get_connection() as conn: + row = conn.execute("SELECT group_id FROM audit_logs WHERE action_type = 'page_edit'").fetchone() + assert str(row["group_id"]) == _group(db, "mh") + + @pytest.mark.db + @pytest.mark.unit + def test_the_snapshot_survives_a_purge_and_keeps_tenant_scope(self, db_connection): + db = db_connection + _doc(db, "wf-gone", group="mh") + _doc(db, "wf-other", group="up", code="other") + for wf in ("wf-gone", "wf-other"): + db.log_audit(workflow_id=wf, document_id="x", action_type="purge_document", entity_type="document") + db.purge_document("wf-gone") + db.purge_document("wf-other") + + with db.get_connection() as conn: + rows = conn.execute( + "SELECT document_id, group_id, workflow_id FROM audit_logs WHERE action_type = 'purge_document'" + ).fetchall() + assert {r["workflow_id"] for r in rows} == {"wf-gone", "wf-other"} + assert all(r["document_id"] is None for r in rows) + assert {str(r["group_id"]) for r in rows} == {_group(db, "mh"), _group(db, "up")} + assert db.get_all_audit_log_count(groups=["mh"]) == 1 + assert db.get_all_audit_log_count(groups=["up"]) == 1 + assert db.get_all_audit_log_count(groups=["gj"]) == 0 + assert [e["workflow_id"] for e in db.get_all_audit_logs(groups=["mh"])] == ["wf-gone"] + assert db.get_all_audit_logs(groups=["mh"])[0]["group"] == "mh" + + @pytest.mark.db + @pytest.mark.unit + def test_system_entries_have_no_group(self, db_connection): + db = db_connection + db.log_audit(workflow_id="__catalog__", document_id="__catalog__", action_type="catalog_rebuild", + entity_type="catalog") + with db.get_connection() as conn: + row = conn.execute("SELECT group_id, document_id FROM audit_logs WHERE action_type = 'catalog_rebuild'").fetchone() + assert row["group_id"] is None and row["document_id"] is None + + +class TestCatalogStatusView: + @staticmethod + def _rows(db): + with db.get_connection() as conn: + return {r["code"]: r for r in conn.execute("SELECT * FROM catalog_status").fetchall()} + + @pytest.mark.db + @pytest.mark.unit + def test_one_row_per_document_with_catalog_group_and_type(self, db_connection): + db = db_connection + _doc(db, "wf-1", group="mh", code="alpha", stage="completed") + _doc(db, "wf-2", group="mh", code="beta", stage="completed") + _doc(db, "wf-3", group="up", code="gamma", stage="completed", kind="advisory") + rows = self._rows(db) + assert set(rows) == {"alpha", "beta", "gamma"} + assert rows["alpha"]["catalog_id"] == rows["beta"]["catalog_id"] != rows["gamma"]["catalog_id"] + assert str(rows["alpha"]["group_id"]) == _group(db, "mh") + assert (rows["alpha"]["group_name"], rows["gamma"]["group_name"]) == ("mh", "up") + assert rows["alpha"]["type"] == "openagrinet:AgricultureResource" + assert rows["gamma"]["type"] == "openagrinet:KnowledgeAdvisory" + assert rows["alpha"]["workflow_id"] == "wf-1" + assert rows["alpha"]["name"] == "alpha" + + @pytest.mark.db + @pytest.mark.unit + def test_status_follows_stage_flags_and_disabling(self, db_connection): + db = db_connection + _doc(db, "wf-live", code="live", stage="completed") + _doc(db, "wf-stale", code="stale", stage="completed") + db.mark_document_reindex_required("wf-stale", True, "edit") + _doc(db, "wf-gate", code="gate", stage="approval_for_prod") + _doc(db, "wf-wip", code="wip", stage="chunk_review") + _doc(db, "wf-off", code="off", stage="completed") + db.set_document_disabled("wf-off", True) + _doc(db, "wf-hidden", code="hidden", stage="completed") + db.update_document_fields("wf-hidden", catalog_visible=False) + _doc(db, "wf-gate-off", code="gate-off", stage="approval_for_prod") + db.set_document_disabled("wf-gate-off", True) + + status = {code: row["status"] for code, row in self._rows(db).items()} + assert status == { + "live": "live", + "stale": "pending_reindex", + "gate": "pending_prod", + "wip": "disabled", + "off": "disabled", + "hidden": "disabled", + "gate-off": "disabled", + } + + @pytest.mark.db + @pytest.mark.unit + def test_chunk_count_is_counted_from_chunks(self, db_connection): + db = db_connection + _doc(db, "wf-c", code="counted", stage="completed") + assert self._rows(db)["counted"]["chunk_count"] == 0 + db.save_chunks("wf-c", [{"chunk_number": n, "original_text": "t"} for n in (1, 2, 3)]) + assert self._rows(db)["counted"]["chunk_count"] == 3 + db.save_chunks("wf-c", [{"chunk_number": 1, "original_text": "t"}]) + assert self._rows(db)["counted"]["chunk_count"] == 1 + + @pytest.mark.db + @pytest.mark.unit + def test_network_visible_defaults_to_true_and_follows_the_flag(self, db_connection): + db = db_connection + _doc(db, "wf-v1", code="visible", stage="completed") + _doc(db, "wf-v2", code="private", stage="completed") + db.update_document_fields("wf-v2", network_visible=False) + rows = self._rows(db) + assert (rows["visible"]["network_visible"], rows["private"]["network_visible"]) == (True, False) + + @pytest.mark.db + @pytest.mark.unit + def test_promoted_at_comes_from_the_prod_index_of_live_documents_only(self, db_connection, monkeypatch): + monkeypatch.setenv("PROD_VECTOR_DB_COLLECTION_NAME", "prod-docs") + db = db_connection + _doc(db, "wf-p1", code="promoted", stage="completed") + _doc(db, "wf-p2", code="not-live", stage="chunk_review") + _doc(db, "wf-p3", code="dev-only", stage="completed") + for wf in ("wf-p1", "wf-p2"): + db.upsert_document_index_status(wf, "prod-docs", last_indexed_at="2026-05-01T10:00:00+00:00") + db.upsert_document_index_status("wf-p3", "documents-index", last_indexed_at="2026-05-02T10:00:00+00:00") + rows = self._rows(db) + assert rows["promoted"]["promoted_at"] is not None + assert rows["promoted"]["promoted_at"].isoformat().startswith("2026-05-01T10:00:00") + assert rows["not-live"]["promoted_at"] is None + assert rows["dev-only"]["promoted_at"] is None + + @pytest.mark.db + @pytest.mark.unit + def test_a_catalog_without_documents_has_no_rows(self, db_connection): + db = db_connection + db.get_or_create_catalog(_group(db), "scheme") + assert self._rows(db) == {} + + +class TestDocumentJsonbShape: + @pytest.mark.db + @pytest.mark.unit + def test_a_new_document_carries_the_default_metadata_keys(self, db_connection): + db = db_connection + _doc(db, "wf-meta") + with db.get_connection() as conn: + meta = conn.execute("SELECT metadata FROM documents").fetchone()["metadata"] + assert meta["index_status"] == {} + assert (meta["catalog_visible"], meta["network_visible"], meta["reindex_required"]) == (True, True, False) + for dropped in ("is_demo", "chunk_count", "display_name"): + assert dropped not in meta + + @pytest.mark.db + @pytest.mark.unit + def test_the_source_has_no_page_count(self, db_connection): + db = db_connection + _doc(db, "wf-src") + db.save_pages("wf-src", [{"page_number": 1, "original_markdown": "a"}]) + with db.get_connection() as conn: + source = conn.execute("SELECT source FROM documents").fetchone()["source"] + assert "page_count" not in source + assert db.get_document("wf-src")["page_count"] == 1 + + @pytest.mark.db + @pytest.mark.unit + def test_there_are_no_instance_kind_or_demo_columns(self, db_connection): + with db_connection.get_connection() as conn: + cols = { + r["column_name"] + for r in conn.execute( + "SELECT column_name FROM information_schema.columns WHERE table_name = 'documents'" + ).fetchall() + } + assert {"instance", "document_kind", "is_demo", "catalog_entry_id", "valid_from", "valid_to", + "index_status", "uploaded_by_user_id"}.isdisjoint(cols) + assert {"catalog_id", "code", "name", "aliases", "source", "is_disabled", "metadata"} <= cols + + @pytest.mark.db + @pytest.mark.unit + def test_aliases_cannot_hold_an_empty_string(self, db_connection): + db = db_connection + _doc(db, "wf-al", aliases=["PM Kisan", "", "pmk"]) + with db.get_connection() as conn: + aliases = conn.execute("SELECT aliases FROM documents").fetchone()["aliases"] + assert aliases == ["PM Kisan", "pmk"] + with db.get_connection() as conn: + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute("UPDATE documents SET aliases = ARRAY['a', '']") + + @pytest.mark.db + @pytest.mark.unit + def test_document_metadata_must_be_an_object(self, db_connection): + db = db_connection + _doc(db, "wf-obj") + with db.get_connection() as conn: + with pytest.raises(psycopg.errors.CheckViolation): + conn.execute("UPDATE documents SET metadata = %s", (Jsonb([1]),)) + + @pytest.mark.db + @pytest.mark.unit + def test_workflow_and_audit_columns_are_plain_text(self, db_connection): + with db_connection.get_connection() as conn: + types = { + (r["table_name"], r["column_name"]): r["data_type"] + for r in conn.execute( + "SELECT table_name, column_name, data_type FROM information_schema.columns " + "WHERE (table_name, column_name) IN (('workflows','workflow_type'), ('workflows','status'), " + "('workflows','current_stage'), ('workflow_artifacts','stage'), ('audit_logs','entity_type'))" + ).fetchall() + } + enums = conn.execute("SELECT count(*) AS n FROM pg_type WHERE typtype = 'e'").fetchone()["n"] + assert len(types) == 5 and set(types.values()) == {"text"} + assert enums == 0 diff --git a/tests/test_scheme_catalog.py b/tests/test_scheme_catalog.py index 6ed3764..baa9e70 100644 --- a/tests/test_scheme_catalog.py +++ b/tests/test_scheme_catalog.py @@ -12,52 +12,71 @@ def catalog_db(db_connection): yield db_connection, scheme_catalog -def test_bootstrap_seeds_13_vector_schemes(catalog_db): +def _doc(db, workflow_id, *, code, kind="scheme", name=None, stage="completed", + group="default", filename=None, **extra): + """Register a document the v2 way: it needs a kind (catalog) and a code.""" + return db.upsert_document( + workflow_id=workflow_id, + document_id=f"doc-{workflow_id}", + filename=filename or f"{workflow_id}.pdf", + filepath=f"/tmp/{workflow_id}.pdf", + stage=stage, + group=group, + kind=kind, + code=code, + name=name, + **extra, + ) + + +def test_catalog_has_no_bootstrap_entries(catalog_db): + """The catalog is documents only; the code-defined lists stay as hints.""" db, scheme_catalog = catalog_db - result = scheme_catalog.bootstrap_catalog_if_empty() - assert result["bootstrapped"] is True - assert result["vector_count"] == 13 + assert not hasattr(scheme_catalog, "bootstrap_catalog_if_empty") snap = scheme_catalog.build_snapshot() - codes = {s["scheme_code"] for s in snap["vector_schemes"]} - assert len(codes) == 13 - assert "nbm" not in codes # legacy-only - assert "pkvy" in codes - assert "pm-ddky" in codes - assert snap["catalog_version"] >= 1 + assert snap["vector_schemes"] == [] + assert snap["catalog_version"] == 0 + # Static lists are untouched: curated alias source and legacy tool list. + assert len(scheme_catalog.BOOTSTRAP_VECTOR_SCHEMES) == 13 + assert "nbm" not in scheme_catalog.BOOTSTRAP_VECTOR_SCHEMES # legacy-only + assert any(s["scheme_code"] == "nbm" for s in snap["legacy_schemes"]) assert "search_schemes_doc" in snap["tool_prompts"] - assert "vector_schemes_bullets_en" in snap["tool_prompts"] assert snap["routing_exceptions"]["pkvy"] == "search_schemes" assert snap["routing_exceptions"]["nbm"] == "get_scheme_info" - # second call is no-op - again = scheme_catalog.bootstrap_catalog_if_empty() - assert again["bootstrapped"] is False + + +def test_snapshot_has_one_entry_per_live_scheme_document(catalog_db): + db, scheme_catalog = catalog_db + _doc(db, "wf-a", code="pkvy", name="Paramparagat Krishi Vikas Yojana", group="mh") + _doc(db, "wf-b", code="mif", name="Micro Irrigation Fund", group="gj") + _doc(db, "wf-adv", code="rain-advice", kind="advisory", name="Rain advisory") + _doc(db, "wf-draft", code="draft", stage="chunk_review") + snap = scheme_catalog.build_snapshot() + by_code = {s["scheme_code"]: s for s in snap["vector_schemes"]} + assert set(by_code) == {"pkvy", "mif"} # advisory kind and non-live docs stay out + assert by_code["pkvy"]["instances"] == ["mh"] + assert by_code["mif"]["instances"] == ["gj"] + assert by_code["pkvy"]["workflow_ids"] == ["wf-a"] + assert by_code["pkvy"]["source"] == "pipeline" + + only_mh = scheme_catalog.build_snapshot(instance="mh") + assert [s["scheme_code"] for s in only_mh["vector_schemes"]] == ["pkvy"] def test_scheme_metadata_and_promote_rebuild(catalog_db): db, scheme_catalog = catalog_db - scheme_catalog.bootstrap_catalog_if_empty() v0 = db.get_catalog_meta()["version"] - - db.upsert_document( - workflow_id="wf-pkvy-1", - document_id="doc-pkvy-1", - filename="pkvy.pdf", - filepath="/tmp/pkvy.pdf", - stage="completed", - page_count=10, - chunk_count=5, - instance="default", - ) + _doc(db, "wf-pkvy-1", code="pkvy") result = scheme_catalog.apply_scheme_metadata( "wf-pkvy-1", - document_kind="scheme", - scheme_code="pkvy", - scheme_name="Paramparagat Krishi Vikas Yojana", - scheme_aliases=["PKVY", "organic"], + kind="scheme", + code="pkvy", + name="Paramparagat Krishi Vikas Yojana", + aliases=["PKVY", "organic"], network_visible=True, catalog_visible=True, ) - assert result["document"]["scheme_code"] == "pkvy" + assert result["document"]["code"] == "pkvy" assert result["catalog_version"] is not None assert result["catalog_version"] > v0 @@ -65,36 +84,22 @@ def test_scheme_metadata_and_promote_rebuild(catalog_db): assert entry is not None assert entry["status"] == "live" assert entry["source"] == "pipeline" - assert "organic" in entry["aliases"] or "PKVY" in entry["aliases"] + assert entry["name"] == "Paramparagat Krishi Vikas Yojana" + assert "organic" in entry["aliases"] and "PKVY" in entry["aliases"] def test_code_rename_pending_reindex(catalog_db): db, scheme_catalog = catalog_db - scheme_catalog.bootstrap_catalog_if_empty() - db.upsert_document( - workflow_id="wf-rename", - document_id="doc-rename", - filename="x.pdf", - filepath="/tmp/x.pdf", - stage="completed", - chunk_count=3, - instance="default", - ) - scheme_catalog.apply_scheme_metadata( - "wf-rename", - document_kind="scheme", - scheme_code="old-code", - scheme_name="Old", - ) + _doc(db, "wf-rename", code="old-code", name="Old") result = scheme_catalog.apply_scheme_metadata( "wf-rename", - scheme_code="new-code", - scheme_name="New", + code="new-code", + name="New", ) assert result["pending_reindex"] is True doc = db.get_document("wf-rename") - assert doc["scheme_code"] == "new-code" - assert int(doc["reindex_required"]) == 1 + assert doc["code"] == "new-code" + assert doc["reindex_required"] is True # New code must not appear in live vector_schemes snap = scheme_catalog.build_snapshot() live_codes = {s["scheme_code"] for s in snap["vector_schemes"]} @@ -102,6 +107,7 @@ def test_code_rename_pending_reindex(catalog_db): entry = db.get_catalog_entry("new-code") assert entry is not None assert entry["status"] == "pending_reindex" + assert db.get_catalog_entry("old-code") is None # the document holds one code def test_derive_scheme_code_acronym(catalog_db): @@ -113,59 +119,56 @@ def test_derive_scheme_code_acronym(catalog_db): def test_derive_scheme_code_collision_appends_suffix(catalog_db): db, scheme_catalog = catalog_db - db.upsert_document( - workflow_id="wf-nmnf-1", - document_id="doc-nmnf-1", - filename="a.pdf", - filepath="/tmp/a.pdf", - stage="completed", - instance="default", - ) - scheme_catalog.apply_scheme_metadata( - "wf-nmnf-1", document_kind="scheme", scheme_code="nmnf", scheme_name="National Mission on Natural Farming", - ) + _doc(db, "wf-nmnf-1", code="nmnf", name="National Mission on Natural Farming") # A second, unrelated document whose title happens to acronym to the same code. assert scheme_catalog.derive_scheme_code("New Model National Farming") == "nmnf-2" + # The collision domain is the kind: an advisory may reuse the code. + assert scheme_catalog.derive_scheme_code("New Model National Farming", kind="advisory") == "nmnf" + # A document's own code does not collide with itself. + assert scheme_catalog.derive_scheme_code( + "National Mission on Natural Farming", exclude_workflow_id="wf-nmnf-1" + ) == "nmnf" -def test_apply_scheme_metadata_auto_derives_code_from_name(catalog_db): +def test_collision_domain_spans_tenants(catalog_db): + """Codes are unique per catalog in SQL, but derivation avoids every tenant's codes.""" db, scheme_catalog = catalog_db - db.upsert_document( - workflow_id="wf-auto-code", - document_id="doc-auto-code", - filename="Guideline_of_NMNF_V2_Revised.pdf", - filepath="/tmp/x.pdf", - stage="ready_for_ingestion", - instance="default", - ) + _doc(db, "wf-mh", code="nmnf", group="mh") + assert scheme_catalog.derive_scheme_code("National Mission on Natural Farming") == "nmnf-2" + # The same explicit code in another tenant's catalog is still allowed by the schema. + _doc(db, "wf-gj", code="nmnf", group="gj") + + +def test_apply_scheme_metadata_keeps_the_code_when_only_the_name_changes(catalog_db): + """Every document has a code from upload, so a name edit never re-derives it.""" + db, scheme_catalog = catalog_db + _doc(db, "wf-name-only", code="gn", stage="ready_for_ingestion") result = scheme_catalog.apply_scheme_metadata( - "wf-auto-code", - document_kind="scheme", - scheme_name="National Mission on Natural Farming", + "wf-name-only", + kind="scheme", + name="National Mission on Natural Farming", ) - # scheme_code was never passed explicitly -> derived from the title, not left blank/hash-based. - assert result["document"]["scheme_code"] == "nmnf" - assert result["document"]["scheme_name"] == "National Mission on Natural Farming" + assert result["document"]["code"] == "gn" + assert result["document"]["name"] == "National Mission on Natural Farming" + assert result["code_renamed"] is False -def test_document_kind_widened_to_advisory_and_custom(catalog_db): +def test_kind_is_limited_to_scheme_and_advisory(catalog_db): db, scheme_catalog = catalog_db - db.upsert_document( - workflow_id="wf-kind", - document_id="doc-kind", - filename="advisory.pdf", - filepath="/tmp/advisory.pdf", - stage="ready_for_ingestion", - instance="default", - ) - result = scheme_catalog.apply_scheme_metadata("wf-kind", document_kind="advisory") - assert result["document"]["document_kind"] == "advisory" + _doc(db, "wf-kind", code="kind-doc", stage="ready_for_ingestion") + assert db.get_document("wf-kind")["kind"] == "scheme" + + result = scheme_catalog.apply_scheme_metadata("wf-kind", kind="advisory") + assert result["document"]["kind"] == "advisory" + assert result["document"]["code"] == "kind-doc" # moved catalogs, kept its code - result = scheme_catalog.apply_scheme_metadata("wf-kind", document_kind="how_to_faq") - assert result["document"]["document_kind"] == "how_to_faq" + result = scheme_catalog.apply_scheme_metadata("wf-kind", kind="scheme") + assert result["document"]["kind"] == "scheme" - with pytest.raises(ValueError): - scheme_catalog.apply_scheme_metadata("wf-kind", document_kind="Not Valid!!") + for bad in ("how_to_faq", "document", "video", "Not Valid!!"): + with pytest.raises(ValueError): + scheme_catalog.apply_scheme_metadata("wf-kind", kind=bad) + assert db.get_document("wf-kind")["kind"] == "scheme" def test_prepare_records_scheme_payload(): @@ -186,11 +189,11 @@ def test_prepare_records_scheme_payload(): "pkvy.pdf", chunks, workflow_id="wf1", - instance="default", - document_kind="scheme", - scheme_code="pkvy", - scheme_name="Paramparagat Krishi Vikas Yojana", - scheme_aliases=["PKVY"], + group="default", + kind="scheme", + code="pkvy", + name="Paramparagat Krishi Vikas Yojana", + aliases=["PKVY"], ) assert len(records) == 1 r = records[0] @@ -222,7 +225,7 @@ def test_prepare_records_document_unchanged(): "doc2", "notes.pdf", chunks, - document_kind="document", + kind="document", ) assert records[0]["type"] == "document" assert "scheme_code" not in records[0] @@ -254,9 +257,11 @@ async def _noop_lifespan(_app): def test_catalog_api_snapshot(asgi_client): - from pipeline import scheme_catalog + from pipeline import db, scheme_catalog - scheme_catalog.bootstrap_catalog_if_empty() + _doc(db, "wf-snap-1", code="pkvy", name="Paramparagat Krishi Vikas Yojana") + _doc(db, "wf-snap-2", code="pm-ddky", name="Deen Dayal Upadhyaya Krishi Kalyan Yojana") + scheme_catalog.rebuild_catalog_from_documents(reason="test") r = asgi_client.get("/catalog/v1/version") assert r.status_code == 200 @@ -267,7 +272,7 @@ def test_catalog_api_snapshot(asgi_client): assert r.status_code == 200 body = r.json() assert body["api_version"] == "1" - assert len(body["vector_schemes"]) == 13 + assert {s["scheme_code"] for s in body["vector_schemes"]} == {"pkvy", "pm-ddky"} assert "tool_prompts" in body assert "legacy_schemes" in body assert body["collections"]["scheme_qdrant"]["name"] == "schemes-index" @@ -278,13 +283,14 @@ def test_catalog_api_snapshot(asgi_client): r = asgi_client.get("/catalog/v1/schemes") assert r.status_code == 200 - assert len(r.json()["vector_schemes"]) == 13 + assert len(r.json()["vector_schemes"]) == 2 -def test_catalog_service_key_auth(asgi_client, monkeypatch): - from pipeline import scheme_catalog +def test_catalog_bootstrap_endpoint_is_gone(asgi_client): + assert asgi_client.post("/catalog/v1/bootstrap").status_code in (404, 405) + - scheme_catalog.bootstrap_catalog_if_empty() +def test_catalog_service_key_auth(asgi_client, monkeypatch): monkeypatch.setenv("CATALOG_SERVICE_API_KEYS", "test-secret-key") r = asgi_client.get( @@ -300,70 +306,43 @@ def test_catalog_service_key_auth(asgi_client, monkeypatch): def test_scheme_metadata_api(asgi_client): from pipeline import db - db.upsert_document( - workflow_id="wf-meta", - document_id="doc-meta", - filename="scheme.pdf", - filepath="/tmp/scheme.pdf", - stage="chunk_review", - instance="default", - ) + _doc(db, "wf-meta", code="meta-draft", stage="chunk_review", filename="scheme.pdf") r = asgi_client.patch( "/documents/wf-meta/scheme-metadata", json={ - "document_kind": "scheme", - "scheme_code": "mif", - "scheme_name": "Micro Irrigation Fund", - "scheme_aliases": ["MIF"], + "kind": "scheme", + "code": "mif", + "name": "Micro Irrigation Fund", + "aliases": ["MIF"], "network_visible": True, }, ) assert r.status_code == 200, r.text body = r.json() - assert body["scheme_code"] == "mif" - assert body["document_kind"] == "scheme" - assert "MIF" in body["scheme_aliases"] + assert body["code"] == "mif" + assert body["kind"] == "scheme" + assert "MIF" in body["aliases"] -def test_reingest_blocked_when_never_classified(asgi_client, mock_temporal_client): +def test_scheme_metadata_api_rejects_an_unknown_kind(asgi_client): from pipeline import db - db.upsert_document( - workflow_id="wf-unclassified", - document_id="doc-unclassified", - filename="x.pdf", - filepath="/tmp/x.pdf", - stage="completed", - instance="default", - ) - db.save_chunks("wf-unclassified", [ - {"chunk_number": 1, "original_text": "some content", "token_count": 5}, - ]) - - r = asgi_client.post("/documents/wf-unclassified/reingest") + _doc(db, "wf-meta-bad", code="meta-bad", stage="chunk_review") + r = asgi_client.patch("/documents/wf-meta-bad/scheme-metadata", json={"kind": "video"}) assert r.status_code == 400 - assert "never been classified" in r.json()["detail"] -def test_reingest_allowed_once_classified(asgi_client, mock_temporal_client): - from pipeline import db, scheme_catalog +@pytest.mark.parametrize("kind", ["scheme", "advisory"]) +def test_reingest_is_allowed_for_every_kind(asgi_client, mock_temporal_client, kind): + """No 'never classified' guard any more: a document always has a kind and a code.""" + from pipeline import db - db.upsert_document( - workflow_id="wf-classified", - document_id="doc-classified", - filename="x.pdf", - filepath="/tmp/x.pdf", - stage="completed", - instance="default", - ) - db.save_chunks("wf-classified", [ + _doc(db, "wf-reingest", code="testcode", kind=kind) + db.save_chunks("wf-reingest", [ {"chunk_number": 1, "original_text": "some content", "token_count": 5}, ]) - scheme_catalog.apply_scheme_metadata( - "wf-classified", document_kind="scheme", scheme_code="testcode", scheme_name="Test Scheme", - ) - r = asgi_client.post("/documents/wf-classified/reingest") + r = asgi_client.post("/documents/wf-reingest/reingest") assert r.status_code == 200, r.text assert r.json()["status"] == "started" @@ -371,14 +350,7 @@ def test_reingest_allowed_once_classified(asgi_client, mock_temporal_client): def test_request_prod_ready_wrong_stage_rejected(asgi_client): from pipeline import db - db.upsert_document( - workflow_id="wf-prod-ready-wrong-stage", - document_id="doc-prod-ready-wrong-stage", - filename="x.pdf", - filepath="/tmp/x.pdf", - stage="chunk_review", - instance="default", - ) + _doc(db, "wf-prod-ready-wrong-stage", code="prw", stage="chunk_review") r = asgi_client.post("/documents/wf-prod-ready-wrong-stage/request-prod-ready") assert r.status_code == 400 @@ -386,14 +358,7 @@ def test_request_prod_ready_wrong_stage_rejected(asgi_client): def test_request_prod_ready_flags_document_and_clears_on_approve(asgi_client, mock_temporal_client): from pipeline import db - db.upsert_document( - workflow_id="wf-prod-ready", - document_id="doc-prod-ready", - filename="x.pdf", - filepath="/tmp/x.pdf", - stage="approval_for_prod", - instance="default", - ) + _doc(db, "wf-prod-ready", code="pr", stage="approval_for_prod") doc = db.get_document("wf-prod-ready") assert doc["prod_ready_requested_at"] is None @@ -495,7 +460,7 @@ def test_aliases_are_deduped_case_insensitively(self): @pytest.mark.unit def test_json_string_aliases_are_accepted(self): - """documents.scheme_aliases_json arrives as a JSON string.""" + """Aliases may arrive as a JSON string.""" from pipeline.scheme_catalog import resolve_scheme_aliases aliases = resolve_scheme_aliases("makhana", "Makhana Scheme", '["fox nut", "lotus seed"]') diff --git a/tests/test_tenancy.py b/tests/test_tenancy.py index 7141780..eb05c23 100644 --- a/tests/test_tenancy.py +++ b/tests/test_tenancy.py @@ -1,56 +1,59 @@ -"""Unit tests for multi-instance (tenant) scoping.""" +"""Unit tests for multi-group (tenant) scoping.""" from __future__ import annotations +import logging +import re + import pytest from fastapi import HTTPException from pipeline.auth.jwt import claims_to_user from pipeline.auth.models import local_bypass_user from pipeline.auth.tenancy import ( - PORTAL_INSTANCE, - allowed_instances, - assert_document_instance_access, - assert_instance_access, + PORTAL_GROUP, + allowed_groups, + assert_document_group_access, + assert_group_access, group_for_instance, - resolve_create_instance, - user_can_access_instance, + resolve_create_group, + user_can_access_group, ) def test_bypass_user_is_unrestricted(): user = local_bypass_user() - assert allowed_instances(user) is None - assert user_can_access_instance(user, "tenant-a") - assert user_can_access_instance(user, "tenant-b") + assert allowed_groups(user) is None + assert user_can_access_group(user, "tenant-a") + assert user_can_access_group(user, "tenant-b") -def test_resolve_create_instance_state_user_defaults_to_only_state(): +def test_resolve_create_group_state_user_defaults_to_only_state(): user = claims_to_user( { "sub": "u1", "groups": ["/states/MH/contributor"], } ) - assert resolve_create_instance(user, None) == "mh" - assert resolve_create_instance(user, "MH") == "mh" + assert resolve_create_group(user, None) == "mh" + assert resolve_create_group(user, "MH") == "mh" with pytest.raises(HTTPException) as exc: - resolve_create_instance(user, "up") + resolve_create_group(user, "up") assert exc.value.status_code == 403 -def test_resolve_create_instance_superadmin_defaults_to_portal(): +def test_resolve_create_group_superadmin_defaults_to_portal(): user = claims_to_user( { "sub": "sa", "groups": ["/global/super-admin"], } ) - assert resolve_create_instance(user, None) == PORTAL_INSTANCE - assert resolve_create_instance(user, "mh") == "mh" + assert resolve_create_group(user, None) == PORTAL_GROUP + assert resolve_create_group(user, "mh") == "mh" -def test_resolve_create_instance_multi_state_requires_choice(): +def test_resolve_create_group_multi_state_requires_choice(): user = claims_to_user( { "sub": "u2", @@ -58,9 +61,9 @@ def test_resolve_create_instance_multi_state_requires_choice(): } ) with pytest.raises(HTTPException) as exc: - resolve_create_instance(user, None) + resolve_create_group(user, None) assert exc.value.status_code == 400 - assert resolve_create_instance(user, "up") == "up" + assert resolve_create_group(user, "up") == "up" def test_user_instances_are_enforced(): @@ -71,11 +74,11 @@ def test_user_instances_are_enforced(): "instances": ["Tenant-A", "tenant-b"], } ) - assert allowed_instances(user) == {"tenant-a", "tenant-b"} - assert user_can_access_instance(user, "tenant-a") - assert not user_can_access_instance(user, "mh") + assert allowed_groups(user) == {"tenant-a", "tenant-b"} + assert user_can_access_group(user, "tenant-a") + assert not user_can_access_group(user, "mh") with pytest.raises(HTTPException) as exc: - assert_instance_access(user, "mh") + assert_group_access(user, "mh") assert exc.value.status_code == 403 @@ -88,11 +91,11 @@ def test_document_access_hides_other_instances(): } ) with pytest.raises(HTTPException) as exc: - assert_document_instance_access(user, {"workflow_id": "wf", "instance": "tenant-b"}) + assert_document_group_access(user, {"workflow_id": "wf", "group": "tenant-b"}) assert exc.value.status_code == 404 -def test_superadmin_token_is_instance_unrestricted_despite_scoped_claim(): +def test_superadmin_token_is_group_unrestricted_despite_scoped_claim(): """Platform superadmin stays unrestricted even with a narrow instances claim.""" for role in ("superadmin", "super_admin", "super-admin"): user = claims_to_user( @@ -104,14 +107,14 @@ def test_superadmin_token_is_instance_unrestricted_despite_scoped_claim(): ) assert user.is_superadmin is True assert user.is_admin is True # alias - assert user.is_instance_unrestricted() is True - # Unrestricted -> allowed_instances is None (every instance visible). - assert allowed_instances(user) is None - assert user_can_access_instance(user, "tenant-b") - doc = assert_document_instance_access( - user, {"workflow_id": "wf", "instance": "tenant-b"} + assert user.is_group_unrestricted() is True + # Unrestricted -> allowed_groups is None (every group visible). + assert allowed_groups(user) is None + assert user_can_access_group(user, "tenant-b") + doc = assert_document_group_access( + user, {"workflow_id": "wf", "group": "tenant-b"} ) - assert doc["instance"] == "tenant-b" + assert doc["group"] == "tenant-b" def test_state_admin_is_scoped_to_instances_claim(): @@ -126,15 +129,15 @@ def test_state_admin_is_scoped_to_instances_claim(): } ) assert user.is_superadmin is False - assert user.is_instance_unrestricted() is False - assert allowed_instances(user) == {"tenant-a"} - assert user_can_access_instance(user, "tenant-a") - assert not user_can_access_instance(user, "tenant-b") + assert user.is_group_unrestricted() is False + assert allowed_groups(user) == {"tenant-a"} + assert user_can_access_group(user, "tenant-a") + assert not user_can_access_group(user, "tenant-b") assert Permission.SEARCH in user.permissions assert Permission.UPLOAD in user.permissions assert Permission.ADMIN not in user.permissions with pytest.raises(HTTPException) as exc: - assert_document_instance_access(user, {"workflow_id": "wf", "instance": "tenant-b"}) + assert_document_group_access(user, {"workflow_id": "wf", "group": "tenant-b"}) assert exc.value.status_code == 404 @@ -148,130 +151,141 @@ def test_content_curator_with_scoped_claim_cannot_cross_tenants(): } ) assert user.is_admin is False - assert user.is_instance_unrestricted() is False - assert allowed_instances(user) == {"tenant-a"} - assert user_can_access_instance(user, "tenant-a") - assert not user_can_access_instance(user, "tenant-b") + assert user.is_group_unrestricted() is False + assert allowed_groups(user) == {"tenant-a"} + assert user_can_access_group(user, "tenant-a") + assert not user_can_access_group(user, "tenant-b") with pytest.raises(HTTPException) as exc: - assert_document_instance_access(user, {"workflow_id": "wf", "instance": "tenant-b"}) + assert_document_group_access(user, {"workflow_id": "wf", "group": "tenant-b"}) assert exc.value.status_code == 404 -def test_list_documents_filters_by_instance(db_connection): +def _register(db, **kwargs): + """upsert_document the v2 way: a new document needs a kind and a code. + + The tenant (``group``) picks the group row; the kind plus that group pick the + catalog, which is created on first use. + """ + kwargs.setdefault("kind", "scheme") + kwargs.setdefault("code", re.sub(r"[^a-z0-9]+", "-", kwargs["workflow_id"].lower()).strip("-")) + return db.upsert_document(**kwargs) + + +def test_list_documents_filters_by_group(db_connection): db = db_connection - db.upsert_document( + _register(db, workflow_id="wf-tenant-a", document_id="d1", filename="a.pdf", filepath="/tmp/a.pdf", stage="completed", - instance="tenant-a", + group="tenant-a", ) - db.upsert_document( + _register(db, workflow_id="wf-tenant-b", document_id="d2", filename="b.pdf", filepath="/tmp/b.pdf", stage="completed", - instance="tenant-b", + group="tenant-b", ) - tenant_a_only = db.list_documents(instances=["tenant-a"]) + tenant_a_only = db.list_documents(groups=["tenant-a"]) assert {d["workflow_id"] for d in tenant_a_only} == {"wf-tenant-a"} - both = db.list_documents(instances=["tenant-a", "tenant-b"]) + both = db.list_documents(groups=["tenant-a", "tenant-b"]) assert {d["workflow_id"] for d in both} == {"wf-tenant-a", "wf-tenant-b"} - none = db.list_documents(instances=[]) + none = db.list_documents(groups=[]) assert none == [] -def test_summary_counts_honor_instance_filter(db_connection): +def test_summary_counts_honor_group_filter(db_connection): db = db_connection - db.upsert_document( + _register(db, workflow_id="wf-tenant-a-2", document_id="d3", filename="c.pdf", filepath="/tmp/c.pdf", stage="completed", - instance="tenant-a", + group="tenant-a", ) - db.upsert_document( + _register(db, workflow_id="wf-tenant-b-2", document_id="d4", filename="d.pdf", filepath="/tmp/d.pdf", stage="failed", - instance="tenant-b", + group="tenant-b", ) - summary = db.get_document_summary_counts(instances=["tenant-a"]) + summary = db.get_document_summary_counts(groups=["tenant-a"]) assert summary["total_documents"] == 1 assert summary["completed_documents"] == 1 assert summary["failed_documents"] == 0 -def test_list_runs_filters_by_instance(db_connection): +def test_list_runs_filters_by_group(db_connection): """Runs must honor the same tenant scope as documents (role-based states).""" db = db_connection - doc_a = db.upsert_document( + doc_a = _register(db, with_workflow=False, workflow_id="wf-run-a", document_id="d-run-a", filename="a.pdf", filepath="/tmp/a.pdf", stage="completed", - instance="tenant-a", + group="tenant-a", ) - doc_b = db.upsert_document( + doc_b = _register(db, with_workflow=False, workflow_id="wf-run-b", document_id="d-run-b", filename="b.pdf", filepath="/tmp/b.pdf", stage="completed", - instance="tenant-b", + group="tenant-b", ) job_a = db.create_document_job(doc_a, "ocr_only", status="completed") job_b = db.create_document_job(doc_b, "ocr_only", status="running") - tenant_a_only = db.list_runs(instances=["tenant-a"]) + tenant_a_only = db.list_runs(groups=["tenant-a"]) assert {r["id"] for r in tenant_a_only} == {job_a} - assert all(r.get("instance") == "tenant-a" for r in tenant_a_only) + assert all(r.get("group") == "tenant-a" for r in tenant_a_only) - both = db.list_runs(instances=["tenant-a", "tenant-b"]) + both = db.list_runs(groups=["tenant-a", "tenant-b"]) assert {r["id"] for r in both} == {job_a, job_b} - none = db.list_runs(instances=[]) + none = db.list_runs(groups=[]) assert none == [] - unrestricted_rows = db.list_runs(instances=None) + unrestricted_rows = db.list_runs(groups=None) assert {r["id"] for r in unrestricted_rows} >= {job_a, job_b} - # Status filter still works under instance scope. - running_a = db.list_runs(status="running", instances=["tenant-a"]) + # Status filter still works under group scope. + running_a = db.list_runs(status="running", groups=["tenant-a"]) assert running_a == [] - running_b = db.list_runs(status="running", instances=["tenant-b"]) + running_b = db.list_runs(status="running", groups=["tenant-b"]) assert {r["id"] for r in running_b} == {job_b} def test_state_user_cannot_see_bv_portal_documents(db_connection): """State-scoped users must not see portal (BV) docs — only their state.""" db = db_connection - db.upsert_document( + _register(db, workflow_id="wf-bv", document_id="d-bv", filename="portal.pdf", filepath="/tmp/portal.pdf", stage="ocr_review", - instance=PORTAL_INSTANCE, + group=PORTAL_GROUP, ) - db.upsert_document( + _register(db, workflow_id="wf-mh", document_id="d-mh", filename="mh.pdf", filepath="/tmp/mh.pdf", stage="ocr_review", - instance="mh", + group="mh", ) db.create_document_job("wf-bv", "ocr_only", status="running") db.create_document_job("wf-mh", "ocr_only", status="running") @@ -285,57 +299,57 @@ def test_state_user_cannot_see_bv_portal_documents(db_connection): } ) assert mh_user.is_superadmin is False - assert allowed_instances(mh_user) == {"mh"} - assert not user_can_access_instance(mh_user, PORTAL_INSTANCE) - assert not user_can_access_instance(mh_user, "bv") + assert allowed_groups(mh_user) == {"mh"} + assert not user_can_access_group(mh_user, PORTAL_GROUP) + assert not user_can_access_group(mh_user, "bv") - mh_docs = db.list_documents(instances=sorted(allowed_instances(mh_user) or [])) + mh_docs = db.list_documents(groups=sorted(allowed_groups(mh_user) or [])) assert {d["workflow_id"] for d in mh_docs} == {"wf-mh"} - mh_runs = db.list_runs(instances=sorted(allowed_instances(mh_user) or [])) - assert all(r.get("instance") == "mh" for r in mh_runs) + mh_runs = db.list_runs(groups=sorted(allowed_groups(mh_user) or [])) + assert all(r.get("group") == "mh" for r in mh_runs) assert {r["workflow_id"] for r in mh_runs} == {"wf-mh"} mh_queue, mh_total = db.list_operations_queue( - instances=sorted(allowed_instances(mh_user) or []) + groups=sorted(allowed_groups(mh_user) or []) ) assert mh_total == 1 assert {r["workflow_id"] for r in mh_queue} == {"wf-mh"} with pytest.raises(HTTPException) as exc: - assert_document_instance_access( - mh_user, {"workflow_id": "wf-bv", "instance": PORTAL_INSTANCE} + assert_document_group_access( + mh_user, {"workflow_id": "wf-bv", "group": PORTAL_GROUP} ) assert exc.value.status_code == 404 # Super-admin sees both sa = claims_to_user({"sub": "sa", "groups": ["/global/super-admin"]}) - assert allowed_instances(sa) is None - assert user_can_access_instance(sa, PORTAL_INSTANCE) - all_docs = db.list_documents(instances=None) + assert allowed_groups(sa) is None + assert user_can_access_group(sa, PORTAL_GROUP) + all_docs = db.list_documents(groups=None) assert {d["workflow_id"] for d in all_docs} >= {"wf-bv", "wf-mh"} -def test_upsert_does_not_reassign_instance(db_connection): +def test_upsert_does_not_reassign_group(db_connection): db = db_connection - db.upsert_document( + _register(db, workflow_id="wf-owned", document_id="d1", filename="a.pdf", filepath="/tmp/a.pdf", stage="registered", - instance="tenant-a", + group="tenant-a", ) - db.upsert_document( + _register(db, workflow_id="wf-owned", document_id="d1", filename="a.pdf", filepath="/tmp/a.pdf", stage="ocr_review", - instance="tenant-b", # must be ignored on update + group="tenant-b", # must be ignored on update ) doc = db.get_document("wf-owned") - assert doc["instance"] == "tenant-a" + assert doc["group"] == "tenant-a" assert doc["stage"] == "ocr_review" @@ -345,13 +359,13 @@ def test_api_helpers_hide_cross_tenant_mutations(db_connection, monkeypatch): from pipeline.api import api monkeypatch.setattr(api, "db", db_mod) - db_mod.upsert_document( + _register(db_mod, workflow_id="wf-tenant-b-doc", document_id="d-tenant-b", filename="b.pdf", filepath="/tmp/b.pdf", stage="ocr_review", - instance="tenant-b", + group="tenant-b", ) user = claims_to_user( { @@ -366,39 +380,116 @@ def test_api_helpers_hide_cross_tenant_mutations(db_connection, monkeypatch): assert api._document_for_user_or_none("wf-tenant-b-doc", user) is None -class TestGroupForInstance: - @pytest.fixture(autouse=True) - def _default_instances(self, monkeypatch): - monkeypatch.delenv("PORTAL_INSTANCE", raising=False) - monkeypatch.delenv("DEFAULT_INSTANCE", raising=False) - - @pytest.mark.unit - @pytest.mark.parametrize( - "instance,expected", - [ - ("mh", "mh"), - (" MH ", "mh"), - ("ka", "ka"), - ("bv", None), - ("BV", None), - ("default", None), - ("", None), - (None, None), - ], - ) - def test_states_are_groups_and_platform_instances_are_not(self, instance, expected): - assert group_for_instance(instance) == expected + PORTAL_INSTANCE, + allowed_instances, + assert_document_instance_access, + assert_instance_access, + group_for_instance, + resolve_create_instance, + user_can_access_instance, - @pytest.mark.unit - def test_follows_a_configured_portal_instance(self, monkeypatch): - monkeypatch.setenv("PORTAL_INSTANCE", "portal") - assert group_for_instance("portal") is None - assert group_for_instance("mh") == "mh" +def test_group_is_reached_through_the_documents_catalog(db_connection): + """No documents.instance column: a document's group is a groups row reached through its catalog.""" + db = db_connection + _register(db, workflow_id="wf-g-a", document_id="g1", filename="a.pdf", + filepath="/tmp/a.pdf", group="tenant-a") + _register(db, workflow_id="wf-g-bv", document_id="g2", filename="b.pdf", + filepath="/tmp/b.pdf", group=PORTAL_GROUP) + _register(db, workflow_id="wf-g-adv", document_id="g3", filename="c.pdf", + filepath="/tmp/c.pdf", group="tenant-a", kind="advisory") + + with db.get_connection() as conn: + groups = {r["name"]: r["path"] for r in conn.execute("SELECT name, path FROM groups")} + catalogs = conn.execute( + "SELECT g.name AS group_name, c.type FROM catalogs c JOIN groups g ON g.id = c.group_id" + ).fetchall() + assert groups == { + "tenant-a": "/states/TENANT-A", + PORTAL_GROUP: "/global" if PORTAL_GROUP == "bv" else f"/states/{PORTAL_GROUP.upper()}", + } + # One catalog per (group, type): tenant-a has scheme + advisory, the portal only scheme. + assert sorted((r["group_name"], r["type"]) for r in catalogs) == sorted([ + ("tenant-a", "openagrinet:AgricultureResource"), + ("tenant-a", "openagrinet:KnowledgeAdvisory"), + (PORTAL_GROUP, "openagrinet:AgricultureResource"), + ]) + assert db.get_document("wf-g-a")["group"] == "tenant-a" + assert db.get_document("wf-g-adv")["kind"] == "advisory" + assert {d["workflow_id"] for d in db.list_documents(groups=["tenant-a"])} == {"wf-g-a", "wf-g-adv"} + + +def test_same_code_is_allowed_in_another_tenants_catalog(db_connection): + db = db_connection + _register(db, workflow_id="wf-c-a", document_id="c1", filename="a.pdf", + filepath="/tmp/a.pdf", group="tenant-a", code="pm-kisan") + _register(db, workflow_id="wf-c-b", document_id="c2", filename="b.pdf", + filepath="/tmp/b.pdf", group="tenant-b", code="pm-kisan") + assert db.get_document("wf-c-a")["code"] == db.get_document("wf-c-b")["code"] == "pm-kisan" + # ...but not twice inside one tenant's catalog of that kind. + with pytest.raises(db.DuplicateDocumentCode): + _register(db, workflow_id="wf-c-a2", document_id="c3", filename="a2.pdf", + filepath="/tmp/a2.pdf", group="tenant-a", code="pm-kisan") + + +def test_audit_keeps_the_tenant_scope_after_the_document_is_purged(db_connection): + """audit_logs snapshots the group, so a purged document's history stays tenant-scoped.""" + db = db_connection + _register(db, workflow_id="wf-aud-a", document_id="a1", filename="a.pdf", + filepath="/tmp/a.pdf", group="tenant-a") + _register(db, workflow_id="wf-aud-b", document_id="a2", filename="b.pdf", + filepath="/tmp/b.pdf", group="tenant-b") + db.log_audit(workflow_id="wf-aud-a", document_id="", action_type="page_edit", entity_type="document") + db.log_audit(workflow_id="wf-aud-b", document_id="", action_type="page_edit", entity_type="document") + db.purge_document("wf-aud-a", keep_audit=True) + + a_rows = db.get_all_audit_logs(groups=["tenant-a"]) + assert [r["workflow_id"] for r in a_rows if r["action_type"] == "page_edit"] == ["wf-aud-a"] + b_rows = db.get_all_audit_logs(groups=["tenant-b"]) + assert [r["workflow_id"] for r in b_rows if r["action_type"] == "page_edit"] == ["wf-aud-b"] + assert db.get_all_audit_log_count(groups=["tenant-a"]) == len(a_rows) + assert db.get_all_audit_logs(groups=[]) == [] + + +def test_user_group_follows_the_first_tenant_in_the_token(db_connection, caplog): + db = db_connection + + def group_of(keycloak_id): + with db.get_connection() as conn: + row = conn.execute( + "SELECT g.name FROM users u LEFT JOIN groups g ON g.id = u.group_id " + "WHERE u.keycloak_user_id = %s", (keycloak_id,)).fetchone() + return row["name"] + + db.upsert_user("kc-single", "single", "single@example.com", ["contributor"], tenants=["MH"]) + assert group_of("kc-single") == "mh" - @pytest.mark.unit - def test_follows_a_configured_default_instance(self, monkeypatch): - monkeypatch.setenv("DEFAULT_INSTANCE", "local") + with caplog.at_level(logging.WARNING, logger="pipeline.db"): + db.upsert_user("kc-multi", "multi", "multi@example.com", ["contributor"], tenants=["up", "mh"]) + assert group_of("kc-multi") == "mh" # sorted, first wins + assert any("several tenants" in rec.message for rec in caplog.records) - assert group_for_instance("local") is None - assert group_for_instance("mh") == "mh" + # No tenants in the token leaves an existing group alone. + db.upsert_user("kc-single", "single", "single@example.com", ["contributor"]) + assert group_of("kc-single") == "mh" + db.upsert_user("kc-none", "none", "none@example.com", ["contributor"]) + assert group_of("kc-none") is None + + +def test_group_path_fills_the_keycloak_id_on_sync(db_connection): + """The SQLite move creates groups without a Keycloak id; a later sync matches by path.""" + db = db_connection + first = db.upsert_group("/states/MH") + with db.get_connection() as conn: + assert conn.execute("SELECT keycloak_group_id FROM groups WHERE id = %s::uuid", + (first,)).fetchone()["keycloak_group_id"] is None + again = db.upsert_group("/states/MH", keycloak_group_id="kc-group-123") + assert again == first + with db.get_connection() as conn: + row = conn.execute("SELECT name, keycloak_group_id FROM groups WHERE id = %s::uuid", (first,)).fetchone() + assert (row["name"], row["keycloak_group_id"]) == ("mh", "kc-group-123") + # A sync without an id never erases one already stored. + assert db.upsert_group("/states/MH") == first + with db.get_connection() as conn: + assert conn.execute("SELECT keycloak_group_id FROM groups WHERE id = %s::uuid", + (first,)).fetchone()["keycloak_group_id"] == "kc-group-123" From f4fa702d6a826b37d36f93729c2a25314a91eac1 Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 11:07:18 +0530 Subject: [PATCH 39/53] feat(ui): select document type at upload and use group/kind/code/name NewDocumentView sends group and kind. The classification panel offers scheme and advisory only and no longer blocks ingestion on a missing type. Instance helpers and badge are renamed to group. --- ui/src/auth/AuthProvider.jsx | 36 ++--- ui/src/components/DocumentHeaderSummary.jsx | 6 +- ui/src/components/GroupBadge.jsx | 21 +++ ui/src/components/InstanceBadge.jsx | 21 --- .../lib/{instanceLabels.js => groupLabels.js} | 18 +-- ui/src/lib/pipelineUi.js | 2 +- ui/src/views/ChunkExplorerView.jsx | 4 +- ui/src/views/DashboardView.jsx | 12 +- .../DocumentClassificationPanel.test.jsx | 12 +- ui/src/views/DocumentOpsView.jsx | 147 +++++------------- ui/src/views/DocumentsView.jsx | 30 ++-- ui/src/views/GlobalAuditView.jsx | 6 +- ui/src/views/NewDocumentView.jsx | 48 ++++-- ui/src/views/QueueView.jsx | 4 +- ui/src/views/RunsView.jsx | 4 +- 15 files changed, 161 insertions(+), 210 deletions(-) create mode 100644 ui/src/components/GroupBadge.jsx delete mode 100644 ui/src/components/InstanceBadge.jsx rename ui/src/lib/{instanceLabels.js => groupLabels.js} (63%) diff --git a/ui/src/auth/AuthProvider.jsx b/ui/src/auth/AuthProvider.jsx index fdf8e27..998f86c 100644 --- a/ui/src/auth/AuthProvider.jsx +++ b/ui/src/auth/AuthProvider.jsx @@ -366,11 +366,11 @@ export function AuthProvider({ children }) { ) /** Role for a state tenant (mh, up, …). Super-admin always returns super_admin. */ - const roleForInstance = useCallback( - (instance) => { + const roleForGroup = useCallback( + (group) => { if (!AUTH_ENABLED || isSuperAdmin) return 'super_admin' - if (!instance) return null - const key = String(instance).trim().toLowerCase() + if (!group) return null + const key = String(group).trim().toLowerCase() // A BH viewer holds no per-state group but reads every state. return stateRoles[key] || (isBhViewer ? 'state_view' : null) }, @@ -378,11 +378,11 @@ export function AuthProvider({ children }) { ) /** Whether the user may see a tenant (state). Super-admin → all. */ - const canAccessInstance = useCallback( - (instance) => { + const canAccessGroup = useCallback( + (group) => { if (!AUTH_ENABLED || isSuperAdmin || isBhViewer) return true - if (!instance) return false - const key = String(instance).trim().toLowerCase() + if (!group) return false + const key = String(group).trim().toLowerCase() if (instances.length === 0) return false return instances.map((i) => String(i).toLowerCase()).includes(key) }, @@ -397,17 +397,17 @@ export function AuthProvider({ children }) { * may approve OCR / translation / chunking) but NOT 'approve_ingestion' * (it may not publish to DEV). */ - const hasPermissionForInstance = useCallback( - (perm, instance) => { + const hasPermissionForGroup = useCallback( + (perm, group) => { if (!AUTH_ENABLED) return true if (isSuperAdmin) return hasPermission(perm) - const role = normalizeProductRole(roleForInstance(instance)) + const role = normalizeProductRole(roleForGroup(group)) if (!role) return false // Platform-level capabilities are never granted by a state role. if (perm === 'admin' || perm === 'manage_users') return false return roleGrants(role, perm) }, - [hasPermission, isSuperAdmin, roleForInstance], + [hasPermission, isSuperAdmin, roleForGroup], ) const value = useMemo( @@ -432,10 +432,10 @@ export function AuthProvider({ children }) { isSuperAdmin, isBhViewer, hasPermission, - hasPermissionForInstance, + hasPermissionForGroup, hasRole, - roleForInstance, - canAccessInstance, + roleForGroup, + canAccessGroup, loginWithSso, loginWithOtpTokens, logout, @@ -460,10 +460,10 @@ export function AuthProvider({ children }) { isSuperAdmin, isBhViewer, hasPermission, - hasPermissionForInstance, + hasPermissionForGroup, hasRole, - roleForInstance, - canAccessInstance, + roleForGroup, + canAccessGroup, loginWithSso, loginWithOtpTokens, logout, diff --git a/ui/src/components/DocumentHeaderSummary.jsx b/ui/src/components/DocumentHeaderSummary.jsx index 5098450..999c654 100644 --- a/ui/src/components/DocumentHeaderSummary.jsx +++ b/ui/src/components/DocumentHeaderSummary.jsx @@ -1,6 +1,6 @@ import { useMemo, useState } from 'react' import { ArrowLeft, Info } from 'lucide-react' -import { formatInstanceLabel, instanceBadgeTitle } from '../lib/instanceLabels' +import { formatGroupLabel, groupBadgeTitle } from '../lib/groupLabels' import { formatCompactDateTime, getDocumentFileLabel, @@ -74,7 +74,7 @@ export function DocumentHeaderSummary({ const pages = pageCount ?? doc?.page_count ?? 0 const chunks = chunkCount ?? doc?.chunk_count ?? 0 const uploader = uploaderName(doc) - const stateLabel = formatInstanceLabel(doc?.instance) + const stateLabel = formatGroupLabel(doc?.group) const roleLabels = useMemo( () => productRoleLabels(doc?.uploaded_by_roles || []), [doc?.uploaded_by_roles], @@ -126,7 +126,7 @@ export function DocumentHeaderSummary({ {stateLabel ? ( - {stateLabel} + {stateLabel} ) : ( '—' )} diff --git a/ui/src/components/GroupBadge.jsx b/ui/src/components/GroupBadge.jsx new file mode 100644 index 0000000..edb2c2e --- /dev/null +++ b/ui/src/components/GroupBadge.jsx @@ -0,0 +1,21 @@ +import { formatGroupLabel, groupBadgeTitle } from '../lib/groupLabels' +import { Badge } from './ui/badge' +import { cn } from '../lib/utils' + +/** Plan 2: show document state / BV portal tag from ``group``. */ +export function GroupBadge({ group, className }) { + const label = formatGroupLabel(group) + if (!label) return null + const isPortal = label === 'BV' + return ( + + {label} + + ) +} + +export default GroupBadge diff --git a/ui/src/components/InstanceBadge.jsx b/ui/src/components/InstanceBadge.jsx deleted file mode 100644 index 87d4b4a..0000000 --- a/ui/src/components/InstanceBadge.jsx +++ /dev/null @@ -1,21 +0,0 @@ -import { formatInstanceLabel, instanceBadgeTitle } from '../lib/instanceLabels' -import { Badge } from './ui/badge' -import { cn } from '../lib/utils' - -/** Plan 2: show document state / BV portal tag from ``instance``. */ -export function InstanceBadge({ instance, className }) { - const label = formatInstanceLabel(instance) - if (!label) return null - const isPortal = label === 'BV' - return ( - - {label} - - ) -} - -export default InstanceBadge diff --git a/ui/src/lib/instanceLabels.js b/ui/src/lib/groupLabels.js similarity index 63% rename from ui/src/lib/instanceLabels.js rename to ui/src/lib/groupLabels.js index ae0e843..95e7840 100644 --- a/ui/src/lib/instanceLabels.js +++ b/ui/src/lib/groupLabels.js @@ -1,13 +1,13 @@ /** - * Plan 2: display labels for documents.instance (state / portal tag). + * Plan 2: display labels for the document group (state / portal tag). * Authorization still comes from Keycloak JWT — this is UI only. */ -export const PORTAL_INSTANCE = 'bv' +export const PORTAL_GROUP = 'bv' /** Short badge label: mh → MH, bv → BV */ -export function formatInstanceLabel(instance) { - const code = String(instance || '') +export function formatGroupLabel(group) { + const code = String(group || '') .trim() .toLowerCase() if (!code || code === 'default') return null @@ -17,8 +17,8 @@ export function formatInstanceLabel(instance) { return code.toUpperCase() } -export function instanceBadgeTitle(instance) { - const code = String(instance || '') +export function groupBadgeTitle(group) { + const code = String(group || '') .trim() .toLowerCase() if (!code || code === 'default') return 'No state tag' @@ -29,11 +29,11 @@ export function instanceBadgeTitle(instance) { } /** - * Default instance for upload form. + * Default group for upload form. * Super admin → portal (bv); single state → that state; multi → first (user can change). */ -export function defaultUploadInstance({ isSuperAdmin, instances = [], portalInstance = PORTAL_INSTANCE }) { - if (isSuperAdmin) return portalInstance || PORTAL_INSTANCE +export function defaultUploadGroup({ isSuperAdmin, instances = [], portalGroup = PORTAL_GROUP }) { + if (isSuperAdmin) return portalGroup || PORTAL_GROUP const list = (instances || []).map((i) => String(i).trim().toLowerCase()).filter(Boolean) if (list.length >= 1) return list[0] return '' diff --git a/ui/src/lib/pipelineUi.js b/ui/src/lib/pipelineUi.js index e4f833a..db7dbe2 100644 --- a/ui/src/lib/pipelineUi.js +++ b/ui/src/lib/pipelineUi.js @@ -206,7 +206,7 @@ export const DEFAULT_SEARCH_SETTINGS = { export function getDocumentListLabel(doc) { return ( - doc?.display_name || + doc?.name || doc?.name_en || doc?.name || doc?.filename || diff --git a/ui/src/views/ChunkExplorerView.jsx b/ui/src/views/ChunkExplorerView.jsx index d1ef3ea..e677724 100644 --- a/ui/src/views/ChunkExplorerView.jsx +++ b/ui/src/views/ChunkExplorerView.jsx @@ -166,7 +166,7 @@ export default function ChunkExplorerView() { const groups = new Map() chunks.forEach(chunk => { const key = chunk.workflow_id || '__unknown__' - const label = chunk.display_name || chunk.filename || key + const label = chunk.name || chunk.filename || key const existing = groups.get(key) || { key, label, chunks: [] } existing.chunks.push(chunk) groups.set(key, existing) @@ -401,7 +401,7 @@ export default function ChunkExplorerView() { openDocumentChunks(chunk.workflow_id) }} > - {chunk.display_name || chunk.filename || chunk.workflow_id} + {chunk.name || chunk.filename || chunk.workflow_id}

Chunk {chunk.chunk_number} · Pages {chunk.page_start}–{chunk.page_end} · {getStageLabel(chunk.stage)} diff --git a/ui/src/views/DashboardView.jsx b/ui/src/views/DashboardView.jsx index becf4a6..2538e72 100644 --- a/ui/src/views/DashboardView.jsx +++ b/ui/src/views/DashboardView.jsx @@ -5,7 +5,7 @@ import { Skeleton } from '../components/ui/skeleton' import { StatCard } from '../components/StatCard' import { StageBadge } from '../components/StageBadge' import { fetchJson, formatCount, getDocumentListLabel, summarizeQueueReason } from '../lib/pipelineUi' -import { formatInstanceLabel } from '../lib/instanceLabels' +import { formatGroupLabel } from '../lib/groupLabels' import { useAuth } from '../auth/AuthProvider' import { AlertTriangle, CheckCircle, FileText, Globe2, ListTodo } from 'lucide-react' @@ -66,8 +66,8 @@ export default function DashboardView() { const reindexCount = summary?.needs_reindex || 0 const displayedQueueItems = queue.slice(0, QUEUE_ROWS_SHOWN) - const byInstance = useMemo(() => (Array.isArray(summary?.by_instance) ? summary.by_instance : []), [summary]) - const stateRows = byInstance + const byGroup = useMemo(() => (Array.isArray(summary?.by_group) ? summary.by_group : []), [summary]) + const stateRows = byGroup const maxStateCount = useMemo( () => stateRows.reduce((max, item) => Math.max(max, item.count || 0), 0), @@ -113,7 +113,7 @@ export default function DashboardView() { /> } /> { const label = row.otherStateCount ? `Others (${row.otherStateCount} states)` - : formatInstanceLabel(row.instance) || 'No state tag' + : formatGroupLabel(row.group) || 'No state tag' const pct = (value) => (maxStateCount ? (value / maxStateCount) * 100 : 0) return ( -

+
{label} {formatCount(row.count)} diff --git a/ui/src/views/DocumentClassificationPanel.test.jsx b/ui/src/views/DocumentClassificationPanel.test.jsx index a670ec7..1b96351 100644 --- a/ui/src/views/DocumentClassificationPanel.test.jsx +++ b/ui/src/views/DocumentClassificationPanel.test.jsx @@ -9,7 +9,7 @@ vi.mock('../lib/pipelineUi', async (importOriginal) => ({ fetchJson: vi.fn(), })) -const baseDoc = { document_kind: 'advisory', filename: 'scan_0042.pdf', display_name: 'Cotton Pest Advisory' } +const baseDoc = { kind: 'advisory', filename: 'scan_0042.pdf', name: 'Cotton Pest Advisory' } function renderPanel(doc = baseDoc, props = {}) { const onSaved = vi.fn().mockResolvedValue() @@ -35,7 +35,7 @@ describe('DocumentClassificationPanel display name', () => { }) it('falls back to the filename without its extension', () => { - renderPanel({ ...baseDoc, display_name: null }) + renderPanel({ ...baseDoc, name: null }) expect(screen.getByLabelText(/display name/i)).toHaveValue('scan_0042') }) @@ -53,7 +53,7 @@ describe('DocumentClassificationPanel display name', () => { await user.click(screen.getByRole('button', { name: /^save$/i })) await waitFor(() => expect(onSaved).toHaveBeenCalled()) - expect(patchBody()).toMatchObject({ document_kind: 'advisory', display_name: 'Bollworm Control' }) + expect(patchBody()).toMatchObject({ kind: 'advisory', name: 'Bollworm Control' }) }) it('blocks saving a blank display name', async () => { @@ -82,16 +82,16 @@ describe('DocumentClassificationPanel display name', () => { it('does not resend an unchanged display name', async () => { const user = userEvent.setup() - const { onSaved } = renderPanel({ ...baseDoc, document_kind: 'scheme', scheme_name: '' }) + const { onSaved } = renderPanel({ ...baseDoc, kind: 'scheme', name: '' }) await user.type(screen.getByPlaceholderText(/national mission/i), 'PM Kisan') await user.click(screen.getByRole('button', { name: /^save$/i })) await waitFor(() => expect(onSaved).toHaveBeenCalled()) - expect(patchBody()).not.toHaveProperty('display_name') + expect(patchBody()).not.toHaveProperty('name') }) it('counts a document without a stored name as already saved', () => { - renderPanel({ ...baseDoc, display_name: null }) + renderPanel({ ...baseDoc, name: null }) expect(screen.getByRole('button', { name: /saved/i })).toBeDisabled() }) diff --git a/ui/src/views/DocumentOpsView.jsx b/ui/src/views/DocumentOpsView.jsx index eba6575..3c99c29 100644 --- a/ui/src/views/DocumentOpsView.jsx +++ b/ui/src/views/DocumentOpsView.jsx @@ -95,22 +95,6 @@ function PanelNotice({ tone = 'error', title, message }) { ) } -// Client-side preview only — matches pipeline/scheme_catalog.py's -// derive_scheme_code() closely enough for instant feedback, but the server -// is the authoritative, collision-checked source of truth for the real code. -const SCHEME_CODE_STOPWORDS = new Set(['a', 'an', 'the', 'of', 'on', 'for', 'and', 'or', 'to', 'in', 'at', 'by', 'with', '&']) - -function previewSchemeCode(title) { - const words = (title || '').match(/[A-Za-z0-9]+/g) || [] - const letters = words - .filter(w => !SCHEME_CODE_STOPWORDS.has(w.toLowerCase())) - .map(w => w[0].toLowerCase()) - .join('') - if (letters.length >= 2) return letters - const slug = (title || '').toLowerCase().replace(/[^a-z0-9]+/g, '') - return slug.slice(0, 8) || 'scheme' -} - /** * Validity the document is searchable in. The start is prefilled from what the * server stamped at upload; the end is deliberately blank-able - an empty end @@ -126,26 +110,14 @@ function validateValidity(validFrom, validTo) { const DOCUMENT_KIND_OPTIONS = [ { value: 'scheme', label: 'Scheme' }, { value: 'advisory', label: 'Advisory' }, - { value: 'video', label: 'Video' }, - { value: '__custom__', label: 'Custom…' }, ] export function DocumentClassificationPanel({ doc, workflowId, canClassify, onSaved }) { - const hasKind = doc.document_kind && doc.document_kind !== 'document' - // 'document' is the unclassified default, not a choice the panel offers, so - // it seeds the select as empty — otherwise the select renders blank while the - // save button reads "Saved", which tells a reviewer the opposite of the truth. - const [kind, setKind] = useState( - hasKind && !DOCUMENT_KIND_OPTIONS.some(o => o.value === doc.document_kind) - ? '__custom__' - : (hasKind ? doc.document_kind : '') - ) - const [customKind, setCustomKind] = useState(hasKind && !DOCUMENT_KIND_OPTIONS.some(o => o.value === doc.document_kind) ? doc.document_kind : '') - const [schemeName, setSchemeName] = useState(doc.scheme_name || '') - const savedDisplayName = doc.display_name || filenameStem(doc.filename) - const [displayName, setDisplayName] = useState(savedDisplayName) - const displayNameChanged = displayName.trim() !== savedDisplayName - const displayNameLocked = !canClassify || Boolean(doc.ingested_at) + const [kind, setKind] = useState(doc.kind || '') + // The display name is documents.name; main's lock rule still applies to it. + const savedName = doc.name || filenameStem(doc.filename) + const [name, setName] = useState(savedName) + const nameLocked = !canClassify || Boolean(doc.ingested_at) const [validFrom, setValidFrom] = useState(doc.valid_from || todayISODate()) // Blank when the document has no end date, which is the normal case - an // empty field here reads as "never expires", not as "not filled in yet". @@ -153,16 +125,14 @@ export function DocumentClassificationPanel({ doc, workflowId, canClassify, onSa const [saving, setSaving] = useState(false) const [error, setError] = useState('') - const effectiveKind = kind === '__custom__' ? customKind.trim().toLowerCase() : kind - const isScheme = effectiveKind === 'scheme' const validityError = DOCUMENT_VALIDITY_ENABLED ? validateValidity(validFrom, validTo) : '' - const nameError = displayNameError(displayName) + const nameError = displayNameError(name) + const nameChanged = name.trim() !== savedName const canSave = - Boolean(effectiveKind) && + Boolean(kind) && !nameError && - (!isScheme || schemeName.trim().length > 0) && !validityError async function save() { @@ -174,9 +144,8 @@ export function DocumentClassificationPanel({ doc, workflowId, canClassify, onSa method: 'PATCH', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ - document_kind: effectiveKind, - ...(displayNameChanged ? { display_name: displayName.trim() } : {}), - ...(isScheme ? { scheme_name: schemeName.trim() } : {}), + kind, + ...(nameChanged ? { name: name.trim() } : {}), // Omitted entirely when the feature is off, so classifying a // document leaves whatever period the upload stamped untouched. ...(DOCUMENT_VALIDITY_ENABLED @@ -193,9 +162,8 @@ export function DocumentClassificationPanel({ doc, workflowId, canClassify, onSa } const alreadySaved = - doc.document_kind === effectiveKind - && !displayNameChanged - && (!isScheme || doc.scheme_name === schemeName.trim()) + doc.kind === kind + && !nameChanged && (!DOCUMENT_VALIDITY_ENABLED || (doc.valid_from === validFrom && (doc.valid_to || '') === validTo)) @@ -214,10 +182,10 @@ export function DocumentClassificationPanel({ doc, workflowId, canClassify, onSa setDisplayName(e.target.value)} + onChange={e => setName(e.target.value)} />
@@ -258,38 +226,12 @@ export function DocumentClassificationPanel({ doc, workflowId, canClassify, onSa
)} - {kind === '__custom__' && ( -
- Custom type - setCustomKind(e.target.value)} - placeholder="e.g. faq" - /> -
- )} - {isScheme && ( - <> -
- Scheme name - setSchemeName(e.target.value)} - placeholder="e.g. National Mission on Natural Farming" - /> -
-
- Scheme code (auto) - - {doc.scheme_code && doc.scheme_name === schemeName.trim() ? doc.scheme_code : (schemeName.trim() ? previewSchemeCode(schemeName) : '—')} - -
- - )} +
+ Code + + {doc.code || '—'} + +
{canClassify && (
) @@ -869,13 +811,6 @@ export default function DocumentOpsView() { && canRunAction(action) ) const canRemoveDocument = canAdmin && (doc?.available_actions || []).includes('disable_document') - // 'document' is never offered as a choice in the classification panel (only - // scheme/advisory/video/custom are) — seeing it here means the panel was - // never used. Mirrors the same check the reingest endpoint enforces server-side. - const isDocClassified = Boolean( - (doc?.document_kind && doc.document_kind !== 'document') || doc?.scheme_code || doc?.scheme_name - ) - const ingestBlockedByClassification = doc?.stage === 'ready_for_ingestion' && !isDocClassified const sortedPages = useMemo(() => [...pages].sort((a, b) => a.page_number - b.page_number), [pages]) const reviewedPages = useMemo(() => pages.filter(p => p.is_reviewed).length, [pages]) const reviewedChunks = useMemo(() => chunks.filter(c => c.is_reviewed).length, [chunks]) @@ -1042,30 +977,18 @@ export default function DocumentOpsView() { networkPublish={doc.network_publish} />
- {visibleActions.slice(0, 4).map(action => { - const blockedByClassification = action === 'approve_ingestion' && ingestBlockedByClassification - return ( - - ) - })} + {visibleActions.slice(0, 4).map(action => ( + + ))} {canRemoveDocument && (
diff --git a/ui/src/views/RunsView.jsx b/ui/src/views/RunsView.jsx index c3a9bfb..d11a44d 100644 --- a/ui/src/views/RunsView.jsx +++ b/ui/src/views/RunsView.jsx @@ -4,7 +4,7 @@ import { Badge } from '../components/ui/badge' import { Button } from '../components/ui/button' import { Skeleton } from '../components/ui/skeleton' import { StageBadge } from '../components/StageBadge' -import { InstanceBadge } from '../components/InstanceBadge' +import { GroupBadge } from '../components/GroupBadge' import PipelineStepper from '../components/PipelineStepper' import { fetchJson, @@ -356,7 +356,7 @@ export default function RunsView() { {title}
- + #{runId} {fileLabel && fileLabel !== title ? ` · ${fileLabel}` : ''} From fdd1c9c3ce8d10d3e1a657246c3d78c6ccf20d4e Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 11:07:18 +0530 Subject: [PATCH 40/53] docs: describe schema v2 and record the groups and catalogs decision --- AGENTS.md | 6 +- docs/ADR/0006-groups-and-catalogs-model.md | 21 ++ docs/POSTGRES_SCHEMA.md | 291 +++++++++++---------- 3 files changed, 181 insertions(+), 137 deletions(-) create mode 100644 docs/ADR/0006-groups-and-catalogs-model.md diff --git a/AGENTS.md b/AGENTS.md index 03b5f33..1a61d7c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,8 +77,8 @@ Core modules (all under `pipeline/`): - `discovery_publish_service.py` — `DiscoveryPublishService`: owns the Publish to Network env vars and the HTTP call. Deliberately Temporal-free - `models.py` — Pydantic models, including `DocumentStage` enum and `PIPELINE_STAGES` (the stepper-UI stage list) - `config.py` — `Config` dataclass reading env vars, with defaults -- `instances.py` — instance/tenant (state) code ↔ human name mapping; `bv` = Bharat Vistaar platform-wide, others are state codes matching Keycloak group paths (`/states/MH/...` → `mh`) -- `scheme_catalog.py` — scheme catalog behind the pull-based `/catalog/v1/*` API, served from the `catalog_entries` table and the computed `catalog` view +- `group_names.py` — tenant (group) code ↔ human name mapping; `bv` = Bharat Vistaar platform-wide, others are state codes matching Keycloak group paths (`/states/MH/...` → `mh`). A tenant code is a `groups` row (name = code); a document's `group` is derived through `documents.catalog_id → catalogs.group_id`, there is no `documents.instance` column +- `scheme_catalog.py` — scheme catalog behind the pull-based `/catalog/v1/*` API, served from the computed `catalog_status` view (one row per document; `catalogs` rows are `(group, type)`, `documents` carry `code`/`name`/`aliases`) - `ocr/`, `translation/`, `chunking/`, `vector_store/` — pluggable provider packages, each with a `base.py` protocol/ABC and a `service.py` (or equivalent) that dispatches to the configured provider - `auth/` — Keycloak JWT validation, group-based tenancy/permissions, email-OTP flow, admin user management @@ -96,7 +96,7 @@ registered → ocr_processing → ocr_review → translation_processing → tran → approval_for_prod → ingesting_prod → completed (or → failed at any point) ``` -`approval_for_prod`/`ingesting_prod` are **PROD-only stages** (`PROD_ONLY_STAGES` in `models.py`); with `DISABLE_PROD_SETTING=true` a document goes straight from `ingesting` to `completed`. That flag is read only at workflow *start* and passed in as an argument — never read inside a running workflow, since that would break Temporal replay determinism if flipped mid-flight (see the docstring in `instances.py`). +`approval_for_prod`/`ingesting_prod` are **PROD-only stages** (`PROD_ONLY_STAGES` in `models.py`); with `DISABLE_PROD_SETTING=true` a document goes straight from `ingesting` to `completed`. That flag is read only at workflow *start* and passed in as an argument — never read inside a running workflow, since that would break Temporal replay determinism if flipped mid-flight (see the docstring in `group_names.py`). ### DEV vs PROD dual indexing diff --git a/docs/ADR/0006-groups-and-catalogs-model.md b/docs/ADR/0006-groups-and-catalogs-model.md new file mode 100644 index 0000000..c610a7b --- /dev/null +++ b/docs/ADR/0006-groups-and-catalogs-model.md @@ -0,0 +1,21 @@ +# Providers are groups, catalogs are (group, type), documents are identified by (catalog, code) + +Context: the first Postgres schema modelled a tenant as a free-text `documents.instance`, a `catalog_entries` table keyed by `(type, code)` with one entry per scheme, and a free-form `document_kind` that had to match the entry's type through a composite foreign key. Review feedback was that the unit published on the network is a provider's catalog, that a provider is a Keycloak group, that each scheme is itself a document, and that database enums and demo flags do not belong in the schema. + +Decision: +- A **group** is a provider: one `groups` row per Keycloak tenant node (`/states/MH`, `/global`). Role leaves are not mirrored; permissions still come from the token. `users.group_id` is a single nullable FK for now (first tenant in the token, sorted, with a warning when there are several); a join table can replace it if multi-group membership is needed. +- A **catalog** is `(group, type)`: `catalogs.group_id` plus `catalogs.type`, unique together. `type` holds the network schema name. The kind-to-schema map lives in `pipeline/network_constants.py` and is a placeholder: `scheme` is `openagrinet:AgricultureResource`, `advisory` is `openagrinet:KnowledgeAdvisory`. Code branches through that map, never on raw `openagrinet:*` strings. Catalog Beckn information (`publish_directives`, `visible_networks`, `criteria`, `network`) sits in `catalogs.metadata`. +- A **document** belongs to one catalog and is identified there by `code`: `documents.catalog_id` NOT NULL, `UNIQUE (catalog_id, code)`, with `name` and `aliases` on the document. The API fields `code`, `name` and `aliases` are those columns. `documents.instance`, `document_kind` and `is_demo` are gone; the API returns the derived `group` (the group name) and `kind` (from the type map). The `/catalog/v1/*` snapshot keys, the Qdrant payload keys and Temporal arguments keep their earlier names (`scheme_code`, `document_kind`, `instance`, ...) because other services and in-flight workflows depend on them. +- Re-uploading an existing code updates the same document (new workflow, `source` replaced, pages and chunks cleared, `reindex_required` set) rather than adding a second one. +- Catalogs are created on upload for the tenant and kind. The upload therefore takes the document type up front. +- Bootstrap schemes (the code-seeded entries with no documents) are dropped. `/catalog/v1/*` is served from the `catalog_status` view, one row per document, and a scheme appears once a document exists. +- Period, uploader and vector index state moved into `documents.metadata`; counts are derived from `pages` and `chunks`. +- Database enums are replaced by plain `TEXT`, validated by `pipeline/models.py` (`DocumentStage`, `WorkflowType`, `WorkflowStatus`, `AuditEntityType`). +- `audit_logs.instance` is replaced by `audit_logs.group_id` (snapshot, `ON DELETE SET NULL`). +- No Postgres held the first schema, so `migrations/0001` was rewritten in place instead of adding an ALTER migration. `scripts/migrate_sqlite_to_postgres.py` produces v2 and skips `kind=document` and `is_demo` documents, reporting the counts. + +Alternatives considered: +- **Keep `documents.instance`.** Rejected: the catalog already carries the provider, and two sources of tenant truth drift. +- **Type as a short kind plus a separate `schema_type` column.** Rejected for now in favour of one `type` column holding the schema name; revisit if scheme and advisory end up sharing a schema (ADR 0003 types both as `KnowledgeAdvisory`). +- **`user_groups` join table now.** Deferred: the team wants one group per user for now. +- **Nullable `catalog_id`/`code` until a reviewer classifies the document.** Rejected: a document without a catalog has no tenant once `instance` is gone. The upload form asks for the type instead. diff --git a/docs/POSTGRES_SCHEMA.md b/docs/POSTGRES_SCHEMA.md index d76307a..168fe4c 100644 --- a/docs/POSTGRES_SCHEMA.md +++ b/docs/POSTGRES_SCHEMA.md @@ -1,6 +1,6 @@ # Postgres Schema — knowledge-provider -Postgres schema behind `pipeline/db.py`. A document is content and metadata. A workflow is one run of processing against a document. Source file and vector index state are stored as JSONB on the document. Translations are stored as JSONB on each page. +Postgres schema behind `pipeline/db.py`. A document is content and metadata. A workflow is one run of processing against a document. A document belongs to a catalog (a provider group plus a resource type) and is identified there by its `code`. Source file and vector index state are stored as JSONB on the document. Translations are stored as JSONB on each page. The schema is created by `migrations/0001_initial_schema.sql` and applied by the api and worker at startup (`pipeline/migrate.py`). `pipeline/db.py` is the only code that reads or writes it. The previous SQLite store is described in `docs/SQLITE_SCHEMA.md` (legacy) and is moved over once by `scripts/migrate_sqlite_to_postgres.py`. @@ -8,11 +8,11 @@ The schema is created by `migrations/0001_initial_schema.sql` and applied by the - Every table has a primary key `id`, a UUID v4 (`DEFAULT gen_random_uuid()`, built in from Postgres 13). - A foreign key is named `_id` (for example `document_id`). A role prefix is added when the role matters (`uploaded_by_user_id`, `started_by_user_id`). -- Text identifiers other systems rely on (`catalog_entries.code`, `temporal_workflow_id`) are ordinary columns, not keys. +- Text identifiers other systems rely on (`documents.code`, `temporal_workflow_id`) are ordinary columns, not keys. - `created_at` and `updated_at` are the last two columns of a table. `updated_at` is maintained by one shared trigger. Append-only or short-lived tables skip `updated_at`, as noted per table. - Timestamps are `TIMESTAMPTZ` (UTC), booleans are `BOOLEAN`, JSON is `JSONB`, lists of strings are `TEXT[]`. - JSONB columns have a `CHECK` that the value is an object (or array). Value rules inside them live in Pydantic models, not in the database. -- Fixed value sets are enums. Open sets (`artifact_type`, `action_type`) stay `TEXT`. +- There are no database enums. Every value set (stages, workflow types and statuses, audit entity types) is plain `TEXT`, validated in code by `pipeline/models.py`; open sets (`artifact_type`, `action_type`) are `TEXT` too. - Counts have `CHECK >= 0`. Slugs and identifiers have pattern `CHECK`s. ### Shared trigger @@ -28,50 +28,74 @@ END $$ LANGUAGE plpgsql; -- FOR EACH ROW EXECUTE FUNCTION set_updated_at(); ``` -### Enums +### Value sets (plain `TEXT`, validated in code) -| Enum | Values | -|---|---| -| `workflow_stage` | `registered`, `ocr_processing`, `ocr_review`, `translation_processing`, `translation_review`, `chunking`, `chunk_review`, `ready_for_ingestion`, `ingesting`, `publishing_to_network`, `approval_for_prod`, `ingesting_prod`, `completed`, `failed` | -| `workflow_type_enum` | `pipeline`, `ocr_only`, `ocr_retry`, `translation_retry`, `chunking_retry`, `reingest`, `promote_prod` | -| `workflow_status_enum` | `running`, `waiting_review`, `completed`, `failed` | -| `audit_entity_type` | `document`, `page`, `chunk`, `setting`, `catalog` | +| Column(s) | Python type (`pipeline/models.py`) | Values | +|---|---|---| +| `workflows.current_stage`, `workflow_artifacts.stage` | `DocumentStage` | `registered`, `ocr_processing`, `ocr_review`, `translation_processing`, `translation_review`, `chunking`, `chunk_review`, `ready_for_ingestion`, `ingesting`, `publishing_to_network`, `approval_for_prod`, `ingesting_prod`, `completed`, `failed` | +| `workflows.workflow_type` | `WorkflowType` | `pipeline`, `ocr_only`, `ocr_retry`, `translation_retry`, `chunking_retry`, `reingest`, `promote_prod` | +| `workflows.status` | `WorkflowStatus` | `running`, `waiting_review`, `completed`, `failed` | +| `audit_logs.entity_type` | `AuditEntityType` | `document`, `page`, `chunk`, `setting`, `catalog` | -Adding a value later needs `ALTER TYPE ... ADD VALUE`. +The database does not check these values, so adding one needs no migration. Raw SQL that writes them must use a value from the table. ## Tables | # | Table | Purpose | |---|---|---| -| 1 | `users` | People who upload, review and act, mirrored from Keycloak | -| 2 | `catalog_entries` | Generic named entries that documents belong to. Type `scheme` today, `advisory` and others later | -| 3 | `documents` | One row per document. Content, metadata, source file, index state | -| 4 | `workflows` | One row per processing run of a document. Stage, status, config | -| 5 | `workflow_artifacts` | Files each workflow produces, stored in MinIO | -| 6 | `pages` | OCR layer and translations of each page | -| 7 | `chunks` | Chunked text and review state | -| 8 | `audit_logs` | Append-only action history | -| 9 | `search_settings` | Single-row search configuration | -| 10 | `email_otps` | Email OTP login codes | -| 11 | `webhook_records` | Records pushed to `POST /webhook` | +| 1 | `groups` | Provider tenant nodes (`/states/MH`, `/global`), mirrored from Keycloak | +| 2 | `users` | People who upload, review and act, mirrored from Keycloak | +| 3 | `catalogs` | One row per provider group and resource type (scheme, advisory) | +| 4 | `documents` | One row per document. Content, metadata, source file, index state | +| 5 | `workflows` | One row per processing run of a document. Stage, status, config | +| 6 | `workflow_artifacts` | Files each workflow produces, stored in MinIO | +| 7 | `pages` | OCR layer and translations of each page | +| 8 | `chunks` | Chunked text and review state | +| 9 | `audit_logs` | Append-only action history | +| 10 | `search_settings` | Single-row search configuration | +| 11 | `email_otps` | Email OTP login codes | +| 12 | `webhook_records` | Records pushed to `POST /webhook` | | — | `schema_migrations` | Applied migration files (created by the runner, not by a migration) | -| — | `catalog` (view) | Computed catalog of all entries. `/catalog/v1/*` serves the `scheme` type | +| — | `catalog_status` (view) | One row per document with its catalog status. `/catalog/v1/*` serves the scheme type | | — | `document_current_workflow` (view) | Latest workflow of each document | Relationships: ``` -catalog_entries 1 ── * documents 1 ── * workflows 1 ── * workflow_artifacts - │ - ├── * pages - └── * chunks +groups 1 ── * users (users.group_id, nullable) +groups 1 ── * catalogs 1 ── * documents 1 ── * workflows 1 ── * workflow_artifacts + │ + ├── * pages + └── * chunks +groups 1 ── * audit_logs (audit_logs.group_id, snapshot, SET NULL) ``` -Dependency order for creation: `users`, `catalog_entries`, `documents`, `workflows`, `workflow_artifacts`, `pages`, `chunks`, `audit_logs`, `search_settings`, `email_otps`, `webhook_records`, views. +Dependency order for creation: `groups`, `users`, `catalogs`, `documents`, `workflows`, `workflow_artifacts`, `pages`, `chunks`, `audit_logs`, `search_settings`, `email_otps`, `webhook_records`, views. + +--- + +## 1. `groups` + +One row per tenant node (a state, the centre, or the platform), mirrored from Keycloak. A group is a provider: it owns catalogs. Rows are created on demand from a tenant code (`get_or_create_group`) or from a Keycloak group path (`upsert_group`). + +Unique: `keycloak_group_id`, `path`. + +| Column | Type | Constraints / Default | Purpose | +|---|---|---|---| +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Internal group ID. | +| keycloak_group_id | TEXT | UNIQUE | Keycloak group ID. NULL until a sync that sees the path fills it (rows made by the SQLite move or by an upload have none). | +| name | TEXT | NOT NULL | Tenant code, lowercase (`mh`, `bv`). This is what the API calls the document's `group`. | +| path | TEXT | NOT NULL UNIQUE | Keycloak group path: `/states/MH`, or `/global` for `bv`. | +| created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Created. | +| updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). | + +Notes +- Only the tenant node is mirrored, not the role leaves (`/states/MH/admin`). Roles and permissions are read from the token, not from this table. +- Role leaves stay in Keycloak; `catalogs.group_id` points at the tenant node. --- -## 1. `users` +## 2. `users` Uploader, reviewer and actor identity, looked up once from the Keycloak token. @@ -84,76 +108,72 @@ Unique: `keycloak_user_id`; unique index on `lower(email)` where `email IS NOT N | username | TEXT | | Current username. | | email | TEXT | | Current email. | | roles | TEXT[] | NOT NULL DEFAULT '{}' | Current roles. | +| group_id | UUID | FK → `groups(id)` ON DELETE SET NULL | The user's group (tenant). Single membership for now. | | first_seen_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | First login or upload. | | last_seen_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Most recent activity. | Notes -- Upserted on each authenticated upload or login. A `local-admin` row is seeded for `AUTH_DISABLED=true`. +- Upserted on each authenticated upload or login. A `local-admin` row is seeded by the migration. +- A token can carry several tenants but `group_id` holds one: `upsert_user(..., tenants=[...])` uses the first code in sorted order and logs a warning when there are several. Authorization still reads the token. +- If users later need several groups, add a `user_groups` join table and drop `group_id`. - Point-in-time identity is kept in `audit_logs.actor`. --- -## 2. `catalog_entries` +## 3. `catalogs` -Generic named entries that documents belong to. One table for every entry type: `scheme` today, `advisory` and others later. A new type adds rows, not tables. +One row per provider group and resource type. A catalog is what a provider publishes on the network: Mahavistar's `scheme` catalog and its `advisory` catalog are two rows. A new type adds rows, not tables. -Unique: `(type, code)` and `(id, type)` (the second is the target of the composite FK from `documents`). +Unique: `(group_id, type)`. | Column | Type | Constraints / Default | Purpose | |---|---|---|---| -| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Entry ID. | -| type | TEXT | NOT NULL, CHECK `^[a-z0-9]+([_-][a-z0-9]+)*$` | Entry type (`scheme`, later `advisory`, and so on). Open set, so not an enum. | -| code | TEXT | NOT NULL, CHECK `^[a-z0-9]+(-[a-z0-9]+)*$` | Short code (`pm-kmy`) that catalog consumers see. Unique within a type. | -| name | TEXT | NOT NULL | Display name, shared by all documents of the entry. | -| aliases | TEXT[] | NOT NULL DEFAULT '{}', CHECK no empty string | Ordered alternate names used to match queries. The first one is the prompt hint. | -| metadata | JSONB | NOT NULL DEFAULT '{"source": "pipeline"}', CHECK object | `source` and any type-specific fields. See keys below. | +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Catalog ID. | +| group_id | UUID | NOT NULL, FK → `groups(id)` ON DELETE RESTRICT | The provider that owns the catalog. | +| type | TEXT | NOT NULL, CHECK `^[a-z0-9]+:[A-Za-z0-9]+$` | Network schema/capability, e.g. `openagrinet:AgricultureResource`. | +| name | TEXT | NOT NULL | Catalog display name. Defaults to ` ` when created on upload. | +| metadata | JSONB | NOT NULL DEFAULT '{}', CHECK object | Beckn publish information. See keys below. | | created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Created. | | updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). | ### `metadata` keys -| Key | Type | Default | Purpose | -|---|---|---|---| -| source | string | `pipeline` | `bootstrap` (seeded from code), `pipeline` (built from documents) or `import`. | +| Key | Type | Purpose | +|---|---|---| +| publish_directives | object | Beckn `publishDirectives`. | +| visible_networks | string[] | Networks the catalog is visible to. | +| criteria | object | Catalog criteria. | +| network | object `{catalog_id, last_published_at}` | Set once the catalog is published on the network. | -Type-specific fields are added as keys, with no migration. Pydantic validates the keys per type. +Pydantic (`CatalogMetadata`) validates the keys; the database only checks for an object. Notes -- Computed catalog values (`status`, `instances`, `workflow_ids`, `chunk_count`, `network_visible`, `promoted_at`) come from the `catalog` view, not stored here. -- Case-insensitive alias uniqueness is enforced in the app (`_dedupe_aliases`). Add a GIN index on `aliases` only if "find entry by alias" is needed in SQL. -- No join table: an entry has many documents and a document has one entry, which `documents.catalog_entry_id` expresses. -- Plain documents (kind `document`) have no entry. +- `type` stores the schema name, and code asks `pipeline/network_constants.py` what it means (`catalog_type_for_kind`, `kind_for_catalog_type`) instead of comparing `openagrinet:*` strings. The mapping is a placeholder: `scheme` is `openagrinet:AgricultureResource`, `advisory` is `openagrinet:KnowledgeAdvisory` (see ADR 0003 and ADR 0006). +- Catalogs are created on upload (`get_or_create_catalog`), not seeded. +- Computed catalog values (status, chunk count, network visibility, `promoted_at`) come from the `catalog_status` view, not stored here. --- -## 3. `documents` +## 4. `documents` -One row per document. 13 columns. +One row per document: one resource (a scheme, an advisory) in one catalog, identified there by `code`. + +Unique: `(catalog_id, code)`. | Column | Type | Constraints / Default | Purpose | |---|---|---|---| | id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Document ID. | -| instance | TEXT | NOT NULL | Tenant or state code. | -| source | JSONB | NOT NULL, CHECK `jsonb_typeof(source) = 'object'`, CHECK `source->>'fingerprint' ~ '^[0-9a-f]{32}$'` | The uploaded file. See shape below. | -| document_kind | TEXT | NOT NULL DEFAULT 'document', CHECK `^[a-z0-9]+([_-][a-z0-9]+)*$` | Knowledge kind (`document`, `scheme`, `advisory`, `video`, or custom). | -| catalog_entry_id | UUID | | Catalog entry (for example a scheme) the document belongs to. Nullable. Composite FK, see below. | -| valid_from | DATE | | Start of the period chunks are searchable. | -| valid_to | DATE | CHECK `valid_from <= valid_to` | End of the period. NULL means no end. | +| catalog_id | UUID | NOT NULL, FK → `catalogs(id)` ON DELETE RESTRICT | Catalog (provider group + type) the document belongs to. | +| code | TEXT | NOT NULL, CHECK `^[a-z0-9]+(-[a-z0-9]+)*$` | Short code (`pm-kmy`) that catalog consumers see. Unique within the catalog. | +| name | TEXT | NOT NULL | Display name. | +| aliases | TEXT[] | NOT NULL DEFAULT '{}', CHECK no empty string | Ordered alternate names used to match queries. The first one is the prompt hint. | +| source | JSONB | NOT NULL, CHECK object, CHECK `source->>'fingerprint' ~ '^[0-9a-f]{32}$'` | The uploaded file. See shape below. | | is_disabled | BOOLEAN | NOT NULL DEFAULT FALSE | Soft delete. Restore flips it back. | -| index_status | JSONB | NOT NULL DEFAULT '{}', CHECK `jsonb_typeof(index_status) = 'object'` | Vector index state per index. Keys `dev` and `prod`. | -| metadata | JSONB | NOT NULL, DEFAULT see below, CHECK object, CHECK `(metadata->>'chunk_count')::int >= 0` | Display and flag fields. See keys below. | -| uploaded_by_user_id | UUID | FK → `users(id)` | Who uploaded. | +| metadata | JSONB | NOT NULL, DEFAULT see below, CHECK object | Period, uploader, index state and flags. See keys below. | | created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Row created. | | updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). | -Composite FK `(catalog_entry_id, document_kind)` → `catalog_entries(id, type)`, `ON UPDATE CASCADE ON DELETE RESTRICT`. It guarantees a document's kind equals its entry's type. When `catalog_entry_id` is NULL the FK is not checked, so plain documents need no entry. - -Suggested indexes: -- `(instance)` -- `(created_at DESC)` -- `(catalog_entry_id) WHERE catalog_entry_id IS NOT NULL` -- `(instance, (source->>'fingerprint'))` for duplicate checks and identifier lookups -- `(((metadata->>'is_demo')::boolean))` for the demo filter on every list +Indexes: `(created_at DESC)`, `(catalog_id, (source->>'fingerprint'))`, plus the unique `(catalog_id, code)`. ### `source` @@ -165,8 +185,7 @@ Facts about the uploaded file, not about processing. "original_filename": "pmkisan.pdf", "storage_uri": "s3://...", "source_type": "document", - "canonical_input_type": "pdf", - "page_count": null + "canonical_input_type": "pdf" } ``` @@ -177,55 +196,55 @@ Facts about the uploaded file, not about processing. | storage_uri | string | Location of the uploaded file. | | source_type | string | `spreadsheet`, `document` or `unknown`. | | canonical_input_type | string | `spreadsheet` or `pdf`. Normalized pipeline path. | -| page_count | integer or null | NULL until OCR runs. | -### `index_status` - -One entry per vector index. DEV and PROD are separate Qdrant collections. - -```json -{ - "dev": {"status": "indexed", "chunk_count_indexed": 42, - "last_indexed_at": "2026-10-19T06:20:00Z", "last_verified_at": "2026-10-19T06:25:00Z", - "schema_version": "passage-v1", "details": {}}, - "prod": {"...": "..."} -} -``` - -- A missing key means the document was never ingested to that index. -- `status` is `unknown`, `indexed` or `missing`. -- The document's key inside the vector index is `source.fingerprint`. -- The key is `prod` for the PROD collection names (`PROD_VECTOR_DB_COLLECTION_NAME`, `PROD_SCHEME_VECTOR_DB_COLLECTION_NAME`) and `dev` for every other collection. The collection name is kept inside the entry as `index_name`, next to `doc_id`, the id the index uses for the document. -- Update one index atomically: `SET index_status = index_status || jsonb_build_object('dev', $1::jsonb)`. -- A third index is a new key, with no migration. +There is no page count here: it is the number of `pages` rows. ### `metadata` | Key | Type | Default | Purpose | |---|---|---|---| -| display_name | string | absent | Human-friendly title. | +| valid_from | string `YYYY-MM-DD` | absent | Start of the period chunks are searchable. | +| valid_to | string `YYYY-MM-DD` | absent | End of the period. Absent means no end. Pydantic requires `valid_from <= valid_to`. | +| uploaded_by_user_id | string (users.id) | absent | Who uploaded. Not an FK, so a deleted user leaves a stale id; `audit_logs.actor_user_id` keeps real history. | +| index_status | object | `{}` | Vector index state per index. Keys `dev` and `prod`. See below. | | catalog_visible | boolean | true | Counts toward the catalog. | | network_visible | boolean | true | May be published to the Beckn network. | -| is_demo | boolean | false | Demo content, hidden by default. | -| chunk_count | integer ≥ 0 | 0 | Cached number of chunks. Updated in the same transaction as chunk writes. | | reindex_required | boolean | false | Approved content changed and the vector index is stale. | | reindex_reason | string | absent | Why a reindex is needed. | | chunking | object `{run_number, provider, model, config, created_at}` | absent | Settings of the run that produced the current chunks. | -Column default: `'{"catalog_visible": true, "network_visible": true, "is_demo": false, "chunk_count": 0, "reindex_required": false}'`. +Column default: `'{"index_status": {}, "catalog_visible": true, "network_visible": true, "reindex_required": false}'`. -### Notes +`index_status` is one entry per vector index (DEV and PROD are separate Qdrant collections): +```json +{ + "dev": {"status": "indexed", "chunk_count_indexed": 42, + "last_indexed_at": "2026-10-19T06:20:00Z", "last_verified_at": "2026-10-19T06:25:00Z", + "schema_version": "passage-v1", "details": {}}, + "prod": {"...": "..."} +} +``` + +- A missing key means the document was never ingested to that index. `status` is `unknown`, `indexed` or `missing`. +- Update one index atomically with `jsonb_set(metadata, '{index_status,dev}', $1::jsonb)`; the `index_status` key always exists, which `jsonb_set` needs. + +### Notes - A document's stage is the `current_stage` of its latest workflow. List and filter queries join the `document_current_workflow` view. -- SQL does not check types inside the JSONB columns. The insert path writes every default `metadata` key, and Pydantic validates `source`, `index_status` and `metadata`. -- `metadata.chunk_count` is rewritten in every chunk transaction and `metadata.is_demo` is filtered on every list. If performance suffers, move these two back to columns first. -- Each document keeps its own `source`. Two uploads of the same content from different paths keep their own `storage_uri`. There is no unique key on `(instance, fingerprint)`; the upload path reuses a document by its Temporal workflow id instead. Add `UNIQUE (instance, (source->>'fingerprint'))` only if one document per file per tenant must be enforced. -- Identifier lookup (`find_document_by_doc_identifier`) matches `documents.id`, `workflows.temporal_workflow_id`, `source->>'fingerprint'` and `source->>'original_filename'`. The legacy `md5(identifier)` check is `md5(source->>'fingerprint') = $1` in SQL. +- The API's `group` is derived: `documents.catalog_id → catalogs.group_id → groups.name`. There is no `documents.instance` column, and tenant filters join `groups`. +- The API's `document_kind` is derived from `catalogs.type` through the map in `network_constants.py`. `scheme_code`, `scheme_name` and `scheme_aliases` are `code`, `name` and `aliases`. +- `chunk_count` and `page_count` are counted from `chunks` and `pages`; they are not stored. +- A new document needs a `document_kind` and a `code`. Upload gets the catalog from the tenant and the kind (created when missing). `DuplicateDocumentCode` is raised when the code is taken in that catalog. +- Re-uploading an existing code updates the same document: a new workflow starts, `source` is replaced, pages and chunks are cleared and `reindex_required` is set (`reset_document_for_reupload`). Earlier workflows and artifacts stay as history. +- Changing a document's kind moves it to the same group's catalog of the new type (`update_document_fields(document_kind=...)`). +- SQL does not check types inside the JSONB columns. The insert path writes the default metadata keys, and Pydantic validates `source`, `metadata` and each `index_status` entry. +- Each document keeps its own `source`. There is no unique key on the fingerprint. +- Identifier lookup (`find_document_by_doc_identifier`) matches `documents.id`, `workflows.temporal_workflow_id`, `source->>'fingerprint'`, `source->>'original_filename'`, an index `doc_id` and the legacy `md5(source->>'fingerprint')`. - No `deleted_at`: soft delete is `is_disabled`, hard delete is `purge_document`. --- -## 4. `workflows` +## 5. `workflows` One row per processing run of a document: the main pipeline run, retries, reingest and PROD promotion. Holds all processing state. @@ -235,10 +254,10 @@ Indexes: `(document_id, started_at DESC)`, `(status, started_at DESC)`, `(tempor |---|---|---|---| | id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Workflow ID. | | document_id | UUID | NOT NULL, FK → `documents(id)` ON DELETE CASCADE | Document the workflow runs against. | -| workflow_type | workflow_type_enum | NOT NULL | Kind of run. | +| workflow_type | TEXT | NOT NULL | Kind of run (`WorkflowType`). | | temporal_workflow_id | TEXT | NOT NULL | Temporal workflow ID. `doc-` for the main run, suffixed for retries. Not unique (a restart reuses the ID). Used by the API, UI and Temporal. | -| status | workflow_status_enum | NOT NULL DEFAULT 'running' | `running`, `waiting_review`, `completed` or `failed`. | -| current_stage | workflow_stage | NOT NULL DEFAULT 'registered' | Stage the workflow is in now. This is the document's stage when the workflow is the latest. | +| status | TEXT | NOT NULL DEFAULT 'running' | `running`, `waiting_review`, `completed` or `failed` (`WorkflowStatus`). | +| current_stage | TEXT | NOT NULL DEFAULT 'registered' | Stage the workflow is in now (`DocumentStage`). This is the document's stage when the workflow is the latest. | | stage_timestamps | JSONB | NOT NULL DEFAULT '{}', CHECK object with only keys `ocr`, `translation`, `chunks` | When each stage first completed in this run, ISO-8601 UTC. Set-once per key. | | stop_after_ocr | BOOLEAN | NOT NULL DEFAULT FALSE | Halt after OCR (OCR-only batch runs). | | error_message | TEXT | | Failure reason when status is `failed`. | @@ -266,12 +285,12 @@ Latest workflow per document. Stores nothing. ```sql SELECT DISTINCT ON (document_id) * FROM workflows -ORDER BY document_id, started_at DESC; +ORDER BY document_id, started_at DESC, id DESC; ``` --- -## 5. `workflow_artifacts` +## 6. `workflow_artifacts` Ledger of every file a workflow produces or stores. Files live in MinIO; this table records where and what. Append-only. @@ -282,7 +301,7 @@ Indexes: `(workflow_id, created_at DESC)`, `(workflow_id, artifact_type, created | id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Artifact ID, used in `/documents/{id}/artifacts/{artifact_id}`. | | workflow_id | UUID | NOT NULL, FK → `workflows(id)` ON DELETE CASCADE | Workflow that produced it. | | artifact_type | TEXT | NOT NULL, CHECK not blank | Kind of output (`original_upload`, `normalized_pdf`, `normalized_spreadsheet`, `ocr_pages_json`, `translation_pages_json`, `chunk_json_export`, `vector_payload_export`, network publish exchange). Open set, so not an enum. | -| stage | workflow_stage | | Stage that created it. | +| stage | TEXT | | Stage that created it (`DocumentStage`). | | storage_uri | TEXT | NOT NULL | MinIO location of the blob. | | mime_type | TEXT | | MIME type. | | filename | TEXT | | File name of the blob. | @@ -302,7 +321,7 @@ Notes --- -## 6. `pages` +## 7. `pages` OCR layer and translations of a page. @@ -362,7 +381,7 @@ Notes --- -## 7. `chunks` +## 8. `chunks` Unique: `(document_id, chunk_number)`. @@ -389,7 +408,7 @@ Unique: `(document_id, chunk_number)`. Notes - Chunking provider, model, config and run number are stored once per document in `documents.metadata.chunking`, not per chunk. - The distinct page numbers a chunk draws from are derived from `source_spans`. -- Re-chunk runs in one transaction: delete the document's chunks, insert the new ones, update `metadata.chunking` (with `run_number` incremented) and `metadata.chunk_count`. Chunk UUIDs change on every re-chunk; nothing references `chunks.id` (audit uses chunk number). +- Re-chunk runs in one transaction: delete the document's chunks, insert the new ones, update `metadata.chunking` (with `run_number` incremented) (the chunk count is counted from `chunks`, not stored). Chunk UUIDs change on every re-chunk; nothing references `chunks.id` (audit uses chunk number). - `search_chunks` does a substring match on `lower(coalesce(edited_text, original_text))`, served by a trigram GIN index on that expression (the migration creates the `pg_trgm` extension). > **Why `pages` and `chunks` stay separate tables:** a page is a physical unit from OCR and a chunk is a semantic unit from the chunker, so their boundaries do not align. One chunk can span several pages and one page can hold several chunks, which makes the relationship many-to-many. Storing page text inside each chunk would copy a page into every chunk it overlaps, so any page edit, OCR retry or translation change would have to be rewritten in all of those chunks. @@ -397,7 +416,7 @@ Notes --- -## 8. `audit_logs` +## 9. `audit_logs` Append-only history of actions. Must outlive documents, so there is no cascade delete. @@ -406,9 +425,9 @@ Append-only history of actions. Must outlive documents, so there is no cascade d | id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Audit entry ID. | | document_id | UUID | FK → `documents(id)` ON DELETE SET NULL | Document the action applied to. NULL for system-level entries and after a purge. | | workflow_id | TEXT | | Temporal workflow ID snapshot. Survives a purge. | -| instance | TEXT | | Tenant snapshot, so tenant scoping works after the document is gone. | +| group_id | UUID | FK → `groups(id)` ON DELETE SET NULL | Provider group snapshot, so tenant scoping works after the document is gone. | | action_type | TEXT | NOT NULL, CHECK not blank | What happened (`page_edit`, `purge_document`, ...). 22 values and growing, so not an enum. | -| entity_type | audit_entity_type | | `document`, `page`, `chunk`, `setting` or `catalog`. | +| entity_type | TEXT | | `document`, `page`, `chunk`, `setting` or `catalog` (`AuditEntityType`). | | entity_number | INTEGER | | Page number or chunk number the action applied to. | | field_name | TEXT | | Field that changed. | | old_value | TEXT | | Value before the change. | @@ -420,21 +439,21 @@ Append-only history of actions. Must outlive documents, so there is no cascade d Table CHECK: `entity_type IN ('setting','catalog') OR workflow_id IS NOT NULL`. -Indexes: `(document_id, created_at DESC)`, `(created_at DESC)`, `(action_type, created_at DESC)`, `(created_at DESC) WHERE entity_type = 'setting'`. +Indexes: `(document_id, created_at DESC)`, `(group_id, created_at DESC)`, `(created_at DESC)`, `(action_type, created_at DESC)`, `(created_at DESC) WHERE entity_type = 'setting'`. Notes - The FK to `documents` is `SET NULL`, not cascade. A cascade would erase the audit trail on purge, and a strict FK would reject setting and catalog entries that have no document. - Setting and catalog entries have NULL `document_id` and `workflow_id`, with `entity_type` set to `setting` or `catalog`. -- `instance` is a snapshot, so a purged document's history stays visible only to its own tenant. +- `group_id` is a snapshot of the document's provider group, so a purged document's history stays visible only to its own tenant. - `entity_number` holds a page or chunk number, not an ID (chunk UUIDs would dangle after a re-chunk). - `get_settings_audit_logs` is `WHERE entity_type = 'setting'`. -- `log_audit` fills `instance`, resolves the actor to a `users` row and builds the `actor` JSON. +- `log_audit` fills `group_id`, resolves the actor to a `users` row and builds the `actor` JSON. - No `updated_at`: entries are immutable. - The table only grows. Monthly partitioning is possible later without changing this design. --- -## 9. `search_settings` +## 10. `search_settings` Single row of search configuration. @@ -464,7 +483,7 @@ Notes --- -## 10. `email_otps` +## 11. `email_otps` Server-side email-OTP login codes. Short-lived. @@ -489,32 +508,33 @@ Notes --- -## View: `catalog` +## View: `catalog_status` -Computed from `catalog_entries`, `documents`, `workflows` and `document_current_workflow`. Stores nothing. The `/catalog/v1/*` snapshot API reads it with `type = 'scheme'`, and its fields are named `scheme_code`, `scheme_name` and so on, so consumers see no change. +Computed from `documents`, `catalogs`, `groups`, `workflows` and `document_current_workflow`. Stores nothing. One row per document. The `/catalog/v1/*` snapshot API reads it for the scheme type (`catalogs.type` of the scheme kind), and its fields are named `scheme_code`, `scheme_name` and so on, so consumers see no change. | Column | Type | How it is computed | |---|---|---| -| entry_id | UUID | From `catalog_entries`. | -| type | TEXT | From `catalog_entries`. | -| code | TEXT | From `catalog_entries`. | -| name | TEXT | From `catalog_entries`. | -| aliases | TEXT[] | From `catalog_entries`. | -| source | TEXT | `catalog_entries.metadata->>'source'`. | -| status | TEXT | `pending_reindex` if a live document needs reindex. Else `live` if any live document exists. Else `pending_prod` if a document awaits PROD approval. Else `live` for a bootstrap entry with no documents. Else `disabled`. | -| instances | TEXT[] | Distinct `instance` of the entry's documents, or `{default}` if none. | -| workflow_ids | TEXT[] | `temporal_workflow_id` of each entry document's main `pipeline` workflow. | -| chunk_count | INTEGER | Sum of `metadata->>'chunk_count'` over the entry's documents. | -| network_visible | BOOLEAN | True if any of its documents has `metadata->>'network_visible'` true. For an entry with no documents, true if bootstrap. | -| promoted_at | TIMESTAMPTZ | Latest `index_status->'prod'->>'last_indexed_at'` of the live documents. | - -"Live document" means the latest workflow's `current_stage = 'completed'`, `is_disabled` false, and `metadata->>'catalog_visible'` true. The entry's `type` already ties it to the document's kind through the composite FK. +| document_id | UUID | `documents.id`. | +| catalog_id | UUID | `documents.catalog_id`. | +| group_id | UUID | `catalogs.group_id`. | +| group_name | TEXT | `groups.name`. | +| type | TEXT | `catalogs.type`. | +| code | TEXT | `documents.code`. | +| name | TEXT | `documents.name`. | +| aliases | TEXT[] | `documents.aliases`. | +| status | TEXT | `pending_reindex` if the document is live and `metadata.reindex_required` is true. Else `live` if live. Else `pending_prod` if the latest workflow's stage is `approval_for_prod` and the document is not disabled. Else `disabled`. | +| workflow_id | TEXT | `temporal_workflow_id` of the document's earliest `pipeline` workflow. | +| chunk_count | INTEGER | Count of the document's `chunks`. | +| network_visible | BOOLEAN | `metadata.network_visible`, true when absent. | +| promoted_at | TIMESTAMPTZ | `metadata.index_status.prod.last_indexed_at`, only for live documents. | + +"Live" means the latest workflow's `current_stage = 'completed'`, `is_disabled` is false and `metadata.catalog_visible` is true. There are no bootstrap rows: a scheme appears only once a document exists. --- -## 11. `webhook_records` +## 12. `webhook_records` -Records pushed to `POST /webhook` (`x-api-key` = `WEBHOOK_API_KEY`). Carried over from the previous store unchanged apart from the types. +Records pushed to `POST /webhook` (`x-api-key` = `WEBHOOK_API_KEY`). Carried over from the previous store unchanged; not part of the groups and catalogs model. Unique: `(source, external_id)`; index on `(source, status)`. @@ -530,6 +550,8 @@ Unique: `(source, external_id)`; index on `(source, status)`. | category | TEXT | | From `domainName`. | | payload | JSONB | NOT NULL, CHECK object | The full record. | | payload_hash | TEXT | | SHA-256 of the canonical payload; an upsert with the same hash writes nothing. | +| indexed_hash | TEXT | | `payload_hash` last written to the DEV collection (migration 0002). | +| promoted_hash | TEXT | | `payload_hash` last written to the PROD collection (migration 0002). | | created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | The payload's `createdAt` when it has one. | | updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | The payload's `updatedAt` when it has one. | @@ -547,7 +569,8 @@ Created by the runner, not by a migration file: `version` (the `NNNN` prefix, pr - **Identifiers.** `db.py` functions take a `workflow_id` argument that may be the document UUID, a Temporal workflow id, the fingerprint, the filename, an index `doc_id` or the legacy `md5(fingerprint)` doc id (`find_document_by_doc_identifier`). Documents, jobs and artifacts come back as flat dicts; `id` is the UUID and `workflow_id` is the Temporal workflow id of the document's main run (the earliest `pipeline` or `ocr_only` workflow). The API rewrites `/documents/{id}/...` to that workflow id before routing, because handlers signal and query Temporal with it. - **Jobs.** The API's "jobs" and "runs" are `workflows` rows; job ids and artifact ids are the UUIDs of `workflows` and `workflow_artifacts`. `latest_job_id`, `original_artifact_id` and `normalized_artifact_id` are derived by join, not stored. +- **Group.** The API returns a `group` string on documents, runs, queue rows and audit entries. It is the lowercase `groups.name` reached through `catalogs`; filters compare `lower(groups.name)`. - **Catalog version.** `/catalog/v1/version` counts `audit_logs` rows with `entity_type = 'catalog'` and `action_type = 'catalog_version_bump'`, so it only grows. `updated_at` is the newest such row. - **Document stage.** The stage a document shows is `document_current_workflow.current_stage`; `update_document_stage` writes to that workflow. - **Page translations.** `save_pages` keeps stored translations unless the page carries `translations` (replaces them) or flat `translated_*` fields (merged into `translation_target_language`). A page whose OCR text changed loses its translations. -- **Users.** Rows are upserted from the auth dependency on write requests (at most once per five minutes per user) and from uploads, workflow starts and audit entries. `local-admin` is seeded by the migration. +- **Users.** Rows are upserted from the auth dependency on write requests (at most once per five minutes per user) and from uploads, workflow starts and audit entries. `local-admin` is seeded by the migration. A user's `group_id` is set from the tenant codes in the token. From fbb3544f5349aa2458070110f623c421119d3294 Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 11:17:26 +0530 Subject: [PATCH 41/53] fix: guard document replace and clean up vectors of replaced files Replace guard (api): an upload now needs upload rights in the target group, not just somewhere. Replacing an existing code also needs review rights in the document's group, is refused while the document is disabled or a run is in progress, and needs approval rights when the document is live in PROD. Orphaned vectors: a re-upload changes the fingerprint, which is the document's key in the vector index. reset_document_for_reupload now records the old key in metadata.superseded_doc_ids for each index that holds it. The next DEV ingest and PROD promote delete those points after the new content is indexed (a failed delete keeps the key for the next run). Disable and purge now delete the current and superseded keys, and the PROD cleanup follows the PROD index entry, so a document promoted as a scheme is cleaned up even if its kind changed. --- pipeline/activities.py | 41 ++++++++++++++ pipeline/api/api.py | 80 +++++++++++++++++++++++---- pipeline/db.py | 47 +++++++++++++++- pipeline/models.py | 5 ++ tests/test_activities.py | 62 +++++++++++++++++++++ tests/test_api.py | 115 ++++++++++++++++++++++++++++++++++----- tests/test_schema_v2.py | 72 ++++++++++++++++++++++++ 7 files changed, 394 insertions(+), 28 deletions(-) diff --git a/pipeline/activities.py b/pipeline/activities.py index ff5adfe..d5bed55 100644 --- a/pipeline/activities.py +++ b/pipeline/activities.py @@ -1361,6 +1361,7 @@ async def promote_document_to_prod_qdrant( store.upsert, prod_collection, records, batch_size=max(batch_size, 8) ) activity.logger.info("PROD promotion complete: %s", result.get("index_stats")) + await _drop_superseded_points("prod", workflow_id, doc, document_id, prod_collection, store=store) db.log_audit( workflow_id=workflow_id, document_id=document_id, @@ -1391,6 +1392,45 @@ async def promote_document_to_prod_qdrant( } +async def _drop_superseded_points( + env: str, + workflow_id: str, + doc: dict, + current_doc_id: str, + collection: str, + store=None, +) -> None: + """Delete points of earlier file versions (see ``metadata.superseded_doc_ids``). + + Runs after the new content is indexed, so search never has a gap. A failed + delete keeps the key for the next run and does not fail the ingest. + """ + from . import db + + stale = [ + doc_id + for doc_id in (doc.get("superseded_doc_ids") or {}).get(env, []) + if doc_id and doc_id != current_doc_id + ] + if not stale: + return + if store is None: + from .vector_store import get_vector_store + + store = get_vector_store() + deleted = [] + for doc_id in stale: + result = await asyncio.to_thread(store.delete_by_doc_id, collection, doc_id) + if result.get("error"): + activity.logger.warning( + "Could not delete superseded %s points for %s: %s", env, doc_id, result["error"] + ) + else: + deleted.append(doc_id) + if deleted: + db.clear_superseded_doc_ids(workflow_id, env, deleted) + + @activity.defn async def ingest_document_from_db( workflow_id: str, @@ -1447,6 +1487,7 @@ async def ingest_document_from_db( status="indexed", details=result.get("index_stats"), ) + await _drop_superseded_points("dev", workflow_id, doc, document_id, index_name) return result diff --git a/pipeline/api/api.py b/pipeline/api/api.py index c2b1603..9d7b406 100644 --- a/pipeline/api/api.py +++ b/pipeline/api/api.py @@ -432,6 +432,29 @@ def _document_title(filename: str) -> str: return re.sub(r"[_\-\s]+", " ", stem).strip() or "Untitled document" +def _assert_can_replace(user: AuthUser, existing: dict) -> None: + """Replacing a document's file clears its pages and chunks, so guard it. + + Needs review rights in the document's group; a disabled document or one with + a run in progress cannot be replaced; a document already promoted to PROD + can only be replaced by someone who may approve ingestion. + """ + doc_group = existing["group"] + if not user.has_permission_for_group(Permission.REVIEW, doc_group): + raise HTTPException(403, f"No permission to replace documents in group: {doc_group}") + if existing.get("is_disabled"): + raise HTTPException(409, "This document is disabled; restore it before replacing its file") + latest_job = db.get_latest_document_job(existing["workflow_id"]) + if latest_job and latest_job.get("status") == "running": + raise HTTPException(409, "A run is still in progress for this document") + if (existing.get("index_status") or {}).get("prod") and not user.has_permission_for_group( + Permission.APPROVE_INGESTION, doc_group + ): + raise HTTPException( + 403, "Replacing a document that is live in PROD needs approval rights in its group" + ) + + def _resolve_document_identity( user: AuthUser, group: str, @@ -444,12 +467,19 @@ def _resolve_document_identity( An explicit ``code`` that is already held in the tenant's catalog of that kind means "update this document": the caller reuses it instead of inserting. Without a code one is derived from the file name. + + Permissions are checked against the target group, not just globally: a user + who may upload in one state must not be able to add to, or replace a + document of, a state where they only hold a view role. Replacing clears the + document's pages and chunks, so it also needs review rights in that group. """ kind = (kind or "").strip().lower() try: catalog_type_for_kind(kind) except ValueError as exc: raise HTTPException(400, str(exc)) from exc + if not user.has_permission_for_group(Permission.UPLOAD, group): + raise HTTPException(403, f"No upload permission in group: {group}") code = (code or "").strip() if code: try: @@ -459,6 +489,7 @@ def _resolve_document_identity( existing = db.find_document_by_code(group, kind, code) if existing: existing = assert_document_group_access(user, existing) + _assert_can_replace(user, existing) return kind, code, existing return kind, scheme_catalog.derive_scheme_code(_document_title(filename), kind=kind), None @@ -1063,10 +1094,28 @@ def _require_catalog_read(request: Request, user: AuthUser) -> None: raise HTTPException(403, "Catalog access requires search or admin permission") +def _vector_doc_ids(doc: dict, env: str) -> list[str]: + """The document's current index key plus keys of earlier file versions still in ``env``.""" + ids = [doc.get("document_id")] + ids.extend((doc.get("superseded_doc_ids") or {}).get(env, [])) + return list(dict.fromkeys(i for i in ids if i)) + + +def _delete_all_doc_ids(store, collection: str, doc_ids: list[str]) -> dict: + results = [store.delete_by_doc_id(collection, doc_id) for doc_id in doc_ids] + if len(results) == 1: + return results[0] + errors = [r["error"] for r in results if r.get("error")] + summary = {"deleted": 0 if errors else -1, "doc_ids": doc_ids, "results": results} + if errors: + summary["error"] = "; ".join(errors) + return summary + + def _delete_document_points_from_prod(doc: dict) -> dict: """Best-effort delete of a promoted (non-scheme) document's vectors from PROD documents-index.""" - doc_id = doc.get("document_id") - if not doc_id: + doc_ids = _vector_doc_ids(doc, "prod") + if not doc_ids: return {"deleted": False, "reason": "no_doc_id"} prod_url = (os.environ.get("PROD_VECTOR_DB_URL") or "").strip() if not prod_url: @@ -1082,15 +1131,15 @@ def _delete_document_points_from_prod(doc: dict) -> dict: client = get_qdrant_client(url=prod_url, api_key=prod_key) store = QdrantVectorStore(client=client) - return store.delete_by_doc_id(collection, doc_id) + return _delete_all_doc_ids(store, collection, doc_ids) except Exception as exc: return {"deleted": False, "error": str(exc), "collection": collection} def _delete_scheme_points_from_prod(doc: dict) -> dict: """Best-effort delete of scheme vectors from PROD schemes-index.""" - doc_id = doc.get("document_id") - if not doc_id: + doc_ids = _vector_doc_ids(doc, "prod") + if not doc_ids: return {"deleted": False, "reason": "no_doc_id"} prod_url = ( (os.environ.get("PROD_SCHEME_VECTOR_DB_URL") or "").strip() @@ -1111,7 +1160,7 @@ def _delete_scheme_points_from_prod(doc: dict) -> dict: client = get_qdrant_client(url=prod_url, api_key=prod_key) store = QdrantVectorStore(client=client) - return store.delete_by_doc_id(collection, doc_id) + return _delete_all_doc_ids(store, collection, doc_ids) except Exception as exc: return {"deleted": False, "error": str(exc), "collection": collection} @@ -1523,16 +1572,25 @@ async def disable_document( # Remove from the DEV vector index if requested if remove_from_search: - doc_id = doc.get("document_id") - if doc_id: + # The current index key plus keys of earlier file versions still in DEV + # (see metadata.superseded_doc_ids). + for index, doc_id in enumerate(_vector_doc_ids(doc, "dev")): vector_result = delete_chunks_from_index(doc_id) - result["vector_deleted"] = vector_result.get("deleted", 0) + if index == 0: + result["vector_deleted"] = vector_result.get("deleted", 0) if "error" in vector_result: result["vector_error"] = vector_result["error"] - # PROD Qdrant cleanup, if this document was ever promoted + # PROD Qdrant cleanup, if this document was ever promoted. A document + # promoted as a scheme lives in the scheme index even if its kind was + # changed since, so the PROD index entry decides, not only the kind. kind = (doc.get("kind") or "").strip().lower() - if kind == "scheme": + prod_entry = (doc.get("index_status") or {}).get("prod") or {} + prod_is_scheme = ( + kind == "scheme" + or prod_entry.get("index_name") == scheme_catalog.scheme_collection_name() + ) + if prod_is_scheme: q_result = _delete_scheme_points_from_prod(doc) result["scheme_qdrant_deleted"] = q_result try: diff --git a/pipeline/db.py b/pipeline/db.py index c90ed21..9bec3a1 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -392,6 +392,7 @@ def _document_from_row(row: Optional[dict]) -> Optional[dict]: "latest_job_id": row["current_workflow_pk"], "catalog_id": row["catalog_id"], "index_status": meta.get("index_status") or {}, + "superseded_doc_ids": meta.get("superseded_doc_ids") or {}, "created_at": row["created_at"], "updated_at": row["updated_at"], } @@ -1096,6 +1097,11 @@ def reset_document_for_reupload( ``source`` is replaced, pages and chunks are cleared for the new run, and the document is flagged ``reindex_required`` because its vector index is now stale. Earlier workflows and artifacts stay as history. + + The fingerprint is the document's key in the vector index, so the old key + is remembered in ``metadata.superseded_doc_ids`` for every index (dev/prod) + that holds it. The next ingest/promote and disable/purge delete those points; + without this they would stay searchable with nothing pointing at them. """ source = DocumentSource.model_validate({ "fingerprint": fingerprint, @@ -1105,6 +1111,19 @@ def reset_document_for_reupload( "canonical_input_type": canonical_input_type, }).model_dump(exclude_none=True) with get_connection() as conn: + current = conn.execute( + "SELECT source->>'fingerprint' AS fingerprint, metadata FROM documents " + "WHERE id = %s::uuid FOR UPDATE", + (doc_uuid,), + ).fetchone() + if current is None: + raise LookupError(f"Document not found: {doc_uuid}") + meta = current["metadata"] or {} + superseded = {k: list(v) for k, v in (meta.get("superseded_doc_ids") or {}).items()} + for env, entry in (meta.get("index_status") or {}).items(): + old_key = (entry or {}).get("doc_id") or current["fingerprint"] + if old_key and old_key != fingerprint and old_key not in superseded.get(env, []): + superseded.setdefault(env, []).append(old_key) conn.execute("DELETE FROM pages WHERE document_id = %s::uuid", (doc_uuid,)) conn.execute("DELETE FROM chunks WHERE document_id = %s::uuid", (doc_uuid,)) conn.execute( @@ -1112,10 +1131,34 @@ def reset_document_for_reupload( UPDATE documents SET source = %s::jsonb, metadata = (metadata - 'chunking') - || jsonb_build_object('reindex_required', TRUE, 'reindex_reason', 'source_replaced') + || jsonb_build_object('reindex_required', TRUE, 'reindex_reason', 'source_replaced', + 'superseded_doc_ids', %s::jsonb) WHERE id = %s::uuid """, - (Jsonb(source), doc_uuid), + (Jsonb(source), Jsonb(superseded), doc_uuid), + ) + + +def clear_superseded_doc_ids(workflow_id: str, env: str, doc_ids: list[str]) -> None: + """Forget superseded index keys once their points are deleted from ``env`` (dev/prod).""" + with get_connection() as conn: + doc_uuid = _resolve_document_uuid(conn, workflow_id) + if doc_uuid is None: + return + row = conn.execute( + "SELECT metadata->'superseded_doc_ids' AS sup FROM documents WHERE id = %s::uuid FOR UPDATE", + (doc_uuid,), + ).fetchone() + superseded = {k: list(v) for k, v in (row["sup"] or {}).items()} + remaining = [i for i in superseded.get(env, []) if i not in set(doc_ids)] + if remaining: + superseded[env] = remaining + else: + superseded.pop(env, None) + conn.execute( + "UPDATE documents SET metadata = metadata || jsonb_build_object('superseded_doc_ids', %s::jsonb) " + "WHERE id = %s::uuid", + (Jsonb(superseded), doc_uuid), ) diff --git a/pipeline/models.py b/pipeline/models.py index 07edd53..39af9f9 100644 --- a/pipeline/models.py +++ b/pipeline/models.py @@ -573,6 +573,11 @@ class DocumentMetadata(BaseModel): reindex_required: bool = False reindex_reason: Optional[str] = None chunking: Optional[ChunkingRun] = None + # Vector-index keys (doc_id) of earlier file versions still present in an + # index, by env (``dev`` / ``prod``). A re-upload changes the fingerprint, so + # the old points would otherwise be orphaned; the next ingest/promote and + # disable/purge remove them. + superseded_doc_ids: dict[str, list[str]] = Field(default_factory=dict) @model_validator(mode="after") def _period_is_ordered(self) -> "DocumentMetadata": diff --git a/tests/test_activities.py b/tests/test_activities.py index e10e205..b8c7eba 100644 --- a/tests/test_activities.py +++ b/tests/test_activities.py @@ -946,3 +946,65 @@ def test_every_defined_activity_is_registered_on_the_worker(self): missing = {name for name in defined_activities if name not in worker_source} assert not missing, f"Activities defined but never registered on the worker: {missing}" + + +class TestSupersededPoints: + """After new content is indexed, points of earlier file versions are deleted.""" + + class _Store: + def __init__(self, failing=()): + self.calls = [] + self.failing = set(failing) + + def delete_by_doc_id(self, collection, doc_id): + self.calls.append((collection, doc_id)) + if doc_id in self.failing: + return {"deleted": 0, "doc_id": doc_id, "error": "boom"} + return {"deleted": -1, "doc_id": doc_id} + + @staticmethod + def _document(db): + doc_uuid = db.upsert_document( + workflow_id="wf-sup-act", + document_id="c" * 32, + filename="a.pdf", + filepath="minio://documents/a.pdf", + kind="scheme", + code="sup-act", + ) + db.upsert_document_index_status("wf-sup-act", "documents-index", doc_id="a" * 32, status="indexed") + db.reset_document_for_reupload( + doc_uuid, filename="b.pdf", filepath="minio://documents/b.pdf", fingerprint="c" * 32 + ) + return doc_uuid + + @pytest.mark.asyncio + @pytest.mark.db + @pytest.mark.unit + async def test_old_keys_are_deleted_and_forgotten(self, db_connection): + from pipeline import activities + + doc_uuid = self._document(db_connection) + store = self._Store() + + await activities._drop_superseded_points( + "dev", "wf-sup-act", db_connection.get_document(doc_uuid), "c" * 32, "documents-index", store=store + ) + + assert store.calls == [("documents-index", "a" * 32)] + assert db_connection.get_document(doc_uuid)["superseded_doc_ids"] == {} + + @pytest.mark.asyncio + @pytest.mark.db + @pytest.mark.unit + async def test_a_failed_delete_keeps_the_key_for_the_next_run(self, db_connection): + from pipeline import activities + + doc_uuid = self._document(db_connection) + store = self._Store(failing={"a" * 32}) + + await activities._drop_superseded_points( + "dev", "wf-sup-act", db_connection.get_document(doc_uuid), "c" * 32, "documents-index", store=store + ) + + assert db_connection.get_document(doc_uuid)["superseded_doc_ids"] == {"dev": ["a" * 32]} diff --git a/tests/test_api.py b/tests/test_api.py index 7fbd9c8..eaed4b2 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1185,6 +1185,10 @@ def test_an_existing_code_updates_the_same_document( before["id"], [{"page_number": 1, "original_markdown": "old text"}] ) db_connection.save_chunks(before["id"], [{"chunk_number": 1, "original_text": "old"}]) + # The first upload's run is still "running"; a replace waits for it to settle. + db_connection.update_document_job( + db_connection.get_latest_document_job(before["id"])["id"], status="waiting_review" + ) second = test_client.post( "/upload", @@ -1222,15 +1226,15 @@ def test_the_same_code_in_another_tenant_is_a_different_document( assert conn.execute("SELECT count(*) AS n FROM documents").fetchone()["n"] == 2 assert conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] == 2 - @pytest.mark.api - @pytest.mark.unit - def test_replacing_a_document_needs_access_to_its_tenant(self, db_connection): - from fastapi import HTTPException - - import pipeline.api as api + @staticmethod + def _user(*group_paths): from pipeline.auth.jwt import claims_to_user - db_connection.upsert_document( + return claims_to_user({"sub": "u-" + "-".join(group_paths), "groups": list(group_paths)}) + + @staticmethod + def _mh_doc(db, **extra): + doc_uuid = db.upsert_document( workflow_id="wf-mh-doc", document_id="d-mh", filename="a.pdf", @@ -1238,20 +1242,101 @@ def test_replacing_a_document_needs_access_to_its_tenant(self, db_connection): group="mh", kind="scheme", code="pm-kisan", + **extra, ) - outsider = claims_to_user( - {"sub": "u1", "realm_access": {"roles": ["content_curator"]}, "instances": ["gj"]} + # A new document starts with a "running" workflow; settle it so the + # document is at rest, as one waiting in review is. + db.update_document_job(db.get_latest_document_job(doc_uuid)["id"], status="waiting_review") + return doc_uuid + + @pytest.mark.api + @pytest.mark.unit + def test_replacing_a_document_needs_rights_in_its_group(self, db_connection): + from fastapi import HTTPException + + import pipeline.api as api + + self._mh_doc(db_connection) + identity = lambda user: api._resolve_document_identity( # noqa: E731 + user, "mh", "scheme", "pm-kisan", "a.pdf" ) - insider = claims_to_user( - {"sub": "u2", "realm_access": {"roles": ["content_curator"]}, "instances": ["mh"]} + + # Another state, and a view-only member of this one, are both turned away + # before anything about the document is revealed. + for outsider in (self._user("/states/GJ/contributor"), self._user("/states/MH/view")): + with pytest.raises(HTTPException) as exc: + identity(outsider) + assert exc.value.status_code == 403 + + # The same state with a role that can upload and review may replace. + kind, code, existing = identity(self._user("/states/MH/contributor")) + assert (kind, code, existing["workflow_id"]) == ("scheme", "pm-kisan", "wf-mh-doc") + + @pytest.mark.api + @pytest.mark.unit + def test_upload_into_a_group_needs_upload_rights_there(self, db_connection): + from fastapi import HTTPException + + import pipeline.api as api + + mixed = self._user("/states/MH/contributor", "/states/GJ/view") + # Allowed where the user can upload, refused where they only view. + api._resolve_document_identity(mixed, "mh", "scheme", "", "new scheme.pdf") + with pytest.raises(HTTPException) as exc: + api._resolve_document_identity(mixed, "gj", "scheme", "", "new scheme.pdf") + assert exc.value.status_code == 403 + + @pytest.mark.api + @pytest.mark.unit + def test_a_disabled_or_running_document_cannot_be_replaced(self, db_connection): + from fastapi import HTTPException + + import pipeline.api as api + + db = db_connection + contributor = self._user("/states/MH/contributor") + doc_uuid = self._mh_doc(db) + + db.create_document_job(doc_uuid, "pipeline", "wf-mh-doc", status="running") + with pytest.raises(HTTPException) as exc: + api._resolve_document_identity(contributor, "mh", "scheme", "pm-kisan", "a.pdf") + assert (exc.value.status_code, "in progress" in exc.value.detail) == (409, True) + + db.update_document_job(db.get_latest_document_job(doc_uuid)["id"], status="waiting_review") + db.set_document_disabled("wf-mh-doc", True) + with pytest.raises(HTTPException) as exc: + api._resolve_document_identity(contributor, "mh", "scheme", "pm-kisan", "a.pdf") + assert (exc.value.status_code, "disabled" in exc.value.detail) == (409, True) + + @pytest.mark.api + @pytest.mark.unit + def test_a_document_live_in_prod_needs_approval_rights_to_replace(self, db_connection): + from fastapi import HTTPException + + import pipeline.api as api + + db = db_connection + self._mh_doc(db) + db.upsert_document_index_status( + "wf-mh-doc", "documents-index", doc_id="d-mh", status="indexed" ) + db.update_document_fields("wf-mh-doc") # no-op, keeps the dev entry only + with db.get_connection() as conn: + conn.execute( + "UPDATE documents SET metadata = jsonb_set(metadata, '{index_status,prod}', " + "'{\"status\": \"indexed\", \"doc_id\": \"d-mh\"}')" + ) with pytest.raises(HTTPException) as exc: - api._resolve_document_identity(outsider, "mh", "scheme", "pm-kisan", "a.pdf") - assert exc.value.status_code == 404 + api._resolve_document_identity( + self._user("/states/MH/contributor"), "mh", "scheme", "pm-kisan", "a.pdf" + ) + assert exc.value.status_code == 403 - kind, code, existing = api._resolve_document_identity(insider, "mh", "scheme", "pm-kisan", "a.pdf") - assert (kind, code, existing["workflow_id"]) == ("scheme", "pm-kisan", "wf-mh-doc") + _, _, existing = api._resolve_document_identity( + self._user("/states/MH/approver"), "mh", "scheme", "pm-kisan", "a.pdf" + ) + assert existing["workflow_id"] == "wf-mh-doc" @pytest.mark.api @pytest.mark.unit diff --git a/tests/test_schema_v2.py b/tests/test_schema_v2.py index df13c54..b52add1 100644 --- a/tests/test_schema_v2.py +++ b/tests/test_schema_v2.py @@ -596,3 +596,75 @@ def test_workflow_and_audit_columns_are_plain_text(self, db_connection): enums = conn.execute("SELECT count(*) AS n FROM pg_type WHERE typtype = 'e'").fetchone()["n"] assert len(types) == 5 and set(types.values()) == {"text"} assert enums == 0 + + +class TestSupersededIndexKeys: + """A replaced file changes the index key; the old key must be remembered for cleanup.""" + + @staticmethod + def _indexed(db, workflow_id, envs): + doc_uuid = _doc(db, workflow_id) + old_key = db.get_document(workflow_id)["document_id"] + for env in envs: + index = "prod-index" if env == "prod" else "documents-index" + db.upsert_document_index_status( + workflow_id, index, doc_id=old_key, chunk_count_indexed=1, status="indexed" + ) + return doc_uuid, old_key + + @pytest.mark.db + @pytest.mark.unit + def test_reupload_remembers_the_old_key_for_every_index_that_holds_it( + self, db_connection, monkeypatch + ): + db = db_connection + monkeypatch.setenv("PROD_VECTOR_DB_COLLECTION_NAME", "prod-index") + doc_uuid, old_key = self._indexed(db, "wf-sup", ["dev", "prod"]) + + db.reset_document_for_reupload( + doc_uuid, filename="n.pdf", filepath="minio://documents/n.pdf", fingerprint="b" * 32 + ) + + assert db.get_document(doc_uuid)["superseded_doc_ids"] == {"dev": [old_key], "prod": [old_key]} + + @pytest.mark.db + @pytest.mark.unit + def test_only_indexes_that_hold_the_old_key_are_tracked(self, db_connection): + db = db_connection + doc_uuid, old_key = self._indexed(db, "wf-sup-dev", ["dev"]) + + db.reset_document_for_reupload( + doc_uuid, filename="n.pdf", filepath="minio://documents/n.pdf", fingerprint="c" * 32 + ) + + assert db.get_document(doc_uuid)["superseded_doc_ids"] == {"dev": [old_key]} + + @pytest.mark.db + @pytest.mark.unit + def test_the_same_file_again_supersedes_nothing(self, db_connection): + db = db_connection + doc_uuid, old_key = self._indexed(db, "wf-sup-same", ["dev"]) + + db.reset_document_for_reupload( + doc_uuid, filename="n.pdf", filepath="minio://documents/n.pdf", fingerprint=old_key + ) + + assert db.get_document(doc_uuid)["superseded_doc_ids"] == {} + + @pytest.mark.db + @pytest.mark.unit + def test_two_replacements_keep_both_old_keys_until_cleared(self, db_connection): + db = db_connection + doc_uuid, first_key = self._indexed(db, "wf-sup-two", ["dev"]) + db.reset_document_for_reupload( + doc_uuid, filename="n.pdf", filepath="minio://documents/n.pdf", fingerprint="d" * 32 + ) + # The second version never reached the index, so its key is not tracked. + db.reset_document_for_reupload( + doc_uuid, filename="m.pdf", filepath="minio://documents/m.pdf", fingerprint="e" * 32 + ) + assert db.get_document(doc_uuid)["superseded_doc_ids"] == {"dev": [first_key]} + + db.clear_superseded_doc_ids(doc_uuid, "dev", [first_key]) + + assert db.get_document(doc_uuid)["superseded_doc_ids"] == {} From de4f72b63fdd83e8b8c62558de44b0bf88ecfbb2 Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 11:23:03 +0530 Subject: [PATCH 42/53] fix: never delete a vector key another document still uses The vector index key is the file fingerprint, so two documents holding the same file share points. Replace cleanup, disable and purge now skip any key that another document uses as its fingerprint or index entry doc_id. Replace cleanup forgets such a key without deleting it. A per-document key would remove the sharing but needs a reindex, so it is left for a separate change. --- pipeline/activities.py | 8 ++++++++ pipeline/api/api.py | 12 ++++++++++-- pipeline/db.py | 22 ++++++++++++++++++++++ tests/test_activities.py | 21 +++++++++++++++++++++ tests/test_api.py | 24 ++++++++++++++++++++++++ tests/test_schema_v2.py | 25 +++++++++++++++++++++++++ 6 files changed, 110 insertions(+), 2 deletions(-) diff --git a/pipeline/activities.py b/pipeline/activities.py index d5bed55..9c7ab6c 100644 --- a/pipeline/activities.py +++ b/pipeline/activities.py @@ -1420,6 +1420,14 @@ async def _drop_superseded_points( store = get_vector_store() deleted = [] for doc_id in stale: + # Two documents holding the same file share one index key; keep the + # points while another document still needs them and just forget the key. + if await asyncio.to_thread(db.doc_id_used_by_other_document, doc_id, doc.get("id")): + activity.logger.info( + "Keeping %s points for %s: another document uses the same file", env, doc_id + ) + deleted.append(doc_id) + continue result = await asyncio.to_thread(store.delete_by_doc_id, collection, doc_id) if result.get("error"): activity.logger.warning( diff --git a/pipeline/api/api.py b/pipeline/api/api.py index 9d7b406..778b791 100644 --- a/pipeline/api/api.py +++ b/pipeline/api/api.py @@ -1095,10 +1095,18 @@ def _require_catalog_read(request: Request, user: AuthUser) -> None: def _vector_doc_ids(doc: dict, env: str) -> list[str]: - """The document's current index key plus keys of earlier file versions still in ``env``.""" + """Index keys to delete for a document: its current key plus keys of earlier file versions in ``env``. + + A key another document also uses (the same file uploaded twice) is left + out, since deleting it would remove that document's vectors too. + """ ids = [doc.get("document_id")] ids.extend((doc.get("superseded_doc_ids") or {}).get(env, [])) - return list(dict.fromkeys(i for i in ids if i)) + return [ + i + for i in dict.fromkeys(i for i in ids if i) + if not db.doc_id_used_by_other_document(i, doc.get("id")) + ] def _delete_all_doc_ids(store, collection: str, doc_ids: list[str]) -> dict: diff --git a/pipeline/db.py b/pipeline/db.py index 9bec3a1..ba90737 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -1139,6 +1139,28 @@ def reset_document_for_reupload( ) +def doc_id_used_by_other_document(doc_id: str, exclude_doc_uuid: Optional[str]) -> bool: + """True when a document other than ``exclude_doc_uuid`` uses ``doc_id`` as its index key. + + The index key is the file fingerprint, so two documents holding the same file + share points in the vector index. Deleting a key another document still needs + would remove that document's vectors too. + """ + with get_connection() as conn: + row = conn.execute( + """ + SELECT 1 FROM documents d + WHERE (%s::uuid IS NULL OR d.id <> %s::uuid) + AND (d.source->>'fingerprint' = %s + OR EXISTS (SELECT 1 FROM jsonb_each(COALESCE(d.metadata->'index_status', '{}'::jsonb)) e + WHERE e.value->>'doc_id' = %s)) + LIMIT 1 + """, + (exclude_doc_uuid, exclude_doc_uuid, doc_id, doc_id), + ).fetchone() + return row is not None + + def clear_superseded_doc_ids(workflow_id: str, env: str, doc_ids: list[str]) -> None: """Forget superseded index keys once their points are deleted from ``env`` (dev/prod).""" with get_connection() as conn: diff --git a/tests/test_activities.py b/tests/test_activities.py index b8c7eba..bd1c624 100644 --- a/tests/test_activities.py +++ b/tests/test_activities.py @@ -1008,3 +1008,24 @@ async def test_a_failed_delete_keeps_the_key_for_the_next_run(self, db_connectio ) assert db_connection.get_document(doc_uuid)["superseded_doc_ids"] == {"dev": ["a" * 32]} + + @pytest.mark.asyncio + @pytest.mark.db + @pytest.mark.unit + async def test_a_key_another_document_still_uses_is_kept_in_the_index(self, db_connection): + from pipeline import activities + + doc_uuid = self._document(db_connection) + # A second document holds the very file the first one replaced. + db_connection.upsert_document( + workflow_id="wf-twin", document_id="a" * 32, filename="a.pdf", + filepath="minio://documents/a.pdf", group="gj", kind="scheme", code="twin", + ) + store = self._Store() + + await activities._drop_superseded_points( + "dev", "wf-sup-act", db_connection.get_document(doc_uuid), "c" * 32, "documents-index", store=store + ) + + assert store.calls == [] + assert db_connection.get_document(doc_uuid)["superseded_doc_ids"] == {} diff --git a/tests/test_api.py b/tests/test_api.py index eaed4b2..810611a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1343,3 +1343,27 @@ def test_a_document_live_in_prod_needs_approval_rights_to_replace(self, db_conne def test_the_demo_and_bootstrap_routes_are_gone(self, test_client): assert test_client.post("/documents/anything/demo").status_code in (404, 405) assert test_client.post("/catalog/v1/bootstrap").status_code in (404, 405) + + +class TestVectorKeysToDelete: + @pytest.mark.api + @pytest.mark.unit + def test_disable_never_deletes_a_key_another_document_uses(self, db_connection): + import pipeline.api as api + + db = db_connection + uuid_a = db.upsert_document( + workflow_id="wf-k-a", document_id="a" * 32, filename="a.pdf", + filepath="minio://documents/a.pdf", group="mh", kind="scheme", code="k-a", + ) + db.upsert_document( + workflow_id="wf-k-b", document_id="a" * 32, filename="a.pdf", + filepath="minio://documents/a.pdf", group="gj", kind="scheme", code="k-b", + ) + db.upsert_document( + workflow_id="wf-k-c", document_id="b" * 32, filename="c.pdf", + filepath="minio://documents/c.pdf", group="mh", kind="scheme", code="k-c", + ) + + assert api._vector_doc_ids(db.get_document(uuid_a), "dev") == [] + assert api._vector_doc_ids(db.get_document("wf-k-c"), "dev") == ["b" * 32] diff --git a/tests/test_schema_v2.py b/tests/test_schema_v2.py index b52add1..25e38a9 100644 --- a/tests/test_schema_v2.py +++ b/tests/test_schema_v2.py @@ -668,3 +668,28 @@ def test_two_replacements_keep_both_old_keys_until_cleared(self, db_connection): db.clear_superseded_doc_ids(doc_uuid, "dev", [first_key]) assert db.get_document(doc_uuid)["superseded_doc_ids"] == {} + + +class TestSharedIndexKeys: + """Two documents holding the same file share an index key; neither may delete it for the other.""" + + @pytest.mark.db + @pytest.mark.unit + def test_a_key_is_in_use_while_another_document_has_that_fingerprint(self, db_connection): + db = db_connection + first = _doc(db, "wf-twin-a", document_id="f" * 32) + _doc(db, "wf-twin-b", document_id="f" * 32, group="gj") + + assert db.doc_id_used_by_other_document("f" * 32, first) is True + assert db.doc_id_used_by_other_document("0" * 32, first) is False + + @pytest.mark.db + @pytest.mark.unit + def test_a_key_is_in_use_while_another_documents_index_entry_holds_it(self, db_connection): + db = db_connection + first = _doc(db, "wf-idx-a") + _doc(db, "wf-idx-b") + db.upsert_document_index_status("wf-idx-b", "documents-index", doc_id="9" * 32, status="indexed") + + assert db.doc_id_used_by_other_document("9" * 32, first) is True + assert db.doc_id_used_by_other_document("9" * 32, db.get_document("wf-idx-b")["id"]) is False From 802701275834b14118ec5ef96185856ff66b0524 Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 11:27:27 +0530 Subject: [PATCH 43/53] fix: only live, same-environment owners protect a shared vector key The shared-key guard counted any document with the same fingerprint, including disabled or never-indexed ones, and forgot a skipped key so its points could stay in the index for good. It now counts only another live document that is indexed under the key in the same environment (dev or prod), and a skipped key stays on metadata.superseded_doc_ids so a later ingest, disable or purge can still remove it. --- pipeline/activities.py | 10 ++++---- pipeline/api/api.py | 7 +++--- pipeline/db.py | 15 ++++++------ tests/test_activities.py | 16 +++++++++++-- tests/test_api.py | 9 +++++++- tests/test_schema_v2.py | 50 ++++++++++++++++++++++++++++++++-------- 6 files changed, 79 insertions(+), 28 deletions(-) diff --git a/pipeline/activities.py b/pipeline/activities.py index 9c7ab6c..2dd8d5c 100644 --- a/pipeline/activities.py +++ b/pipeline/activities.py @@ -1420,13 +1420,13 @@ async def _drop_superseded_points( store = get_vector_store() deleted = [] for doc_id in stale: - # Two documents holding the same file share one index key; keep the - # points while another document still needs them and just forget the key. - if await asyncio.to_thread(db.doc_id_used_by_other_document, doc_id, doc.get("id")): + # Two documents holding the same file share one index key. While another + # live document is indexed under it the points stay, and the key stays on + # the list so a later ingest, disable or purge can still remove it. + if await asyncio.to_thread(db.doc_id_used_by_other_document, doc_id, doc.get("id"), env): activity.logger.info( - "Keeping %s points for %s: another document uses the same file", env, doc_id + "Keeping %s points for %s: another document is indexed under the same key", env, doc_id ) - deleted.append(doc_id) continue result = await asyncio.to_thread(store.delete_by_doc_id, collection, doc_id) if result.get("error"): diff --git a/pipeline/api/api.py b/pipeline/api/api.py index 778b791..46feb45 100644 --- a/pipeline/api/api.py +++ b/pipeline/api/api.py @@ -1097,15 +1097,16 @@ def _require_catalog_read(request: Request, user: AuthUser) -> None: def _vector_doc_ids(doc: dict, env: str) -> list[str]: """Index keys to delete for a document: its current key plus keys of earlier file versions in ``env``. - A key another document also uses (the same file uploaded twice) is left - out, since deleting it would remove that document's vectors too. + A key another live document is indexed under in ``env`` (the same file + uploaded twice) is left out, since deleting it would remove that document's + vectors too. """ ids = [doc.get("document_id")] ids.extend((doc.get("superseded_doc_ids") or {}).get(env, [])) return [ i for i in dict.fromkeys(i for i in ids if i) - if not db.doc_id_used_by_other_document(i, doc.get("id")) + if not db.doc_id_used_by_other_document(i, doc.get("id"), env) ] diff --git a/pipeline/db.py b/pipeline/db.py index ba90737..9be8ec9 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -1139,24 +1139,25 @@ def reset_document_for_reupload( ) -def doc_id_used_by_other_document(doc_id: str, exclude_doc_uuid: Optional[str]) -> bool: - """True when a document other than ``exclude_doc_uuid`` uses ``doc_id`` as its index key. +def doc_id_used_by_other_document(doc_id: str, exclude_doc_uuid: Optional[str], env: str) -> bool: + """True when another *live* document is indexed in ``env`` (dev/prod) under ``doc_id``. The index key is the file fingerprint, so two documents holding the same file share points in the vector index. Deleting a key another document still needs - would remove that document's vectors too. + would remove that document's vectors too. Only a document that is not + disabled and whose own ``index_status[env]`` holds this key counts: one that + was merely uploaded, or is disabled, has nothing in the index to protect. """ with get_connection() as conn: row = conn.execute( """ SELECT 1 FROM documents d WHERE (%s::uuid IS NULL OR d.id <> %s::uuid) - AND (d.source->>'fingerprint' = %s - OR EXISTS (SELECT 1 FROM jsonb_each(COALESCE(d.metadata->'index_status', '{}'::jsonb)) e - WHERE e.value->>'doc_id' = %s)) + AND NOT d.is_disabled + AND d.metadata->'index_status'->%s::text->>'doc_id' = %s LIMIT 1 """, - (exclude_doc_uuid, exclude_doc_uuid, doc_id, doc_id), + (exclude_doc_uuid, exclude_doc_uuid, env, doc_id), ).fetchone() return row is not None diff --git a/tests/test_activities.py b/tests/test_activities.py index bd1c624..56aae71 100644 --- a/tests/test_activities.py +++ b/tests/test_activities.py @@ -1012,20 +1012,32 @@ async def test_a_failed_delete_keeps_the_key_for_the_next_run(self, db_connectio @pytest.mark.asyncio @pytest.mark.db @pytest.mark.unit - async def test_a_key_another_document_still_uses_is_kept_in_the_index(self, db_connection): + async def test_a_key_another_live_document_is_indexed_under_is_kept_and_remembered(self, db_connection): from pipeline import activities doc_uuid = self._document(db_connection) - # A second document holds the very file the first one replaced. + # A second document is indexed under the very key the first one replaced. db_connection.upsert_document( workflow_id="wf-twin", document_id="a" * 32, filename="a.pdf", filepath="minio://documents/a.pdf", group="gj", kind="scheme", code="twin", ) + db_connection.upsert_document_index_status( + "wf-twin", "documents-index", doc_id="a" * 32, status="indexed" + ) store = self._Store() await activities._drop_superseded_points( "dev", "wf-sup-act", db_connection.get_document(doc_uuid), "c" * 32, "documents-index", store=store ) + # Points stay for the twin, and the key stays on the list for a later cleanup. assert store.calls == [] + assert db_connection.get_document(doc_uuid)["superseded_doc_ids"] == {"dev": ["a" * 32]} + + # Once the twin is gone the same cleanup removes them. + db_connection.set_document_disabled("wf-twin", True) + await activities._drop_superseded_points( + "dev", "wf-sup-act", db_connection.get_document(doc_uuid), "c" * 32, "documents-index", store=store + ) + assert store.calls == [("documents-index", "a" * 32)] assert db_connection.get_document(doc_uuid)["superseded_doc_ids"] == {} diff --git a/tests/test_api.py b/tests/test_api.py index 810611a..9c02646 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1348,7 +1348,7 @@ def test_the_demo_and_bootstrap_routes_are_gone(self, test_client): class TestVectorKeysToDelete: @pytest.mark.api @pytest.mark.unit - def test_disable_never_deletes_a_key_another_document_uses(self, db_connection): + def test_disable_never_deletes_a_key_another_live_document_is_indexed_under(self, db_connection): import pipeline.api as api db = db_connection @@ -1365,5 +1365,12 @@ def test_disable_never_deletes_a_key_another_document_uses(self, db_connection): filepath="minio://documents/c.pdf", group="mh", kind="scheme", code="k-c", ) + # The twin was only uploaded, never indexed, so nothing of it is at risk. + assert api._vector_doc_ids(db.get_document(uuid_a), "dev") == ["a" * 32] + + db.upsert_document_index_status("wf-k-b", "documents-index", doc_id="a" * 32, status="indexed") assert api._vector_doc_ids(db.get_document(uuid_a), "dev") == [] assert api._vector_doc_ids(db.get_document("wf-k-c"), "dev") == ["b" * 32] + + db.set_document_disabled("wf-k-b", True) + assert api._vector_doc_ids(db.get_document(uuid_a), "dev") == ["a" * 32] diff --git a/tests/test_schema_v2.py b/tests/test_schema_v2.py index 25e38a9..d0ac976 100644 --- a/tests/test_schema_v2.py +++ b/tests/test_schema_v2.py @@ -673,23 +673,53 @@ def test_two_replacements_keep_both_old_keys_until_cleared(self, db_connection): class TestSharedIndexKeys: """Two documents holding the same file share an index key; neither may delete it for the other.""" + @staticmethod + def _indexed_twin(db, workflow_id, key, env="dev", **kwargs): + _doc(db, workflow_id, document_id=key, group="gj", **kwargs) + index = "prod-index" if env == "prod" else "documents-index" + db.upsert_document_index_status(workflow_id, index, doc_id=key, status="indexed") + @pytest.mark.db @pytest.mark.unit - def test_a_key_is_in_use_while_another_document_has_that_fingerprint(self, db_connection): + def test_a_key_is_in_use_while_another_live_document_is_indexed_under_it( + self, db_connection, monkeypatch + ): db = db_connection + monkeypatch.setenv("PROD_VECTOR_DB_COLLECTION_NAME", "prod-index") first = _doc(db, "wf-twin-a", document_id="f" * 32) - _doc(db, "wf-twin-b", document_id="f" * 32, group="gj") + self._indexed_twin(db, "wf-twin-b", "f" * 32, env="dev") + + assert db.doc_id_used_by_other_document("f" * 32, first, "dev") is True + # Another environment is checked on its own index entry. + assert db.doc_id_used_by_other_document("f" * 32, first, "prod") is False + assert db.doc_id_used_by_other_document("0" * 32, first, "dev") is False + + @pytest.mark.db + @pytest.mark.unit + def test_an_uploaded_but_unindexed_twin_does_not_block_deletion(self, db_connection): + db = db_connection + first = _doc(db, "wf-up-a", document_id="e" * 32) + _doc(db, "wf-up-b", document_id="e" * 32, group="gj") # same file, never indexed + + assert db.doc_id_used_by_other_document("e" * 32, first, "dev") is False + + @pytest.mark.db + @pytest.mark.unit + def test_a_disabled_twin_does_not_block_deletion(self, db_connection): + db = db_connection + first = _doc(db, "wf-dis-a", document_id="d" * 32) + self._indexed_twin(db, "wf-dis-b", "d" * 32) + assert db.doc_id_used_by_other_document("d" * 32, first, "dev") is True + + db.set_document_disabled("wf-dis-b", True) - assert db.doc_id_used_by_other_document("f" * 32, first) is True - assert db.doc_id_used_by_other_document("0" * 32, first) is False + assert db.doc_id_used_by_other_document("d" * 32, first, "dev") is False @pytest.mark.db @pytest.mark.unit - def test_a_key_is_in_use_while_another_documents_index_entry_holds_it(self, db_connection): + def test_a_document_never_blocks_its_own_key(self, db_connection): db = db_connection - first = _doc(db, "wf-idx-a") - _doc(db, "wf-idx-b") - db.upsert_document_index_status("wf-idx-b", "documents-index", doc_id="9" * 32, status="indexed") + only = _doc(db, "wf-own", document_id="c" * 32) + db.upsert_document_index_status("wf-own", "documents-index", doc_id="c" * 32, status="indexed") - assert db.doc_id_used_by_other_document("9" * 32, first) is True - assert db.doc_id_used_by_other_document("9" * 32, db.get_document("wf-idx-b")["id"]) is False + assert db.doc_id_used_by_other_document("c" * 32, only, "dev") is False From ff08f2c3ff3608d9aa3ef33312338b023368cabb Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 11:34:08 +0530 Subject: [PATCH 44/53] fix(scripts): use the v2 upload parameters in helper scripts and ENV.md /upload, /documents and /documents/batch now require kind (scheme|advisory) and accept group and code. The upload helpers take DOC_KIND/PIPELINE_GROUP (shell, PowerShell) or --kind/--group (Python), the validity check calls upsert_document and _prepare_records with group/kind/code, list_failed_workflows drops the removed include_demo, and ENV.md drops the bootstrap endpoint and lists the new PATCH body. --- ENV.md | 2 +- scripts/ingest_test_docs.sh | 10 ++++++++-- scripts/list_failed_workflows.py | 2 +- scripts/start_ocr_only_batch.py | 11 ++++++++--- scripts/test_pipeline_e2e.py | 18 +++++++++++++++--- scripts/upload_test_docs.ps1 | 6 +++++- scripts/verify_document_validity_e2e.py | 7 +++++-- 7 files changed, 43 insertions(+), 13 deletions(-) diff --git a/ENV.md b/ENV.md index c8ffacb..e72da63 100644 --- a/ENV.md +++ b/ENV.md @@ -242,7 +242,7 @@ Exposes `/catalog/v1/*` so **bharat-oan-api** and **bharat-provider-backend** ca | `PROD_SCHEME_VECTOR_DB_API_KEY` | `PROD_VECTOR_DB_API_KEY` | Scheme collection API key | | `PROD_SCHEME_VECTOR_DB_COLLECTION_NAME` | `schemes-index` | Must **not** equal documents collection | -**Endpoints:** `GET /catalog/v1/snapshot`, `/version`, `/schemes`, `/tool-prompt`; `POST /catalog/v1/rebuild`, `/bootstrap`; `PATCH /documents/{id}/scheme-metadata`. +**Endpoints:** `GET /catalog/v1/snapshot`, `/version`, `/schemes`, `/tool-prompt`; `POST /catalog/v1/rebuild`; `PATCH /documents/{id}/scheme-metadata` (body: `kind`, `code`, `name`, `aliases`, `tool_routing`, `catalog_visible`, `network_visible`, `valid_from`, `valid_to`). The catalog is computed from documents (one row per document), so there is no bootstrap endpoint. `/upload`, `/documents` and `/documents/batch` take a required `kind` (`scheme` or `advisory`) and an optional `group` and `code`. ### Publish to Network diff --git a/scripts/ingest_test_docs.sh b/scripts/ingest_test_docs.sh index 03f7bf8..3db67cd 100644 --- a/scripts/ingest_test_docs.sh +++ b/scripts/ingest_test_docs.sh @@ -1,9 +1,15 @@ #!/usr/bin/env bash # Ingest all PDFs from ~/test-docs via H100 pipeline API. +# DOC_KIND: scheme (default) or advisory. PIPELINE_GROUP: tenant group code (optional). set -euo pipefail DOCS_DIR="${1:-$HOME/test-docs}" API="${PIPELINE_API:-http://127.0.0.1:8001}" +DOC_KIND="${DOC_KIND:-scheme}" +QUERY="auto_approve=true&kind=${DOC_KIND}" +if [[ -n "${PIPELINE_GROUP:-}" ]]; then + QUERY="${QUERY}&group=${PIPELINE_GROUP}" +fi if [[ ! -d "$DOCS_DIR" ]]; then echo "Missing directory: $DOCS_DIR" @@ -18,7 +24,7 @@ if [[ ${#pdfs[@]} -eq 0 ]]; then fi echo "API: $API" -echo "Files: ${#pdfs[@]}" +echo "Files: ${#pdfs[@]} kind: $DOC_KIND" curl -sf "$API/health" | python3 -m json.tool || { echo "API not reachable"; exit 1; } echo "" @@ -26,7 +32,7 @@ for f in "${pdfs[@]}"; do name="$(basename "$f")" echo "========================================" echo "UPLOAD: $name" - resp="$(curl -s -X POST "$API/upload?auto_approve=true" -F "file=@$f")" + resp="$(curl -s -X POST "$API/upload?$QUERY" -F "file=@$f")" echo "$resp" | python3 -m json.tool || echo "$resp" wf="$(echo "$resp" | python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('workflow_id',''))" 2>/dev/null || true)" if [[ -n "$wf" ]]; then diff --git a/scripts/list_failed_workflows.py b/scripts/list_failed_workflows.py index 1a19271..75bbb12 100755 --- a/scripts/list_failed_workflows.py +++ b/scripts/list_failed_workflows.py @@ -95,7 +95,7 @@ async def get_temporal_error_details(workflow_id: str, client: Client) -> dict: def query_failed_workflows_from_db() -> list: """Query the database for failed workflows (including demo and disabled documents).""" return db.list_documents( - stage="failed", limit=1_000_000, include_demo=True, include_disabled=True + stage="failed", limit=1_000_000, include_disabled=True ) diff --git a/scripts/start_ocr_only_batch.py b/scripts/start_ocr_only_batch.py index 701b0c0..a48fb47 100644 --- a/scripts/start_ocr_only_batch.py +++ b/scripts/start_ocr_only_batch.py @@ -45,8 +45,11 @@ def already_has_ocr(filepath: str, fingerprint: str) -> bool: return bool(db.get_saved_page_numbers(doc["workflow_id"])) -def post_document(api_base: str, filepath: str) -> dict: - query = urllib.parse.urlencode({"stop_after_ocr": "true"}) +def post_document(api_base: str, filepath: str, kind: str, group: str = "") -> dict: + params = {"stop_after_ocr": "true", "kind": kind} + if group: + params["group"] = group + query = urllib.parse.urlencode(params) url = f"{api_base.rstrip('/')}/documents?{query}" payload = json.dumps({"filepath": filepath}).encode("utf-8") req = urllib.request.Request( @@ -64,6 +67,8 @@ def main() -> int: parser.add_argument("directory", help="Directory containing source docs") parser.add_argument("--api-base", default="http://localhost:8001", help="Docs-pipeline API base URL") parser.add_argument("--limit", type=int, default=0, help="Optional max files to submit") + parser.add_argument("--kind", choices=["scheme", "advisory"], default="scheme", help="Document kind (catalog)") + parser.add_argument("--group", default="", help="Tenant group code (optional; defaults to your group)") args = parser.parse_args() directory = Path(args.directory) @@ -89,7 +94,7 @@ def main() -> int: print(f"SKIP {path.name}: OCR already present") continue try: - result = post_document(args.api_base, str(path)) + result = post_document(args.api_base, str(path), args.kind, args.group) submitted += 1 print(f"SUBMIT {path.name}: {result.get('workflow_id')}") except urllib.error.HTTPError as exc: diff --git a/scripts/test_pipeline_e2e.py b/scripts/test_pipeline_e2e.py index 5c77f86..66d8eeb 100644 --- a/scripts/test_pipeline_e2e.py +++ b/scripts/test_pipeline_e2e.py @@ -37,10 +37,17 @@ def check_health(name: str, url: str) -> bool: return False -def upload_pdf(api: str, pdf_path: Path, *, auto_approve: bool, stop_after_ocr: bool) -> str: +def upload_pdf( + api: str, pdf_path: Path, *, auto_approve: bool, stop_after_ocr: bool, kind: str, group: str = "" +) -> str: import subprocess - query = f"auto_approve={'true' if auto_approve else 'false'}&stop_after_ocr={'true' if stop_after_ocr else 'false'}" + query = ( + f"auto_approve={'true' if auto_approve else 'false'}" + f"&stop_after_ocr={'true' if stop_after_ocr else 'false'}&kind={kind}" + ) + if group: + query += f"&group={group}" url = f"{api.rstrip('/')}/upload?{query}" result = subprocess.run( ["curl", "-sf", "-X", "POST", url, "-F", f"file=@{pdf_path}"], @@ -92,6 +99,8 @@ def main() -> int: parser.add_argument("--chandra-health", default=os.environ.get("CHANDRA_HEALTH_URL", "http://127.0.0.1:8010/health")) parser.add_argument("--gemma-health", default=os.environ.get("TRANSLATION_VLLM_BASE_URL", "http://localhost:8020/v1")) parser.add_argument("--mode", choices=["ocr", "full"], default="ocr", help="ocr=stop after OCR; full=entire pipeline") + parser.add_argument("--kind", choices=["scheme", "advisory"], default="scheme", help="Document kind (catalog)") + parser.add_argument("--group", default="", help="Tenant group code (optional; defaults to your group)") parser.add_argument("--timeout", type=int, default=1800, help="Max seconds to wait for workflow") parser.add_argument("--poll-interval", type=float, default=5.0) args = parser.parse_args() @@ -121,7 +130,10 @@ def main() -> int: stop_after_ocr = args.mode == "ocr" print(f"==> Upload ({'OCR only' if stop_after_ocr else 'full pipeline'})") - workflow_id = upload_pdf(args.api, args.pdf, auto_approve=True, stop_after_ocr=stop_after_ocr) + workflow_id = upload_pdf( + args.api, args.pdf, auto_approve=True, stop_after_ocr=stop_after_ocr, + kind=args.kind, group=args.group, + ) print("==> Polling workflow") doc = poll_document(args.api, workflow_id, timeout_seconds=args.timeout, interval=args.poll_interval) diff --git a/scripts/upload_test_docs.ps1 b/scripts/upload_test_docs.ps1 index df3e934..ad80769 100644 --- a/scripts/upload_test_docs.ps1 +++ b/scripts/upload_test_docs.ps1 @@ -2,6 +2,10 @@ # Prerequisite: ssh -L 18001:127.0.0.1:8001 -N docs-pipeline-host $ErrorActionPreference = "Stop" $Api = if ($env:PIPELINE_API) { $env:PIPELINE_API } else { "http://127.0.0.1:18001" } +# DOC_KIND: scheme (default) or advisory. PIPELINE_GROUP: tenant group code (optional). +$Kind = if ($env:DOC_KIND) { $env:DOC_KIND } else { "scheme" } +$Query = "auto_approve=true&kind=$Kind" +if ($env:PIPELINE_GROUP) { $Query = "$Query&group=$($env:PIPELINE_GROUP)" } $DocsDir = Join-Path $PSScriptRoot ".." "test docs" | Resolve-Path Write-Host "API: $Api" @@ -31,7 +35,7 @@ Get-ChildItem -Path $DocsDir -Filter "*.pdf" | ForEach-Object { ) $body = $bodyLines -join $LF $bodyBytes = [System.Text.Encoding]::GetEncoding("iso-8859-1").GetBytes($body) - $resp = Invoke-RestMethod -Uri "$Api/upload?auto_approve=true" -Method Post ` + $resp = Invoke-RestMethod -Uri "$Api/upload?$Query" -Method Post ` -ContentType "multipart/form-data; boundary=$boundary" -Body $bodyBytes $resp | ConvertTo-Json -Depth 5 } diff --git a/scripts/verify_document_validity_e2e.py b/scripts/verify_document_validity_e2e.py index f249d58..3b5d839 100644 --- a/scripts/verify_document_validity_e2e.py +++ b/scripts/verify_document_validity_e2e.py @@ -75,6 +75,9 @@ def make_document(workflow_id, text, valid_from=None, valid_to=None, legacy=Fals stage="chunk_review", valid_from=valid_from, valid_to=valid_to, + group="bv", + kind="scheme", + code=workflow_id, ) if legacy: # A document that predates the columns: NULL, as the migration leaves it. @@ -87,7 +90,7 @@ def make_document(workflow_id, text, valid_from=None, valid_to=None, legacy=Fals filename=f"{workflow_id}.pdf", chunks=chunks, workflow_id=workflow_id, - instance="bv", + group="bv", **kwargs, ) @@ -228,7 +231,7 @@ def search_docs(**kwargs): filename="wf-active.pdf", chunks=[{"chunk_number": 1, "original_text": "Kisan credit card subsidy for active season", "token_count": 8}], workflow_id="wf-active", - instance="bv", + group="bv", **_validity_fields_from_doc(doc), ) store.upsert(COLLECTION, again, batch_size=8) From a2afa820f43f609034e3469fe6d9f50b2db5725b Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 28 Oct 2026 12:56:38 +0530 Subject: [PATCH 45/53] fix(ui): use themed Select for document type on ingest page The native select did not match the console theme or the Select used on the Documents filters. Switch to the shared Select component. --- ui/src/views/NewDocumentView.jsx | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/ui/src/views/NewDocumentView.jsx b/ui/src/views/NewDocumentView.jsx index 770746d..bd8b85b 100644 --- a/ui/src/views/NewDocumentView.jsx +++ b/ui/src/views/NewDocumentView.jsx @@ -3,6 +3,7 @@ import { useNavigate } from 'react-router-dom' import { Button } from '../components/ui/button' import { Badge } from '../components/ui/badge' import { Input } from '../components/ui/input' +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '../components/ui/select' import { Upload, FileText, @@ -242,20 +243,18 @@ export default function NewDocumentView() { - + Required
Date: Wed, 28 Oct 2026 16:07:20 +0530 Subject: [PATCH 46/53] feat: add video type back and make document type read-only after upload Video returns as a third type with a placeholder catalog schema name (openagrinet:VideoResource), offered on the upload page. Custom types stay removed. The type decides which catalog a document lives in and is chosen at upload, so the document page now shows it read-only and no longer sends kind in the metadata PATCH. Tests that used video as the example of an unknown kind now use podcast. --- docs/ADR/0006-groups-and-catalogs-model.md | 2 +- docs/POSTGRES_SCHEMA.md | 4 +-- pipeline/network_constants.py | 2 ++ tests/test_api.py | 2 +- tests/test_network_constants.py | 25 +++++++++++++++++ tests/test_schema_v2.py | 2 +- tests/test_scheme_catalog.py | 11 +++++--- ui/src/views/DocumentOpsView.jsx | 31 ++++++++-------------- ui/src/views/NewDocumentView.jsx | 1 + 9 files changed, 51 insertions(+), 29 deletions(-) create mode 100644 tests/test_network_constants.py diff --git a/docs/ADR/0006-groups-and-catalogs-model.md b/docs/ADR/0006-groups-and-catalogs-model.md index c610a7b..474b8cd 100644 --- a/docs/ADR/0006-groups-and-catalogs-model.md +++ b/docs/ADR/0006-groups-and-catalogs-model.md @@ -4,7 +4,7 @@ Context: the first Postgres schema modelled a tenant as a free-text `documents.i Decision: - A **group** is a provider: one `groups` row per Keycloak tenant node (`/states/MH`, `/global`). Role leaves are not mirrored; permissions still come from the token. `users.group_id` is a single nullable FK for now (first tenant in the token, sorted, with a warning when there are several); a join table can replace it if multi-group membership is needed. -- A **catalog** is `(group, type)`: `catalogs.group_id` plus `catalogs.type`, unique together. `type` holds the network schema name. The kind-to-schema map lives in `pipeline/network_constants.py` and is a placeholder: `scheme` is `openagrinet:AgricultureResource`, `advisory` is `openagrinet:KnowledgeAdvisory`. Code branches through that map, never on raw `openagrinet:*` strings. Catalog Beckn information (`publish_directives`, `visible_networks`, `criteria`, `network`) sits in `catalogs.metadata`. +- A **catalog** is `(group, type)`: `catalogs.group_id` plus `catalogs.type`, unique together. `type` holds the network schema name. The kind-to-schema map lives in `pipeline/network_constants.py` and is a placeholder: `scheme` is `openagrinet:AgricultureResource`, `advisory` is `openagrinet:KnowledgeAdvisory`, `video` is `openagrinet:VideoResource`. Code branches through that map, never on raw `openagrinet:*` strings. Catalog Beckn information (`publish_directives`, `visible_networks`, `criteria`, `network`) sits in `catalogs.metadata`. - A **document** belongs to one catalog and is identified there by `code`: `documents.catalog_id` NOT NULL, `UNIQUE (catalog_id, code)`, with `name` and `aliases` on the document. The API fields `code`, `name` and `aliases` are those columns. `documents.instance`, `document_kind` and `is_demo` are gone; the API returns the derived `group` (the group name) and `kind` (from the type map). The `/catalog/v1/*` snapshot keys, the Qdrant payload keys and Temporal arguments keep their earlier names (`scheme_code`, `document_kind`, `instance`, ...) because other services and in-flight workflows depend on them. - Re-uploading an existing code updates the same document (new workflow, `source` replaced, pages and chunks cleared, `reindex_required` set) rather than adding a second one. - Catalogs are created on upload for the tenant and kind. The upload therefore takes the document type up front. diff --git a/docs/POSTGRES_SCHEMA.md b/docs/POSTGRES_SCHEMA.md index 168fe4c..025f995 100644 --- a/docs/POSTGRES_SCHEMA.md +++ b/docs/POSTGRES_SCHEMA.md @@ -45,7 +45,7 @@ The database does not check these values, so adding one needs no migration. Raw |---|---|---| | 1 | `groups` | Provider tenant nodes (`/states/MH`, `/global`), mirrored from Keycloak | | 2 | `users` | People who upload, review and act, mirrored from Keycloak | -| 3 | `catalogs` | One row per provider group and resource type (scheme, advisory) | +| 3 | `catalogs` | One row per provider group and resource type (scheme, advisory, video) | | 4 | `documents` | One row per document. Content, metadata, source file, index state | | 5 | `workflows` | One row per processing run of a document. Stage, status, config | | 6 | `workflow_artifacts` | Files each workflow produces, stored in MinIO | @@ -148,7 +148,7 @@ Unique: `(group_id, type)`. Pydantic (`CatalogMetadata`) validates the keys; the database only checks for an object. Notes -- `type` stores the schema name, and code asks `pipeline/network_constants.py` what it means (`catalog_type_for_kind`, `kind_for_catalog_type`) instead of comparing `openagrinet:*` strings. The mapping is a placeholder: `scheme` is `openagrinet:AgricultureResource`, `advisory` is `openagrinet:KnowledgeAdvisory` (see ADR 0003 and ADR 0006). +- `type` stores the schema name, and code asks `pipeline/network_constants.py` what it means (`catalog_type_for_kind`, `kind_for_catalog_type`) instead of comparing `openagrinet:*` strings. The mapping is a placeholder: `scheme` is `openagrinet:AgricultureResource`, `advisory` is `openagrinet:KnowledgeAdvisory`, `video` is `openagrinet:VideoResource` (see ADR 0003 and ADR 0006). - Catalogs are created on upload (`get_or_create_catalog`), not seeded. - Computed catalog values (status, chunk count, network visibility, `promoted_at`) come from the `catalog_status` view, not stored here. diff --git a/pipeline/network_constants.py b/pipeline/network_constants.py index fdc4575..c4e6b39 100644 --- a/pipeline/network_constants.py +++ b/pipeline/network_constants.py @@ -42,9 +42,11 @@ # placeholder (ADR 0003 types schemes as KnowledgeAdvisory) - change it here only. KIND_SCHEME = "scheme" KIND_ADVISORY = "advisory" +KIND_VIDEO = "video" KIND_TO_CATALOG_TYPE = { KIND_SCHEME: "openagrinet:AgricultureResource", KIND_ADVISORY: "openagrinet:KnowledgeAdvisory", + KIND_VIDEO: "openagrinet:VideoResource", } CATALOG_TYPE_TO_KIND = {schema: kind for kind, schema in KIND_TO_CATALOG_TYPE.items()} diff --git a/tests/test_api.py b/tests/test_api.py index 9c02646..2ee888a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1097,7 +1097,7 @@ def test_kind_is_required(self, test_client, sample_pdf_content): @pytest.mark.api @pytest.mark.unit - @pytest.mark.parametrize("kind", ["document", "video", "", "Scheme-X"]) + @pytest.mark.parametrize("kind", ["document", "podcast", "", "Scheme-X"]) def test_an_unknown_kind_is_rejected(self, test_client, sample_pdf_content, kind): response = test_client.post( "/upload", diff --git a/tests/test_network_constants.py b/tests/test_network_constants.py new file mode 100644 index 0000000..d4603c0 --- /dev/null +++ b/tests/test_network_constants.py @@ -0,0 +1,25 @@ +import pytest + +from pipeline.network_constants import ( + KIND_TO_CATALOG_TYPE, + catalog_type_for_kind, + kind_for_catalog_type, +) + + +@pytest.mark.parametrize("kind", ["scheme", "advisory", "video"]) +def test_every_kind_round_trips_through_its_catalog_type(kind): + assert kind_for_catalog_type(catalog_type_for_kind(kind)) == kind + + +def test_each_kind_has_its_own_catalog_type(): + assert len(set(KIND_TO_CATALOG_TYPE.values())) == len(KIND_TO_CATALOG_TYPE) + + +def test_video_is_a_known_kind(): + assert catalog_type_for_kind(" Video ") == "openagrinet:VideoResource" + + +def test_unknown_kind_is_rejected(): + with pytest.raises(ValueError): + catalog_type_for_kind("custom") diff --git a/tests/test_schema_v2.py b/tests/test_schema_v2.py index d0ac976..f14a46a 100644 --- a/tests/test_schema_v2.py +++ b/tests/test_schema_v2.py @@ -115,7 +115,7 @@ def test_get_or_create_catalog_is_idempotent_per_group_and_kind(self, db_connect @pytest.mark.unit def test_a_kind_without_a_catalog_type_is_rejected(self, db_connection): with pytest.raises(ValueError): - db_connection.get_or_create_catalog(_group(db_connection), "video") + db_connection.get_or_create_catalog(_group(db_connection), "podcast") @pytest.mark.db @pytest.mark.unit diff --git a/tests/test_scheme_catalog.py b/tests/test_scheme_catalog.py index baa9e70..5e560aa 100644 --- a/tests/test_scheme_catalog.py +++ b/tests/test_scheme_catalog.py @@ -153,7 +153,7 @@ def test_apply_scheme_metadata_keeps_the_code_when_only_the_name_changes(catalog assert result["code_renamed"] is False -def test_kind_is_limited_to_scheme_and_advisory(catalog_db): +def test_kind_is_limited_to_the_known_kinds(catalog_db): db, scheme_catalog = catalog_db _doc(db, "wf-kind", code="kind-doc", stage="ready_for_ingestion") assert db.get_document("wf-kind")["kind"] == "scheme" @@ -162,10 +162,13 @@ def test_kind_is_limited_to_scheme_and_advisory(catalog_db): assert result["document"]["kind"] == "advisory" assert result["document"]["code"] == "kind-doc" # moved catalogs, kept its code + result = scheme_catalog.apply_scheme_metadata("wf-kind", kind="video") + assert result["document"]["kind"] == "video" + result = scheme_catalog.apply_scheme_metadata("wf-kind", kind="scheme") assert result["document"]["kind"] == "scheme" - for bad in ("how_to_faq", "document", "video", "Not Valid!!"): + for bad in ("how_to_faq", "document", "podcast", "Not Valid!!"): with pytest.raises(ValueError): scheme_catalog.apply_scheme_metadata("wf-kind", kind=bad) assert db.get_document("wf-kind")["kind"] == "scheme" @@ -328,11 +331,11 @@ def test_scheme_metadata_api_rejects_an_unknown_kind(asgi_client): from pipeline import db _doc(db, "wf-meta-bad", code="meta-bad", stage="chunk_review") - r = asgi_client.patch("/documents/wf-meta-bad/scheme-metadata", json={"kind": "video"}) + r = asgi_client.patch("/documents/wf-meta-bad/scheme-metadata", json={"kind": "podcast"}) assert r.status_code == 400 -@pytest.mark.parametrize("kind", ["scheme", "advisory"]) +@pytest.mark.parametrize("kind", ["scheme", "advisory", "video"]) def test_reingest_is_allowed_for_every_kind(asgi_client, mock_temporal_client, kind): """No 'never classified' guard any more: a document always has a kind and a code.""" from pipeline import db diff --git a/ui/src/views/DocumentOpsView.jsx b/ui/src/views/DocumentOpsView.jsx index 3c99c29..7b04f75 100644 --- a/ui/src/views/DocumentOpsView.jsx +++ b/ui/src/views/DocumentOpsView.jsx @@ -40,7 +40,6 @@ import { Badge } from '../components/ui/badge' import { Button } from '../components/ui/button' import { Checkbox } from '../components/ui/checkbox' import { Input } from '../components/ui/input' -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '../components/ui/select' import { Skeleton } from '../components/ui/skeleton' import { Tabs, TabsContent, TabsList, TabsTrigger } from '../components/ui/tabs' import { Textarea } from '../components/ui/textarea' @@ -107,13 +106,15 @@ function validateValidity(validFrom, validTo) { return '' } -const DOCUMENT_KIND_OPTIONS = [ - { value: 'scheme', label: 'Scheme' }, - { value: 'advisory', label: 'Advisory' }, -] +// The type is chosen at upload and fixed afterwards: it decides the catalog the +// document lives in, so this panel only displays it. +const DOCUMENT_KIND_LABELS = { + scheme: 'Scheme', + advisory: 'Advisory', + video: 'Video', +} export function DocumentClassificationPanel({ doc, workflowId, canClassify, onSaved }) { - const [kind, setKind] = useState(doc.kind || '') // The display name is documents.name; main's lock rule still applies to it. const savedName = doc.name || filenameStem(doc.filename) const [name, setName] = useState(savedName) @@ -131,7 +132,6 @@ export function DocumentClassificationPanel({ doc, workflowId, canClassify, onSa const nameError = displayNameError(name) const nameChanged = name.trim() !== savedName const canSave = - Boolean(kind) && !nameError && !validityError @@ -144,7 +144,6 @@ export function DocumentClassificationPanel({ doc, workflowId, canClassify, onSa method: 'PATCH', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ - kind, ...(nameChanged ? { name: name.trim() } : {}), // Omitted entirely when the feature is off, so classifying a // document leaves whatever period the upload stamped untouched. @@ -162,8 +161,7 @@ export function DocumentClassificationPanel({ doc, workflowId, canClassify, onSa } const alreadySaved = - doc.kind === kind - && !nameChanged + !nameChanged && (!DOCUMENT_VALIDITY_ENABLED || (doc.valid_from === validFrom && (doc.valid_to || '') === validTo)) @@ -190,16 +188,9 @@ export function DocumentClassificationPanel({ doc, workflowId, canClassify, onSa
Type - + + {DOCUMENT_KIND_LABELS[doc.kind] || doc.kind || '—'} +
{DOCUMENT_VALIDITY_ENABLED && ( <> diff --git a/ui/src/views/NewDocumentView.jsx b/ui/src/views/NewDocumentView.jsx index bd8b85b..41acaed 100644 --- a/ui/src/views/NewDocumentView.jsx +++ b/ui/src/views/NewDocumentView.jsx @@ -35,6 +35,7 @@ const FILE_TYPE_CHIPS = ['PDF', 'Word'] const DOCUMENT_TYPE_OPTIONS = [ { value: 'scheme', label: 'Scheme' }, { value: 'advisory', label: 'Advisory' }, + { value: 'video', label: 'Video' }, ] const PIPELINE_STEPS = [ From 06e763c1a08812b2bf233887efb277b2832c84a2 Mon Sep 17 00:00:00 2001 From: Naveen Date: Thu, 29 Oct 2026 11:18:01 +0530 Subject: [PATCH 47/53] feat: create a group's catalogs with the group and fix the kind after upload A group row now gets one catalog per supported type (scheme, advisory, video) in the same transaction, named " catalog". Startup runs an idempotent sync so a type added to the kind map reaches existing groups. Upload keeps get-or-create as a fallback for a missing catalog. The kind is fixed by the catalog chosen at upload, so it is removed from the scheme-metadata PATCH and from update_document_fields. --- docs/POSTGRES_SCHEMA.md | 4 +- pipeline/api/api.py | 3 +- pipeline/db.py | 77 +++++++++++++++++++------- pipeline/models.py | 3 +- pipeline/scheme_catalog.py | 9 +--- tests/test_api.py | 8 +-- tests/test_db.py | 13 ++--- tests/test_document_repository.py | 16 ++++-- tests/test_schema_v2.py | 90 ++++++++++++++++++++++++------- tests/test_scheme_catalog.py | 33 +++++------- tests/test_tenancy.py | 14 ++--- 11 files changed, 174 insertions(+), 96 deletions(-) diff --git a/docs/POSTGRES_SCHEMA.md b/docs/POSTGRES_SCHEMA.md index 025f995..fbda3fa 100644 --- a/docs/POSTGRES_SCHEMA.md +++ b/docs/POSTGRES_SCHEMA.md @@ -149,7 +149,7 @@ Pydantic (`CatalogMetadata`) validates the keys; the database only checks for an Notes - `type` stores the schema name, and code asks `pipeline/network_constants.py` what it means (`catalog_type_for_kind`, `kind_for_catalog_type`) instead of comparing `openagrinet:*` strings. The mapping is a placeholder: `scheme` is `openagrinet:AgricultureResource`, `advisory` is `openagrinet:KnowledgeAdvisory`, `video` is `openagrinet:VideoResource` (see ADR 0003 and ADR 0006). -- Catalogs are created on upload (`get_or_create_catalog`), not seeded. +- Every group gets one catalog per supported type when the group row is created (`upsert_group` calls `ensure_group_catalogs` in the same transaction). Startup (`init_db`) runs `sync_catalogs_for_all_groups`, so a type added to `KIND_TO_CATALOG_TYPE` reaches existing groups. Upload keeps `get_or_create_catalog` as a fallback for a catalog that is missing. Names are ` catalog`, for example `MH scheme catalog`. - Computed catalog values (status, chunk count, network visibility, `promoted_at`) come from the `catalog_status` view, not stored here. --- @@ -234,7 +234,7 @@ Column default: `'{"index_status": {}, "catalog_visible": true, "network_visible - The API's `group` is derived: `documents.catalog_id → catalogs.group_id → groups.name`. There is no `documents.instance` column, and tenant filters join `groups`. - The API's `document_kind` is derived from `catalogs.type` through the map in `network_constants.py`. `scheme_code`, `scheme_name` and `scheme_aliases` are `code`, `name` and `aliases`. - `chunk_count` and `page_count` are counted from `chunks` and `pages`; they are not stored. -- A new document needs a `document_kind` and a `code`. Upload gets the catalog from the tenant and the kind (created when missing). `DuplicateDocumentCode` is raised when the code is taken in that catalog. +- A new document needs a `document_kind` and a `code`. Upload gets the catalog from the tenant and the kind (seeded with the group; created when missing). A document's kind is fixed after upload, so there is no kind change. `DuplicateDocumentCode` is raised when the code is taken in that catalog. - Re-uploading an existing code updates the same document: a new workflow starts, `source` is replaced, pages and chunks are cleared and `reindex_required` is set (`reset_document_for_reupload`). Earlier workflows and artifacts stay as history. - Changing a document's kind moves it to the same group's catalog of the new type (`update_document_fields(document_kind=...)`). - SQL does not check types inside the JSONB columns. The insert path writes the default metadata keys, and Pydantic validates `source`, `metadata` and each `index_status` entry. diff --git a/pipeline/api/api.py b/pipeline/api/api.py index 46feb45..a50900f 100644 --- a/pipeline/api/api.py +++ b/pipeline/api/api.py @@ -1663,7 +1663,7 @@ async def patch_scheme_metadata( user: RequireReview, ): """ - Set kind / code / name / aliases used by Master Catalog and Qdrant scheme promote. + Set code / name / aliases / validity used by Master Catalog and Qdrant scheme promote. On completed docs, renaming the code sets pending_reindex (excluded from live vector_schemes until re-promote). Other metadata edits bump catalog version only. @@ -1672,7 +1672,6 @@ async def patch_scheme_metadata( try: result = scheme_catalog.apply_scheme_metadata( workflow_id, - kind=body.kind, code=body.code, name=body.name, aliases=body.aliases, diff --git a/pipeline/db.py b/pipeline/db.py index 9be8ec9..0dff900 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -41,7 +41,7 @@ ResultSettings, RetrievalSettings, ) -from .network_constants import catalog_type_for_kind, kind_for_catalog_type +from .network_constants import KIND_TO_CATALOG_TYPE, catalog_type_for_kind, kind_for_catalog_type logger = logging.getLogger(__name__) @@ -103,6 +103,9 @@ def init_db() -> None: current = migrate.current_version(database_url()) logger.info("db migrations current=%s applied_now=%s", current, applied) get_pool() + created = sync_catalogs_for_all_groups() + if created: + logger.info("catalog sync created %d missing catalogs", created) def check_health() -> dict: @@ -440,7 +443,8 @@ def upsert_group(path: str, keycloak_group_id: Optional[str] = None, conn=None) """Create or refresh the ``groups`` row for a Keycloak group path; returns its UUID. Rows made by the SQLite move have no Keycloak id yet; the first sync that - sees the path fills it in. + sees the path fills it in. The group's catalogs (one per supported type) are + ensured in the same transaction, so a group never exists without them. """ if conn is None: with get_connection() as own: @@ -454,7 +458,9 @@ def upsert_group(path: str, keycloak_group_id: Optional[str] = None, conn=None) """, (keycloak_group_id, _tenant_code_from_path(path), path), ).fetchone() - return str(row["id"]) + group_id = str(row["id"]) + ensure_group_catalogs(group_id, conn=conn) + return group_id def get_or_create_group(group: Optional[str], conn=None) -> str: @@ -463,8 +469,54 @@ def get_or_create_group(group: Optional[str], conn=None) -> str: return upsert_group(_tenant_path(code), conn=conn) +def _catalog_seed_rows() -> tuple[list[str], list[str]]: + """Parallel ``(types, kinds)`` lists for every supported catalog type.""" + kinds = list(KIND_TO_CATALOG_TYPE) + return [KIND_TO_CATALOG_TYPE[k] for k in kinds], kinds + + +def ensure_group_catalogs(group_id: str, conn=None) -> None: + """Create the group's missing catalogs, one per supported type; existing rows are untouched.""" + if conn is None: + with get_connection() as own: + return ensure_group_catalogs(group_id, conn=own) + types, kinds = _catalog_seed_rows() + conn.execute( + """ + INSERT INTO catalogs (group_id, type, name) + SELECT g.id, t.type, upper(g.name) || ' ' || t.kind || ' catalog' + FROM groups g, unnest(%s::text[], %s::text[]) AS t(type, kind) + WHERE g.id = %s::uuid + ON CONFLICT (group_id, type) DO NOTHING + """, + (types, kinds, group_id), + ) + + +def sync_catalogs_for_all_groups(conn=None) -> int: + """Add every missing ``(group, type)`` catalog; returns how many rows were created. + + Idempotent. Run at startup so a type added to the kind map reaches groups + that already exist. + """ + if conn is None: + with get_connection() as own: + return sync_catalogs_for_all_groups(conn=own) + types, kinds = _catalog_seed_rows() + cur = conn.execute( + """ + INSERT INTO catalogs (group_id, type, name) + SELECT g.id, t.type, upper(g.name) || ' ' || t.kind || ' catalog' + FROM groups g, unnest(%s::text[], %s::text[]) AS t(type, kind) + ON CONFLICT (group_id, type) DO NOTHING + """, + (types, kinds), + ) + return cur.rowcount + + def get_or_create_catalog(group_id: str, kind: str, conn=None) -> dict: - """The ``(group, kind)`` catalog row, created with a default name when missing.""" + """The ``(group, kind)`` catalog row; created when missing (normally seeded with the group).""" if conn is None: with get_connection() as own: return get_or_create_catalog(group_id, kind, conn=own) @@ -472,7 +524,7 @@ def get_or_create_catalog(group_id: str, kind: str, conn=None) -> dict: row = conn.execute( """ INSERT INTO catalogs (group_id, type, name) - SELECT g.id, %s, g.name || ' ' || %s FROM groups g WHERE g.id = %s::uuid + SELECT g.id, %s, upper(g.name) || ' ' || %s || ' catalog' FROM groups g WHERE g.id = %s::uuid ON CONFLICT (group_id, type) DO UPDATE SET type = EXCLUDED.type RETURNING id, group_id, type, name, metadata """, @@ -1609,7 +1661,7 @@ def update_document_fields(workflow_id: str, **updates: object) -> Optional[dict Flat names are mapped onto the tables: ``stage``/``error_message`` act on the current workflow, ``code``/``name``/``aliases`` on the document columns, - ``kind`` moves the document to the group's catalog of that kind, and flags, + a document's kind is fixed by its catalog at upload, and flags, period and uploader go into ``metadata``. Names derived from other rows (counts, artifact and job pointers, completion timestamps) are accepted and ignored. """ @@ -1688,19 +1740,6 @@ def update_document_fields(workflow_id: str, **updates: object) -> Optional[dict conn=conn, ) - if updates.get("kind"): - current = conn.execute( - "SELECT c.group_id, c.type FROM documents d JOIN catalogs c ON c.id = d.catalog_id " - "WHERE d.id = %s::uuid", - (doc_uuid,), - ).fetchone() - if current["type"] != catalog_type_for_kind(str(updates["kind"])): - target = get_or_create_catalog( - str(current["group_id"]), str(updates["kind"]), conn=conn - ) - doc_sets.append("catalog_id = %s::uuid") - doc_params.append(target["id"]) - if source_patch: doc_sets.append("source = source || %s::jsonb") doc_params.append(Jsonb({k: v for k, v in source_patch.items() if v is not None})) diff --git a/pipeline/models.py b/pipeline/models.py index 39af9f9..79ba659 100644 --- a/pipeline/models.py +++ b/pipeline/models.py @@ -257,9 +257,8 @@ class SchemeMetadataUpdate(BaseModel): period is part of classifying a document, not a separate errand. Omitting both leaves the stored period alone; the dates are validated in `document_validity.parse_period` so a bad period comes back as the same - 400 a bad kind does, not a Pydantic 422. + 400 other bad input does, not a Pydantic 422. """ - kind: Optional[str] = None # scheme | advisory code: Optional[str] = None name: Optional[str] = None aliases: Optional[list[str]] = None diff --git a/pipeline/scheme_catalog.py b/pipeline/scheme_catalog.py index 5657199..937c82f 100644 --- a/pipeline/scheme_catalog.py +++ b/pipeline/scheme_catalog.py @@ -16,7 +16,6 @@ from . import db, document_validity from . import display_name as display_name_rules -from .network_constants import catalog_type_for_kind SCHEME_CODE_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$") @@ -515,7 +514,6 @@ def build_snapshot( def apply_scheme_metadata( workflow_id: str, *, - kind: Optional[str] = None, code: Optional[str] = None, name: Optional[str] = None, aliases: Optional[list[str]] = None, @@ -549,12 +547,7 @@ def apply_scheme_metadata( display_name_rules.ensure_editable(doc) updates["name"] = display_name_rules.normalize(display_name) - if kind is not None: - new_kind = kind.strip().lower() - catalog_type_for_kind(new_kind) # ValueError for a kind with no catalog type - updates["kind"] = new_kind - else: - new_kind = old_kind + new_kind = old_kind # the kind is fixed at upload; it is the document's catalog new_code = old_code if code is not None: diff --git a/tests/test_api.py b/tests/test_api.py index 2ee888a..5350b66 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1138,7 +1138,9 @@ def test_upload_creates_the_catalog_and_derives_the_code_from_the_filename( assert doc["name"] == "National Mission on Natural Farming" with db_connection.get_connection() as conn: catalog = conn.execute( - "SELECT c.type, g.name, g.path FROM catalogs c JOIN groups g ON g.id = c.group_id" + "SELECT c.type, g.name, g.path FROM documents d JOIN catalogs c ON c.id = d.catalog_id " + "JOIN groups g ON g.id = c.group_id WHERE d.code = %s", + (doc["code"],), ).fetchall() assert [(r["type"], r["name"], r["path"]) for r in catalog] == [ ("openagrinet:AgricultureResource", "mh", "/states/MH") @@ -1167,7 +1169,7 @@ def test_a_second_upload_reuses_the_catalog_and_gets_a_fresh_code( } assert codes == {"ca", "ca-2"} with db_connection.get_connection() as conn: - assert conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] == 1 + assert conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] == 3 # seeded with the group @pytest.mark.api @pytest.mark.unit @@ -1224,7 +1226,7 @@ def test_the_same_code_in_another_tenant_is_a_different_document( with db_connection.get_connection() as conn: assert conn.execute("SELECT count(*) AS n FROM documents").fetchone()["n"] == 2 - assert conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] == 2 + assert conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] == 6 # 3 per group @staticmethod def _user(*group_paths): diff --git a/tests/test_db.py b/tests/test_db.py index 2d2d5b6..d98a0e0 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -1100,18 +1100,13 @@ def test_other_kinds_stay_out_of_the_scheme_listing(self, db_connection): @pytest.mark.db @pytest.mark.unit - def test_changing_the_kind_moves_the_document_and_rejects_unknown_kinds(self, db_connection): + def test_the_kind_is_fixed_by_the_catalog_chosen_at_upload(self, db_connection): _new_doc(db_connection, "wf-move", kind="scheme", code="movable", name="Movable") before = db_connection.get_document("wf-move")["catalog_id"] - db_connection.update_document_fields("wf-move", kind="advisory") + db_connection.update_document_fields("wf-move", kind="advisory", name="Renamed") doc = db_connection.get_document("wf-move") - assert (doc["kind"], doc["code"], doc["name"]) == ( - "advisory", "movable", "Movable", - ) - assert doc["catalog_id"] != before - with pytest.raises(ValueError): - db_connection.update_document_fields("wf-move", kind="document") - assert db_connection.get_document("wf-move")["kind"] == "advisory" + assert (doc["kind"], doc["code"], doc["name"]) == ("scheme", "movable", "Renamed") + assert doc["catalog_id"] == before @pytest.mark.db @pytest.mark.unit diff --git a/tests/test_document_repository.py b/tests/test_document_repository.py index 7888493..3335332 100644 --- a/tests/test_document_repository.py +++ b/tests/test_document_repository.py @@ -81,10 +81,18 @@ def test_reads_the_kind_of_the_documents_catalog(self, db_connection, scheme_doc @pytest.mark.unit @pytest.mark.db - def test_reads_a_kind_written_through_the_db_layer(self, db_connection, scheme_document): - db_connection.update_document_fields(scheme_document, kind="advisory") - - assert DocumentRepository().get_document_kind(scheme_document) == "advisory" + def test_reads_a_kind_written_through_the_db_layer(self, db_connection): + db_connection.upsert_document( + workflow_id="repo-wf-adv", + document_id="repo-doc-adv", + filename="adv.pdf", + filepath="/app/books/adv.pdf", + stage="registered", + kind="advisory", + code="repo-advisory", + ) + + assert DocumentRepository().get_document_kind("repo-wf-adv") == "advisory" @pytest.mark.unit @pytest.mark.db diff --git a/tests/test_schema_v2.py b/tests/test_schema_v2.py index f14a46a..d1ce3b2 100644 --- a/tests/test_schema_v2.py +++ b/tests/test_schema_v2.py @@ -91,12 +91,77 @@ def test_deleting_a_group_clears_users_and_audit_snapshots(self, db_connection): "INSERT INTO audit_logs (action_type, entity_type, group_id) VALUES ('x', 'setting', %s::uuid)", (group_id,), ) + conn.execute("DELETE FROM catalogs WHERE group_id = %s::uuid", (group_id,)) conn.execute("DELETE FROM groups WHERE id = %s::uuid", (group_id,)) assert conn.execute("SELECT group_id FROM audit_logs WHERE action_type = 'x'").fetchone()["group_id"] is None assert db.get_user_by_keycloak_id("kc-zz")["group_id"] is None class TestCatalogs: + @pytest.mark.db + @pytest.mark.unit + def test_a_new_group_gets_one_catalog_per_supported_type(self, db_connection): + from pipeline.network_constants import KIND_TO_CATALOG_TYPE + + db = db_connection + mh = _group(db, "mh") + with db.get_connection() as conn: + rows = conn.execute( + "SELECT type, name FROM catalogs WHERE group_id = %s::uuid ORDER BY type", (mh,) + ).fetchall() + assert [r["type"] for r in rows] == sorted(KIND_TO_CATALOG_TYPE.values()) + assert {r["name"] for r in rows} == { + "MH scheme catalog", "MH advisory catalog", "MH video catalog", + } + + @pytest.mark.db + @pytest.mark.unit + def test_upload_uses_the_seeded_catalog_and_creates_none(self, db_connection): + db = db_connection + mh = _group(db, "mh") + with db.get_connection() as conn: + seeded = conn.execute( + "SELECT id FROM catalogs WHERE group_id = %s::uuid AND type = 'openagrinet:AgricultureResource'", + (mh,), + ).fetchone()["id"] + before = conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] + _doc(db, "wf-seeded", group="mh", code="seeded") + assert db.get_document("wf-seeded")["catalog_id"] == str(seeded) + with db.get_connection() as conn: + assert conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] == before + + @pytest.mark.db + @pytest.mark.unit + def test_a_missing_catalog_is_created_on_demand(self, db_connection): + db = db_connection + mh = _group(db, "mh") + with db.get_connection() as conn: + conn.execute( + "DELETE FROM catalogs WHERE group_id = %s::uuid AND type = 'openagrinet:KnowledgeAdvisory'", + (mh,), + ) + _doc(db, "wf-fallback", group="mh", code="fallback", kind="advisory") + assert db.get_document("wf-fallback")["kind"] == "advisory" + + @pytest.mark.db + @pytest.mark.unit + def test_startup_sync_fills_missing_catalogs_for_every_group_and_is_idempotent(self, db_connection): + db = db_connection + mh, up = _group(db, "mh"), _group(db, "up") + with db.get_connection() as conn: + conn.execute("DELETE FROM catalogs WHERE type = 'openagrinet:VideoResource'") + conn.execute( + "DELETE FROM catalogs WHERE group_id = %s::uuid AND type = 'openagrinet:KnowledgeAdvisory'", (up,) + ) + assert db.sync_catalogs_for_all_groups() == 3 + assert db.sync_catalogs_for_all_groups() == 0 + with db.get_connection() as conn: + counts = { + str(r["group_id"]): r["n"] + for r in conn.execute("SELECT group_id, count(*) AS n FROM catalogs GROUP BY group_id") + } + assert counts == {mh: 3, up: 3} + @pytest.mark.db @pytest.mark.unit def test_get_or_create_catalog_is_idempotent_per_group_and_kind(self, db_connection): @@ -105,7 +170,7 @@ def test_get_or_create_catalog_is_idempotent_per_group_and_kind(self, db_connect scheme = db.get_or_create_catalog(mh, "scheme") assert db.get_or_create_catalog(mh, "scheme")["id"] == scheme["id"] assert scheme["type"] == "openagrinet:AgricultureResource" - assert scheme["name"] == "mh scheme" + assert scheme["name"] == "MH scheme catalog" advisory = db.get_or_create_catalog(mh, "advisory") assert advisory["id"] != scheme["id"] assert advisory["type"] == "openagrinet:KnowledgeAdvisory" @@ -240,28 +305,13 @@ def test_find_document_by_code(self, db_connection): @pytest.mark.db @pytest.mark.unit - def test_changing_the_kind_moves_the_document_within_its_group(self, db_connection): + def test_the_kind_does_not_change_after_the_document_exists(self, db_connection): db = db_connection - _doc(db, "wf-kind", group="mh", code="movable") + _doc(db, "wf-kind", group="mh", code="fixed") + before = db.get_document("wf-kind")["catalog_id"] db.update_document_fields("wf-kind", kind="advisory") doc = db.get_document("wf-kind") - assert doc["kind"] == "advisory" - with db.get_connection() as conn: - row = conn.execute( - "SELECT c.group_id, c.type FROM catalogs c WHERE c.id = %s::uuid", (doc["catalog_id"],) - ).fetchone() - assert str(row["group_id"]) == _group(db, "mh") - assert row["type"] == "openagrinet:KnowledgeAdvisory" - - @pytest.mark.db - @pytest.mark.unit - def test_moving_to_a_kind_that_already_holds_the_code_is_a_duplicate(self, db_connection): - db = db_connection - _doc(db, "wf-s", group="mh", code="same") - _doc(db, "wf-a", group="mh", code="same", kind="advisory") - with pytest.raises(db.DuplicateDocumentCode): - db.update_document_fields("wf-s", kind="advisory") - assert db.get_document("wf-s")["kind"] == "scheme" + assert (doc["kind"], doc["catalog_id"]) == ("scheme", before) class TestReupload: diff --git a/tests/test_scheme_catalog.py b/tests/test_scheme_catalog.py index 5e560aa..0ff85ca 100644 --- a/tests/test_scheme_catalog.py +++ b/tests/test_scheme_catalog.py @@ -69,7 +69,6 @@ def test_scheme_metadata_and_promote_rebuild(catalog_db): _doc(db, "wf-pkvy-1", code="pkvy") result = scheme_catalog.apply_scheme_metadata( "wf-pkvy-1", - kind="scheme", code="pkvy", name="Paramparagat Krishi Vikas Yojana", aliases=["PKVY", "organic"], @@ -145,7 +144,6 @@ def test_apply_scheme_metadata_keeps_the_code_when_only_the_name_changes(catalog _doc(db, "wf-name-only", code="gn", stage="ready_for_ingestion") result = scheme_catalog.apply_scheme_metadata( "wf-name-only", - kind="scheme", name="National Mission on Natural Farming", ) assert result["document"]["code"] == "gn" @@ -153,25 +151,14 @@ def test_apply_scheme_metadata_keeps_the_code_when_only_the_name_changes(catalog assert result["code_renamed"] is False -def test_kind_is_limited_to_the_known_kinds(catalog_db): +def test_kind_is_fixed_after_upload(catalog_db): db, scheme_catalog = catalog_db _doc(db, "wf-kind", code="kind-doc", stage="ready_for_ingestion") - assert db.get_document("wf-kind")["kind"] == "scheme" - - result = scheme_catalog.apply_scheme_metadata("wf-kind", kind="advisory") - assert result["document"]["kind"] == "advisory" - assert result["document"]["code"] == "kind-doc" # moved catalogs, kept its code - - result = scheme_catalog.apply_scheme_metadata("wf-kind", kind="video") - assert result["document"]["kind"] == "video" - - result = scheme_catalog.apply_scheme_metadata("wf-kind", kind="scheme") + result = scheme_catalog.apply_scheme_metadata("wf-kind", name="Renamed") assert result["document"]["kind"] == "scheme" - - for bad in ("how_to_faq", "document", "podcast", "Not Valid!!"): - with pytest.raises(ValueError): - scheme_catalog.apply_scheme_metadata("wf-kind", kind=bad) assert db.get_document("wf-kind")["kind"] == "scheme" + with pytest.raises(TypeError): + scheme_catalog.apply_scheme_metadata("wf-kind", kind="advisory") def test_prepare_records_scheme_payload(): @@ -327,12 +314,16 @@ def test_scheme_metadata_api(asgi_client): assert "MIF" in body["aliases"] -def test_scheme_metadata_api_rejects_an_unknown_kind(asgi_client): +def test_scheme_metadata_api_ignores_a_kind_in_the_body(asgi_client): from pipeline import db - _doc(db, "wf-meta-bad", code="meta-bad", stage="chunk_review") - r = asgi_client.patch("/documents/wf-meta-bad/scheme-metadata", json={"kind": "podcast"}) - assert r.status_code == 400 + _doc(db, "wf-meta-kind", code="meta-kind", stage="chunk_review") + r = asgi_client.patch( + "/documents/wf-meta-kind/scheme-metadata", json={"kind": "advisory", "name": "Still a scheme"} + ) + assert r.status_code == 200 + assert r.json()["kind"] == "scheme" + assert r.json()["name"] == "Still a scheme" @pytest.mark.parametrize("kind", ["scheme", "advisory", "video"]) diff --git a/tests/test_tenancy.py b/tests/test_tenancy.py index eb05c23..1a9dc11 100644 --- a/tests/test_tenancy.py +++ b/tests/test_tenancy.py @@ -408,12 +408,14 @@ def test_group_is_reached_through_the_documents_catalog(db_connection): "tenant-a": "/states/TENANT-A", PORTAL_GROUP: "/global" if PORTAL_GROUP == "bv" else f"/states/{PORTAL_GROUP.upper()}", } - # One catalog per (group, type): tenant-a has scheme + advisory, the portal only scheme. - assert sorted((r["group_name"], r["type"]) for r in catalogs) == sorted([ - ("tenant-a", "openagrinet:AgricultureResource"), - ("tenant-a", "openagrinet:KnowledgeAdvisory"), - (PORTAL_GROUP, "openagrinet:AgricultureResource"), - ]) + # Every group is seeded with one catalog per supported type. + from pipeline.network_constants import KIND_TO_CATALOG_TYPE + + assert sorted((r["group_name"], r["type"]) for r in catalogs) == sorted( + (group, catalog_type) + for group in ("tenant-a", PORTAL_GROUP) + for catalog_type in KIND_TO_CATALOG_TYPE.values() + ) assert db.get_document("wf-g-a")["group"] == "tenant-a" assert db.get_document("wf-g-adv")["kind"] == "advisory" assert {d["workflow_id"] for d in db.list_documents(groups=["tenant-a"])} == {"wf-g-a", "wf-g-adv"} From f12e83c962d4c0d8ee719adb5fa61dbeac353243 Mon Sep 17 00:00:00 2001 From: Naveen Date: Thu, 29 Oct 2026 16:50:10 +0530 Subject: [PATCH 48/53] feat: write knowledge kind as Qdrant payload type, drop document_kind type is now scheme/advisory/video (or the kind slug) instead of scheme/document. Existing points need a backfill before readers that filter on type=advisory are deployed. --- pipeline/activities.py | 3 +-- pipeline/advisory_chunks.py | 3 +-- pipeline/vector_store/qdrant_store.py | 3 +-- tests/test_activities.py | 15 ++++++++------- tests/test_advisory_chunks.py | 4 ++-- tests/test_advisory_index.py | 5 +++-- tests/test_provenance.py | 1 + tests/test_scheme_catalog.py | 14 ++++++++++++++ 8 files changed, 31 insertions(+), 17 deletions(-) diff --git a/pipeline/activities.py b/pipeline/activities.py index 2dd8d5c..cd8d861 100644 --- a/pipeline/activities.py +++ b/pipeline/activities.py @@ -702,8 +702,7 @@ def _prepare_records( "instance": resolved_group, # vector payload key, kept for consumers "instance_name": group_name, "provider_id": provider_id, - "type": "scheme" if is_scheme else "document", - "document_kind": kind, + "type": kind, "source": "docs-pipeline", "filename": external_slug, "name_gu": name_gu, diff --git a/pipeline/advisory_chunks.py b/pipeline/advisory_chunks.py index 2cd248d..cf496e2 100644 --- a/pipeline/advisory_chunks.py +++ b/pipeline/advisory_chunks.py @@ -77,8 +77,7 @@ def build_chunks( base = { "doc_id": doc_id, "workflow_id": "", - "type": "document", - "document_kind": "advisory", + "type": "advisory", "source": "webhook", "title_en": title, "slug": clean_value(row.get("slug")), diff --git a/pipeline/vector_store/qdrant_store.py b/pipeline/vector_store/qdrant_store.py index e89abe2..7e1596e 100644 --- a/pipeline/vector_store/qdrant_store.py +++ b/pipeline/vector_store/qdrant_store.py @@ -54,8 +54,7 @@ "scheme_aliases", "chunk_id", "chunk_index", - # Advisory payload (document_kind=advisory; webhook records and PDF advisory documents) - "document_kind", + # Advisory payload (type=advisory; webhook records and PDF advisory documents) "external_id", "pop_id", "slug", diff --git a/tests/test_activities.py b/tests/test_activities.py index 56aae71..6ee1c7e 100644 --- a/tests/test_activities.py +++ b/tests/test_activities.py @@ -173,8 +173,8 @@ def test_prepare_records_carries_instance_name(self): assert records[0]["instance_name"] == expected @pytest.mark.unit - def test_prepare_records_carries_document_kind(self): - """Every record carries its document_kind; only scheme documents get type=scheme.""" + def test_prepare_records_type_carries_kind(self): + """Every record's payload type is its kind; document_kind is not written.""" from pipeline.activities import _prepare_records from pipeline.vector_store.qdrant_store import _record_payload @@ -183,12 +183,13 @@ def test_prepare_records_carries_document_kind(self): advisory = _prepare_records( document_id="test-doc", filename="test.pdf", chunks=chunks, kind="advisory" ) - assert advisory[0]["document_kind"] == "advisory" - assert advisory[0]["type"] == "document" - assert _record_payload(advisory[0])["document_kind"] == "advisory" + assert advisory[0]["type"] == "advisory" + assert "document_kind" not in advisory[0] + assert _record_payload(advisory[0])["type"] == "advisory" + assert "document_kind" not in _record_payload(advisory[0]) default = _prepare_records(document_id="test-doc", filename="test.pdf", chunks=chunks) - assert default[0]["document_kind"] == "document" + assert default[0]["type"] == "document" scheme = _prepare_records( document_id="test-doc", @@ -199,7 +200,7 @@ def test_prepare_records_carries_document_kind(self): name="PM-KISAN", ) assert scheme[0]["type"] == "scheme" - assert scheme[0]["document_kind"] == "scheme" + assert "document_kind" not in scheme[0] @pytest.mark.unit def test_prepare_records_fills_scheme_aliases(self): diff --git a/tests/test_advisory_chunks.py b/tests/test_advisory_chunks.py index 913a805..016d2c6 100644 --- a/tests/test_advisory_chunks.py +++ b/tests/test_advisory_chunks.py @@ -184,8 +184,8 @@ def test_ids_are_stable_and_unique(self): def test_metadata_fields(self): chunk = build_chunks(make_row(), lambda t: 250)[0] - assert chunk["document_kind"] == "advisory" - assert chunk["type"] == "document" + assert "document_kind" not in chunk + assert chunk["type"] == "advisory" assert chunk["source"] == "webhook" assert chunk["workflow_id"] == "" assert chunk["doc_id"] == "row-1" diff --git a/tests/test_advisory_index.py b/tests/test_advisory_index.py index ab71be5..440ee04 100644 --- a/tests/test_advisory_index.py +++ b/tests/test_advisory_index.py @@ -144,9 +144,10 @@ def test_payload_whitelist_keeps_advisory_fields(): chunk = build_chunks(make_row(), _count)[0] payload = _record_payload(chunk) for key in ( - "document_kind", "external_id", "pop_id", "slug", "crop", "domain", "subdomain", + "type", "external_id", "pop_id", "slug", "crop", "domain", "subdomain", "institute", "creator", "problem_type", "problem", "season", "start_period", "end_period", "updated_at", "section", "chunk_id", "text", "doc_id", "title_en", ): assert key in payload, key - assert payload["document_kind"] == "advisory" + assert payload["type"] == "advisory" + assert "document_kind" not in payload diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 5b52532..6e1cd66 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -50,6 +50,7 @@ def test_prepare_records_includes_provenance_fields(): assert record["name_en"] == "Sample Circular" assert record["source"] == "docs-pipeline" assert record["type"] == "document" + assert "document_kind" not in record assert "text" in record assert record["is_reference"] is False diff --git a/tests/test_scheme_catalog.py b/tests/test_scheme_catalog.py index 0ff85ca..9ecda68 100644 --- a/tests/test_scheme_catalog.py +++ b/tests/test_scheme_catalog.py @@ -218,9 +218,23 @@ def test_prepare_records_document_unchanged(): kind="document", ) assert records[0]["type"] == "document" + assert "document_kind" not in records[0] assert "scheme_code" not in records[0] +@pytest.mark.parametrize( + "kind,expected", + [("advisory", "advisory"), ("video", "video"), (" Advisory ", "advisory")], +) +def test_prepare_records_type_equals_kind(kind, expected): + from pipeline.activities import _prepare_records + + chunks = [{"chunk_number": 0, "original_text": "Some text.", "token_count": 3, "is_excluded": False}] + records = _prepare_records("doc3", "x.pdf", chunks, kind=kind) + assert records[0]["type"] == expected + assert "document_kind" not in records[0] + + @pytest.fixture def asgi_client(db_connection, mock_temporal_client, mock_minio_client): """HTTP client that does not run real Temporal/MinIO lifespan.""" From 4d80b943af8a1b3d334bf6543015c19c313e621d Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 4 Nov 2026 10:50:15 +0530 Subject: [PATCH 49/53] fix: adapt rebased kp-migrations to main (group/kind/name keys, routers, tests) Follow-up to the rebase of kp-migrations onto main: port kind/code/group handling into the upload and documents routers, fix stale instance/ document_kind keys in tests, update the route snapshot and auth allowlist for /webhook, and let the migration script into the Docker build context. --- .dockerignore | 1 + pipeline/activities.py | 6 ++-- pipeline/api/api.py | 2 +- pipeline/api/documents.py | 9 ++++-- tests/test_activities.py | 6 ++-- tests/test_api_routes.py | 6 ++-- tests/test_auth.py | 4 +++ tests/test_discovery_publish_service.py | 4 +-- tests/test_display_name.py | 18 +++++------ tests/test_document_repository.py | 2 +- tests/test_network_group.py | 4 +-- tests/test_provider_id_on_chunks.py | 18 +++++------ tests/test_tenancy.py | 43 +++++++++++++++++++++---- 13 files changed, 81 insertions(+), 42 deletions(-) diff --git a/.dockerignore b/.dockerignore index 79adb55..215e875 100644 --- a/.dockerignore +++ b/.dockerignore @@ -67,6 +67,7 @@ test_ocr_output*/ # Scripts (not needed at runtime inside the container) scripts/ +!scripts/migrate_sqlite_to_postgres.py # Claude / AI tooling .claude/ diff --git a/pipeline/activities.py b/pipeline/activities.py index cd8d861..4effc46 100644 --- a/pipeline/activities.py +++ b/pipeline/activities.py @@ -1235,7 +1235,7 @@ async def prepare_for_ingestion( group=(doc or {}).get("group"), **_validity_fields_from_doc(doc), provider_id=_provider_id_from_doc(doc), - display_name=(doc or {}).get("display_name"), + display_name=(doc or {}).get("name"), ) activity.logger.info(f"Prepared {len(records)} records") return records @@ -1347,7 +1347,7 @@ async def promote_document_to_prod_qdrant( **scheme_kwargs, **_validity_fields_from_doc(doc), provider_id=_provider_id_from_doc(doc), - display_name=doc.get("display_name"), + display_name=doc.get("name"), ) activity.logger.info( "Promoting %s records to PROD Qdrant collection %s (kind=%s)", @@ -1460,7 +1460,7 @@ async def ingest_document_from_db( **_scheme_fields_from_doc(doc), **_validity_fields_from_doc(doc), provider_id=_provider_id_from_doc(doc), - display_name=doc.get("display_name"), + display_name=doc.get("name"), ) payload_path = _write_json_temp(records) try: diff --git a/pipeline/api/api.py b/pipeline/api/api.py index a50900f..d42b79b 100644 --- a/pipeline/api/api.py +++ b/pipeline/api/api.py @@ -9,6 +9,7 @@ import json import logging import os +import re import time from contextlib import asynccontextmanager from datetime import datetime @@ -1712,7 +1713,6 @@ async def patch_scheme_metadata( "network_visible": bool(int(doc["network_visible"])) if doc.get("network_visible") is not None else True, "valid_from": doc.get("valid_from"), "valid_to": doc.get("valid_to"), - "display_name": doc.get("display_name"), "catalog_version": result.get("catalog_version"), "requires_reindex": result.get("requires_reindex"), "pending_reindex": result.get("pending_reindex"), diff --git a/pipeline/api/documents.py b/pipeline/api/documents.py index a35607b..aae3ceb 100644 --- a/pipeline/api/documents.py +++ b/pipeline/api/documents.py @@ -58,6 +58,9 @@ async def start_document_workflow( # Validate file path to prevent path traversal attacks filepath = _api.validate_file_path(data.filepath) source_filename = _api.get_filename_from_path(filepath) + doc_kind, doc_code, replace_doc = _api._resolve_document_identity( + user, create_group, kind, code, source_filename + ) source_file_fingerprint = _api._compute_file_fingerprint(filepath) canonical_document_id = source_file_fingerprint @@ -284,7 +287,7 @@ async def list_documents( - offset: Skip first N documents (default 0) """ stage_filter = stage.value if stage else None - include_disabled = x_include_disabled and x_include_disabled.lower() == "true" + include_disabled = bool(x_include_disabled and x_include_disabled.lower() == "true") # Use Postgres only for fast listing - no Temporal queries docs = db.list_documents( @@ -304,7 +307,7 @@ async def get_documents_summary( x_include_disabled: Optional[str] = Header(None, alias="X-Include-Disabled") ): """Return aggregate Postgres counts for dashboard totals and migration planning.""" - include_disabled = x_include_disabled and x_include_disabled.lower() == "true" + include_disabled = bool(x_include_disabled and x_include_disabled.lower() == "true") group_scope = _api._group_scope_for_user(user) summary = db.get_document_summary_counts( include_disabled=include_disabled, @@ -323,7 +326,7 @@ async def get_document_cohorts( x_include_disabled: Optional[str] = Header(None, alias="X-Include-Disabled") ): """Return machine-friendly cohort counts for queueing and orchestration.""" - include_disabled = x_include_disabled and x_include_disabled.lower() == "true" + include_disabled = bool(x_include_disabled and x_include_disabled.lower() == "true") group_scope = _api._group_scope_for_user(user) summary = db.get_document_summary_counts( include_disabled=include_disabled, diff --git a/tests/test_activities.py b/tests/test_activities.py index 6ee1c7e..09d590b 100644 --- a/tests/test_activities.py +++ b/tests/test_activities.py @@ -859,7 +859,7 @@ async def test_passes_the_documents_group(self, monkeypatch, instance, group): seen = {} self._wire( monkeypatch, - {"document_kind": "advisory", "instance": instance}, + {"kind": "advisory", "group": instance}, self._PUBLISHED, seen, ) @@ -883,7 +883,7 @@ async def test_unmapped_kind_records_a_skip(self, monkeypatch): } recorded = self._wire( monkeypatch, - {"document_kind": "document", "instance": "mh"}, + {"kind": "document", "group": "mh"}, skipped, {}, ) @@ -908,7 +908,7 @@ def publish(self, transaction_id, document_kind, workflow_id=None, group=None): raise InvalidProviderIdError("Provider id 'advisory.ladakh_ut.provider.oan.dev' is invalid") monkeypatch.setattr(activities, "DiscoveryPublishService", FakeService) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory", "instance": "ladakh_ut"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory", "group": "ladakh_ut"}) recorded = {} monkeypatch.setattr(db, "add_document_artifact", lambda **kwargs: recorded.update(kwargs)) diff --git a/tests/test_api_routes.py b/tests/test_api_routes.py index 351cc35..e8fc2a3 100644 --- a/tests/test_api_routes.py +++ b/tests/test_api_routes.py @@ -56,6 +56,7 @@ ("GET", "/runs/{job_id}"), ("GET", "/settings/search"), ("GET", "/settings/search/audit"), + ("GET", "/webhook/advisory/status"), ("PATCH", "/documents/{workflow_id}/chunks/{chunk_num}"), ("PATCH", "/documents/{workflow_id}/pages/{page_num}"), ("PATCH", "/documents/{workflow_id}/scheme-metadata"), @@ -66,7 +67,6 @@ ("POST", "/auth/otp/verify"), ("POST", "/auth/session/refresh"), ("POST", "/auth/sso/exchange"), - ("POST", "/catalog/v1/bootstrap"), ("POST", "/catalog/v1/rebuild"), ("POST", "/documents"), ("POST", "/documents/batch"), @@ -82,7 +82,6 @@ ("POST", "/documents/{workflow_id}/approve-translation"), ("POST", "/documents/{workflow_id}/chunks/{chunk_num}/reset"), ("POST", "/documents/{workflow_id}/clear-reindex-required"), - ("POST", "/documents/{workflow_id}/demo"), ("POST", "/documents/{workflow_id}/mark-reindex-required"), ("POST", "/documents/{workflow_id}/pages/{page_num}/reset"), ("POST", "/documents/{workflow_id}/reconcile"), @@ -96,6 +95,9 @@ ("POST", "/search"), ("POST", "/settings/search/reset"), ("POST", "/upload"), + ("POST", "/webhook"), + ("POST", "/webhook/advisory/promote"), + ("POST", "/webhook/advisory/reindex"), ("PUT", "/admin/users/{user_id}/access"), ("PUT", "/settings/search"), ] diff --git a/tests/test_auth.py b/tests/test_auth.py index 5f00285..5f7fdb0 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -200,6 +200,10 @@ def test_every_route_is_gated_or_explicitly_classified(): # credential, and the access token has usually expired by then. ("POST", "/auth/session/refresh"), ("POST", "/auth/otp/refresh"), + # Inbound webhook for external systems: gated by its own x-api-key + # (require_webhook_key, fails closed when WEBHOOK_API_KEY is unset), + # not by a user token. + ("POST", "/webhook"), } def dependency_calls(dependant): calls = {dependant.call} diff --git a/tests/test_discovery_publish_service.py b/tests/test_discovery_publish_service.py index 834f437..06f4522 100644 --- a/tests/test_discovery_publish_service.py +++ b/tests/test_discovery_publish_service.py @@ -465,7 +465,7 @@ def _publish(monkeypatch, kind, group, sender_id="provider.oan.dev"): service = DiscoveryPublishService() client = _client_returning(_on_publish_response("any", "ACCEPTED")) with patch("pipeline.discovery_publish_service.httpx.Client", return_value=client): - result = service.publish(transaction_id="txn", document_kind=kind, group=group) + result = service.publish(transaction_id="txn", kind=kind, group=group) return client.post.call_args.kwargs["json"], result @@ -550,7 +550,7 @@ def test_unmapped_kind_is_skipped_for_a_group(self, monkeypatch): monkeypatch.setenv("NETWORK_SENDER_ID", "provider.oan.dev") service = DiscoveryPublishService() - result = service.publish(transaction_id="txn-doc", document_kind="document", group="ka") + result = service.publish(transaction_id="txn-doc", kind="document", group="ka") assert result["skipped"] is True assert result["envelope"] is None diff --git a/tests/test_display_name.py b/tests/test_display_name.py index 3e3308a..99716ca 100644 --- a/tests/test_display_name.py +++ b/tests/test_display_name.py @@ -127,7 +127,7 @@ def capture(*args, **kwargs): monkeypatch.setattr(activities, "_prepare_records", capture) monkeypatch.setattr(db, "get_chunks", lambda *a, **k: [CHUNK]) monkeypatch.setattr( - db, "get_document", lambda workflow_id: {"display_name": "PM Kisan Guidelines", "instance": "mh"} + db, "get_document", lambda workflow_id: {"name": "PM Kisan Guidelines", "group": "mh"} ) return seen @@ -174,7 +174,7 @@ def test_stores_the_trimmed_display_name(self, test_client, sample_pdf_content): response = test_client.post( "/upload", files={"file": ("scan_0042.pdf", sample_pdf_content, "application/pdf")}, - data={"display_name": " PM Kisan Guidelines "}, + data={"name": " PM Kisan Guidelines "}, ) assert response.status_code == 200 @@ -200,7 +200,7 @@ def test_rejects_an_invalid_display_name(self, test_client, sample_pdf_content, response = test_client.post( "/upload", files={"file": ("bad-name.pdf", sample_pdf_content, "application/pdf")}, - data={"display_name": value}, + data={"name": value}, ) assert response.status_code == 400 @@ -223,7 +223,7 @@ def test_sets_the_display_name_before_ingest(self, test_client, db_connection): response = test_client.patch( "/documents/display-001/scheme-metadata", - json={"display_name": " Cotton Pest Advisory "}, + json={"name": " Cotton Pest Advisory "}, ) assert response.status_code == 200 @@ -235,7 +235,7 @@ def test_sets_the_display_name_before_ingest(self, test_client, db_connection): def test_rejects_a_blank_display_name(self, test_client, db_connection): self._document(db_connection, "display-002") - response = test_client.patch("/documents/display-002/scheme-metadata", json={"display_name": " "}) + response = test_client.patch("/documents/display-002/scheme-metadata", json={"name": " "}) assert response.status_code == 400 @@ -246,7 +246,7 @@ def test_refuses_once_ingest_has_started(self, test_client, db_connection): response = test_client.patch( "/documents/display-003/scheme-metadata", - json={"display_name": "Too Late"}, + json={"name": "Too Late"}, ) assert response.status_code == 409 @@ -259,7 +259,7 @@ def test_other_metadata_still_editable_after_ingest(self, test_client, db_connec response = test_client.patch( "/documents/display-004/scheme-metadata", - json={"document_kind": "advisory"}, + json={"kind": "advisory"}, ) assert response.status_code == 200 @@ -272,7 +272,7 @@ def test_resending_the_unchanged_name_on_a_locked_document_is_accepted(self, tes response = test_client.patch( "/documents/display-005/scheme-metadata", - json={"document_kind": "advisory", "display_name": " Cotton Pest Advisory "}, + json={"kind": "advisory", "name": " Cotton Pest Advisory "}, ) assert response.status_code == 200 @@ -283,6 +283,6 @@ def test_resending_the_unchanged_name_on_a_locked_document_is_accepted(self, tes def test_a_locked_document_reports_the_lock_before_validation(self, test_client, db_connection): self._document(db_connection, "display-006", stage="completed") - response = test_client.patch("/documents/display-006/scheme-metadata", json={"display_name": " "}) + response = test_client.patch("/documents/display-006/scheme-metadata", json={"name": " "}) assert response.status_code == 409 diff --git a/tests/test_document_repository.py b/tests/test_document_repository.py index 3335332..1d9e301 100644 --- a/tests/test_document_repository.py +++ b/tests/test_document_repository.py @@ -112,7 +112,7 @@ def _default_no_group_instances(self, monkeypatch): [("mh", "mh"), ("MH", "mh"), ("bv", None), ("default", None), (None, None)], ) def test_group_comes_from_the_documents_instance(self, instance, group): - repo = DocumentRepository(FakeDb(document={"instance": instance})) + repo = DocumentRepository(FakeDb(document={"group": instance})) assert repo.get_document_group("wf-1") == group diff --git a/tests/test_network_group.py b/tests/test_network_group.py index 9c0d13e..14f17cd 100644 --- a/tests/test_network_group.py +++ b/tests/test_network_group.py @@ -2,7 +2,7 @@ import pytest -from pipeline.instances import INSTANCE_NAMES +from pipeline.group_names import GROUP_NAMES from pipeline.network_group import ( DEFAULT_PROVIDER_CONFIG_OVERRIDE_PATH, PROVIDER_CONFIG_OVERRIDE_PATH_ENV, @@ -212,7 +212,7 @@ def test_id_template_must_render_a_valid_id(self, monkeypatch, template): @pytest.mark.unit def test_every_known_instance_is_checked_at_boot(self, monkeypatch): - monkeypatch.setitem(INSTANCE_NAMES, "ladakh_ut", "Ladakh") + monkeypatch.setitem(GROUP_NAMES, "ladakh_ut", "Ladakh") with pytest.raises(NetworkProviderConfigError, match="provider id"): load_provider_naming() diff --git a/tests/test_provider_id_on_chunks.py b/tests/test_provider_id_on_chunks.py index 7a9ded0..2f31fee 100644 --- a/tests/test_provider_id_on_chunks.py +++ b/tests/test_provider_id_on_chunks.py @@ -44,14 +44,14 @@ class TestProviderIdFromDoc: def test_published_kinds_get_the_provider_id(self, kind, instance, expected): from pipeline.activities import _provider_id_from_doc - assert _provider_id_from_doc({"document_kind": kind, "instance": instance}) == expected + assert _provider_id_from_doc({"kind": kind, "group": instance}) == expected @pytest.mark.unit @pytest.mark.parametrize("kind", ["document", "video", "how_to_faq", None]) def test_kinds_that_never_publish_get_none(self, kind): from pipeline.activities import _provider_id_from_doc - assert _provider_id_from_doc({"document_kind": kind, "instance": "mh"}) is None + assert _provider_id_from_doc({"kind": kind, "group": "mh"}) is None @pytest.mark.unit def test_follows_the_overrides_file(self, monkeypatch, tmp_path): @@ -59,9 +59,9 @@ def test_follows_the_overrides_file(self, monkeypatch, tmp_path): _use_overrides(monkeypatch, tmp_path, '{"ka": "ka.example.com"}') - assert _provider_id_from_doc({"document_kind": "advisory", "instance": "ka"}) == "ka.example.com" - assert _provider_id_from_doc({"document_kind": "scheme", "instance": "ka"}) == "ka.example.com" - assert _provider_id_from_doc({"document_kind": "scheme", "instance": "mh"}) == "mh.provider.oan.dev" + assert _provider_id_from_doc({"kind": "advisory", "group": "ka"}) == "ka.example.com" + assert _provider_id_from_doc({"kind": "scheme", "group": "ka"}) == "ka.example.com" + assert _provider_id_from_doc({"kind": "scheme", "group": "mh"}) == "mh.provider.oan.dev" @pytest.mark.unit def test_no_network_id_means_no_provider_id(self, monkeypatch): @@ -69,14 +69,14 @@ def test_no_network_id_means_no_provider_id(self, monkeypatch): monkeypatch.delenv("NETWORK_SENDER_ID", raising=False) - assert _provider_id_from_doc({"document_kind": "advisory", "instance": "mh"}) is None + assert _provider_id_from_doc({"kind": "advisory", "group": "mh"}) is None @pytest.mark.unit def test_an_invalid_provider_id_fails_without_retry(self): from pipeline.activities import _provider_id_from_doc with pytest.raises(ApplicationError, match="ladakh_ut") as excinfo: - _provider_id_from_doc({"document_kind": "advisory", "instance": "ladakh_ut"}) + _provider_id_from_doc({"kind": "advisory", "group": "ladakh_ut"}) assert excinfo.value.non_retryable is True @@ -101,7 +101,7 @@ def test_matches_the_provider_id_sent_in_publish(self, monkeypatch, tmp_path, ki DiscoveryPublishService().publish("txn", kind, "wf", group_for_instance(instance)) published = client.post.call_args.kwargs["json"]["message"]["catalogs"][0]["provider"]["id"] - assert _provider_id_from_doc({"document_kind": kind, "instance": instance}) == published + assert _provider_id_from_doc({"kind": kind, "group": instance}) == published class TestProviderIdOnRecords: @@ -147,7 +147,7 @@ def capture(*args, **kwargs): monkeypatch.setattr(activities, "_prepare_records", capture) monkeypatch.setattr(db, "get_chunks", lambda *a, **k: [CHUNK]) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"document_kind": "scheme", "instance": "mh"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "scheme", "group": "mh"}) return seen @pytest.mark.unit diff --git a/tests/test_tenancy.py b/tests/test_tenancy.py index 1a9dc11..f93354d 100644 --- a/tests/test_tenancy.py +++ b/tests/test_tenancy.py @@ -380,13 +380,42 @@ def test_api_helpers_hide_cross_tenant_mutations(db_connection, monkeypatch): assert api._document_for_user_or_none("wf-tenant-b-doc", user) is None - PORTAL_INSTANCE, - allowed_instances, - assert_document_instance_access, - assert_instance_access, - group_for_instance, - resolve_create_instance, - user_can_access_instance, +class TestGroupForInstance: + @pytest.fixture(autouse=True) + def _default_instances(self, monkeypatch): + monkeypatch.delenv("PORTAL_INSTANCE", raising=False) + monkeypatch.delenv("DEFAULT_INSTANCE", raising=False) + + @pytest.mark.unit + @pytest.mark.parametrize( + "instance,expected", + [ + ("mh", "mh"), + (" MH ", "mh"), + ("ka", "ka"), + ("bv", None), + ("BV", None), + ("default", None), + ("", None), + (None, None), + ], + ) + def test_states_are_groups_and_platform_instances_are_not(self, instance, expected): + assert group_for_instance(instance) == expected + + @pytest.mark.unit + def test_follows_a_configured_portal_instance(self, monkeypatch): + monkeypatch.setenv("PORTAL_INSTANCE", "portal") + + assert group_for_instance("portal") is None + assert group_for_instance("mh") == "mh" + + @pytest.mark.unit + def test_follows_a_configured_default_instance(self, monkeypatch): + monkeypatch.setenv("DEFAULT_INSTANCE", "local") + + assert group_for_instance("local") is None + assert group_for_instance("mh") == "mh" def test_group_is_reached_through_the_documents_catalog(db_connection): From b4f7abe86ebe3e764c4b9057390f2a6dc628f5b5 Mon Sep 17 00:00:00 2001 From: Naveen Date: Wed, 4 Nov 2026 16:04:50 +0530 Subject: [PATCH 50/53] test: update tests for the api package split and the v2 upload and name flow - patch module state on pipeline.api.api (limiter, temporal_client, _vector_doc_ids, WEBHOOK_MAX_RECORDS), not the pipeline.api package - send kind with uploads, create fixture documents with kind and code, and use the display_name field (stored as documents.name) in the display name tests - UI: document type is read-only after upload and must be chosen before upload --- tests/test_advisory_api.py | 8 ++-- tests/test_api.py | 12 +++--- tests/test_display_name.py | 37 +++++++++++-------- tests/test_webhook.py | 4 +- .../DocumentClassificationPanel.test.jsx | 17 +++++---- ui/src/views/NewDocumentView.test.jsx | 15 +++++++- 6 files changed, 57 insertions(+), 36 deletions(-) diff --git a/tests/test_advisory_api.py b/tests/test_advisory_api.py index b0b7f8c..f045998 100644 --- a/tests/test_advisory_api.py +++ b/tests/test_advisory_api.py @@ -23,7 +23,7 @@ def _rec(content_id, published=1, status="Published"): def client(db_connection, monkeypatch): from fastapi.testclient import TestClient - from pipeline import api + from pipeline.api import api monkeypatch.setattr(api.limiter, "enabled", False) # many posts per test process with patch.dict(os.environ, {"WEBHOOK_API_KEY": TEST_KEY, "AUTH_DISABLED": "true"}): @@ -32,7 +32,7 @@ def client(db_connection, monkeypatch): @pytest.fixture def temporal(monkeypatch): - from pipeline import api + from pipeline.api import api mock = AsyncMock() monkeypatch.setattr(api, "temporal_client", mock) @@ -95,7 +95,7 @@ def test_still_200_when_temporal_fails(self, client, db_connection, temporal): assert len(db_connection.get_unindexed_webhook_record_ids()) == 1 def test_still_200_when_temporal_client_is_none(self, client, db_connection, monkeypatch): - from pipeline import api + from pipeline.api import api monkeypatch.setattr(api, "temporal_client", None) assert client.post("/webhook", json=[_rec(1)], headers=H).status_code == 200 @@ -161,7 +161,7 @@ def test_status_returns_counts(self, client, db_connection, temporal): def test_endpoints_require_a_signed_in_admin(self, db_connection, temporal, monkeypatch): from fastapi.testclient import TestClient - from pipeline import api + from pipeline.api import api env = { "AUTH_DISABLED": "false", diff --git a/tests/test_api.py b/tests/test_api.py index 5350b66..ec690fc 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1075,7 +1075,7 @@ class TestUploadCatalogIdentity: @pytest.fixture(autouse=True) def _fresh_rate_limit(self): # /upload is limited per IP; these tests post more often than one minute allows. - import pipeline.api as api + from pipeline.api import api api.limiter.reset() yield @@ -1256,7 +1256,7 @@ def _mh_doc(db, **extra): def test_replacing_a_document_needs_rights_in_its_group(self, db_connection): from fastapi import HTTPException - import pipeline.api as api + from pipeline.api import api self._mh_doc(db_connection) identity = lambda user: api._resolve_document_identity( # noqa: E731 @@ -1279,7 +1279,7 @@ def test_replacing_a_document_needs_rights_in_its_group(self, db_connection): def test_upload_into_a_group_needs_upload_rights_there(self, db_connection): from fastapi import HTTPException - import pipeline.api as api + from pipeline.api import api mixed = self._user("/states/MH/contributor", "/states/GJ/view") # Allowed where the user can upload, refused where they only view. @@ -1293,7 +1293,7 @@ def test_upload_into_a_group_needs_upload_rights_there(self, db_connection): def test_a_disabled_or_running_document_cannot_be_replaced(self, db_connection): from fastapi import HTTPException - import pipeline.api as api + from pipeline.api import api db = db_connection contributor = self._user("/states/MH/contributor") @@ -1315,7 +1315,7 @@ def test_a_disabled_or_running_document_cannot_be_replaced(self, db_connection): def test_a_document_live_in_prod_needs_approval_rights_to_replace(self, db_connection): from fastapi import HTTPException - import pipeline.api as api + from pipeline.api import api db = db_connection self._mh_doc(db) @@ -1351,7 +1351,7 @@ class TestVectorKeysToDelete: @pytest.mark.api @pytest.mark.unit def test_disable_never_deletes_a_key_another_live_document_is_indexed_under(self, db_connection): - import pipeline.api as api + from pipeline.api import api db = db_connection uuid_a = db.upsert_document( diff --git a/tests/test_display_name.py b/tests/test_display_name.py index 99716ca..14f6c20 100644 --- a/tests/test_display_name.py +++ b/tests/test_display_name.py @@ -174,24 +174,26 @@ def test_stores_the_trimmed_display_name(self, test_client, sample_pdf_content): response = test_client.post( "/upload", files={"file": ("scan_0042.pdf", sample_pdf_content, "application/pdf")}, - data={"name": " PM Kisan Guidelines "}, + params={"kind": "scheme"}, + data={"display_name": " PM Kisan Guidelines "}, ) assert response.status_code == 200 doc = test_client.get(f"/documents/{response.json()['workflow_id']}").json() - assert doc["display_name"] == "PM Kisan Guidelines" + assert doc["name"] == "PM Kisan Guidelines" @pytest.mark.api @pytest.mark.unit - def test_omitting_it_stores_nothing(self, test_client, sample_pdf_content): + def test_omitting_it_defaults_to_the_file_title(self, test_client, sample_pdf_content): response = test_client.post( "/upload", files={"file": ("no-name.pdf", sample_pdf_content + b" ", "application/pdf")}, + params={"kind": "scheme"}, ) assert response.status_code == 200 doc = test_client.get(f"/documents/{response.json()['workflow_id']}").json() - assert doc["display_name"] is None + assert doc["name"] == "no name" @pytest.mark.api @pytest.mark.unit @@ -200,7 +202,8 @@ def test_rejects_an_invalid_display_name(self, test_client, sample_pdf_content, response = test_client.post( "/upload", files={"file": ("bad-name.pdf", sample_pdf_content, "application/pdf")}, - data={"name": value}, + params={"kind": "scheme"}, + data={"display_name": value}, ) assert response.status_code == 400 @@ -214,6 +217,8 @@ def _document(self, db_connection, workflow_id, stage="ready_for_ingestion"): filename="scan_0042.pdf", filepath="/app/books/scan_0042.pdf", stage=stage, + kind="scheme", + code=workflow_id, ) @pytest.mark.api @@ -223,19 +228,19 @@ def test_sets_the_display_name_before_ingest(self, test_client, db_connection): response = test_client.patch( "/documents/display-001/scheme-metadata", - json={"name": " Cotton Pest Advisory "}, + json={"display_name": " Cotton Pest Advisory "}, ) assert response.status_code == 200 - assert response.json()["display_name"] == "Cotton Pest Advisory" - assert db_connection.get_document("display-001")["display_name"] == "Cotton Pest Advisory" + assert response.json()["name"] == "Cotton Pest Advisory" + assert db_connection.get_document("display-001")["name"] == "Cotton Pest Advisory" @pytest.mark.api @pytest.mark.unit def test_rejects_a_blank_display_name(self, test_client, db_connection): self._document(db_connection, "display-002") - response = test_client.patch("/documents/display-002/scheme-metadata", json={"name": " "}) + response = test_client.patch("/documents/display-002/scheme-metadata", json={"display_name": " "}) assert response.status_code == 400 @@ -246,11 +251,11 @@ def test_refuses_once_ingest_has_started(self, test_client, db_connection): response = test_client.patch( "/documents/display-003/scheme-metadata", - json={"name": "Too Late"}, + json={"display_name": "Too Late"}, ) assert response.status_code == 409 - assert db_connection.get_document("display-003")["display_name"] is None + assert db_connection.get_document("display-003")["name"] != "Too Late" @pytest.mark.api @pytest.mark.unit @@ -259,7 +264,7 @@ def test_other_metadata_still_editable_after_ingest(self, test_client, db_connec response = test_client.patch( "/documents/display-004/scheme-metadata", - json={"kind": "advisory"}, + json={"aliases": ["cotton"]}, ) assert response.status_code == 200 @@ -268,21 +273,21 @@ def test_other_metadata_still_editable_after_ingest(self, test_client, db_connec @pytest.mark.unit def test_resending_the_unchanged_name_on_a_locked_document_is_accepted(self, test_client, db_connection): self._document(db_connection, "display-005", stage="ready_for_ingestion") - db_connection.update_document_fields("display-005", display_name="Cotton Pest Advisory", ingested_at="2026-11-01T10:00:00") + db_connection.update_document_fields("display-005", name="Cotton Pest Advisory", ingested_at="2026-11-01T10:00:00") response = test_client.patch( "/documents/display-005/scheme-metadata", - json={"kind": "advisory", "name": " Cotton Pest Advisory "}, + json={"aliases": ["cotton"], "display_name": " Cotton Pest Advisory "}, ) assert response.status_code == 200 - assert db_connection.get_document("display-005")["document_kind"] == "advisory" + assert db_connection.get_document("display-005")["name"] == "Cotton Pest Advisory" @pytest.mark.api @pytest.mark.unit def test_a_locked_document_reports_the_lock_before_validation(self, test_client, db_connection): self._document(db_connection, "display-006", stage="completed") - response = test_client.patch("/documents/display-006/scheme-metadata", json={"name": " "}) + response = test_client.patch("/documents/display-006/scheme-metadata", json={"display_name": " "}) assert response.status_code == 409 diff --git a/tests/test_webhook.py b/tests/test_webhook.py index 47177b8..920e9f3 100644 --- a/tests/test_webhook.py +++ b/tests/test_webhook.py @@ -30,7 +30,7 @@ def test_client(db_connection): """TestClient without lifespan (no Temporal/MinIO needed for /webhook).""" from fastapi.testclient import TestClient - from pipeline import api + from pipeline.api import api return TestClient(api.app) @@ -120,7 +120,7 @@ def test_non_object_item_422(self, test_client, db_connection, key_env): assert r.status_code == 422 def test_too_many_records_413(self, test_client, db_connection, key_env): - from pipeline import api + from pipeline.api import api with patch.object(api, "WEBHOOK_MAX_RECORDS", 1): body = SAMPLE + [{**SAMPLE[0], "contentID": 102}] diff --git a/ui/src/views/DocumentClassificationPanel.test.jsx b/ui/src/views/DocumentClassificationPanel.test.jsx index 1b96351..3dc6270 100644 --- a/ui/src/views/DocumentClassificationPanel.test.jsx +++ b/ui/src/views/DocumentClassificationPanel.test.jsx @@ -53,7 +53,9 @@ describe('DocumentClassificationPanel display name', () => { await user.click(screen.getByRole('button', { name: /^save$/i })) await waitFor(() => expect(onSaved).toHaveBeenCalled()) - expect(patchBody()).toMatchObject({ kind: 'advisory', name: 'Bollworm Control' }) + expect(patchBody()).toMatchObject({ name: 'Bollworm Control' }) + // Document type is read-only after upload, so it is never sent back. + expect(patchBody()).not.toHaveProperty('kind') }) it('blocks saving a blank display name', async () => { @@ -80,14 +82,15 @@ describe('DocumentClassificationPanel display name', () => { expect(screen.getByLabelText(/display name/i)).toBeDisabled() }) - it('does not resend an unchanged display name', async () => { + it('does not offer to save a display name retyped unchanged', async () => { const user = userEvent.setup() - const { onSaved } = renderPanel({ ...baseDoc, kind: 'scheme', name: '' }) - await user.type(screen.getByPlaceholderText(/national mission/i), 'PM Kisan') - await user.click(screen.getByRole('button', { name: /^save$/i })) + renderPanel() + const field = screen.getByLabelText(/display name/i) + await user.clear(field) + await user.type(field, 'Cotton Pest Advisory') - await waitFor(() => expect(onSaved).toHaveBeenCalled()) - expect(patchBody()).not.toHaveProperty('name') + expect(screen.getByRole('button', { name: /saved/i })).toBeDisabled() + expect(fetchJson).not.toHaveBeenCalled() }) it('counts a document without a stored name as already saved', () => { diff --git a/ui/src/views/NewDocumentView.test.jsx b/ui/src/views/NewDocumentView.test.jsx index c3685b7..8e49916 100644 --- a/ui/src/views/NewDocumentView.test.jsx +++ b/ui/src/views/NewDocumentView.test.jsx @@ -1,7 +1,7 @@ import { render, screen, waitFor } from '@testing-library/react' import userEvent from '@testing-library/user-event' import { MemoryRouter } from 'react-router-dom' -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' import NewDocumentView from './NewDocumentView' const apiFetch = vi.fn() @@ -33,6 +33,18 @@ async function pickFile(user, name = 'PM Kisan Guidelines.pdf') { await user.upload(input, new File(['%PDF-1.4'], name, { type: 'application/pdf' })) } +// Radix Select needs these browser APIs, which jsdom lacks. +beforeAll(() => { + window.HTMLElement.prototype.hasPointerCapture ??= () => false + window.HTMLElement.prototype.releasePointerCapture ??= () => {} + window.HTMLElement.prototype.scrollIntoView ??= () => {} +}) + +async function pickKind(user, label = 'Advisory') { + await user.click(screen.getByRole('combobox', { name: /document type/i })) + await user.click(await screen.findByRole('option', { name: label })) +} + describe('NewDocumentView display name', () => { beforeEach(() => { apiFetch.mockReset() @@ -55,6 +67,7 @@ describe('NewDocumentView display name', () => { const user = userEvent.setup() renderView() await pickFile(user) + await pickKind(user) const field = screen.getByLabelText(/display name/i) await user.clear(field) await user.type(field, 'Cotton Pest Advisory') From a0744408aeabb09f97163ec1d3ce83e0b756aad8 Mon Sep 17 00:00:00 2001 From: Ravi Mula Date: Wed, 11 Nov 2026 13:00:19 +0530 Subject: [PATCH 51/53] fix: use instance for knowledge provider ownership --- README.md | 2 +- docs/ADR/0006-groups-and-catalogs-model.md | 2 +- docs/openapi-search.yaml | 28 +++---- pipeline/api/api.py | 22 ++++-- pipeline/api/documents.py | 16 ++-- pipeline/api/search.py | 16 ++-- pipeline/api/upload.py | 10 +-- pipeline/auth/keycloak_admin.py | 4 +- pipeline/db.py | 2 +- pipeline/models.py | 10 +-- scripts/start_ocr_only_batch.py | 10 +-- scripts/test_pipeline_e2e.py | 10 +-- tests/test_api.py | 86 +++++++++++++-------- tests/test_migrate_sqlite_to_postgres.py | 2 +- tests/test_schema_v2.py | 2 +- tests/test_search_group_integration.py | 26 +++---- tests/test_search_provider_integration.py | 4 +- ui/src/components/DocumentHeaderSummary.jsx | 4 +- ui/src/views/DashboardView.jsx | 6 +- ui/src/views/DocumentsView.jsx | 6 +- ui/src/views/GlobalAuditView.jsx | 4 +- ui/src/views/NewDocumentView.jsx | 4 +- ui/src/views/RunsView.jsx | 2 +- 23 files changed, 152 insertions(+), 126 deletions(-) diff --git a/README.md b/README.md index 81d055f..7a78c7b 100644 --- a/README.md +++ b/README.md @@ -518,7 +518,7 @@ Translation data is surfaced through the page model and review endpoints. - `GET /documents/{workflow_id}/qdrant` - `GET /documents/{workflow_id}/qdrant/chunks` - `POST /documents/{workflow_id}/reingest` -- `POST /search` — only answers from documents whose validity period covers today; `valid_on` asks about another day and `include_expired` drops the filter. Optional `group` (e.g. `mh`) limits results to one group's documents; omitted means every group, and the response's `effective_config.group` echoes what was applied (see `docs/openapi-search.yaml`) +- `POST /search` — only answers from documents whose validity period covers today; `valid_on` asks about another day and `include_expired` drops the filter. Optional `instance` (e.g. `mh`) limits results to one instance's documents; omitted means every instance, and the response's `effective_config.instance` echoes what was applied (see `docs/openapi-search.yaml`) - `GET /indexes/summary` - `GET /indexes/{index_name}/settings` - `GET /indexes/{index_name}/stats` diff --git a/docs/ADR/0006-groups-and-catalogs-model.md b/docs/ADR/0006-groups-and-catalogs-model.md index 474b8cd..74e1f53 100644 --- a/docs/ADR/0006-groups-and-catalogs-model.md +++ b/docs/ADR/0006-groups-and-catalogs-model.md @@ -5,7 +5,7 @@ Context: the first Postgres schema modelled a tenant as a free-text `documents.i Decision: - A **group** is a provider: one `groups` row per Keycloak tenant node (`/states/MH`, `/global`). Role leaves are not mirrored; permissions still come from the token. `users.group_id` is a single nullable FK for now (first tenant in the token, sorted, with a warning when there are several); a join table can replace it if multi-group membership is needed. - A **catalog** is `(group, type)`: `catalogs.group_id` plus `catalogs.type`, unique together. `type` holds the network schema name. The kind-to-schema map lives in `pipeline/network_constants.py` and is a placeholder: `scheme` is `openagrinet:AgricultureResource`, `advisory` is `openagrinet:KnowledgeAdvisory`, `video` is `openagrinet:VideoResource`. Code branches through that map, never on raw `openagrinet:*` strings. Catalog Beckn information (`publish_directives`, `visible_networks`, `criteria`, `network`) sits in `catalogs.metadata`. -- A **document** belongs to one catalog and is identified there by `code`: `documents.catalog_id` NOT NULL, `UNIQUE (catalog_id, code)`, with `name` and `aliases` on the document. The API fields `code`, `name` and `aliases` are those columns. `documents.instance`, `document_kind` and `is_demo` are gone; the API returns the derived `group` (the group name) and `kind` (from the type map). The `/catalog/v1/*` snapshot keys, the Qdrant payload keys and Temporal arguments keep their earlier names (`scheme_code`, `document_kind`, `instance`, ...) because other services and in-flight workflows depend on them. +- A **document** belongs to one catalog and is identified there by `code`: `documents.catalog_id` NOT NULL, `UNIQUE (catalog_id, code)`, with `name` and `aliases` on the document. The API fields `code`, `name` and `aliases` are those columns. `documents.instance`, `document_kind` and `is_demo` are gone; the API returns the derived `instance` (the group name) and `kind` (from the type map). The `/catalog/v1/*` snapshot keys, the Qdrant payload keys and Temporal arguments keep their earlier names (`scheme_code`, `document_kind`, `instance`, ...) because other services and in-flight workflows depend on them. - Re-uploading an existing code updates the same document (new workflow, `source` replaced, pages and chunks cleared, `reindex_required` set) rather than adding a second one. - Catalogs are created on upload for the tenant and kind. The upload therefore takes the document type up front. - Bootstrap schemes (the code-seeded entries with no documents) are dropped. `/catalog/v1/*` is served from the `catalog_status` view, one row per document, and a scheme appears once a document exists. diff --git a/docs/openapi-search.yaml b/docs/openapi-search.yaml index 5115ef8..220f079 100644 --- a/docs/openapi-search.yaml +++ b/docs/openapi-search.yaml @@ -52,11 +52,11 @@ paths: top_k: 10 hybrid_alpha: 0.6 rerank_mode: "none" - one_group: - summary: Only one group's documents (e.g. a state's DSS) + one_instance: + summary: Only one instance's documents (e.g. a state's DSS) value: query: "cotton pest advisory" - group: "mh" + instance: "mh" one_provider: summary: Only the chunks one network provider publishes value: @@ -70,7 +70,7 @@ paths: schema: $ref: '#/components/schemas/SearchResponse' '400': - description: Missing/empty query, malformed valid_on, non-string group, non-string provider_id, or the underlying vector-store search failed + description: Missing/empty query, malformed valid_on, non-string instance, non-string provider_id, or the underlying vector-store search failed content: application/json: schema: @@ -263,21 +263,21 @@ components: what search would have returned, or will return, on that date. A malformed value returns 400. Ignored when include_expired is true. default: "(today)" - group: + instance: type: string nullable: true description: | - Limit results to chunks tagged with this group code (for example + Limit results to chunks tagged with this instance code (for example `mh` for Maharashtra, `bv` for Bharat Vistaar). Case-insensitive and trimmed: `MH` and ` mh ` both mean `mh`. One value only — - a caller belongs to one group; fallback to another group is the - caller's second request. Omitted, null or empty means no group - filter: chunks from every group are searched as one pool. An - unknown group is not an error: + a caller belongs to one instance; fallback to another instance is the + caller's second request. Omitted, null or empty means no instance + filter: chunks from every instance are searched as one pool. An + unknown instance is not an error: the response is a success with zero hits. A non-string value returns 400. Applies in every search mode. A filter, not an access check: this endpoint applies no tenant scoping, so any - search-capable caller may ask for any group. + search-capable caller may ask for any instance. example: "mh" provider_id: type: string @@ -289,7 +289,7 @@ components: empty means no provider filter; an unknown id returns zero hits; a non-string value returns 400. Only advisory and scheme chunks carry a provider id, so a provider filter leaves other kinds out. - Combines with `group` and validity. Any other type returns 400. + Combines with `instance` and validity. Any other type returns 400. A filter, not an access check. example: "mh.provider.oan.dev" @@ -424,10 +424,10 @@ components: type: boolean description: True if the query-expansion profile actually rewrote the query. rerank_mode: { type: string } - group: + instance: type: string nullable: true - description: The normalised group filter that was applied (e.g. `mh`), or null when the search spanned every group. + description: The normalised instance filter that was applied (e.g. `mh`), or null when the search spanned every instance. provider_id: type: string nullable: true diff --git a/pipeline/api/api.py b/pipeline/api/api.py index d42b79b..d277584 100644 --- a/pipeline/api/api.py +++ b/pipeline/api/api.py @@ -418,13 +418,15 @@ def _i(key: str, default: int = 0) -> int: "ready_for_ingestion": _i("ready_for_ingestion_documents"), "failed": _i("failed_documents"), }, - "by_group": by_group or [], + "by_instance": [ + {**row, "instance": row["group"]} for row in (by_group or []) + ], } -def _resolve_create_group(user: AuthUser, requested: Optional[str] = None) -> str: - """Pick the document's group from Keycloak-allowed states / portal.""" - return resolve_create_group(user, requested) +def _resolve_upload_instance(user: AuthUser, instance: str) -> str: + """Resolve the upload owner using the existing Keycloak tenancy rules.""" + return resolve_create_group(user, instance) def _document_title(filename: str) -> str: @@ -558,7 +560,7 @@ def _document_summary_from_row(doc: dict, current_job: Optional[dict] = None) -> source_manifest_name=doc.get("source_manifest_name"), source_file_fingerprint=doc.get("source_file_fingerprint"), authoritative=bool(doc.get("source_manifest_name")), - group=normalize_group(doc.get("group")), + instance=normalize_group(doc.get("group")), stage=DocumentStage(doc["stage"]), page_count=doc.get("page_count") or 0, chunk_count=doc.get("chunk_count") or 0, @@ -804,7 +806,7 @@ async def auth_me(user: CurrentUser): "state_roles": dict(user.state_roles or {}), "is_super_admin": bool(user.is_superadmin), # Portal group code for BV / platform docs (not a state). - "portal_group": PORTAL_GROUP, + "portal_instance": PORTAL_GROUP, "auth_disabled": user.token_disabled_mode, } @@ -945,13 +947,17 @@ async def list_runs( Restricted users only see jobs whose document ``group`` is in their allowed states. Super-admin / bypass users see all tenants. """ - return db.list_runs( + runs = db.list_runs( limit=limit, offset=offset, status=status, groups=_group_scope_for_user(user), ) + for run in runs: + run["instance"] = run.pop("group") + return runs + @app.get("/runs/{job_id}") async def get_run(job_id: str, user: RequireSearch): @@ -1010,7 +1016,7 @@ def _build_document_detail(doc: dict) -> DocumentDetail: source_manifest_name=doc.get("source_manifest_name"), source_file_fingerprint=doc.get("source_file_fingerprint"), authoritative=bool(doc.get("source_manifest_name")), - group=normalize_group(doc.get("group")), + instance=normalize_group(doc.get("group")), filepath=doc["filepath"], stage=DocumentStage(doc["stage"]), page_count=doc.get("page_count", 0), diff --git a/pipeline/api/documents.py b/pipeline/api/documents.py index aae3ceb..0432ebf 100644 --- a/pipeline/api/documents.py +++ b/pipeline/api/documents.py @@ -36,7 +36,7 @@ async def start_document_workflow( min_tokens: int = 100, index_name: str = "documents-index", stop_after_ocr: bool = False, - group: str = "", + instance: str = "", kind: str = Query(..., description="scheme or advisory"), code: str = "", ): @@ -54,7 +54,7 @@ async def start_document_workflow( Rate limited to 10 requests/minute per IP. Requires permission: upload (no-op while AUTH_DISABLED=true). """ - create_group = _api._resolve_create_group(user, group) + create_group = _api._resolve_upload_instance(user, instance) # Validate file path to prevent path traversal attacks filepath = _api.validate_file_path(data.filepath) source_filename = _api.get_filename_from_path(filepath) @@ -85,7 +85,7 @@ async def start_document_workflow( source_filename=source_filename, source_file_fingerprint=source_file_fingerprint, authoritative=bool(existing_doc.get("source_manifest_name")) if existing_doc else False, - group=normalize_group(existing_doc.get("group")), + instance=normalize_group(existing_doc.get("group")), stage=DocumentStage(state.get("stage", "registered")), page_count=state.get("page_count", 0), chunk_count=state.get("chunk_count", 0), @@ -182,7 +182,7 @@ async def start_document_workflow( new_value=source_filename, metadata={ "source": "register_endpoint", - "group": create_group, + "instance": create_group, "kind": doc_kind, "code": doc_code, "replaced_existing": bool(replace_doc), @@ -198,7 +198,7 @@ async def start_document_workflow( source_filename=source_filename, source_file_fingerprint=source_file_fingerprint, authoritative=False, - group=create_group, + instance=create_group, stage=DocumentStage.REGISTERED, page_count=0, chunk_count=0, @@ -220,11 +220,11 @@ async def start_batch_workflows( chunk_overlap: int = 128, min_tokens: int = 100, stop_after_ocr: bool = False, - group: str = "", + instance: str = "", kind: str = Query(..., description="scheme or advisory"), ): """Start workflows for all supported documents in a directory.""" - create_group = _api._resolve_create_group(user, group) + create_group = _api._resolve_upload_instance(user, instance) directory = Path(data.directory) if not directory.exists(): raise HTTPException(404, f"Directory not found: {data.directory}") @@ -245,7 +245,7 @@ async def start_batch_workflows( chunk_overlap=chunk_overlap, min_tokens=min_tokens, stop_after_ocr=stop_after_ocr, - group=create_group, + instance=create_group, kind=kind, code="", ) diff --git a/pipeline/api/search.py b/pipeline/api/search.py index 70beb7c..a621082 100644 --- a/pipeline/api/search.py +++ b/pipeline/api/search.py @@ -170,10 +170,10 @@ def _rerank_hits(query: str, hits: list[dict], rerank_mode: str) -> list[dict]: -def _requested_group(payload: dict) -> str | None: - raw = payload.get("group") +def _requested_instance(payload: dict) -> str | None: + raw = payload.get("instance") if raw is not None and not isinstance(raw, str): - raise HTTPException(400, "group must be a string") + raise HTTPException(400, "instance must be a string") return (raw or "").strip().lower() or None @@ -188,9 +188,9 @@ def _requested_provider_id(payload: dict) -> str | None: async def run_search(payload: dict, user: RequireSearch): """Search the vector index. - `group` is a filter, not an access check: this endpoint applies no tenant + `instance` is a filter, not an access check: this endpoint applies no tenant scoping to its results (unlike `GET /chunks/search`), so any - search-capable caller may ask for any group. + search-capable caller may ask for any instance. """ from ..vector_store import get_default_index_name, get_vector_store @@ -247,7 +247,7 @@ async def run_search(payload: dict, user: RequireSearch): valid_on = document_validity.parse_day(valid_on, "valid_on") except document_validity.DocumentValidityError as exc: raise HTTPException(400, str(exc)) from None - group = _requested_group(payload) + instance = _requested_instance(payload) provider_id = _requested_provider_id(payload) expanded_query = _expand_query(query, query_expansion_profile) # Qdrant embeddings apply E5 prefixes internally; don't pre-prefix here. @@ -270,7 +270,7 @@ async def run_search(payload: dict, user: RequireSearch): ef_search=ef_search, apply_validity=not include_expired, valid_on=valid_on, - instance=group, + instance=instance, provider_id=provider_id, ) except Exception as error: @@ -320,7 +320,7 @@ async def run_search(payload: dict, user: RequireSearch): "rerank_mode": rerank_mode, "apply_validity": not include_expired, "valid_on": result.get("valid_on"), - "group": group, + "instance": instance, "provider_id": provider_id, "filter_string": None, }, diff --git a/pipeline/api/upload.py b/pipeline/api/upload.py index f4fb5c7..dd62f41 100644 --- a/pipeline/api/upload.py +++ b/pipeline/api/upload.py @@ -31,7 +31,7 @@ async def upload_and_process( min_tokens: int = 100, index_name: str = "documents-index", stop_after_ocr: bool = False, - group: str = "", + instance: str = "", kind: str = Query(..., description="scheme or advisory"), code: str = "", display_name: Optional[str] = Form(None), @@ -44,7 +44,7 @@ async def upload_and_process( Rate limited to 10 requests/minute per IP. Requires permission: upload (no-op while AUTH_DISABLED=true). """ - create_group = _api._resolve_create_group(user, group) + create_group = _api._resolve_upload_instance(user, instance) doc_kind, doc_code, replace_doc = _api._resolve_document_identity( user, create_group, kind, code, file.filename or "" ) @@ -104,7 +104,7 @@ async def upload_and_process( source_filename=file.filename, source_file_fingerprint=file_hash, authoritative=bool(existing_doc.get("source_manifest_name")) if existing_doc else False, - group=normalize_group(existing_doc.get("group")), + instance=normalize_group(existing_doc.get("group")), stage=DocumentStage(state.get("stage", "registered")), page_count=state.get("page_count", 0), chunk_count=state.get("chunk_count", 0), @@ -211,7 +211,7 @@ async def upload_and_process( new_value=file.filename, metadata={ "source": "upload_endpoint", - "group": create_group, + "instance": create_group, "kind": doc_kind, "code": doc_code, "replaced_existing": bool(replace_doc), @@ -237,7 +237,7 @@ async def upload_and_process( source_filename=file.filename, source_file_fingerprint=file_hash, authoritative=False, - group=create_group, + instance=create_group, stage=DocumentStage.REGISTERED, page_count=0, chunk_count=0, diff --git a/pipeline/auth/keycloak_admin.py b/pipeline/auth/keycloak_admin.py index f4b1a2c..d1bbdc8 100644 --- a/pipeline/auth/keycloak_admin.py +++ b/pipeline/auth/keycloak_admin.py @@ -506,7 +506,7 @@ def provision_user( "roles": [r.get("name") for r in (roles or []) if isinstance(r, dict)], "app_url": ui_url, "login_url": login_url, - "portal_group": PORTAL_GROUP, + "portal_instance": PORTAL_GROUP, "share_message": "\n".join(share_lines), } @@ -517,7 +517,7 @@ def list_access_options() -> dict[str, Any]: return { "keycloak_admin_configured": cfg.configured, "realm": cfg.realm, - "portal_group": PORTAL_GROUP, + "portal_instance": PORTAL_GROUP, "access_types": [ { "id": "super_admin", diff --git a/pipeline/db.py b/pipeline/db.py index 0dff900..f8d9b78 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -2612,7 +2612,7 @@ def _audit_row_to_legacy(row: dict) -> dict: "actor_roles": ",".join(roles) if roles else None, "filename": row["filename"], "name": row["document_name"], - "group": row["doc_group"] or row["audit_group"], + "instance": row["doc_group"] or row["audit_group"], "uploaded_by_username": row["uploaded_by_username"], "uploaded_by_email": row["uploaded_by_email"], } diff --git a/pipeline/models.py b/pipeline/models.py index 79ba659..011b101 100644 --- a/pipeline/models.py +++ b/pipeline/models.py @@ -218,7 +218,7 @@ class DocumentSummary(BaseModel): source_manifest_name: Optional[str] = None source_file_fingerprint: Optional[str] = None authoritative: bool = False - group: str = "default" # tenant code (groups.name) of the document's catalog + instance: str = "default" # tenant code of the document's catalog stage: DocumentStage page_count: int chunk_count: int @@ -352,8 +352,8 @@ class DocumentGraph(BaseModel): runtime: dict[str, Any] -class GroupDocumentCount(BaseModel): - group: str +class InstanceDocumentCount(BaseModel): + instance: str count: int success: int = 0 failed: int = 0 @@ -368,7 +368,7 @@ class DocumentCohortsResponse(BaseModel): review_queue: int failed_documents: int by_stage: dict[str, int] - by_group: list[GroupDocumentCount] = [] + by_instance: list[InstanceDocumentCount] = [] needs_reindex: int running_jobs: int = 0 @@ -445,7 +445,7 @@ class AuditLogEntry(BaseModel): # strips them from the response and the UI can only show a raw doc_id hash. filename: Optional[str] = None name: Optional[str] = None - group: Optional[str] = None + instance: Optional[str] = None uploaded_by_username: Optional[str] = None uploaded_by_email: Optional[str] = None diff --git a/scripts/start_ocr_only_batch.py b/scripts/start_ocr_only_batch.py index a48fb47..234fa61 100644 --- a/scripts/start_ocr_only_batch.py +++ b/scripts/start_ocr_only_batch.py @@ -45,10 +45,10 @@ def already_has_ocr(filepath: str, fingerprint: str) -> bool: return bool(db.get_saved_page_numbers(doc["workflow_id"])) -def post_document(api_base: str, filepath: str, kind: str, group: str = "") -> dict: +def post_document(api_base: str, filepath: str, kind: str, instance: str = "") -> dict: params = {"stop_after_ocr": "true", "kind": kind} - if group: - params["group"] = group + if instance: + params["instance"] = instance query = urllib.parse.urlencode(params) url = f"{api_base.rstrip('/')}/documents?{query}" payload = json.dumps({"filepath": filepath}).encode("utf-8") @@ -68,7 +68,7 @@ def main() -> int: parser.add_argument("--api-base", default="http://localhost:8001", help="Docs-pipeline API base URL") parser.add_argument("--limit", type=int, default=0, help="Optional max files to submit") parser.add_argument("--kind", choices=["scheme", "advisory"], default="scheme", help="Document kind (catalog)") - parser.add_argument("--group", default="", help="Tenant group code (optional; defaults to your group)") + parser.add_argument("--instance", default="", help="Tenant instance code (optional; defaults to your instance)") args = parser.parse_args() directory = Path(args.directory) @@ -94,7 +94,7 @@ def main() -> int: print(f"SKIP {path.name}: OCR already present") continue try: - result = post_document(args.api_base, str(path), args.kind, args.group) + result = post_document(args.api_base, str(path), args.kind, args.instance) submitted += 1 print(f"SUBMIT {path.name}: {result.get('workflow_id')}") except urllib.error.HTTPError as exc: diff --git a/scripts/test_pipeline_e2e.py b/scripts/test_pipeline_e2e.py index 66d8eeb..eda4ff7 100644 --- a/scripts/test_pipeline_e2e.py +++ b/scripts/test_pipeline_e2e.py @@ -38,7 +38,7 @@ def check_health(name: str, url: str) -> bool: def upload_pdf( - api: str, pdf_path: Path, *, auto_approve: bool, stop_after_ocr: bool, kind: str, group: str = "" + api: str, pdf_path: Path, *, auto_approve: bool, stop_after_ocr: bool, kind: str, instance: str = "" ) -> str: import subprocess @@ -46,8 +46,8 @@ def upload_pdf( f"auto_approve={'true' if auto_approve else 'false'}" f"&stop_after_ocr={'true' if stop_after_ocr else 'false'}&kind={kind}" ) - if group: - query += f"&group={group}" + if instance: + query += f"&instance={instance}" url = f"{api.rstrip('/')}/upload?{query}" result = subprocess.run( ["curl", "-sf", "-X", "POST", url, "-F", f"file=@{pdf_path}"], @@ -100,7 +100,7 @@ def main() -> int: parser.add_argument("--gemma-health", default=os.environ.get("TRANSLATION_VLLM_BASE_URL", "http://localhost:8020/v1")) parser.add_argument("--mode", choices=["ocr", "full"], default="ocr", help="ocr=stop after OCR; full=entire pipeline") parser.add_argument("--kind", choices=["scheme", "advisory"], default="scheme", help="Document kind (catalog)") - parser.add_argument("--group", default="", help="Tenant group code (optional; defaults to your group)") + parser.add_argument("--instance", default="", help="Tenant instance code (optional; defaults to your instance)") parser.add_argument("--timeout", type=int, default=1800, help="Max seconds to wait for workflow") parser.add_argument("--poll-interval", type=float, default=5.0) args = parser.parse_args() @@ -132,7 +132,7 @@ def main() -> int: print(f"==> Upload ({'OCR only' if stop_after_ocr else 'full pipeline'})") workflow_id = upload_pdf( args.api, args.pdf, auto_approve=True, stop_after_ocr=stop_after_ocr, - kind=args.kind, group=args.group, + kind=args.kind, instance=args.instance, ) print("==> Polling workflow") diff --git a/tests/test_api.py b/tests/test_api.py index ec690fc..ef0bda9 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -615,72 +615,68 @@ def test_rejects_a_malformed_valid_on(self, test_client): assert "YYYY-MM-DD" in response.json()["detail"] -class TestSearchGroup: - """`group` narrows a search to one group's chunks (story #202). - - The API says `group`; the store and the chunk payload say `instance`. - The endpoint is the only place that maps one name to the other. - """ +class TestSearchInstance: + """`instance` narrows a search to one instance's chunks.""" @pytest.mark.api @pytest.mark.unit - def test_passes_the_group_to_the_store_as_instance(self, test_client, capturing_store): - response = test_client.post("/search", json={"query": "kisan", "group": "mh"}) + def test_passes_the_instance_to_the_store(self, test_client, capturing_store): + response = test_client.post("/search", json={"query": "kisan", "instance": "mh"}) assert response.status_code == 200 assert capturing_store.captured["instance"] == "mh" - assert response.json()["effective_config"]["group"] == "mh" + assert response.json()["effective_config"]["instance"] == "mh" @pytest.mark.api @pytest.mark.unit @pytest.mark.parametrize("raw", ["MH", " mh ", " Mh\n"]) def test_normalises_case_and_whitespace(self, test_client, capturing_store, raw): - response = test_client.post("/search", json={"query": "kisan", "group": raw}) + response = test_client.post("/search", json={"query": "kisan", "instance": raw}) assert capturing_store.captured["instance"] == "mh" - assert response.json()["effective_config"]["group"] == "mh" + assert response.json()["effective_config"]["instance"] == "mh" @pytest.mark.api @pytest.mark.unit @pytest.mark.parametrize("body", [ {"query": "kisan"}, - {"query": "kisan", "group": ""}, - {"query": "kisan", "group": " "}, - {"query": "kisan", "group": None}, + {"query": "kisan", "instance": ""}, + {"query": "kisan", "instance": " "}, + {"query": "kisan", "instance": None}, ]) - def test_absent_or_empty_group_means_no_filter(self, test_client, capturing_store, body): + def test_absent_or_empty_instance_means_no_filter(self, test_client, capturing_store, body): response = test_client.post("/search", json=body) assert response.status_code == 200 assert capturing_store.captured["instance"] is None - assert response.json()["effective_config"]["group"] is None + assert response.json()["effective_config"]["instance"] is None @pytest.mark.api @pytest.mark.unit @pytest.mark.parametrize("bad", [["mh"], 7, {"code": "mh"}]) - def test_rejects_a_non_string_group(self, test_client, capturing_store, bad): - response = test_client.post("/search", json={"query": "kisan", "group": bad}) + def test_rejects_a_non_string_instance(self, test_client, capturing_store, bad): + response = test_client.post("/search", json={"query": "kisan", "instance": bad}) assert response.status_code == 400 - assert response.json()["detail"] == "group must be a string" + assert response.json()["detail"] == "instance must be a string" assert capturing_store.captured == {}, "the store must not be called for a bad request" @pytest.mark.api @pytest.mark.unit - def test_unknown_group_is_an_empty_success(self, test_client, capturing_store): - response = test_client.post("/search", json={"query": "kisan", "group": "xx"}) + def test_unknown_instance_is_an_empty_success(self, test_client, capturing_store): + response = test_client.post("/search", json={"query": "kisan", "instance": "xx"}) assert response.status_code == 200 body = response.json() assert body["hits"] == [] assert body["final_count"] == 0 - assert body["effective_config"]["group"] == "xx" + assert body["effective_config"]["instance"] == "xx" @pytest.mark.api @pytest.mark.unit - def test_group_composes_with_include_expired(self, test_client, capturing_store): + def test_instance_composes_with_include_expired(self, test_client, capturing_store): test_client.post( - "/search", json={"query": "kisan", "group": "mh", "include_expired": True} + "/search", json={"query": "kisan", "instance": "mh", "include_expired": True} ) assert capturing_store.captured["instance"] == "mh" @@ -730,8 +726,8 @@ def test_rejects_anything_but_a_string(self, test_client, capturing_store, bad): @pytest.mark.api @pytest.mark.unit - def test_composes_with_group(self, test_client, capturing_store): - test_client.post("/search", json={"query": "kisan", "group": "mh", "provider_id": "mh.provider.oan.dev"}) + def test_composes_with_instance(self, test_client, capturing_store): + test_client.post("/search", json={"query": "kisan", "instance": "mh", "provider_id": "mh.provider.oan.dev"}) assert capturing_store.captured["instance"] == "mh" assert capturing_store.captured["provider_id"] == "mh.provider.oan.dev" @@ -1126,14 +1122,16 @@ def test_upload_creates_the_catalog_and_derives_the_code_from_the_filename( ): response = test_client.post( "/upload", - params={"kind": "scheme", "group": "mh"}, + params={"kind": "scheme", "instance": "mh"}, files={"file": ("National Mission on Natural Farming.pdf", sample_pdf_content, "application/pdf")}, ) assert response.status_code == 200 doc = test_client.get(f"/documents/{response.json()['workflow_id']}").json() assert doc["kind"] == "scheme" - assert doc["group"] == "mh" + assert response.json()["instance"] == "mh" + assert doc["instance"] == "mh" + assert "group" not in doc assert doc["code"] == "nmnf" assert doc["name"] == "National Mission on Natural Farming" with db_connection.get_connection() as conn: @@ -1146,6 +1144,28 @@ def test_upload_creates_the_catalog_and_derives_the_code_from_the_filename( ("openagrinet:AgricultureResource", "mh", "/states/MH") ] + @pytest.mark.api + @pytest.mark.unit + def test_upload_rejects_an_unauthorized_instance( + self, test_client, db_connection, sample_pdf_content, monkeypatch + ): + from pipeline.api.api import app + from pipeline.auth.deps import get_current_user + + monkeypatch.setitem( + app.dependency_overrides, get_current_user, + lambda: self._user("/states/MH/contributor"), + ) + response = test_client.post( + "/upload", + params={"kind": "scheme", "instance": "gj"}, + files={"file": ("a.pdf", sample_pdf_content, "application/pdf")}, + ) + + assert response.status_code == 403 + with db_connection.get_connection() as conn: + assert conn.execute("SELECT count(*) AS n FROM documents").fetchone()["n"] == 0 + @pytest.mark.api @pytest.mark.unit def test_a_second_upload_reuses_the_catalog_and_gets_a_fresh_code( @@ -1153,12 +1173,12 @@ def test_a_second_upload_reuses_the_catalog_and_gets_a_fresh_code( ): first = test_client.post( "/upload", - params={"kind": "advisory", "group": "mh"}, + params={"kind": "advisory", "instance": "mh"}, files={"file": ("Crop Advisory.pdf", self._pdf(sample_pdf_content, "one"), "application/pdf")}, ) second = test_client.post( "/upload", - params={"kind": "advisory", "group": "mh"}, + params={"kind": "advisory", "instance": "mh"}, files={"file": ("Crop Advisory.pdf", self._pdf(sample_pdf_content, "two"), "application/pdf")}, ) @@ -1178,7 +1198,7 @@ def test_an_existing_code_updates_the_same_document( ): first = test_client.post( "/upload", - params={"kind": "scheme", "group": "mh", "code": "pm-kisan"}, + params={"kind": "scheme", "instance": "mh", "code": "pm-kisan"}, files={"file": ("pmkisan.pdf", self._pdf(sample_pdf_content, "v1"), "application/pdf")}, ) assert first.status_code == 200 @@ -1194,7 +1214,7 @@ def test_an_existing_code_updates_the_same_document( second = test_client.post( "/upload", - params={"kind": "scheme", "group": "mh", "code": "pm-kisan"}, + params={"kind": "scheme", "instance": "mh", "code": "pm-kisan"}, files={"file": ("pmkisan-2026.pdf", self._pdf(sample_pdf_content, "v2"), "application/pdf")}, ) @@ -1219,7 +1239,7 @@ def test_the_same_code_in_another_tenant_is_a_different_document( for group, tag in (("mh", "a"), ("gj", "b")): response = test_client.post( "/upload", - params={"kind": "scheme", "group": group, "code": "pm-kisan"}, + params={"kind": "scheme", "instance": group, "code": "pm-kisan"}, files={"file": ("p.pdf", self._pdf(sample_pdf_content, tag), "application/pdf")}, ) assert response.status_code == 200 diff --git a/tests/test_migrate_sqlite_to_postgres.py b/tests/test_migrate_sqlite_to_postgres.py index 99b2884..ab56a1b 100644 --- a/tests/test_migrate_sqlite_to_postgres.py +++ b/tests/test_migrate_sqlite_to_postgres.py @@ -375,7 +375,7 @@ def test_a_full_run_maps_every_table(self, db_connection, postgres_url, legacy_d assert [e["action_type"] for e in purged] == ["purge_document"] demo_trail = db.get_audit_logs(WF5) # kept for the skipped demo document, tied to its group assert [e["action_type"] for e in demo_trail] == ["stage_change"] - assert demo_trail[0]["group"] == "mh" + assert demo_trail[0]["instance"] == "mh" settings_log = db.get_settings_audit_logs() assert settings_log[0]["field_name"] == "retrieval.alpha" diff --git a/tests/test_schema_v2.py b/tests/test_schema_v2.py index d1ce3b2..fa6b053 100644 --- a/tests/test_schema_v2.py +++ b/tests/test_schema_v2.py @@ -463,7 +463,7 @@ def test_the_snapshot_survives_a_purge_and_keeps_tenant_scope(self, db_connectio assert db.get_all_audit_log_count(groups=["up"]) == 1 assert db.get_all_audit_log_count(groups=["gj"]) == 0 assert [e["workflow_id"] for e in db.get_all_audit_logs(groups=["mh"])] == ["wf-gone"] - assert db.get_all_audit_logs(groups=["mh"])[0]["group"] == "mh" + assert db.get_all_audit_logs(groups=["mh"])[0]["instance"] == "mh" @pytest.mark.db @pytest.mark.unit diff --git a/tests/test_search_group_integration.py b/tests/test_search_group_integration.py index 505de70..93da320 100644 --- a/tests/test_search_group_integration.py +++ b/tests/test_search_group_integration.py @@ -1,9 +1,9 @@ -"""Integration: ``POST /search`` with ``group`` through the real Qdrant engine. +"""Integration: ``POST /search`` with ``instance`` through the real Qdrant engine. qdrant-client ships an in-process mode (``QdrantClient(":memory:")``) that runs the real filtering and vector engine, so this exercises the route, the store and the engine together without Docker. Only the embedding model is -faked: search quality is not under test here, the group filter is. +faked: search quality is not under test here, the instance filter is. """ import zlib @@ -14,7 +14,7 @@ from pipeline.vector_store import qdrant_store from tests.test_qdrant_validity import TODAY, clock_at -INDEX = "group-integration" +INDEX = "instance-integration" VECTOR_SIZE = 8 CHUNKS = [ @@ -67,27 +67,27 @@ def test_every_chunk_is_indexed(local_store): @pytest.mark.integration @pytest.mark.api @pytest.mark.parametrize("mode", ["HYBRID", "TENSOR", "LEXICAL"]) -def test_group_limits_hits_to_that_group_in_every_mode(local_store, test_client, mode): - body = search(test_client, group="mh", search_mode=mode) +def test_instance_limits_hits_to_that_group_in_every_mode(local_store, test_client, mode): + body = search(test_client, instance="mh", search_mode=mode) - assert body["effective_config"]["group"] == "mh" + assert body["effective_config"]["instance"] == "mh" assert body["hits"], "the mh chunk must be found" assert {hit["instance"] for hit in body["hits"]} == {"mh"} @pytest.mark.integration @pytest.mark.api -def test_no_group_searches_every_group(local_store, test_client): +def test_no_instance_searches_every_group(local_store, test_client): body = search(test_client) - assert body["effective_config"]["group"] is None + assert body["effective_config"]["instance"] is None assert {hit["instance"] for hit in body["hits"]} == {"mh", "ka", "bv"} @pytest.mark.integration @pytest.mark.api -def test_unknown_group_is_an_empty_success(local_store, test_client): - body = search(test_client, group="xx") +def test_unknown_instance_is_an_empty_success(local_store, test_client): + body = search(test_client, instance="xx") assert body["hits"] == [] assert body["final_count"] == 0 @@ -95,9 +95,9 @@ def test_unknown_group_is_an_empty_success(local_store, test_client): @pytest.mark.integration @pytest.mark.api -def test_group_and_validity_compose_in_the_engine(local_store, test_client): - current_only = search(test_client, group="mh") - with_expired = search(test_client, group="mh", include_expired=True) +def test_instance_and_validity_compose_in_the_engine(local_store, test_client): + current_only = search(test_client, instance="mh") + with_expired = search(test_client, instance="mh", include_expired=True) assert {hit["doc_id"] for hit in current_only["hits"]} == {"mh-current"} assert {hit["doc_id"] for hit in with_expired["hits"]} == {"mh-current", "mh-expired"} diff --git a/tests/test_search_provider_integration.py b/tests/test_search_provider_integration.py index bd28f3f..9589206 100644 --- a/tests/test_search_provider_integration.py +++ b/tests/test_search_provider_integration.py @@ -60,7 +60,7 @@ def test_no_provider_searches_every_chunk(local_store, test_client): @pytest.mark.integration @pytest.mark.api -def test_provider_composes_with_group(local_store, test_client): - body = search(test_client, provider_id="bv.provider.oan.dev", group="mh") +def test_provider_composes_with_instance(local_store, test_client): + body = search(test_client, provider_id="bv.provider.oan.dev", instance="mh") assert body["hits"] == [] diff --git a/ui/src/components/DocumentHeaderSummary.jsx b/ui/src/components/DocumentHeaderSummary.jsx index 999c654..3af75e9 100644 --- a/ui/src/components/DocumentHeaderSummary.jsx +++ b/ui/src/components/DocumentHeaderSummary.jsx @@ -74,7 +74,7 @@ export function DocumentHeaderSummary({ const pages = pageCount ?? doc?.page_count ?? 0 const chunks = chunkCount ?? doc?.chunk_count ?? 0 const uploader = uploaderName(doc) - const stateLabel = formatGroupLabel(doc?.group) + const stateLabel = formatGroupLabel(doc?.instance) const roleLabels = useMemo( () => productRoleLabels(doc?.uploaded_by_roles || []), [doc?.uploaded_by_roles], @@ -126,7 +126,7 @@ export function DocumentHeaderSummary({ {stateLabel ? ( - {stateLabel} + {stateLabel} ) : ( '—' )} diff --git a/ui/src/views/DashboardView.jsx b/ui/src/views/DashboardView.jsx index 2538e72..2cfd9d4 100644 --- a/ui/src/views/DashboardView.jsx +++ b/ui/src/views/DashboardView.jsx @@ -66,7 +66,7 @@ export default function DashboardView() { const reindexCount = summary?.needs_reindex || 0 const displayedQueueItems = queue.slice(0, QUEUE_ROWS_SHOWN) - const byGroup = useMemo(() => (Array.isArray(summary?.by_group) ? summary.by_group : []), [summary]) + const byGroup = useMemo(() => (Array.isArray(summary?.by_instance) ? summary.by_instance : []), [summary]) const stateRows = byGroup const maxStateCount = useMemo( @@ -172,10 +172,10 @@ export default function DashboardView() { {stateRows.map((row) => { const label = row.otherStateCount ? `Others (${row.otherStateCount} states)` - : formatGroupLabel(row.group) || 'No state tag' + : formatGroupLabel(row.instance) || 'No state tag' const pct = (value) => (maxStateCount ? (value / maxStateCount) * 100 : 0) return ( -
+
{label} {formatCount(row.count)} diff --git a/ui/src/views/DocumentsView.jsx b/ui/src/views/DocumentsView.jsx index 47024c9..f164edb 100644 --- a/ui/src/views/DocumentsView.jsx +++ b/ui/src/views/DocumentsView.jsx @@ -48,7 +48,7 @@ export default function DocumentsView() { const stageOptions = useMemo(() => ['all', ...new Set(documents.map(doc => doc.stage))], [documents]) const groupOptions = useMemo(() => { - const fromDocs = documents.map((d) => (d.group || '').trim().toLowerCase()).filter(Boolean) + const fromDocs = documents.map((d) => (d.instance || '').trim().toLowerCase()).filter(Boolean) return ['all', ...new Set(fromDocs)] }, [documents]) @@ -60,7 +60,7 @@ export default function DocumentsView() { if (authFilter === 'authoritative' && !doc.authoritative) return false if (authFilter === 'legacy' && doc.authoritative) return false if (groupFilter !== 'all') { - const code = (doc.group || 'default').trim().toLowerCase() + const code = (doc.instance || 'default').trim().toLowerCase() if (code !== groupFilter) return false } if (showFailed && !doc.failed && doc.stage !== 'failed') return false @@ -233,7 +233,7 @@ export default function DocumentsView() {
{getDocumentMetaLabel(doc)}
- + diff --git a/ui/src/views/GlobalAuditView.jsx b/ui/src/views/GlobalAuditView.jsx index 82bd6c5..a3aed6c 100644 --- a/ui/src/views/GlobalAuditView.jsx +++ b/ui/src/views/GlobalAuditView.jsx @@ -120,10 +120,10 @@ export default function GlobalAuditView() { onClick={() => navigate(`/documents/${entry.workflow_id}`)} title={entry.document_id || entry.workflow_id} > - {entry.display_name || entry.filename || entry.document_id || entry.workflow_id} + {entry.name || entry.filename || entry.document_id || entry.workflow_id} - {entry.instance && ( - {entry.instance} + {entry.group && ( + {entry.group} )} {/* What actually changed, in words — the raw before/after stays diff --git a/ui/src/views/NewDocumentView.jsx b/ui/src/views/NewDocumentView.jsx index 06d7346..770746d 100644 --- a/ui/src/views/NewDocumentView.jsx +++ b/ui/src/views/NewDocumentView.jsx @@ -23,13 +23,18 @@ import { import { API_BASE } from '../config' import { apiFetch } from '../auth/keycloak' import { useAuth } from '../auth/AuthProvider' -import { defaultUploadInstance, PORTAL_INSTANCE } from '../lib/instanceLabels' +import { defaultUploadGroup, PORTAL_GROUP } from '../lib/groupLabels' import { fetchJson, formatCompactDateTime, getDocumentListLabel, summarizeIngestStatus } from '../lib/pipelineUi' import { displayNameError, filenameStem } from '../lib/displayName' const SUPPORTED_TYPES = ['.pdf', '.doc', '.docx'] const MAX_SIZE_MB = 100 const FILE_TYPE_CHIPS = ['PDF', 'Word'] +// The catalog a document joins is (your group, type), so the type is chosen up front. +const DOCUMENT_TYPE_OPTIONS = [ + { value: 'scheme', label: 'Scheme' }, + { value: 'advisory', label: 'Advisory' }, +] const PIPELINE_STEPS = [ { id: 'ocr', label: 'OCR', icon: ScanText }, @@ -55,14 +60,15 @@ export default function NewDocumentView() { const navigate = useNavigate() const { hasPermission, instances, isSuperAdmin, user } = useAuth() const canUpload = hasPermission('upload') - // Instance from login (Keycloak) — not user-selected on this form. - const documentInstance = defaultUploadInstance({ + // Group from login (Keycloak) — not user-selected on this form. + const documentGroup = defaultUploadGroup({ isSuperAdmin, instances, - portalInstance: user?.portal_instance || PORTAL_INSTANCE, + portalGroup: user?.portal_group || PORTAL_GROUP, }) const [file, setFile] = useState(null) const [displayName, setDisplayName] = useState('') + const [documentKind, setDocumentKind] = useState('') const [dragging, setDragging] = useState(false) const [uploading, setUploading] = useState(false) const [uploadSuccess, setUploadSuccess] = useState(false) @@ -126,8 +132,8 @@ export default function NewDocumentView() { const nameError = file ? displayNameError(displayName) : '' async function handleSubmit() { - if (!file || !canUpload || nameError) return - if (!documentInstance && !isSuperAdmin) { + if (!file || !documentKind || !canUpload || nameError) return + if (!documentGroup && !isSuperAdmin) { setUploadError( 'No state on your account. Ask an admin to assign a Keycloak group (e.g. /states/MH/admin).', ) @@ -140,8 +146,8 @@ export default function NewDocumentView() { formData.append('file', file) formData.append('display_name', displayName.trim()) // State users: Keycloak group state only. Super-admin: portal BV when unset. - const uploadInstance = documentInstance || (isSuperAdmin ? PORTAL_INSTANCE : '') - if (!uploadInstance) { + const uploadGroup = documentGroup || (isSuperAdmin ? PORTAL_GROUP : '') + if (!uploadGroup) { setUploadError( 'No state on your account. Ask an admin to assign a Keycloak group (e.g. /states/MH/admin).', ) @@ -149,7 +155,8 @@ export default function NewDocumentView() { } const params = new URLSearchParams({ auto_approve: 'false', - instance: uploadInstance, + group: uploadGroup, + kind: documentKind, }) const response = await apiFetch(`${API_BASE}/upload?${params}`, { method: 'POST', body: formData }) const data = await response.json() @@ -169,6 +176,7 @@ export default function NewDocumentView() { setDisplayName('') setUploadSuccess(false) setLastWorkflowId('') + setDocumentKind('') setUploadError('') setValidationError('') } @@ -230,6 +238,26 @@ export default function NewDocumentView() {
{/* Left column */}
+
+ + + Required +
{uploading ? ( <> diff --git a/ui/src/views/QueueView.jsx b/ui/src/views/QueueView.jsx index 83119eb..8a5fe0d 100644 --- a/ui/src/views/QueueView.jsx +++ b/ui/src/views/QueueView.jsx @@ -204,12 +204,12 @@ export default function QueueView() {
{canMutateQueue ? ( - toggleSelect(item.workflow_id)} aria-label={`Select ${item.display_name || item.filename || item.workflow_id}`} /> + toggleSelect(item.workflow_id)} aria-label={`Select ${item.name || item.filename || item.workflow_id}`} /> ) : null} navigate(`/documents/${item.workflow_id}`)}> - {item.display_name || item.filename} + {item.name || item.filename}
{item.workflow_id}
 - + diff --git a/ui/src/views/GlobalAuditView.jsx b/ui/src/views/GlobalAuditView.jsx index a3aed6c..c2aaa93 100644 --- a/ui/src/views/GlobalAuditView.jsx +++ b/ui/src/views/GlobalAuditView.jsx @@ -122,8 +122,8 @@ export default function GlobalAuditView() { > {entry.name || entry.filename || entry.document_id || entry.workflow_id} - {entry.group && ( - {entry.group} + {entry.instance && ( + {entry.instance} )} {/* What actually changed, in words — the raw before/after stays diff --git a/ui/src/views/NewDocumentView.jsx b/ui/src/views/NewDocumentView.jsx index 41acaed..f12be02 100644 --- a/ui/src/views/NewDocumentView.jsx +++ b/ui/src/views/NewDocumentView.jsx @@ -66,7 +66,7 @@ export default function NewDocumentView() { const documentGroup = defaultUploadGroup({ isSuperAdmin, instances, - portalGroup: user?.portal_group || PORTAL_GROUP, + portalGroup: user?.portal_instance || PORTAL_GROUP, }) const [file, setFile] = useState(null) const [displayName, setDisplayName] = useState('') @@ -157,7 +157,7 @@ export default function NewDocumentView() { } const params = new URLSearchParams({ auto_approve: 'false', - group: uploadGroup, + instance: uploadGroup, kind: documentKind, }) const response = await apiFetch(`${API_BASE}/upload?${params}`, { method: 'POST', body: formData }) diff --git a/ui/src/views/RunsView.jsx b/ui/src/views/RunsView.jsx index d11a44d..727f2df 100644 --- a/ui/src/views/RunsView.jsx +++ b/ui/src/views/RunsView.jsx @@ -356,7 +356,7 @@ export default function RunsView() { {title}
- + #{runId} {fileLabel && fileLabel !== title ? ` · ${fileLabel}` : ''} From ebd93f24ba588c7f7717b1349472573ca5b0458c Mon Sep 17 00:00:00 2001 From: Ravi Mula Date: Wed, 11 Nov 2026 13:45:21 +0530 Subject: [PATCH 52/53] fix: restore instance naming throughout knowledge provider --- AGENTS.md | 6 +- ...d => 0006-instances-and-catalogs-model.md} | 12 +- docs/POSTGRES_SCHEMA.md | 68 ++--- docs/SYSTEM_DESIGN.md | 12 +- migrations/0001_initial_schema.sql | 24 +- pipeline/activities.py | 28 +- pipeline/api/api.py | 105 ++++--- pipeline/api/documents.py | 46 +-- pipeline/api/upload.py | 18 +- pipeline/auth/groups.py | 10 +- pipeline/auth/keycloak_admin.py | 10 +- pipeline/auth/models.py | 26 +- pipeline/auth/tenancy.py | 76 ++--- pipeline/catalog_builder.py | 16 +- pipeline/db.py | 270 +++++++++--------- pipeline/discovery_publish_service.py | 16 +- pipeline/document_repository.py | 2 +- pipeline/{group_names.py => instances.py} | 20 +- pipeline/network_group.py | 12 +- pipeline/scheme_catalog.py | 18 +- scripts/ingest_test_docs.sh | 6 +- scripts/migrate_sqlite_to_postgres.py | 34 +-- scripts/upload_test_docs.ps1 | 4 +- scripts/verify_document_validity_e2e.py | 6 +- tests/conftest.py | 2 +- tests/test_activities.py | 14 +- tests/test_api.py | 14 +- tests/test_auth.py | 2 +- tests/test_db.py | 38 +-- tests/test_discovery_publish_service.py | 36 +-- tests/test_display_name.py | 2 +- tests/test_document_repository.py | 2 +- tests/test_groups.py | 46 +-- ...{test_group_names.py => test_instances.py} | 28 +- tests/test_migrate_sqlite_to_postgres.py | 12 +- tests/test_network_group.py | 4 +- tests/test_provider_id_on_chunks.py | 18 +- tests/test_schema_v2.py | 220 +++++++------- tests/test_scheme_catalog.py | 14 +- tests/test_tenancy.py | 220 +++++++------- ui/src/auth/AuthProvider.jsx | 36 +-- ui/src/components/DocumentHeaderSummary.jsx | 6 +- ui/src/components/GroupBadge.jsx | 21 -- ui/src/components/InstanceBadge.jsx | 21 ++ .../lib/{groupLabels.js => instanceLabels.js} | 18 +- ui/src/views/DashboardView.jsx | 10 +- ui/src/views/DocumentsView.jsx | 26 +- ui/src/views/NewDocumentView.jsx | 18 +- ui/src/views/RunsView.jsx | 4 +- 49 files changed, 837 insertions(+), 840 deletions(-) rename docs/ADR/{0006-groups-and-catalogs-model.md => 0006-instances-and-catalogs-model.md} (61%) rename pipeline/{group_names.py => instances.py} (84%) rename tests/{test_group_names.py => test_instances.py} (81%) delete mode 100644 ui/src/components/GroupBadge.jsx create mode 100644 ui/src/components/InstanceBadge.jsx rename ui/src/lib/{groupLabels.js => instanceLabels.js} (63%) diff --git a/AGENTS.md b/AGENTS.md index 153a32e..a549ba8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -79,8 +79,8 @@ Core modules (all under `pipeline/`): - `models.py` — Pydantic models, including `DocumentStage` enum and `PIPELINE_STAGES` (the stepper-UI stage list) - `config.py` — `Config` dataclass reading env vars, with defaults - `observability.py` — bridges root-logger `INFO`+ lines to OTLP when `OTEL_EXPORTER_OTLP_ENDPOINT` is set (ClickHouse/HyperDX); no-op otherwise. See ADR 0007 -- `group_names.py` — tenant (group) code ↔ human name mapping; `bv` = Bharat Vistaar platform-wide, others are state codes matching Keycloak group paths (`/states/MH/...` → `mh`). A tenant code is a `groups` row (name = code); a document's `instance` is derived through `documents.catalog_id → catalogs.group_id`, there is no `documents.instance` column -- `scheme_catalog.py` — scheme catalog behind the pull-based `/catalog/v1/*` API, served from the computed `catalog_status` view (one row per document; `catalogs` rows are `(group, type)`, `documents` carry `code`/`name`/`aliases`) +- `instances.py` — tenant instance code ↔ human name mapping; `bv` = Bharat Vistaar platform-wide, others are state codes matching Keycloak group paths (`/states/MH/...` → `mh`). A tenant code is an `instances` row (name = code); a document's `instance` is derived through `documents.catalog_id → catalogs.instance_id`, there is no `documents.instance` column +- `scheme_catalog.py` — scheme catalog behind the pull-based `/catalog/v1/*` API, served from the computed `catalog_status` view (one row per document; `catalogs` rows are `(instance, type)`, `documents` carry `code`/`name`/`aliases`) - `ocr/`, `translation/`, `chunking/`, `vector_store/` — pluggable provider packages, each with a `base.py` protocol/ABC and a `service.py` (or equivalent) that dispatches to the configured provider - `auth/` — Keycloak JWT validation, group-based tenancy/permissions, email-OTP flow, admin user management @@ -98,7 +98,7 @@ registered → ocr_processing → ocr_review → translation_processing → tran → approval_for_prod → ingesting_prod → completed (or → failed at any point) ``` -`approval_for_prod`/`ingesting_prod` are **PROD-only stages** (`PROD_ONLY_STAGES` in `models.py`); with `DISABLE_PROD_SETTING=true` a document goes straight from `ingesting` to `completed`. That flag is read only at workflow *start* and passed in as an argument — never read inside a running workflow, since that would break Temporal replay determinism if flipped mid-flight (see the docstring in `group_names.py`). +`approval_for_prod`/`ingesting_prod` are **PROD-only stages** (`PROD_ONLY_STAGES` in `models.py`); with `DISABLE_PROD_SETTING=true` a document goes straight from `ingesting` to `completed`. That flag is read only at workflow *start* and passed in as an argument — never read inside a running workflow, since that would break Temporal replay determinism if flipped mid-flight (see the docstring in `instances.py`). ### DEV vs PROD dual indexing diff --git a/docs/ADR/0006-groups-and-catalogs-model.md b/docs/ADR/0006-instances-and-catalogs-model.md similarity index 61% rename from docs/ADR/0006-groups-and-catalogs-model.md rename to docs/ADR/0006-instances-and-catalogs-model.md index 74e1f53..91f4190 100644 --- a/docs/ADR/0006-groups-and-catalogs-model.md +++ b/docs/ADR/0006-instances-and-catalogs-model.md @@ -1,21 +1,21 @@ -# Providers are groups, catalogs are (group, type), documents are identified by (catalog, code) +# Providers are instances, catalogs are (instance, type), documents are identified by (catalog, code) Context: the first Postgres schema modelled a tenant as a free-text `documents.instance`, a `catalog_entries` table keyed by `(type, code)` with one entry per scheme, and a free-form `document_kind` that had to match the entry's type through a composite foreign key. Review feedback was that the unit published on the network is a provider's catalog, that a provider is a Keycloak group, that each scheme is itself a document, and that database enums and demo flags do not belong in the schema. Decision: -- A **group** is a provider: one `groups` row per Keycloak tenant node (`/states/MH`, `/global`). Role leaves are not mirrored; permissions still come from the token. `users.group_id` is a single nullable FK for now (first tenant in the token, sorted, with a warning when there are several); a join table can replace it if multi-group membership is needed. -- A **catalog** is `(group, type)`: `catalogs.group_id` plus `catalogs.type`, unique together. `type` holds the network schema name. The kind-to-schema map lives in `pipeline/network_constants.py` and is a placeholder: `scheme` is `openagrinet:AgricultureResource`, `advisory` is `openagrinet:KnowledgeAdvisory`, `video` is `openagrinet:VideoResource`. Code branches through that map, never on raw `openagrinet:*` strings. Catalog Beckn information (`publish_directives`, `visible_networks`, `criteria`, `network`) sits in `catalogs.metadata`. -- A **document** belongs to one catalog and is identified there by `code`: `documents.catalog_id` NOT NULL, `UNIQUE (catalog_id, code)`, with `name` and `aliases` on the document. The API fields `code`, `name` and `aliases` are those columns. `documents.instance`, `document_kind` and `is_demo` are gone; the API returns the derived `instance` (the group name) and `kind` (from the type map). The `/catalog/v1/*` snapshot keys, the Qdrant payload keys and Temporal arguments keep their earlier names (`scheme_code`, `document_kind`, `instance`, ...) because other services and in-flight workflows depend on them. +- An **instance** owns provider catalogs. Use one `instances` row per Keycloak tenant node (`/states/MH`, `/global`). Role leaves are not mirrored; permissions still come from the token. `users.instance_id` is a single nullable FK for now (first tenant in the token, sorted, with a warning when there are several); a join table can replace it if multi-instance membership is needed. +- A **catalog** is `(instance, type)`: `catalogs.instance_id` plus `catalogs.type`, unique together. `type` holds the network schema name. The kind-to-schema map lives in `pipeline/network_constants.py` and is a placeholder: `scheme` is `openagrinet:AgricultureResource`, `advisory` is `openagrinet:KnowledgeAdvisory`, `video` is `openagrinet:VideoResource`. Code branches through that map, never on raw `openagrinet:*` strings. Catalog Beckn information (`publish_directives`, `visible_networks`, `criteria`, `network`) sits in `catalogs.metadata`. +- A **document** belongs to one catalog and is identified there by `code`: `documents.catalog_id` NOT NULL, `UNIQUE (catalog_id, code)`, with `name` and `aliases` on the document. The API fields `code`, `name` and `aliases` are those columns. `documents.instance`, `document_kind` and `is_demo` are gone; the API returns the derived `instance` (the instance name) and `kind` (from the type map). The `/catalog/v1/*` snapshot keys, the Qdrant payload keys and Temporal arguments keep their earlier names (`scheme_code`, `document_kind`, `instance`, ...) because other services and in-flight workflows depend on them. - Re-uploading an existing code updates the same document (new workflow, `source` replaced, pages and chunks cleared, `reindex_required` set) rather than adding a second one. - Catalogs are created on upload for the tenant and kind. The upload therefore takes the document type up front. - Bootstrap schemes (the code-seeded entries with no documents) are dropped. `/catalog/v1/*` is served from the `catalog_status` view, one row per document, and a scheme appears once a document exists. - Period, uploader and vector index state moved into `documents.metadata`; counts are derived from `pages` and `chunks`. - Database enums are replaced by plain `TEXT`, validated by `pipeline/models.py` (`DocumentStage`, `WorkflowType`, `WorkflowStatus`, `AuditEntityType`). -- `audit_logs.instance` is replaced by `audit_logs.group_id` (snapshot, `ON DELETE SET NULL`). +- `audit_logs.instance` is replaced by `audit_logs.instance_id` (snapshot, `ON DELETE SET NULL`). - No Postgres held the first schema, so `migrations/0001` was rewritten in place instead of adding an ALTER migration. `scripts/migrate_sqlite_to_postgres.py` produces v2 and skips `kind=document` and `is_demo` documents, reporting the counts. Alternatives considered: - **Keep `documents.instance`.** Rejected: the catalog already carries the provider, and two sources of tenant truth drift. - **Type as a short kind plus a separate `schema_type` column.** Rejected for now in favour of one `type` column holding the schema name; revisit if scheme and advisory end up sharing a schema (ADR 0003 types both as `KnowledgeAdvisory`). -- **`user_groups` join table now.** Deferred: the team wants one group per user for now. +- **`user_instances` join table now.** Deferred: the team wants one instance per user for now. - **Nullable `catalog_id`/`code` until a reviewer classifies the document.** Rejected: a document without a catalog has no tenant once `instance` is gone. The upload form asks for the type instead. diff --git a/docs/POSTGRES_SCHEMA.md b/docs/POSTGRES_SCHEMA.md index fbda3fa..f9e260e 100644 --- a/docs/POSTGRES_SCHEMA.md +++ b/docs/POSTGRES_SCHEMA.md @@ -1,6 +1,6 @@ # Postgres Schema — knowledge-provider -Postgres schema behind `pipeline/db.py`. A document is content and metadata. A workflow is one run of processing against a document. A document belongs to a catalog (a provider group plus a resource type) and is identified there by its `code`. Source file and vector index state are stored as JSONB on the document. Translations are stored as JSONB on each page. +Postgres schema behind `pipeline/db.py`. A document is content and metadata. A workflow is one run of processing against a document. A document belongs to a catalog (a provider instance plus a resource type) and is identified there by its `code`. Source file and vector index state are stored as JSONB on the document. Translations are stored as JSONB on each page. The schema is created by `migrations/0001_initial_schema.sql` and applied by the api and worker at startup (`pipeline/migrate.py`). `pipeline/db.py` is the only code that reads or writes it. The previous SQLite store is described in `docs/SQLITE_SCHEMA.md` (legacy) and is moved over once by `scripts/migrate_sqlite_to_postgres.py`. @@ -43,9 +43,9 @@ The database does not check these values, so adding one needs no migration. Raw | # | Table | Purpose | |---|---|---| -| 1 | `groups` | Provider tenant nodes (`/states/MH`, `/global`), mirrored from Keycloak | +| 1 | `instances` | Provider tenant nodes (`/states/MH`, `/global`), mirrored from Keycloak | | 2 | `users` | People who upload, review and act, mirrored from Keycloak | -| 3 | `catalogs` | One row per provider group and resource type (scheme, advisory, video) | +| 3 | `catalogs` | One row per provider instance and resource type (scheme, advisory, video) | | 4 | `documents` | One row per document. Content, metadata, source file, index state | | 5 | `workflows` | One row per processing run of a document. Stage, status, config | | 6 | `workflow_artifacts` | Files each workflow produces, stored in MinIO | @@ -62,36 +62,36 @@ The database does not check these values, so adding one needs no migration. Raw Relationships: ``` -groups 1 ── * users (users.group_id, nullable) -groups 1 ── * catalogs 1 ── * documents 1 ── * workflows 1 ── * workflow_artifacts +instances 1 ── * users (users.instance_id, nullable) +instances 1 ── * catalogs 1 ── * documents 1 ── * workflows 1 ── * workflow_artifacts │ ├── * pages └── * chunks -groups 1 ── * audit_logs (audit_logs.group_id, snapshot, SET NULL) +instances 1 ── * audit_logs (audit_logs.instance_id, snapshot, SET NULL) ``` -Dependency order for creation: `groups`, `users`, `catalogs`, `documents`, `workflows`, `workflow_artifacts`, `pages`, `chunks`, `audit_logs`, `search_settings`, `email_otps`, `webhook_records`, views. +Dependency order for creation: `instances`, `users`, `catalogs`, `documents`, `workflows`, `workflow_artifacts`, `pages`, `chunks`, `audit_logs`, `search_settings`, `email_otps`, `webhook_records`, views. --- -## 1. `groups` +## 1. `instances` -One row per tenant node (a state, the centre, or the platform), mirrored from Keycloak. A group is a provider: it owns catalogs. Rows are created on demand from a tenant code (`get_or_create_group`) or from a Keycloak group path (`upsert_group`). +One row per tenant node (a state, the centre, or the platform), mirrored from Keycloak. An instance is a provider: it owns catalogs. Rows are created on demand from a tenant code (`get_or_create_instance`) or from a Keycloak group path (`upsert_instance`). Unique: `keycloak_group_id`, `path`. | Column | Type | Constraints / Default | Purpose | |---|---|---|---| -| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Internal group ID. | +| id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Internal instance ID. | | keycloak_group_id | TEXT | UNIQUE | Keycloak group ID. NULL until a sync that sees the path fills it (rows made by the SQLite move or by an upload have none). | -| name | TEXT | NOT NULL | Tenant code, lowercase (`mh`, `bv`). This is what the API calls the document's `group`. | +| name | TEXT | NOT NULL | Tenant code, lowercase (`mh`, `bv`). This is what the API calls the document's `instance`. | | path | TEXT | NOT NULL UNIQUE | Keycloak group path: `/states/MH`, or `/global` for `bv`. | | created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Created. | | updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). | Notes - Only the tenant node is mirrored, not the role leaves (`/states/MH/admin`). Roles and permissions are read from the token, not from this table. -- Role leaves stay in Keycloak; `catalogs.group_id` points at the tenant node. +- Role leaves stay in Keycloak; `catalogs.instance_id` points at the tenant node. --- @@ -108,30 +108,30 @@ Unique: `keycloak_user_id`; unique index on `lower(email)` where `email IS NOT N | username | TEXT | | Current username. | | email | TEXT | | Current email. | | roles | TEXT[] | NOT NULL DEFAULT '{}' | Current roles. | -| group_id | UUID | FK → `groups(id)` ON DELETE SET NULL | The user's group (tenant). Single membership for now. | +| instance_id | UUID | FK → `instances(id)` ON DELETE SET NULL | The user's instance (tenant). Single membership for now. | | first_seen_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | First login or upload. | | last_seen_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Most recent activity. | Notes - Upserted on each authenticated upload or login. A `local-admin` row is seeded by the migration. -- A token can carry several tenants but `group_id` holds one: `upsert_user(..., tenants=[...])` uses the first code in sorted order and logs a warning when there are several. Authorization still reads the token. -- If users later need several groups, add a `user_groups` join table and drop `group_id`. +- A token can carry several tenants but `instance_id` holds one: `upsert_user(..., tenants=[...])` uses the first code in sorted order and logs a warning when there are several. Authorization still reads the token. +- If users later need several instances, add a `user_instances` join table and drop `instance_id`. - Point-in-time identity is kept in `audit_logs.actor`. --- ## 3. `catalogs` -One row per provider group and resource type. A catalog is what a provider publishes on the network: Mahavistar's `scheme` catalog and its `advisory` catalog are two rows. A new type adds rows, not tables. +One row per provider instance and resource type. A catalog is what a provider publishes on the network: Mahavistar's `scheme` catalog and its `advisory` catalog are two rows. A new type adds rows, not tables. -Unique: `(group_id, type)`. +Unique: `(instance_id, type)`. | Column | Type | Constraints / Default | Purpose | |---|---|---|---| | id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Catalog ID. | -| group_id | UUID | NOT NULL, FK → `groups(id)` ON DELETE RESTRICT | The provider that owns the catalog. | +| instance_id | UUID | NOT NULL, FK → `instances(id)` ON DELETE RESTRICT | The provider that owns the catalog. | | type | TEXT | NOT NULL, CHECK `^[a-z0-9]+:[A-Za-z0-9]+$` | Network schema/capability, e.g. `openagrinet:AgricultureResource`. | -| name | TEXT | NOT NULL | Catalog display name. Defaults to ` ` when created on upload. | +| name | TEXT | NOT NULL | Catalog display name. Defaults to ` ` when created on upload. | | metadata | JSONB | NOT NULL DEFAULT '{}', CHECK object | Beckn publish information. See keys below. | | created_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Created. | | updated_at | TIMESTAMPTZ | NOT NULL DEFAULT now() | Last change (trigger). | @@ -149,7 +149,7 @@ Pydantic (`CatalogMetadata`) validates the keys; the database only checks for an Notes - `type` stores the schema name, and code asks `pipeline/network_constants.py` what it means (`catalog_type_for_kind`, `kind_for_catalog_type`) instead of comparing `openagrinet:*` strings. The mapping is a placeholder: `scheme` is `openagrinet:AgricultureResource`, `advisory` is `openagrinet:KnowledgeAdvisory`, `video` is `openagrinet:VideoResource` (see ADR 0003 and ADR 0006). -- Every group gets one catalog per supported type when the group row is created (`upsert_group` calls `ensure_group_catalogs` in the same transaction). Startup (`init_db`) runs `sync_catalogs_for_all_groups`, so a type added to `KIND_TO_CATALOG_TYPE` reaches existing groups. Upload keeps `get_or_create_catalog` as a fallback for a catalog that is missing. Names are ` catalog`, for example `MH scheme catalog`. +- Every instance gets one catalog per supported type when the instance row is created (`upsert_instance` calls `ensure_instance_catalogs` in the same transaction). Startup (`init_db`) runs `sync_catalogs_for_all_instances`, so a type added to `KIND_TO_CATALOG_TYPE` reaches existing instances. Upload keeps `get_or_create_catalog` as a fallback for a catalog that is missing. Names are ` catalog`, for example `MH scheme catalog`. - Computed catalog values (status, chunk count, network visibility, `promoted_at`) come from the `catalog_status` view, not stored here. --- @@ -163,7 +163,7 @@ Unique: `(catalog_id, code)`. | Column | Type | Constraints / Default | Purpose | |---|---|---|---| | id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Document ID. | -| catalog_id | UUID | NOT NULL, FK → `catalogs(id)` ON DELETE RESTRICT | Catalog (provider group + type) the document belongs to. | +| catalog_id | UUID | NOT NULL, FK → `catalogs(id)` ON DELETE RESTRICT | Catalog (provider instance + type) the document belongs to. | | code | TEXT | NOT NULL, CHECK `^[a-z0-9]+(-[a-z0-9]+)*$` | Short code (`pm-kmy`) that catalog consumers see. Unique within the catalog. | | name | TEXT | NOT NULL | Display name. | | aliases | TEXT[] | NOT NULL DEFAULT '{}', CHECK no empty string | Ordered alternate names used to match queries. The first one is the prompt hint. | @@ -231,12 +231,12 @@ Column default: `'{"index_status": {}, "catalog_visible": true, "network_visible ### Notes - A document's stage is the `current_stage` of its latest workflow. List and filter queries join the `document_current_workflow` view. -- The API's `group` is derived: `documents.catalog_id → catalogs.group_id → groups.name`. There is no `documents.instance` column, and tenant filters join `groups`. +- The API's `instance` is derived: `documents.catalog_id → catalogs.instance_id → instances.name`. There is no `documents.instance` column, and tenant filters join `instances`. - The API's `document_kind` is derived from `catalogs.type` through the map in `network_constants.py`. `scheme_code`, `scheme_name` and `scheme_aliases` are `code`, `name` and `aliases`. - `chunk_count` and `page_count` are counted from `chunks` and `pages`; they are not stored. -- A new document needs a `document_kind` and a `code`. Upload gets the catalog from the tenant and the kind (seeded with the group; created when missing). A document's kind is fixed after upload, so there is no kind change. `DuplicateDocumentCode` is raised when the code is taken in that catalog. +- A new document needs a `document_kind` and a `code`. Upload gets the catalog from the tenant and the kind (seeded with the instance; created when missing). A document's kind is fixed after upload, so there is no kind change. `DuplicateDocumentCode` is raised when the code is taken in that catalog. - Re-uploading an existing code updates the same document: a new workflow starts, `source` is replaced, pages and chunks are cleared and `reindex_required` is set (`reset_document_for_reupload`). Earlier workflows and artifacts stay as history. -- Changing a document's kind moves it to the same group's catalog of the new type (`update_document_fields(document_kind=...)`). +- Changing a document's kind moves it to the same instance's catalog of the new type (`update_document_fields(document_kind=...)`). - SQL does not check types inside the JSONB columns. The insert path writes the default metadata keys, and Pydantic validates `source`, `metadata` and each `index_status` entry. - Each document keeps its own `source`. There is no unique key on the fingerprint. - Identifier lookup (`find_document_by_doc_identifier`) matches `documents.id`, `workflows.temporal_workflow_id`, `source->>'fingerprint'`, `source->>'original_filename'`, an index `doc_id` and the legacy `md5(source->>'fingerprint')`. @@ -425,7 +425,7 @@ Append-only history of actions. Must outlive documents, so there is no cascade d | id | UUID | PRIMARY KEY, DEFAULT gen_random_uuid() | Audit entry ID. | | document_id | UUID | FK → `documents(id)` ON DELETE SET NULL | Document the action applied to. NULL for system-level entries and after a purge. | | workflow_id | TEXT | | Temporal workflow ID snapshot. Survives a purge. | -| group_id | UUID | FK → `groups(id)` ON DELETE SET NULL | Provider group snapshot, so tenant scoping works after the document is gone. | +| instance_id | UUID | FK → `instances(id)` ON DELETE SET NULL | Provider instance snapshot, so tenant scoping works after the document is gone. | | action_type | TEXT | NOT NULL, CHECK not blank | What happened (`page_edit`, `purge_document`, ...). 22 values and growing, so not an enum. | | entity_type | TEXT | | `document`, `page`, `chunk`, `setting` or `catalog` (`AuditEntityType`). | | entity_number | INTEGER | | Page number or chunk number the action applied to. | @@ -439,15 +439,15 @@ Append-only history of actions. Must outlive documents, so there is no cascade d Table CHECK: `entity_type IN ('setting','catalog') OR workflow_id IS NOT NULL`. -Indexes: `(document_id, created_at DESC)`, `(group_id, created_at DESC)`, `(created_at DESC)`, `(action_type, created_at DESC)`, `(created_at DESC) WHERE entity_type = 'setting'`. +Indexes: `(document_id, created_at DESC)`, `(instance_id, created_at DESC)`, `(created_at DESC)`, `(action_type, created_at DESC)`, `(created_at DESC) WHERE entity_type = 'setting'`. Notes - The FK to `documents` is `SET NULL`, not cascade. A cascade would erase the audit trail on purge, and a strict FK would reject setting and catalog entries that have no document. - Setting and catalog entries have NULL `document_id` and `workflow_id`, with `entity_type` set to `setting` or `catalog`. -- `group_id` is a snapshot of the document's provider group, so a purged document's history stays visible only to its own tenant. +- `instance_id` is a snapshot of the document's provider instance, so a purged document's history stays visible only to its own tenant. - `entity_number` holds a page or chunk number, not an ID (chunk UUIDs would dangle after a re-chunk). - `get_settings_audit_logs` is `WHERE entity_type = 'setting'`. -- `log_audit` fills `group_id`, resolves the actor to a `users` row and builds the `actor` JSON. +- `log_audit` fills `instance_id`, resolves the actor to a `users` row and builds the `actor` JSON. - No `updated_at`: entries are immutable. - The table only grows. Monthly partitioning is possible later without changing this design. @@ -510,14 +510,14 @@ Notes ## View: `catalog_status` -Computed from `documents`, `catalogs`, `groups`, `workflows` and `document_current_workflow`. Stores nothing. One row per document. The `/catalog/v1/*` snapshot API reads it for the scheme type (`catalogs.type` of the scheme kind), and its fields are named `scheme_code`, `scheme_name` and so on, so consumers see no change. +Computed from `documents`, `catalogs`, `instances`, `workflows` and `document_current_workflow`. Stores nothing. One row per document. The `/catalog/v1/*` snapshot API reads it for the scheme type (`catalogs.type` of the scheme kind), and its fields are named `scheme_code`, `scheme_name` and so on, so consumers see no change. | Column | Type | How it is computed | |---|---|---| | document_id | UUID | `documents.id`. | | catalog_id | UUID | `documents.catalog_id`. | -| group_id | UUID | `catalogs.group_id`. | -| group_name | TEXT | `groups.name`. | +| instance_id | UUID | `catalogs.instance_id`. | +| instance_name | TEXT | `instances.name`. | | type | TEXT | `catalogs.type`. | | code | TEXT | `documents.code`. | | name | TEXT | `documents.name`. | @@ -534,7 +534,7 @@ Computed from `documents`, `catalogs`, `groups`, `workflows` and `document_curre ## 12. `webhook_records` -Records pushed to `POST /webhook` (`x-api-key` = `WEBHOOK_API_KEY`). Carried over from the previous store unchanged; not part of the groups and catalogs model. +Records pushed to `POST /webhook` (`x-api-key` = `WEBHOOK_API_KEY`). Carried over from the previous store unchanged; not part of the instances and catalogs model. Unique: `(source, external_id)`; index on `(source, status)`. @@ -569,8 +569,8 @@ Created by the runner, not by a migration file: `version` (the `NNNN` prefix, pr - **Identifiers.** `db.py` functions take a `workflow_id` argument that may be the document UUID, a Temporal workflow id, the fingerprint, the filename, an index `doc_id` or the legacy `md5(fingerprint)` doc id (`find_document_by_doc_identifier`). Documents, jobs and artifacts come back as flat dicts; `id` is the UUID and `workflow_id` is the Temporal workflow id of the document's main run (the earliest `pipeline` or `ocr_only` workflow). The API rewrites `/documents/{id}/...` to that workflow id before routing, because handlers signal and query Temporal with it. - **Jobs.** The API's "jobs" and "runs" are `workflows` rows; job ids and artifact ids are the UUIDs of `workflows` and `workflow_artifacts`. `latest_job_id`, `original_artifact_id` and `normalized_artifact_id` are derived by join, not stored. -- **Group.** The API returns a `group` string on documents, runs, queue rows and audit entries. It is the lowercase `groups.name` reached through `catalogs`; filters compare `lower(groups.name)`. +- **Instance.** The API returns an `instance` string on documents, runs, queue rows and audit entries. It is the lowercase `instances.name` reached through `catalogs`; filters compare `lower(instances.name)`. - **Catalog version.** `/catalog/v1/version` counts `audit_logs` rows with `entity_type = 'catalog'` and `action_type = 'catalog_version_bump'`, so it only grows. `updated_at` is the newest such row. - **Document stage.** The stage a document shows is `document_current_workflow.current_stage`; `update_document_stage` writes to that workflow. - **Page translations.** `save_pages` keeps stored translations unless the page carries `translations` (replaces them) or flat `translated_*` fields (merged into `translation_target_language`). A page whose OCR text changed loses its translations. -- **Users.** Rows are upserted from the auth dependency on write requests (at most once per five minutes per user) and from uploads, workflow starts and audit entries. `local-admin` is seeded by the migration. A user's `group_id` is set from the tenant codes in the token. +- **Users.** Rows are upserted from the auth dependency on write requests (at most once per five minutes per user) and from uploads, workflow starts and audit entries. `local-admin` is seeded by the migration. A user's `instance_id` is set from the tenant codes in the token. diff --git a/docs/SYSTEM_DESIGN.md b/docs/SYSTEM_DESIGN.md index 6cb6229..7aa88e4 100644 --- a/docs/SYSTEM_DESIGN.md +++ b/docs/SYSTEM_DESIGN.md @@ -36,7 +36,7 @@ gate before **PROD**. |---|---| | Multilingual document → searchable chunks | OCR → translate → chunk → embed → index | | Human quality control | Temporal **signals** pause at OCR / translation / chunk / pre-ingest | -| Multi-state tenancy | `documents.instance` + JWT groups; lists/search scoped | +| Multi-state tenancy | Catalog-owned instance + JWT groups; document access scoped | | DEV before PROD | Stage 9 writes DEV index; stage 10 waits; Super Admin promotes to PROD | | Operability | Runs, queue, audit, reconcile, artifacts UI | | Durable long work | Temporal retries + Postgres state mirror | @@ -255,12 +255,12 @@ Supporting workflows: `OcrOnly`, `TranslationOnly`, `ChunkingOnly`, ```text documents (id UUID PK) - ├── instance # tenant / state code + ├── catalog_id # → catalogs.instance_id → instances.name ├── source (JSONB) # fingerprint, filename, storage_uri, page_count ├── index_status (JSONB) # dev / prod vector index state ├── metadata (JSONB) # chunk_count, reindex_required, is_demo, ... ├── uploaded_by_user_id # → users - └── is_disabled / catalog_entry_id + └── is_disabled workflows (one per processing run; the latest gives the document's stage) └── workflow_type, status, current_stage, temporal_workflow_id → Runs UI @@ -277,6 +277,8 @@ audit_logs └── approvals, edits, promote_to_prod (outlives the document) ``` +The [legacy SQLite schema](SQLITE_SCHEMA.md) describes the importer’s source database. + ### 7.3 Tenant rule - `instance` set **once** at create (not reassigned on upsert). - Restricted users: `list_*` / get / search filtered by allowed instances. @@ -399,7 +401,7 @@ Routes: Dashboard, Documents, New Document, Document Ops, Runs, Search, Users (S ### Runtime properties - API + worker: **same image**, different entrypoint -- Shared Postgres database (pooled connections per process; schema applied from `migrations/` at startup) +- Shared Postgres database (pooled connections per process; schema applied from `migrations/` at startup) - Horizontal scale: Temporal workers and API replicas scale out against the same database --- @@ -450,7 +452,7 @@ Routes: Dashboard, Documents, New Document, Document Ops, Runs, Search, Users (S ## 17. Evolution / extension points 1. Explicit **“Request PROD approval”** action (today: auto-enter queue after DEV). -2. Moving `chunk_count` / `is_demo` out of `documents.metadata` into columns if list queries slow down. +2. Moving `chunk_count` / `is_demo` out of `documents.metadata` into columns if list queries slow down. 3. Stricter per-state role on every mutation (not only global permission). 4. Automated quality checks before Super Admin sees queue. diff --git a/migrations/0001_initial_schema.sql b/migrations/0001_initial_schema.sql index 6ea17de..b20e900 100644 --- a/migrations/0001_initial_schema.sql +++ b/migrations/0001_initial_schema.sql @@ -6,8 +6,8 @@ BEGIN RETURN NEW; END $$ LANGUAGE plpgsql; --- 1. groups: one row per tenant node (provider), mirrored from Keycloak -CREATE TABLE groups ( +-- 1. instances: one row per tenant node (provider), mirrored from Keycloak +CREATE TABLE instances ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), keycloak_group_id TEXT UNIQUE, name TEXT NOT NULL, @@ -15,7 +15,7 @@ CREATE TABLE groups ( created_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now() ); -CREATE TRIGGER trg_groups_updated_at BEFORE UPDATE ON groups +CREATE TRIGGER trg_instances_updated_at BEFORE UPDATE ON instances FOR EACH ROW EXECUTE FUNCTION set_updated_at(); -- 2. users @@ -25,7 +25,7 @@ CREATE TABLE users ( username TEXT, email TEXT, roles TEXT[] NOT NULL DEFAULT '{}', - group_id UUID REFERENCES groups(id) ON DELETE SET NULL, + instance_id UUID REFERENCES instances(id) ON DELETE SET NULL, first_seen_at TIMESTAMPTZ NOT NULL DEFAULT now(), last_seen_at TIMESTAMPTZ NOT NULL DEFAULT now() ); @@ -34,16 +34,16 @@ CREATE UNIQUE INDEX users_email_lower_key ON users (lower(email)) WHERE email IS INSERT INTO users (keycloak_user_id, username, roles) VALUES ('local-admin', 'local-admin', '{superadmin}'); --- 3. catalogs: one row per provider group and resource type +-- 3. catalogs: one row per provider instance and resource type CREATE TABLE catalogs ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), - group_id UUID NOT NULL REFERENCES groups(id) ON DELETE RESTRICT, + instance_id UUID NOT NULL REFERENCES instances(id) ON DELETE RESTRICT, type TEXT NOT NULL CHECK (type ~ '^[a-z0-9]+:[A-Za-z0-9]+$'), name TEXT NOT NULL, metadata JSONB NOT NULL DEFAULT '{}' CHECK (jsonb_typeof(metadata) = 'object'), created_at TIMESTAMPTZ NOT NULL DEFAULT now(), updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), - UNIQUE (group_id, type) + UNIQUE (instance_id, type) ); CREATE TRIGGER trg_catalogs_updated_at BEFORE UPDATE ON catalogs FOR EACH ROW EXECUTE FUNCTION set_updated_at(); @@ -165,7 +165,7 @@ CREATE TABLE audit_logs ( id UUID PRIMARY KEY DEFAULT gen_random_uuid(), document_id UUID REFERENCES documents(id) ON DELETE SET NULL, workflow_id TEXT, - group_id UUID REFERENCES groups(id) ON DELETE SET NULL, + instance_id UUID REFERENCES instances(id) ON DELETE SET NULL, action_type TEXT NOT NULL CHECK (btrim(action_type) <> ''), entity_type TEXT, entity_number INTEGER, @@ -179,7 +179,7 @@ CREATE TABLE audit_logs ( CHECK (entity_type IN ('setting', 'catalog') OR workflow_id IS NOT NULL) ); CREATE INDEX audit_logs_document_idx ON audit_logs (document_id, created_at DESC); -CREATE INDEX audit_logs_group_idx ON audit_logs (group_id, created_at DESC); +CREATE INDEX audit_logs_instance_idx ON audit_logs (instance_id, created_at DESC); CREATE INDEX audit_logs_created_idx ON audit_logs (created_at DESC); CREATE INDEX audit_logs_action_idx ON audit_logs (action_type, created_at DESC); CREATE INDEX audit_logs_settings_idx ON audit_logs (created_at DESC) WHERE entity_type = 'setting'; @@ -233,8 +233,8 @@ CREATE VIEW catalog_status AS SELECT d.id AS document_id, c.id AS catalog_id, - c.group_id, - g.name AS group_name, + c.instance_id, + g.name AS instance_name, c.type, d.code, d.name, @@ -254,7 +254,7 @@ SELECT THEN (d.metadata->'index_status'->'prod'->>'last_indexed_at')::timestamptz END AS promoted_at FROM documents d JOIN catalogs c ON c.id = d.catalog_id -JOIN groups g ON g.id = c.group_id +JOIN instances g ON g.id = c.instance_id LEFT JOIN document_current_workflow cw ON cw.document_id = d.id CROSS JOIN LATERAL ( SELECT (cw.current_stage = 'completed' AND NOT d.is_disabled diff --git a/pipeline/activities.py b/pipeline/activities.py index 4effc46..cac2215 100644 --- a/pipeline/activities.py +++ b/pipeline/activities.py @@ -29,7 +29,7 @@ from .chunking import chunk_pages, load_chunking_config from .discovery_publish_service import DiscoveryPublishService, PublishConfigError from .document_repository import DocumentRepository -from .group_names import group_display_name, publish_to_network_enabled +from .instances import instance_display_name, publish_to_network_enabled from .models import PUBLISH_SKIPPED_DISABLED, PUBLISH_SKIPPED_UNMAPPED_KIND from .network_constants import RESULT_REJECTED from .network_group import load_provider_naming, named_group, provider_for @@ -623,8 +623,8 @@ def clean_text_for_ingestion(text: str) -> str: return result.strip() -def _normalize_group(value: str | None) -> str: - """Ingest-side group normalizer (mirrors auth.tenancy without importing FastAPI).""" +def _normalize_instance(value: str | None) -> str: + """Ingest-side instance normalizer (mirrors auth.tenancy without importing FastAPI).""" text = (value or "").strip().lower() if text: return text @@ -640,7 +640,7 @@ def _prepare_records( name_en: str | None = None, description: str | None = None, include_e5_prefix_field: bool = True, - group: str | None = None, + instance: str | None = None, kind: str = "document", code: str | None = None, name: str | None = None, @@ -651,7 +651,7 @@ def _prepare_records( display_name: str | None = None, ) -> list[dict]: metadata = _get_doc_metadata(filename) - resolved_group = _normalize_group(group) + resolved_instance = _normalize_instance(instance) llm_doc_description = _get_doc_description(filename) doc_hash = hashlib.md5(document_id.encode()).hexdigest() @@ -677,7 +677,7 @@ def _prepare_records( if is_scheme else [] ) - group_name = group_display_name(resolved_group) + instance_name = instance_display_name(resolved_instance) # One period for the whole document, resolved before the loop: every chunk # of a document expires together, and re-deriving it per chunk would let a # midnight boundary split one document across two periods. @@ -699,8 +699,8 @@ def _prepare_records( "_id": point_id, "doc_id": document_id, "workflow_id": workflow_id or "", - "instance": resolved_group, # vector payload key, kept for consumers - "instance_name": group_name, + "instance": resolved_instance, # vector payload key, kept for consumers + "instance_name": instance_name, "provider_id": provider_id, "type": kind, "source": "docs-pipeline", @@ -758,7 +758,7 @@ def _scheme_fields_from_doc(doc: dict | None) -> dict: def _provider_id_from_doc(doc: dict | None) -> str | None: doc = doc or {} - group = named_group(group_for_instance(doc.get("group"))) + group = named_group(group_for_instance(doc.get("instance"))) try: provider = provider_for(load_provider_naming(), doc.get("kind"), group) except InvalidProviderIdError as exc: @@ -791,7 +791,7 @@ def prepare_ingestion_records( name_gu: str | None = None, name_en: str | None = None, description: str | None = None, - group: str | None = None, + instance: str | None = None, ) -> list[dict]: """Public helper used by tests and scripts when preparing vector-index payloads.""" return _prepare_records( @@ -802,7 +802,7 @@ def prepare_ingestion_records( name_gu=name_gu, name_en=name_en, description=description, - group=group, + instance=instance, ) @@ -1232,7 +1232,7 @@ async def prepare_for_ingestion( name_gu=name_gu, name_en=name_en, description=description, - group=(doc or {}).get("group"), + instance=(doc or {}).get("instance"), **_validity_fields_from_doc(doc), provider_id=_provider_id_from_doc(doc), display_name=(doc or {}).get("name"), @@ -1343,7 +1343,7 @@ async def promote_document_to_prod_qdrant( filename, chunks, workflow_id=workflow_id, - group=doc.get("group"), + instance=doc.get("instance"), **scheme_kwargs, **_validity_fields_from_doc(doc), provider_id=_provider_id_from_doc(doc), @@ -1456,7 +1456,7 @@ async def ingest_document_from_db( filename, chunks, workflow_id=workflow_id, - group=doc.get("group"), + instance=doc.get("instance"), **_scheme_fields_from_doc(doc), **_validity_fields_from_doc(doc), provider_id=_provider_id_from_doc(doc), diff --git a/pipeline/api/api.py b/pipeline/api/api.py index 2d8a4fc..4304a00 100644 --- a/pipeline/api/api.py +++ b/pipeline/api/api.py @@ -69,15 +69,15 @@ from ..auth.models import AuthUser from ..auth.permissions import Permission from ..auth.tenancy import ( - PORTAL_GROUP, - allowed_groups, - assert_document_group_access, - normalize_group, - resolve_create_group, + PORTAL_INSTANCE, + allowed_instances, + assert_document_instance_access, + normalize_instance, + resolve_create_instance, unrestricted, - user_can_access_group, + user_can_access_instance, ) -from ..group_names import prod_stage_disabled +from ..instances import prod_stage_disabled from ..models import ( PIPELINE_STAGES, PROD_ONLY_STAGES, @@ -160,7 +160,7 @@ async def _start_services(): if auth_cfg.disabled: logging.warning( "WARNING: AUTH_DISABLED=true — every caller is treated as synthetic " - "master_admin with unrestricted group access. Do not expose this " + "master_admin with unrestricted instance access. Do not expose this " "API beyond the internal network or set AUTH_DISABLED=false until " "the maintainer UI sends Bearer tokens." ) @@ -407,15 +407,15 @@ def _compute_file_fingerprint(filepath: Path) -> str: return md5.hexdigest() -def _group_scope_for_user(user: AuthUser) -> Optional[list[str]]: - """None = unrestricted; otherwise only these group codes.""" - allowed = allowed_groups(user) +def _instance_scope_for_user(user: AuthUser) -> Optional[list[str]]: + """None = unrestricted; otherwise only these instance codes.""" + allowed = allowed_instances(user) if allowed is None: return None return sorted(allowed) -def _document_cohorts_payload(summary: dict, by_group: Optional[list[dict]] = None) -> dict: +def _document_cohorts_payload(summary: dict, by_instance: Optional[list[dict]] = None) -> dict: """Shape Postgres aggregate counts for DocumentCohortsResponse (all ints).""" def _i(key: str, default: int = 0) -> int: val = summary.get(key, default) @@ -439,15 +439,13 @@ def _i(key: str, default: int = 0) -> int: "ready_for_ingestion": _i("ready_for_ingestion_documents"), "failed": _i("failed_documents"), }, - "by_instance": [ - {**row, "instance": row["group"]} for row in (by_group or []) - ], + "by_instance": by_instance or [], } -def _resolve_upload_instance(user: AuthUser, instance: str) -> str: +def _resolve_create_instance(user: AuthUser, requested: Optional[str] = None) -> str: """Resolve the upload owner using the existing Keycloak tenancy rules.""" - return resolve_create_group(user, instance) + return resolve_create_instance(user, requested) def _document_title(filename: str) -> str: @@ -459,29 +457,29 @@ def _document_title(filename: str) -> str: def _assert_can_replace(user: AuthUser, existing: dict) -> None: """Replacing a document's file clears its pages and chunks, so guard it. - Needs review rights in the document's group; a disabled document or one with + Needs review rights in the document's instance; a disabled document or one with a run in progress cannot be replaced; a document already promoted to PROD can only be replaced by someone who may approve ingestion. """ - doc_group = existing["group"] - if not user.has_permission_for_group(Permission.REVIEW, doc_group): - raise HTTPException(403, f"No permission to replace documents in group: {doc_group}") + doc_instance = existing["instance"] + if not user.has_permission_for_instance(Permission.REVIEW, doc_instance): + raise HTTPException(403, f"No permission to replace documents in instance: {doc_instance}") if existing.get("is_disabled"): raise HTTPException(409, "This document is disabled; restore it before replacing its file") latest_job = db.get_latest_document_job(existing["workflow_id"]) if latest_job and latest_job.get("status") == "running": raise HTTPException(409, "A run is still in progress for this document") - if (existing.get("index_status") or {}).get("prod") and not user.has_permission_for_group( - Permission.APPROVE_INGESTION, doc_group + if (existing.get("index_status") or {}).get("prod") and not user.has_permission_for_instance( + Permission.APPROVE_INGESTION, doc_instance ): raise HTTPException( - 403, "Replacing a document that is live in PROD needs approval rights in its group" + 403, "Replacing a document that is live in PROD needs approval rights in its instance" ) def _resolve_document_identity( user: AuthUser, - group: str, + instance: str, kind: str, code: str, filename: str, @@ -492,41 +490,41 @@ def _resolve_document_identity( kind means "update this document": the caller reuses it instead of inserting. Without a code one is derived from the file name. - Permissions are checked against the target group, not just globally: a user + Permissions are checked against the target instance, not just globally: a user who may upload in one state must not be able to add to, or replace a document of, a state where they only hold a view role. Replacing clears the - document's pages and chunks, so it also needs review rights in that group. + document's pages and chunks, so it also needs review rights in that instance. """ kind = (kind or "").strip().lower() try: catalog_type_for_kind(kind) except ValueError as exc: raise HTTPException(400, str(exc)) from exc - if not user.has_permission_for_group(Permission.UPLOAD, group): - raise HTTPException(403, f"No upload permission in group: {group}") + if not user.has_permission_for_instance(Permission.UPLOAD, instance): + raise HTTPException(403, f"No upload permission in instance: {instance}") code = (code or "").strip() if code: try: code = scheme_catalog.validate_scheme_code(code) except ValueError as exc: raise HTTPException(400, str(exc)) from exc - existing = db.find_document_by_code(group, kind, code) + existing = db.find_document_by_code(instance, kind, code) if existing: - existing = assert_document_group_access(user, existing) + existing = assert_document_instance_access(user, existing) _assert_can_replace(user, existing) return kind, code, existing return kind, scheme_catalog.derive_scheme_code(_document_title(filename), kind=kind), None def _require_document_for_user(workflow_id: str, user: AuthUser) -> dict: - """Load a document or 404 if missing / outside the caller's group scope.""" - return assert_document_group_access(user, db.get_document(workflow_id)) + """Load a document or 404 if missing / outside the caller's instance scope.""" + return assert_document_instance_access(user, db.get_document(workflow_id)) def _document_for_user_or_none(workflow_id: str, user: AuthUser) -> Optional[dict]: """Like _require_document_for_user but returns None instead of raising (bulk paths).""" doc = db.get_document(workflow_id) - if not doc or not user_can_access_group(user, doc.get("group")): + if not doc or not user_can_access_instance(user, doc.get("instance")): return None return doc @@ -581,7 +579,7 @@ def _document_summary_from_row(doc: dict, current_job: Optional[dict] = None) -> source_manifest_name=doc.get("source_manifest_name"), source_file_fingerprint=doc.get("source_file_fingerprint"), authoritative=bool(doc.get("source_manifest_name")), - instance=normalize_group(doc.get("group")), + instance=normalize_instance(doc.get("instance")), stage=DocumentStage(doc["stage"]), page_count=doc.get("page_count") or 0, chunk_count=doc.get("chunk_count") or 0, @@ -828,8 +826,8 @@ async def auth_me(user: CurrentUser): "groups": list(user.groups or []), "state_roles": dict(user.state_roles or {}), "is_super_admin": bool(user.is_superadmin), - # Portal group code for BV / platform docs (not a state). - "portal_instance": PORTAL_GROUP, + # Portal instance code for BV / platform docs (not a state). + "portal_instance": PORTAL_INSTANCE, "auth_disabled": user.token_disabled_mode, } @@ -931,7 +929,7 @@ async def get_operations_queue( ): """Return documents that currently need operator or agent action. - Results are limited to groups the caller can access (state users never + Results are limited to instances the caller can access (state users never see portal ``bv`` or other states' documents). """ include_disabled = x_include_disabled and x_include_disabled.lower() == "true" @@ -939,7 +937,7 @@ async def get_operations_queue( limit=limit, offset=offset, include_disabled=include_disabled, - groups=_group_scope_for_user(user), + instances=_instance_scope_for_user(user), ) items = [ OperationQueueEntry( @@ -967,19 +965,16 @@ async def list_runs( ): """List recent document jobs visible to the caller (tenant/role scoped). - Restricted users only see jobs whose document ``group`` is in their + Restricted users only see jobs whose document ``instance`` is in their allowed states. Super-admin / bypass users see all tenants. """ - runs = db.list_runs( + return db.list_runs( limit=limit, offset=offset, status=status, - groups=_group_scope_for_user(user), + instances=_instance_scope_for_user(user), ) - for run in runs: - run["instance"] = run.pop("group") - return runs @app.get("/runs/{job_id}") @@ -988,7 +983,7 @@ async def get_run(job_id: str, user: RequireSearch): run = db.get_document_job(job_id) if not run: raise HTTPException(404, f"Run not found: {job_id}") - # Enforce same group scope as list: hide other tenants' runs as 404. + # Enforce same instance scope as list: hide other tenants' runs as 404. workflow_id = run.get("workflow_id") if workflow_id: _require_document_for_user(str(workflow_id), user) @@ -1039,7 +1034,7 @@ def _build_document_detail(doc: dict) -> DocumentDetail: source_manifest_name=doc.get("source_manifest_name"), source_file_fingerprint=doc.get("source_file_fingerprint"), authoritative=bool(doc.get("source_manifest_name")), - instance=normalize_group(doc.get("group")), + instance=normalize_instance(doc.get("instance")), filepath=doc["filepath"], stage=DocumentStage(doc["stage"]), page_count=doc.get("page_count", 0), @@ -1564,7 +1559,7 @@ async def disable_document( doc = _require_document_for_user(workflow_id, user) if not user.has_permission(Permission.ADMIN): - if not user.has_permission_for_group(Permission.DELETE_OWN, doc.get("group")): + if not user.has_permission_for_instance(Permission.DELETE_OWN, doc.get("instance")): raise HTTPException(403, "Missing permission: delete_own") if purge: raise HTTPException( @@ -2703,14 +2698,14 @@ async def get_all_audit_logs( Each entry includes the document filename for context. """ - scope = _group_scope_for_user(user) + scope = _instance_scope_for_user(user) logs = db.get_all_audit_logs( action_type=action_type, limit=limit, offset=offset, - groups=scope, + instances=scope, ) - total = db.get_all_audit_log_count(action_type, groups=scope) + total = db.get_all_audit_log_count(action_type, instances=scope) return AuditLogResponse( logs=logs, @@ -2944,14 +2939,14 @@ async def search_chunks_across_documents( stage: Optional[DocumentStage] = Query(None, description="Optional document stage filter"), ): """Search chunks across all documents for KB maintainer workflows.""" - # Tenant-scope via the owning document's group (None = unrestricted admin). + # Tenant-scope via the owning document's instance (None = unrestricted admin). chunks, total = db.search_chunks( query=q, limit=limit, offset=offset, include_excluded=include_excluded, stage=stage.value if stage else None, - groups=_group_scope_for_user(user), + instances=_instance_scope_for_user(user), ) return { "items": chunks, @@ -3643,11 +3638,11 @@ async def reconcile_document_states(user: RequirePipeline): 'ready_for_ingestion', 'ingesting' ] - # Scope to caller's groups (None = unrestricted bypass / all tenants). + # Scope to caller's instances (None = unrestricted bypass / all tenants). docs = db.list_documents( limit=1000, include_disabled=True, - groups=_group_scope_for_user(user), + instances=_instance_scope_for_user(user), ) active_docs = [d for d in docs if d.get('stage') in active_stages] diff --git a/pipeline/api/documents.py b/pipeline/api/documents.py index 0432ebf..b431755 100644 --- a/pipeline/api/documents.py +++ b/pipeline/api/documents.py @@ -9,8 +9,8 @@ from .. import db, document_validity from ..auth.deps import CurrentUser, RequireUpload -from ..auth.tenancy import assert_document_group_access, normalize_group -from ..group_names import prod_stage_disabled +from ..auth.tenancy import assert_document_instance_access, normalize_instance +from ..instances import prod_stage_disabled from ..models import ( DocumentCohortsResponse, DocumentStage, @@ -54,12 +54,12 @@ async def start_document_workflow( Rate limited to 10 requests/minute per IP. Requires permission: upload (no-op while AUTH_DISABLED=true). """ - create_group = _api._resolve_upload_instance(user, instance) + create_instance = _api._resolve_create_instance(user, instance) # Validate file path to prevent path traversal attacks filepath = _api.validate_file_path(data.filepath) source_filename = _api.get_filename_from_path(filepath) doc_kind, doc_code, replace_doc = _api._resolve_document_identity( - user, create_group, kind, code, source_filename + user, create_instance, kind, code, source_filename ) source_file_fingerprint = _api._compute_file_fingerprint(filepath) canonical_document_id = source_file_fingerprint @@ -72,7 +72,7 @@ async def start_document_workflow( existing_doc = db.get_document(workflow_id) if existing_doc: # Same fingerprint/path must not leak or restart another tenant's doc. - existing_doc = assert_document_group_access(user, existing_doc) + existing_doc = assert_document_instance_access(user, existing_doc) try: handle = _api.temporal_client.get_workflow_handle(workflow_id) state = await handle.query("get_state") @@ -85,7 +85,7 @@ async def start_document_workflow( source_filename=source_filename, source_file_fingerprint=source_file_fingerprint, authoritative=bool(existing_doc.get("source_manifest_name")) if existing_doc else False, - instance=normalize_group(existing_doc.get("group")), + instance=normalize_instance(existing_doc.get("instance")), stage=DocumentStage(state.get("stage", "registered")), page_count=state.get("page_count", 0), chunk_count=state.get("chunk_count", 0), @@ -143,7 +143,7 @@ async def start_document_workflow( filepath=str(filepath), stage="registered", stop_after_ocr=stop_after_ocr, - group=create_group, + instance=create_instance, uploaded_by_user_id=actor.get("user_id") or None, uploaded_by_username=actor.get("username") or None, uploaded_by_email=actor.get("email") or None, @@ -182,7 +182,7 @@ async def start_document_workflow( new_value=source_filename, metadata={ "source": "register_endpoint", - "instance": create_group, + "instance": create_instance, "kind": doc_kind, "code": doc_code, "replaced_existing": bool(replace_doc), @@ -198,7 +198,7 @@ async def start_document_workflow( source_filename=source_filename, source_file_fingerprint=source_file_fingerprint, authoritative=False, - instance=create_group, + instance=create_instance, stage=DocumentStage.REGISTERED, page_count=0, chunk_count=0, @@ -224,7 +224,7 @@ async def start_batch_workflows( kind: str = Query(..., description="scheme or advisory"), ): """Start workflows for all supported documents in a directory.""" - create_group = _api._resolve_upload_instance(user, instance) + create_instance = _api._resolve_create_instance(user, instance) directory = Path(data.directory) if not directory.exists(): raise HTTPException(404, f"Directory not found: {data.directory}") @@ -245,7 +245,7 @@ async def start_batch_workflows( chunk_overlap=chunk_overlap, min_tokens=min_tokens, stop_after_ocr=stop_after_ocr, - instance=create_group, + instance=create_instance, kind=kind, code="", ) @@ -280,7 +280,7 @@ async def list_documents( Uses Postgres for fast listing (no Temporal queries for performance). Disabled (soft-deleted) documents are excluded by default - use X-Include-Disabled: true to show them. - Results are limited to groups the caller can access. + Results are limited to instances the caller can access. Pagination: - limit: Max documents to return (default 100, max 500) @@ -295,7 +295,7 @@ async def list_documents( limit=limit, offset=offset, include_disabled=include_disabled, - groups=_api._group_scope_for_user(user), + instances=_api._instance_scope_for_user(user), ) return [_api._document_summary_from_row(doc) for doc in docs] @@ -308,16 +308,16 @@ async def get_documents_summary( ): """Return aggregate Postgres counts for dashboard totals and migration planning.""" include_disabled = bool(x_include_disabled and x_include_disabled.lower() == "true") - group_scope = _api._group_scope_for_user(user) + instance_scope = _api._instance_scope_for_user(user) summary = db.get_document_summary_counts( include_disabled=include_disabled, - groups=group_scope, + instances=instance_scope, ) - by_group = db.get_document_counts_by_group( + by_instance = db.get_document_counts_by_instance( include_disabled=include_disabled, - groups=group_scope, + instances=instance_scope, ) - return _api._document_cohorts_payload(summary, by_group) + return _api._document_cohorts_payload(summary, by_instance) @router.get("/cohorts", response_model=DocumentCohortsResponse) @@ -327,13 +327,13 @@ async def get_document_cohorts( ): """Return machine-friendly cohort counts for queueing and orchestration.""" include_disabled = bool(x_include_disabled and x_include_disabled.lower() == "true") - group_scope = _api._group_scope_for_user(user) + instance_scope = _api._instance_scope_for_user(user) summary = db.get_document_summary_counts( include_disabled=include_disabled, - groups=group_scope, + instances=instance_scope, ) - by_group = db.get_document_counts_by_group( + by_instance = db.get_document_counts_by_instance( include_disabled=include_disabled, - groups=group_scope, + instances=instance_scope, ) - return _api._document_cohorts_payload(summary, by_group) + return _api._document_cohorts_payload(summary, by_instance) diff --git a/pipeline/api/upload.py b/pipeline/api/upload.py index dd62f41..b7d4be1 100644 --- a/pipeline/api/upload.py +++ b/pipeline/api/upload.py @@ -10,8 +10,8 @@ from .. import db, document_validity from .. import display_name as display_name_rules from ..auth.deps import RequireUpload -from ..auth.tenancy import assert_document_group_access, normalize_group -from ..group_names import prod_stage_disabled +from ..auth.tenancy import assert_document_instance_access, normalize_instance +from ..instances import prod_stage_disabled from ..models import DocumentStage, DocumentSummary from ..workflows import DocumentPipelineWorkflow from . import api as _api @@ -44,9 +44,9 @@ async def upload_and_process( Rate limited to 10 requests/minute per IP. Requires permission: upload (no-op while AUTH_DISABLED=true). """ - create_group = _api._resolve_upload_instance(user, instance) + create_instance = _api._resolve_create_instance(user, instance) doc_kind, doc_code, replace_doc = _api._resolve_document_identity( - user, create_group, kind, code, file.filename or "" + user, create_instance, kind, code, file.filename or "" ) suffix = Path(file.filename or "").suffix.lower() if suffix not in _api.ALLOWED_EXTENSIONS: @@ -91,7 +91,7 @@ async def upload_and_process( # If Postgres was purged, avoid returning stale Temporal state and create a fresh run ID. existing_doc = db.get_document(workflow_id) if existing_doc: - existing_doc = assert_document_group_access(user, existing_doc) + existing_doc = assert_document_instance_access(user, existing_doc) try: handle = _api.temporal_client.get_workflow_handle(workflow_id) state = await handle.query("get_state") @@ -104,7 +104,7 @@ async def upload_and_process( source_filename=file.filename, source_file_fingerprint=file_hash, authoritative=bool(existing_doc.get("source_manifest_name")) if existing_doc else False, - instance=normalize_group(existing_doc.get("group")), + instance=normalize_instance(existing_doc.get("instance")), stage=DocumentStage(state.get("stage", "registered")), page_count=state.get("page_count", 0), chunk_count=state.get("chunk_count", 0), @@ -160,7 +160,7 @@ async def upload_and_process( filepath=minio_path, stage="registered", stop_after_ocr=stop_after_ocr, - group=create_group, + instance=create_instance, uploaded_by_user_id=actor.get("user_id") or None, uploaded_by_username=actor.get("username") or None, uploaded_by_email=actor.get("email") or None, @@ -211,7 +211,7 @@ async def upload_and_process( new_value=file.filename, metadata={ "source": "upload_endpoint", - "instance": create_group, + "instance": create_instance, "kind": doc_kind, "code": doc_code, "replaced_existing": bool(replace_doc), @@ -237,7 +237,7 @@ async def upload_and_process( source_filename=file.filename, source_file_fingerprint=file_hash, authoritative=False, - instance=create_group, + instance=create_instance, stage=DocumentStage.REGISTERED, page_count=0, chunk_count=0, diff --git a/pipeline/auth/groups.py b/pipeline/auth/groups.py index 4104b6e..f907c02 100644 --- a/pipeline/auth/groups.py +++ b/pipeline/auth/groups.py @@ -217,17 +217,17 @@ def extract_groups_claim(claims: dict[str, Any]) -> list[str]: return [] -def role_for_group(access: GroupAccess, group: str | None) -> str | None: - """Return the user's role for a state group, or SUPER_ADMIN when global. +def role_for_instance(access: GroupAccess, instance: str | None) -> str | None: + """Return the user's role for a state instance, or SUPER_ADMIN when global. A ``bh_viewer`` has no per-state group, but reads every state — so any - group resolves to STATE_VIEW unless a stronger state role is held. + instance resolves to STATE_VIEW unless a stronger state role is held. """ if access.is_super_admin: return ROLE_SUPER_ADMIN - if not group: + if not instance: return None - role = access.state_roles.get(normalize_state_code(group)) + role = access.state_roles.get(normalize_state_code(instance)) if role: return role return ROLE_STATE_VIEW if access.is_bh_viewer else None diff --git a/pipeline/auth/keycloak_admin.py b/pipeline/auth/keycloak_admin.py index d1bbdc8..0b67380 100644 --- a/pipeline/auth/keycloak_admin.py +++ b/pipeline/auth/keycloak_admin.py @@ -28,7 +28,7 @@ group_leaf_for_role, normalize_role, ) -from .tenancy import PORTAL_GROUP +from .tenancy import PORTAL_INSTANCE # Realm-role descriptions written when a role is auto-created. STATE_ROLE_DESCRIPTIONS: dict[str, str] = { @@ -506,7 +506,7 @@ def provision_user( "roles": [r.get("name") for r in (roles or []) if isinstance(r, dict)], "app_url": ui_url, "login_url": login_url, - "portal_instance": PORTAL_GROUP, + "portal_instance": PORTAL_INSTANCE, "share_message": "\n".join(share_lines), } @@ -517,7 +517,7 @@ def list_access_options() -> dict[str, Any]: return { "keycloak_admin_configured": cfg.configured, "realm": cfg.realm, - "portal_instance": PORTAL_GROUP, + "portal_instance": PORTAL_INSTANCE, "access_types": [ { "id": "super_admin", @@ -579,9 +579,9 @@ def list_access_options() -> dict[str, Any]: "notes": [ "Users sign in with Google SSO using the same email you enter.", "No app password is required for normal SSO login.", - "State codes must match document group codes (mh, up, …).", + "State codes must match document instance tags (mh, up, …).", "A centre is modelled as just another state code.", - "Super admin (BV) documents can use portal group 'bv'.", + "Super admin (BV) documents can use portal instance 'bv'.", "Access comes from group membership only — no direct role assignment.", "Keycloak groups: /global/super-admin, /global/bh-viewer, " "/states/{STATE}/{admin|approver|contributor|view}", diff --git a/pipeline/auth/models.py b/pipeline/auth/models.py index 6a3ab40..947d21c 100644 --- a/pipeline/auth/models.py +++ b/pipeline/auth/models.py @@ -29,7 +29,7 @@ # Roles not limited by the JWT ``instances`` claim. # State-level roles are restricted to their claimed instances (tenants/states). -GROUP_UNRESTRICTED_ROLES = SUPERADMIN_ROLES | GLOBAL_READ_ROLES +INSTANCE_UNRESTRICTED_ROLES = SUPERADMIN_ROLES | GLOBAL_READ_ROLES @dataclass @@ -73,44 +73,44 @@ def is_admin(self) -> bool: """ return self.is_superadmin - def is_group_unrestricted(self) -> bool: - """True when the caller may access every group (all tenants/states). + def is_instance_unrestricted(self) -> bool: + """True when the caller may access every instance (all tenants/states). Two cases: local bypass mode with no scoped claim, or platform superadmin even when the token carries a narrow ``instances`` claim. """ if self.token_disabled_mode and not self.instances: return True - return bool(GROUP_UNRESTRICTED_ROLES & self._role_keys) + return bool(INSTANCE_UNRESTRICTED_ROLES & self._role_keys) - def has_group(self, group: str) -> bool: + def has_instance(self, instance: str) -> bool: if not self.instances: # Empty list means "no tenant restriction yet" only in disabled mode. return self.token_disabled_mode - return group.strip().lower() in {i.lower() for i in self.instances} + return instance.strip().lower() in {i.lower() for i in self.instances} def has_env(self, env: str) -> bool: if not self.envs: return self.token_disabled_mode return env.strip().lower() in {e.lower() for e in self.envs} - def role_for_group(self, group: str | None) -> str | None: + def role_for_instance(self, instance: str | None) -> str | None: """Role in a given state; super_admin always returns super_admin. - A ``bh_viewer`` reads every state, so any group falls back to + A ``bh_viewer`` reads every state, so any instance falls back to STATE_VIEW when no stronger per-state role is held. """ if self.is_superadmin: return ROLE_SUPER_ADMIN - if not group: + if not instance: return None - role = self.state_roles.get(normalize_state_code(group)) + role = self.state_roles.get(normalize_state_code(instance)) if role: return role return ROLE_STATE_VIEW if self.is_bh_viewer else None - def has_permission_for_group( - self, permission: Permission | str, group: str | None + def has_permission_for_instance( + self, permission: Permission | str, instance: str | None ) -> bool: """Permission check scoped to a state when per-state roles are present. @@ -126,7 +126,7 @@ def has_permission_for_group( if not self.state_roles: return needed in self.permissions - role = self.role_for_group(group) + role = self.role_for_instance(instance) if not role: return False from .permissions import permissions_for_roles diff --git a/pipeline/auth/tenancy.py b/pipeline/auth/tenancy.py index a9b0f27..e507bac 100644 --- a/pipeline/auth/tenancy.py +++ b/pipeline/auth/tenancy.py @@ -1,14 +1,14 @@ -"""Group (tenant / state) access helpers — Keycloak groups + document groups. +"""Instance (tenant / state) access helpers — Keycloak groups + document instances. Who-sees-what-state comes from the JWT (Keycloak groups). The database only -stores which group a *document* belongs to (``groups`` via -``documents.catalog_id -> catalogs.group_id``); the document dict carries its -tenant code as ``group``. +stores which instance a *document* belongs to (``instances`` via +``documents.catalog_id -> catalogs.instance_id``); the document dict carries its +tenant code as ``instance``. Conventions: - State codes: lowercase (``mh``, ``up``, …) matching Keycloak ``/states/MH/...`` - Portal / BV platform docs: ``bv`` (Bharat Vistaar) - - Super admin (``/global/super-admin``): unrestricted groups + - Super admin (``/global/super-admin``): unrestricted instances """ from __future__ import annotations @@ -20,64 +20,64 @@ from .models import AuthUser # Bharat Vistaar portal / platform-wide documents (not a state). -PORTAL_GROUP = "bv" +PORTAL_INSTANCE = "bv" -def default_group() -> str: - """Fallback group code for new docs and legacy rows without a value.""" +def default_instance() -> str: + """Fallback instance code for new docs and legacy rows without a value.""" return (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" -def portal_group() -> str: - return (os.environ.get("PORTAL_INSTANCE") or PORTAL_GROUP).strip().lower() or PORTAL_GROUP +def portal_instance() -> str: + return (os.environ.get("PORTAL_INSTANCE") or PORTAL_INSTANCE).strip().lower() or PORTAL_INSTANCE -def group_for_instance(group: str | None) -> str | None: - """The named network group of a tenant code, or None for the portal / default tenant.""" - code = (group or "").strip().lower() - if not code or code in {portal_group(), default_group()}: +def group_for_instance(instance: str | None) -> str | None: + """The named network instance of a tenant code, or None for the portal / default tenant.""" + code = (instance or "").strip().lower() + if not code or code in {portal_instance(), default_instance()}: return None return code -def normalize_group(value: str | None) -> str: +def normalize_instance(value: str | None) -> str: text = (value or "").strip().lower() - return text or default_group() + return text or default_instance() def unrestricted(user: AuthUser) -> bool: - """True when the caller may see all groups (super admin / local bypass).""" - return user.is_group_unrestricted() + """True when the caller may see all instances (super admin / local bypass).""" + return user.is_instance_unrestricted() -def allowed_groups(user: AuthUser) -> set[str] | None: +def allowed_instances(user: AuthUser) -> set[str] | None: """ - Return the set of group codes the user may access, or None if unrestricted. + Return the set of instance codes the user may access, or None if unrestricted. """ if unrestricted(user): return None - return {normalize_group(i) for i in user.instances if str(i).strip()} + return {normalize_instance(i) for i in user.instances if str(i).strip()} -def user_can_access_group(user: AuthUser, group: str | None) -> bool: - allowed = allowed_groups(user) +def user_can_access_instance(user: AuthUser, instance: str | None) -> bool: + allowed = allowed_instances(user) if allowed is None: return True if not allowed: return False - return normalize_group(group) in allowed + return normalize_instance(instance) in allowed -def assert_group_access(user: AuthUser, group: str | None) -> str: - """Raise 403 if user cannot access group; return normalized group code.""" - normalized = normalize_group(group) - if not user_can_access_group(user, normalized): - raise HTTPException(403, f"No access to group: {normalized}") +def assert_instance_access(user: AuthUser, instance: str | None) -> str: + """Raise 403 if user cannot access instance; return normalized instance code.""" + normalized = normalize_instance(instance) + if not user_can_access_instance(user, normalized): + raise HTTPException(403, f"No access to instance: {normalized}") return normalized -def resolve_create_group(user: AuthUser, requested: str | None = None) -> str: - """Pick the document's ``group`` at create/upload time. +def resolve_create_instance(user: AuthUser, requested: str | None = None) -> str: + """Pick the document's ``instance`` at create/upload time. - Super admin / bypass: use requested, else portal ``bv``, else DEFAULT_INSTANCE. - Single-state user: that state (requested must match if provided). @@ -88,11 +88,11 @@ def resolve_create_group(user: AuthUser, requested: str | None = None) -> str: if unrestricted(user): if requested_norm: - return normalize_group(requested_norm) + return normalize_instance(requested_norm) # Prefer portal tag for platform operators when nothing chosen. - return normalize_group(portal_group()) + return normalize_instance(portal_instance()) - allowed = allowed_groups(user) or set() + allowed = allowed_instances(user) or set() if not allowed: raise HTTPException( 403, @@ -101,7 +101,7 @@ def resolve_create_group(user: AuthUser, requested: str | None = None) -> str: if requested_norm: if requested_norm not in allowed: - raise HTTPException(403, f"No access to group: {requested_norm}") + raise HTTPException(403, f"No access to instance: {requested_norm}") return requested_norm if len(allowed) == 1: @@ -114,13 +114,13 @@ def resolve_create_group(user: AuthUser, requested: str | None = None) -> str: ) -def assert_document_group_access(user: AuthUser, doc: dict | None) -> dict: +def assert_document_instance_access(user: AuthUser, doc: dict | None) -> dict: """ - Ensure the document exists and the user may access its group. + Ensure the document exists and the user may access its instance. Missing / forbidden both return 404 to avoid leaking other tenants' ids. """ if not doc: raise HTTPException(404, "Document not found") - if not user_can_access_group(user, doc.get("group")): + if not user_can_access_instance(user, doc.get("instance")): raise HTTPException(404, "Document not found") return doc diff --git a/pipeline/catalog_builder.py b/pipeline/catalog_builder.py index fd8d3d4..4b74978 100644 --- a/pipeline/catalog_builder.py +++ b/pipeline/catalog_builder.py @@ -72,28 +72,28 @@ PUBLISHABLE_DOCUMENT_KINDS = tuple(_KNOWLEDGE_KIND_CATALOGS) -def normalize_document_kind(kind: Optional[str]) -> str: - """Coerce a document's knowledge kind (from its catalog type) to its lookup form.""" - return (kind or DEFAULT_DOCUMENT_KIND).strip().lower() +def normalize_document_kind(document_kind: Optional[str]) -> str: + """Coerce a stored `documents.document_kind` to its lookup form.""" + return (document_kind or DEFAULT_DOCUMENT_KIND).strip().lower() -def is_publishable_kind(kind: Optional[str]) -> bool: - return normalize_document_kind(kind) in _KNOWLEDGE_KIND_CATALOGS +def is_publishable_kind(document_kind: Optional[str]) -> bool: + return normalize_document_kind(document_kind) in _KNOWLEDGE_KIND_CATALOGS def build_catalog( - kind: Optional[str], + document_kind: Optional[str], provider: dict, catalog_name: str, group: Optional[Group] = None, ) -> Optional[dict]: - """Return the catalog to announce for `kind`, or None. + """Return the catalog to announce for `document_kind`, or None. None means this kind has nothing to announce - `document`, `video` and any operator-entered custom slug. Callers must skip the publish rather than send an empty `catalogs` array, which the spec rejects (`minItems: 1`). """ - spec = _KNOWLEDGE_KIND_CATALOGS.get(normalize_document_kind(kind)) + spec = _KNOWLEDGE_KIND_CATALOGS.get(normalize_document_kind(document_kind)) if spec is None: return None diff --git a/pipeline/db.py b/pipeline/db.py index f8d9b78..b8a70e8 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -103,7 +103,7 @@ def init_db() -> None: current = migrate.current_version(database_url()) logger.info("db migrations current=%s applied_now=%s", current, applied) get_pool() - created = sync_catalogs_for_all_groups() + created = sync_catalogs_for_all_instances() if created: logger.info("catalog sync created %d missing catalogs", created) @@ -140,13 +140,13 @@ def check_health() -> dict: DEV_INDEX_KEY = "dev" -def _default_group() -> str: +def _default_instance() -> str: return (os.environ.get("DEFAULT_INSTANCE") or "default").strip().lower() or "default" -def _normalize_groups(groups: list[str]) -> list[str]: - default = _default_group() - return sorted({(i or "").strip().lower() or default for i in groups}) +def _normalize_instances(instances: list[str]) -> list[str]: + default = _default_instance() + return sorted({(i or "").strip().lower() or default for i in instances}) def _iso_now() -> str: @@ -274,7 +274,7 @@ def _require_document_uuid(conn, identifier: str) -> str: SELECT d.id, d.catalog_id, d.code, d.name, d.aliases, d.source, d.is_disabled, d.metadata, d.created_at, d.updated_at, - c.type AS catalog_type, g.name AS group_name, + c.type AS catalog_type, g.name AS instance_name, (SELECT count(*) FROM pages p WHERE p.document_id = d.id) AS page_total, (SELECT count(*) FROM chunks ch WHERE ch.document_id = d.id) AS chunk_total, root.temporal_workflow_id AS root_workflow_id, @@ -293,7 +293,7 @@ def _require_document_uuid(conn, identifier: str) -> str: na.id AS normalized_artifact_id FROM documents d JOIN catalogs c ON c.id = d.catalog_id - JOIN groups g ON g.id = c.group_id + JOIN instances g ON g.id = c.instance_id LEFT JOIN document_current_workflow cw ON cw.document_id = d.id LEFT JOIN LATERAL ( SELECT w.temporal_workflow_id, w.stop_after_ocr FROM workflows w @@ -360,7 +360,7 @@ def _document_from_row(row: Optional[dict]) -> Optional[dict]: "source_manifest_name": None, "source_file_fingerprint": fingerprint, "filepath": source.get("storage_uri") or "", - "group": (row["group_name"] or "").lower(), + "instance": (row["instance_name"] or "").lower(), "stage": row["current_stage"] or "registered", "page_count": int(row["page_total"] or 0), "chunk_count": int(row["chunk_total"] or 0), @@ -411,62 +411,62 @@ def _fetch_document(conn, doc_uuid: Optional[str]) -> Optional[dict]: def _document_filters( include_disabled: bool, - groups: Optional[list[str]], + instances: Optional[list[str]], alias: str = "d", - group_alias: str = "g", + instance_alias: str = "g", ) -> tuple[list[str], list]: - """WHERE clauses for document lists. The query must join ``groups`` as ``group_alias``.""" + """WHERE clauses for document lists. The query must join ``instances`` as ``instance_alias``.""" clauses: list[str] = [] params: list = [] if not include_disabled: clauses.append(f"NOT {alias}.is_disabled") - if groups is not None: - clauses.append(f"lower({group_alias}.name) = ANY(%s)") - params.append(_normalize_groups(groups)) + if instances is not None: + clauses.append(f"lower({instance_alias}.name) = ANY(%s)") + params.append(_normalize_instances(instances)) return clauses, params # ============================================================================= -# Groups +# Instances # ============================================================================= -def _tenant_path(group: str) -> str: +def _tenant_path(instance: str) -> str: """Keycloak group path of a tenant code: ``bv`` is ``/global``, others ``/states/``.""" - return "/global" if group == "bv" else f"/states/{group.upper()}" + return "/global" if instance == "bv" else f"/states/{instance.upper()}" def _tenant_code_from_path(path: str) -> str: return "bv" if path == "/global" else path.rstrip("/").rsplit("/", 1)[-1].lower() -def upsert_group(path: str, keycloak_group_id: Optional[str] = None, conn=None) -> str: - """Create or refresh the ``groups`` row for a Keycloak group path; returns its UUID. +def upsert_instance(path: str, keycloak_group_id: Optional[str] = None, conn=None) -> str: + """Create or refresh the ``instances`` row for a Keycloak group path; returns its UUID. Rows made by the SQLite move have no Keycloak id yet; the first sync that - sees the path fills it in. The group's catalogs (one per supported type) are - ensured in the same transaction, so a group never exists without them. + sees the path fills it in. The instance's catalogs (one per supported type) are + ensured in the same transaction, so an instance never exists without them. """ if conn is None: with get_connection() as own: - return upsert_group(path, keycloak_group_id, conn=own) + return upsert_instance(path, keycloak_group_id, conn=own) row = conn.execute( """ - INSERT INTO groups (keycloak_group_id, name, path) VALUES (%s, %s, %s) + INSERT INTO instances (keycloak_group_id, name, path) VALUES (%s, %s, %s) ON CONFLICT (path) DO UPDATE SET - keycloak_group_id = COALESCE(EXCLUDED.keycloak_group_id, groups.keycloak_group_id) + keycloak_group_id = COALESCE(EXCLUDED.keycloak_group_id, instances.keycloak_group_id) RETURNING id """, (keycloak_group_id, _tenant_code_from_path(path), path), ).fetchone() - group_id = str(row["id"]) - ensure_group_catalogs(group_id, conn=conn) - return group_id + instance_id = str(row["id"]) + ensure_instance_catalogs(instance_id, conn=conn) + return instance_id -def get_or_create_group(group: Optional[str], conn=None) -> str: - """UUID of the group for a tenant code (``bv``, ``mh``, ...), created when missing.""" - code = (group or _default_group()).strip().lower() or _default_group() - return upsert_group(_tenant_path(code), conn=conn) +def get_or_create_instance(instance: Optional[str], conn=None) -> str: + """UUID of the instance for a tenant code (``bv``, ``mh``, ...), created when missing.""" + code = (instance or _default_instance()).strip().lower() or _default_instance() + return upsert_instance(_tenant_path(code), conn=conn) def _catalog_seed_rows() -> tuple[list[str], list[str]]: @@ -475,60 +475,60 @@ def _catalog_seed_rows() -> tuple[list[str], list[str]]: return [KIND_TO_CATALOG_TYPE[k] for k in kinds], kinds -def ensure_group_catalogs(group_id: str, conn=None) -> None: - """Create the group's missing catalogs, one per supported type; existing rows are untouched.""" +def ensure_instance_catalogs(instance_id: str, conn=None) -> None: + """Create the instance's missing catalogs, one per supported type; existing rows are untouched.""" if conn is None: with get_connection() as own: - return ensure_group_catalogs(group_id, conn=own) + return ensure_instance_catalogs(instance_id, conn=own) types, kinds = _catalog_seed_rows() conn.execute( """ - INSERT INTO catalogs (group_id, type, name) + INSERT INTO catalogs (instance_id, type, name) SELECT g.id, t.type, upper(g.name) || ' ' || t.kind || ' catalog' - FROM groups g, unnest(%s::text[], %s::text[]) AS t(type, kind) + FROM instances g, unnest(%s::text[], %s::text[]) AS t(type, kind) WHERE g.id = %s::uuid - ON CONFLICT (group_id, type) DO NOTHING + ON CONFLICT (instance_id, type) DO NOTHING """, - (types, kinds, group_id), + (types, kinds, instance_id), ) -def sync_catalogs_for_all_groups(conn=None) -> int: - """Add every missing ``(group, type)`` catalog; returns how many rows were created. +def sync_catalogs_for_all_instances(conn=None) -> int: + """Add every missing ``(instance, type)`` catalog; returns how many rows were created. - Idempotent. Run at startup so a type added to the kind map reaches groups + Idempotent. Run at startup so a type added to the kind map reaches instances that already exist. """ if conn is None: with get_connection() as own: - return sync_catalogs_for_all_groups(conn=own) + return sync_catalogs_for_all_instances(conn=own) types, kinds = _catalog_seed_rows() cur = conn.execute( """ - INSERT INTO catalogs (group_id, type, name) + INSERT INTO catalogs (instance_id, type, name) SELECT g.id, t.type, upper(g.name) || ' ' || t.kind || ' catalog' - FROM groups g, unnest(%s::text[], %s::text[]) AS t(type, kind) - ON CONFLICT (group_id, type) DO NOTHING + FROM instances g, unnest(%s::text[], %s::text[]) AS t(type, kind) + ON CONFLICT (instance_id, type) DO NOTHING """, (types, kinds), ) return cur.rowcount -def get_or_create_catalog(group_id: str, kind: str, conn=None) -> dict: - """The ``(group, kind)`` catalog row; created when missing (normally seeded with the group).""" +def get_or_create_catalog(instance_id: str, kind: str, conn=None) -> dict: + """The ``(instance, kind)`` catalog row; created when missing (normally seeded with the instance).""" if conn is None: with get_connection() as own: - return get_or_create_catalog(group_id, kind, conn=own) + return get_or_create_catalog(instance_id, kind, conn=own) catalog_type = catalog_type_for_kind(kind) row = conn.execute( """ - INSERT INTO catalogs (group_id, type, name) - SELECT g.id, %s, upper(g.name) || ' ' || %s || ' catalog' FROM groups g WHERE g.id = %s::uuid - ON CONFLICT (group_id, type) DO UPDATE SET type = EXCLUDED.type - RETURNING id, group_id, type, name, metadata + INSERT INTO catalogs (instance_id, type, name) + SELECT g.id, %s, upper(g.name) || ' ' || %s || ' catalog' FROM instances g WHERE g.id = %s::uuid + ON CONFLICT (instance_id, type) DO UPDATE SET type = EXCLUDED.type + RETURNING id, instance_id, type, name, metadata """, - (catalog_type, kind, group_id), + (catalog_type, kind, instance_id), ).fetchone() return _clean_row(row) @@ -548,7 +548,7 @@ def upsert_user( """Create or refresh the ``users`` row for a Keycloak subject; returns its UUID. ``tenants`` are the tenant codes in the user's token. A user has one - ``group_id``, so the first code (sorted) wins and a warning is logged when + ``instance_id``, so the first code (sorted) wins and a warning is logged when there are several. Authorization still reads the token, not this column. """ keycloak_user_id = (keycloak_user_id or "").strip() @@ -562,10 +562,10 @@ def upsert_user( tenant_codes = sorted({(t or "").strip().lower() for t in (tenants or []) if (t or "").strip()}) if len(tenant_codes) > 1: logger.warning( - "user %s has several tenants %s; users.group_id uses %s", + "user %s has several tenants %s; users.instance_id uses %s", keycloak_user_id, tenant_codes, tenant_codes[0], ) - group_uuid = get_or_create_group(tenant_codes[0], conn=conn) if tenant_codes else None + instance_uuid = get_or_create_instance(tenant_codes[0], conn=conn) if tenant_codes else None email = (email or "").strip() or None username = (username or "").strip() or None for candidate_email in ([email, None] if email else [None]): @@ -573,18 +573,18 @@ def upsert_user( with conn.transaction(): row = conn.execute( """ - INSERT INTO users (keycloak_user_id, username, email, roles, group_id) + INSERT INTO users (keycloak_user_id, username, email, roles, instance_id) VALUES (%s, %s, %s, %s, %s::uuid) ON CONFLICT (keycloak_user_id) DO UPDATE SET username = COALESCE(EXCLUDED.username, users.username), email = COALESCE(EXCLUDED.email, users.email), roles = CASE WHEN cardinality(EXCLUDED.roles) > 0 THEN EXCLUDED.roles ELSE users.roles END, - group_id = COALESCE(EXCLUDED.group_id, users.group_id), + instance_id = COALESCE(EXCLUDED.instance_id, users.instance_id), last_seen_at = now() RETURNING id """, - (keycloak_user_id, username, candidate_email, role_list, group_uuid), + (keycloak_user_id, username, candidate_email, role_list, instance_uuid), ).fetchone() return str(row["id"]) except psycopg.errors.UniqueViolation: @@ -890,9 +890,9 @@ def list_document_codes(kind: str) -> set[str]: def _catalog_row(row: dict) -> dict: cleaned = _clean_row(row) - cleaned.pop("group_name", None) + cleaned.pop("instance_name", None) cleaned["aliases"] = list(row["aliases"] or []) - cleaned["groups"] = [(row["group_name"] or "").lower()] + cleaned["instances"] = [(row["instance_name"] or "").lower()] cleaned["workflow_ids"] = [row["workflow_id"]] if row["workflow_id"] else [] cleaned["source"] = "pipeline" return cleaned @@ -951,15 +951,15 @@ def list_scheme_catalog_entries_for_workflow(workflow_id: str) -> list[dict]: return [{"code": r["code"], "workflow_ids": [r["workflow_id"]]} for r in rows] -def find_document_by_code(group: str, kind: str, code: str) -> Optional[dict]: - """The document holding ``code`` in the group's ``kind`` catalog, if any.""" - tenant = (group or _default_group()).strip().lower() or _default_group() +def find_document_by_code(instance: str, kind: str, code: str) -> Optional[dict]: + """The document holding ``code`` in the instance's ``kind`` catalog, if any.""" + tenant = (instance or _default_instance()).strip().lower() or _default_instance() with get_connection() as conn: row = conn.execute( """ SELECT d.id FROM documents d JOIN catalogs c ON c.id = d.catalog_id - JOIN groups g ON g.id = c.group_id + JOIN instances g ON g.id = c.instance_id WHERE lower(g.name) = %s AND c.type = %s AND d.code = %s """, (tenant, catalog_type_for_kind(kind), code), @@ -999,7 +999,7 @@ def upsert_document( original_artifact_id: Optional[str] = None, normalized_artifact_id: Optional[str] = None, latest_job_id: Optional[str] = None, - group: Optional[str] = None, + instance: Optional[str] = None, uploaded_by_user_id: Optional[str] = None, uploaded_by_username: Optional[str] = None, uploaded_by_email: Optional[str] = None, @@ -1014,12 +1014,12 @@ def upsert_document( """Insert or update a document record; returns the document UUID. A new document needs ``kind`` (``scheme`` / ``advisory``) and ``code``: - ``group`` (a tenant code) and the kind pick its ``(group, kind)`` catalog, + ``instance`` (a tenant code) and the kind pick its ``(instance, kind)`` catalog, which is created when missing. ``name`` becomes ``documents.name`` (the code when absent). ``DuplicateDocumentCode`` is raised when another document already holds ``code`` in that catalog. - ``group`` and the catalog are applied on INSERT only. Updates leave them + ``instance`` and the catalog are applied on INSERT only. Updates leave them alone so a re-upload / restart cannot silently reassign tenants. Uploader identity is set on INSERT, and filled on UPDATE only when currently null. @@ -1037,7 +1037,7 @@ def upsert_document( it themselves with ``create_document_job`` (addressing the document by the returned UUID, since no workflow id resolves to it yet). """ - group_value = (group or _default_group()).strip().lower() or _default_group() + instance_value = (instance or _default_instance()).strip().lower() or _default_instance() fingerprint = _fingerprint_for( source_file_fingerprint or canonical_document_id, document_id, workflow_id ) @@ -1055,7 +1055,7 @@ def upsert_document( DocumentSource.model_validate({**_load(conn, doc_uuid, "source"), **patch}) uploader_uuid = ( upsert_user(uploaded_by_user_id, uploaded_by_username, uploaded_by_email, - uploaded_by_roles, conn=conn, tenants=[group_value]) + uploaded_by_roles, conn=conn, tenants=[instance_value]) if uploaded_by_user_id else None ) conn.execute( @@ -1092,13 +1092,13 @@ def upsert_document( ).model_dump(exclude_none=True) user_uuid = ( upsert_user(uploaded_by_user_id, uploaded_by_username, uploaded_by_email, - uploaded_by_roles, conn=conn, tenants=[group_value]) + uploaded_by_roles, conn=conn, tenants=[instance_value]) if uploaded_by_user_id else None ) metadata = DocumentMetadata( valid_from=valid_from, valid_to=valid_to, uploaded_by_user_id=user_uuid ).model_dump(exclude_none=True) - catalog = get_or_create_catalog(get_or_create_group(group_value, conn=conn), kind, conn=conn) + catalog = get_or_create_catalog(get_or_create_instance(instance_value, conn=conn), kind, conn=conn) try: with conn.transaction(): row = conn.execute( @@ -1115,7 +1115,7 @@ def upsert_document( except psycopg.errors.UniqueViolation as exc: if exc.diag.constraint_name == "documents_catalog_id_code_key": raise DuplicateDocumentCode( - f"Code '{code}' is already used in the {kind} catalog of {group_value}" + f"Code '{code}' is already used in the {kind} catalog of {instance_value}" ) from exc raise if with_workflow: @@ -1443,20 +1443,20 @@ def list_documents( limit: int = 100, offset: int = 0, include_disabled: bool = False, - groups: Optional[list[str]] = None, + instances: Optional[list[str]] = None, ) -> list[dict]: - """List documents with optional stage / group filters. + """List documents with optional stage / instance filters. Args: stage: Filter by document stage limit: Max documents to return offset: Pagination offset include_disabled: If False (default), excludes soft-deleted documents from results - groups: If provided, only return docs whose group is in this list + instances: If provided, only return docs whose instance is in this list """ - if groups is not None and not _normalize_groups(groups): + if instances is not None and not _normalize_instances(instances): return [] - clauses, params = _document_filters(include_disabled, groups) + clauses, params = _document_filters(include_disabled, instances) if stage: clauses.append("cw.current_stage = %s") params.append(stage) @@ -1491,12 +1491,12 @@ def list_documents( def get_document_summary_counts( include_disabled: bool = False, - groups: Optional[list[str]] = None, + instances: Optional[list[str]] = None, ) -> dict: """Return aggregate document counts for dashboard and migration planning.""" - if groups is not None and not _normalize_groups(groups): + if instances is not None and not _normalize_instances(instances): return {key: 0 for key in _SUMMARY_KEYS} - clauses, params = _document_filters(include_disabled, groups) + clauses, params = _document_filters(include_disabled, instances) where = f"WHERE {' AND '.join(clauses)}" if clauses else "" with get_connection() as conn: row = conn.execute( @@ -1520,7 +1520,7 @@ def get_document_summary_counts( WHERE j.document_id = d.id AND j.status = 'running')), 0) AS running_jobs FROM documents d JOIN catalogs c ON c.id = d.catalog_id - JOIN groups g ON g.id = c.group_id + JOIN instances g ON g.id = c.instance_id LEFT JOIN document_current_workflow cw ON cw.document_id = d.id {where} """, @@ -1529,20 +1529,20 @@ def get_document_summary_counts( return {key: int(row[key] or 0) for key in _SUMMARY_KEYS} -def get_document_counts_by_group( +def get_document_counts_by_instance( include_disabled: bool = False, - groups: Optional[list[str]] = None, + instances: Optional[list[str]] = None, ) -> list[dict]: - """Return document counts grouped by group (state), for dashboard breakdowns.""" - if groups is not None and not _normalize_groups(groups): + """Return document counts grouped by instance (state), for dashboard breakdowns.""" + if instances is not None and not _normalize_instances(instances): return [] - clauses, params = _document_filters(include_disabled, groups) + clauses, params = _document_filters(include_disabled, instances) where = f"WHERE {' AND '.join(clauses)}" if clauses else "" with get_connection() as conn: rows = conn.execute( f""" SELECT - lower(g.name) AS "group", + lower(g.name) AS "instance", COUNT(*) AS count, COUNT(*) FILTER (WHERE cw.current_stage = 'completed') AS success, COUNT(*) FILTER (WHERE cw.current_stage = 'failed') AS failed, @@ -1550,17 +1550,17 @@ def get_document_counts_by_group( ('ocr_review', 'translation_review', 'chunk_review', 'approval_for_prod')) AS dev_approval FROM documents d JOIN catalogs c ON c.id = d.catalog_id - JOIN groups g ON g.id = c.group_id + JOIN instances g ON g.id = c.instance_id LEFT JOIN document_current_workflow cw ON cw.document_id = d.id {where} GROUP BY lower(g.name) - ORDER BY count DESC, "group" ASC + ORDER BY count DESC, "instance" ASC """, params, ).fetchall() return [ { - "group": row["group"], + "instance": row["instance"], "count": int(row["count"] or 0), "success": int(row["success"] or 0), "failed": int(row["failed"] or 0), @@ -1601,7 +1601,7 @@ def purge_document(workflow_id: str, keep_audit: bool = True) -> dict: Audit rows are kept by default: they are the record that the deletion happened, not part of the document itself. Their ``document_id`` becomes - NULL; the ``workflow_id`` and ``group`` snapshots stay. + NULL; the ``workflow_id`` and ``instance`` snapshots stay. Returns a per-table count of deleted rows. """ @@ -2026,18 +2026,18 @@ def list_operations_queue( limit: int = 100, offset: int = 0, include_disabled: bool = False, - groups: Optional[list[str]] = None, + instances: Optional[list[str]] = None, ) -> tuple[list[dict], int]: - """List documents needing action, optionally scoped to tenant groups. + """List documents needing action, optionally scoped to tenant instances. - ``groups``: + ``instances``: - ``None`` - unrestricted (super admin / bypass) - ``[]`` - restricted user with no states -> empty result - - non-empty - only those document.group values + - non-empty - only those document.instance values """ - if groups is not None and not _normalize_groups(groups): + if instances is not None and not _normalize_instances(instances): return [], 0 - clauses, params = _document_filters(include_disabled, groups) + clauses, params = _document_filters(include_disabled, instances) clauses.append( "(cw.current_stage IN ('ocr_review', 'translation_review', 'chunk_review', " "'ready_for_ingestion', 'approval_for_prod', 'failed') " @@ -2048,7 +2048,7 @@ def list_operations_queue( from_sql = """ FROM documents d JOIN catalogs c ON c.id = d.catalog_id - JOIN groups g ON g.id = c.group_id + JOIN instances g ON g.id = c.instance_id JOIN document_current_workflow cw ON cw.document_id = d.id """ with get_connection() as conn: @@ -2062,7 +2062,7 @@ def list_operations_queue( LIMIT 1) AS workflow_id, d.source->>'original_filename' AS filename, cw.current_stage AS stage, - lower(g.name) AS "group", + lower(g.name) AS "instance", cw.error_message, COALESCE((d.metadata->>'reindex_required')::boolean, FALSE) AS reindex_required, d.metadata->>'reindex_reason' AS reindex_reason, @@ -2084,12 +2084,12 @@ def list_runs( limit: int = 100, offset: int = 0, status: Optional[str] = None, - groups: Optional[list[str]] = None, + instances: Optional[list[str]] = None, ) -> list[dict]: """List workflows with document title fields for the Runs UI. Args: - groups: If provided, only return runs whose document group is in + instances: If provided, only return runs whose document instance is in this list. Empty list returns no rows. None = unrestricted. """ where_parts: list[str] = [] @@ -2097,8 +2097,8 @@ def list_runs( if status: where_parts.append("w.status = %s") params.append(status) - if groups is not None: - normalized = _normalize_groups(groups) + if instances is not None: + normalized = _normalize_instances(instances) if not normalized: return [] where_parts.append("lower(g.name) = ANY(%s)") @@ -2116,11 +2116,11 @@ def list_runs( d.name AS name, d.source->>'original_filename' AS source_filename, cw.current_stage AS document_stage, - lower(g.name) AS "group" + lower(g.name) AS "instance" FROM workflows w JOIN documents d ON d.id = w.document_id JOIN catalogs c ON c.id = d.catalog_id - JOIN groups g ON g.id = c.group_id + JOIN instances g ON g.id = c.instance_id LEFT JOIN document_current_workflow cw ON cw.document_id = d.id {where_sql} ORDER BY w.started_at DESC, w.id DESC @@ -2132,7 +2132,7 @@ def list_runs( for row in rows: job = _job_from_row(row) job["document_uuid"] = str(row["document_id"]) - for extra in ("filename", "name", "source_filename", "document_stage", "group"): + for extra in ("filename", "name", "source_filename", "document_stage", "instance"): job[extra] = row[extra] result.append(job) return result @@ -2467,15 +2467,15 @@ def log_audit( roles = _csv_roles(actor_roles) with get_connection() as conn: doc_uuid = None if system_entry else _resolve_document_uuid(conn, workflow_id) - group_uuid = None + instance_uuid = None stored_workflow_id = None if system_entry else workflow_id if doc_uuid: doc_row = conn.execute( - "SELECT c.group_id FROM documents d JOIN catalogs c ON c.id = d.catalog_id " + "SELECT c.instance_id FROM documents d JOIN catalogs c ON c.id = d.catalog_id " "WHERE d.id = %s::uuid", (doc_uuid,), ).fetchone() - group_uuid = str(doc_row["group_id"]) + instance_uuid = str(doc_row["instance_id"]) if _is_uuid(workflow_id): stored_workflow_id = _root_workflow_id(conn, doc_uuid) or workflow_id if stored_workflow_id is None and entity not in ("setting", "catalog"): @@ -2486,14 +2486,14 @@ def log_audit( ) row = conn.execute( """ - INSERT INTO audit_logs (document_id, workflow_id, group_id, action_type, entity_type, + INSERT INTO audit_logs (document_id, workflow_id, instance_id, action_type, entity_type, entity_number, field_name, old_value, new_value, metadata, actor_user_id, actor) VALUES (%s::uuid, %s, %s::uuid, %s, %s, %s, %s, %s, %s, %s, %s::uuid, %s) RETURNING id """, ( - doc_uuid, stored_workflow_id, group_uuid, action_type, entity, entity_id, + doc_uuid, stored_workflow_id, instance_uuid, action_type, entity, entity_id, field_name, old_value, new_value, Jsonb(metadata or {}), actor_uuid, Jsonb({"username": actor_username or None, "email": actor_email or None, "roles": roles}) if (actor_user_id or actor_username or actor_email or roles) else Jsonb({}), @@ -2565,18 +2565,18 @@ def _normalize_audit_row(row) -> dict: _AUDIT_SELECT = """ SELECT a.id, a.workflow_id, a.action_type, a.entity_type, a.entity_number, a.field_name, - a.old_value, a.new_value, a.metadata, a.created_at, a.actor, ag.name AS audit_group, + a.old_value, a.new_value, a.metadata, a.created_at, a.actor, ag.name AS audit_instance, au.keycloak_user_id AS actor_keycloak_id, d.source->>'fingerprint' AS doc_fingerprint, d.source->>'original_filename' AS filename, d.name AS document_name, - dg.name AS doc_group, + dg.name AS doc_instance, uu.username AS uploaded_by_username, uu.email AS uploaded_by_email FROM audit_logs a LEFT JOIN documents d ON d.id = a.document_id - LEFT JOIN groups ag ON ag.id = a.group_id + LEFT JOIN instances ag ON ag.id = a.instance_id LEFT JOIN catalogs dc ON dc.id = d.catalog_id - LEFT JOIN groups dg ON dg.id = dc.group_id + LEFT JOIN instances dg ON dg.id = dc.instance_id LEFT JOIN users au ON au.id = a.actor_user_id LEFT JOIN users uu ON uu.id = NULLIF(d.metadata->>'uploaded_by_user_id', '')::uuid """ @@ -2612,7 +2612,7 @@ def _audit_row_to_legacy(row: dict) -> dict: "actor_roles": ",".join(roles) if roles else None, "filename": row["filename"], "name": row["document_name"], - "instance": row["doc_group"] or row["audit_group"], + "instance": row["doc_instance"] or row["audit_instance"], "uploaded_by_username": row["uploaded_by_username"], "uploaded_by_email": row["uploaded_by_email"], } @@ -2669,16 +2669,16 @@ def get_audit_log_count(workflow_id: str, action_type: Optional[str] = None) -> def _global_audit_filters( - action_type: Optional[str], groups: Optional[list[str]] + action_type: Optional[str], instances: Optional[list[str]] ) -> tuple[list[str], list]: where_parts: list[str] = ["a.action_type <> %s"] params: list = [CATALOG_VERSION_ACTION] if action_type: where_parts.append("a.action_type = %s") params.append(action_type) - if groups is not None: + if instances is not None: where_parts.append("lower(COALESCE(ag.name, dg.name, %s)) = ANY(%s)") - params.extend([_default_group(), _normalize_groups(groups)]) + params.extend([_default_instance(), _normalize_instances(instances)]) return where_parts, params @@ -2686,7 +2686,7 @@ def get_all_audit_logs( action_type: Optional[str] = None, limit: int = 100, offset: int = 0, - groups: Optional[list[str]] = None, + instances: Optional[list[str]] = None, ) -> list[dict]: """ Get all audit logs across all documents. @@ -2695,14 +2695,14 @@ def get_all_audit_logs( action_type: Optional filter by action type limit: Maximum number of entries to return offset: Offset for pagination - groups: Tenant scope (None = all, [] = none) + instances: Tenant scope (None = all, [] = none) Returns: List of audit log entries as dicts, with document filename included """ - if groups is not None and not _normalize_groups(groups): + if instances is not None and not _normalize_instances(instances): return [] - where_parts, params = _global_audit_filters(action_type, groups) + where_parts, params = _global_audit_filters(action_type, instances) with get_connection() as conn: rows = conn.execute( f"{_AUDIT_SELECT} WHERE {' AND '.join(where_parts)} " @@ -2714,19 +2714,19 @@ def get_all_audit_logs( def get_all_audit_log_count( action_type: Optional[str] = None, - groups: Optional[list[str]] = None, + instances: Optional[list[str]] = None, ) -> int: """Get total count of all audit logs (optionally tenant-scoped).""" - if groups is not None and not _normalize_groups(groups): + if instances is not None and not _normalize_instances(instances): return 0 - where_parts, params = _global_audit_filters(action_type, groups) + where_parts, params = _global_audit_filters(action_type, instances) with get_connection() as conn: row = conn.execute( "SELECT COUNT(*) AS cnt FROM audit_logs a " "LEFT JOIN documents d ON d.id = a.document_id " - "LEFT JOIN groups ag ON ag.id = a.group_id " + "LEFT JOIN instances ag ON ag.id = a.instance_id " "LEFT JOIN catalogs dc ON dc.id = d.catalog_id " - "LEFT JOIN groups dg ON dg.id = dc.group_id " + "LEFT JOIN instances dg ON dg.id = dc.instance_id " f"WHERE {' AND '.join(where_parts)}", params, ).fetchone() @@ -3163,12 +3163,12 @@ def search_chunks( offset: int = 0, include_excluded: bool = False, stage: Optional[str] = None, - groups: Optional[list[str]] = None, + instances: Optional[list[str]] = None, ) -> tuple[list[dict], int]: """Search chunks across documents for maintainer workflows. - groups: If provided, only return chunks whose owning document's group - is in this list (tenant scoping). ``None`` means no group restriction. + instances: If provided, only return chunks whose owning document's instance + is in this list (tenant scoping). ``None`` means no instance restriction. """ where_clauses = ["TRUE"] params: list = [] @@ -3178,8 +3178,8 @@ def search_chunks( if stage: where_clauses.append("cw.current_stage = %s") params.append(stage) - if groups is not None: - normalized = _normalize_groups(groups) + if instances is not None: + normalized = _normalize_instances(instances) if not normalized: where_clauses.append("FALSE") else: @@ -3193,7 +3193,7 @@ def search_chunks( FROM chunks c JOIN documents d ON d.id = c.document_id JOIN catalogs cat ON cat.id = d.catalog_id - JOIN groups g ON g.id = cat.group_id + JOIN instances g ON g.id = cat.instance_id LEFT JOIN document_current_workflow cw ON cw.document_id = d.id WHERE {' AND '.join(where_clauses)} """ diff --git a/pipeline/discovery_publish_service.py b/pipeline/discovery_publish_service.py index 88b2b43..175fdb7 100644 --- a/pipeline/discovery_publish_service.py +++ b/pipeline/discovery_publish_service.py @@ -50,12 +50,12 @@ def __init__( def publish( self, transaction_id: str, - kind: Optional[str], + document_kind: Optional[str], workflow_id: Optional[str] = None, group: Optional[str] = None, ) -> dict: """ - POST a catalog/publish envelope for `kind`. + POST a catalog/publish envelope for `document_kind`. `transaction_id` is supplied by the caller (generated once per publish, reused across retries) - only `messageId` is generated fresh here, per @@ -65,18 +65,18 @@ def publish( with `skipped=True`: the spec declares `message.catalogs` as `minItems: 1`, so there is no valid "publish nothing" request to send. """ - if not is_publishable_kind(kind): + if not is_publishable_kind(document_kind): logger.info( - "workflow_id=%s kind=%s network_publish_skipped=True", + "workflow_id=%s document_kind=%s network_publish_skipped=True", workflow_id, - kind, + document_kind, ) return self._skipped() catalog_group = named_group(group) - provider = provider_for(self.provider_naming, kind, catalog_group) - catalog_name = self.provider_naming.catalog_name(normalize_document_kind(kind), catalog_group) - catalog = build_catalog(kind, provider, catalog_name, catalog_group) + provider = provider_for(self.provider_naming, document_kind, catalog_group) + catalog_name = self.provider_naming.catalog_name(normalize_document_kind(document_kind), catalog_group) + catalog = build_catalog(document_kind, provider, catalog_name, catalog_group) envelope = { "context": { diff --git a/pipeline/document_repository.py b/pipeline/document_repository.py index 654570e..8c989e3 100644 --- a/pipeline/document_repository.py +++ b/pipeline/document_repository.py @@ -34,4 +34,4 @@ def get_document_group(self, workflow_id: str) -> Optional[str]: doc = self._db.get_document(workflow_id) if not doc: return None - return group_for_instance(doc.get("group")) + return group_for_instance(doc.get("instance")) diff --git a/pipeline/group_names.py b/pipeline/instances.py similarity index 84% rename from pipeline/group_names.py rename to pipeline/instances.py index 2dee0bf..0bc0777 100644 --- a/pipeline/group_names.py +++ b/pipeline/instances.py @@ -1,9 +1,9 @@ -"""Group (state / tenant) code → human-readable name. +"""Instance (state / tenant) code → human-readable name. -A group code is a short lowercase ``groups.name`` — ``bv`` for the Bharat +An instance code is a short lowercase ``instances.name`` — ``bv`` for the Bharat Vistaar portal, otherwise the state code matching the Keycloak group path -(``/states/MH/...`` → ``mh``). A document's group is derived through its -catalog (``documents.catalog_id`` → ``catalogs.group_id``). That code is what +(``/states/MH/...`` → ``mh``). A document's instance is derived through its +catalog (``documents.catalog_id`` → ``catalogs.instance_id``). That code is what the pipeline filters on, but it is not what a reader wants to see: consumers of the vector payload and the master catalog get ``instance_name`` (their field name, unchanged) alongside the code so they can show "Maharashtra" without @@ -19,7 +19,7 @@ import os # Portal / platform-wide documents (not a state). -PORTAL_GROUP = "bv" +PORTAL_INSTANCE = "bv" def publish_to_network_enabled() -> bool: @@ -42,7 +42,7 @@ def prod_stage_disabled() -> bool: return (os.environ.get("DISABLE_PROD_SETTING", "false").strip().lower() in {"true", "1", "yes"}) -GROUP_NAMES: dict[str, str] = { +INSTANCE_NAMES: dict[str, str] = { # Platform "bv": "Bharat Vistaar", "default": "Default", @@ -90,15 +90,15 @@ def prod_stage_disabled() -> bool: } -def group_display_name(code: str | None) -> str: - """Human-readable name for a group code (``mh`` → ``Maharashtra``). +def instance_display_name(code: str | None) -> str: + """Human-readable name for an instance code (``mh`` → ``Maharashtra``). Unknown codes are title-cased so the field is never empty. """ normalized = (code or "").strip().lower() if not normalized: - return GROUP_NAMES["default"] - known = GROUP_NAMES.get(normalized) + return INSTANCE_NAMES["default"] + known = INSTANCE_NAMES.get(normalized) if known: return known # Unknown code: make it presentable rather than failing the ingest. diff --git a/pipeline/network_group.py b/pipeline/network_group.py index edd71f2..ce2aad0 100644 --- a/pipeline/network_group.py +++ b/pipeline/network_group.py @@ -6,9 +6,9 @@ from pathlib import Path from typing import Optional -from .auth.tenancy import portal_group +from .auth.tenancy import portal_instance from .catalog_builder import normalize_document_kind -from .group_names import GROUP_NAMES, group_display_name +from .instances import INSTANCE_NAMES, instance_display_name from .network_constants import ( ADVISORY_CATALOG_NAME, GROUP_CATALOG_NAME_TEMPLATE, @@ -117,7 +117,7 @@ def _checked_id_template(shared_instance: str, network_id: Optional[str]) -> str instance=shared_instance, network_id=sample_network_id, ) - for instance in dict.fromkeys((shared_instance, *GROUP_NAMES)): + for instance in dict.fromkeys((shared_instance, *INSTANCE_NAMES)): _check_provider_id( template.format(instance=instance, network_id=sample_network_id), f"{PROVIDER_ID_TEMPLATE_ENV}={template!r} with {NETWORK_ID_ENV}={sample_network_id!r}", @@ -126,11 +126,11 @@ def _checked_id_template(shared_instance: str, network_id: Optional[str]) -> str def load_provider_naming(network_id: Optional[str] = None) -> ProviderNaming: - shared_instance = portal_group() + shared_instance = portal_instance() network_id = (network_id or os.environ.get(NETWORK_ID_ENV) or "").strip() or None return ProviderNaming( shared_instance=shared_instance, - shared_name=group_display_name(shared_instance), + shared_name=instance_display_name(shared_instance), network_id=network_id, id_template=_checked_id_template(shared_instance, network_id), group_catalog_name_template=_checked_template( @@ -155,7 +155,7 @@ def validate_network_provider_config() -> None: def named_group(code: Optional[str]) -> Optional[Group]: if code is None: return None - return Group(code=code, name=group_display_name(code)) + return Group(code=code, name=instance_display_name(code)) def provider_for(naming: ProviderNaming, document_kind: Optional[str], group: Optional[Group]) -> Optional[dict]: diff --git a/pipeline/scheme_catalog.py b/pipeline/scheme_catalog.py index 937c82f..624c4bf 100644 --- a/pipeline/scheme_catalog.py +++ b/pipeline/scheme_catalog.py @@ -187,14 +187,14 @@ def utc_now_iso() -> str: return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") -def central_groups() -> set[str]: +def central_instances() -> set[str]: raw = (os.environ.get("CENTRAL_INSTANCES") or "default").strip() return {p.strip().lower() for p in raw.split(",") if p.strip()} or {"default"} -def default_network_visible(group: str | None) -> bool: - code = (group or "default").strip().lower() or "default" - return code in central_groups() +def default_network_visible(instance: str | None) -> bool: + code = (instance or "default").strip().lower() or "default" + return code in central_instances() def scheme_collection_name() -> str: @@ -372,7 +372,7 @@ def rebuild_catalog_from_documents(*, reason: str = "rebuild") -> int: """ Bump the catalog version. - Status, groups, chunk counts and the rest are computed by the + Status, instances, chunk counts and the rest are computed by the ``catalog_status`` view, so there is nothing to recompute here. """ return db.bump_catalog_version(reason) @@ -454,10 +454,10 @@ def build_snapshot( # Live snapshot for warmers: only status=live (pending excluded unless include_pending) if not include_pending and st != "live": continue - groups = e["groups"] + instances = e["instances"] if instance: inst = instance.strip().lower() - if inst not in {g.lower() for g in groups} and groups: + if inst not in {g.lower() for g in instances} and instances: continue nv = bool(e.get("network_visible")) if network_visible is not None and nv != network_visible: @@ -469,7 +469,7 @@ def build_snapshot( "scheme_aliases": e["aliases"], "tool": "search_schemes", "tool_routing": "qdrant", - "instances": groups or ["default"], + "instances": instances or ["default"], "network_visible": nv, "chunk_count": int(e.get("chunk_count") or 0), "workflow_ids": e["workflow_ids"], @@ -586,7 +586,7 @@ def apply_scheme_metadata( if network_visible is not None: updates["network_visible"] = 1 if network_visible else 0 elif new_kind == "scheme" and doc.get("network_visible") is None: - updates["network_visible"] = 1 if default_network_visible(doc.get("group")) else 0 + updates["network_visible"] = 1 if default_network_visible(doc.get("instance")) else 0 if valid_from is not None or valid_to is not None: # Validated here rather than at the SQL layer, and against the stored diff --git a/scripts/ingest_test_docs.sh b/scripts/ingest_test_docs.sh index 3db67cd..e6fa840 100644 --- a/scripts/ingest_test_docs.sh +++ b/scripts/ingest_test_docs.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash # Ingest all PDFs from ~/test-docs via H100 pipeline API. -# DOC_KIND: scheme (default) or advisory. PIPELINE_GROUP: tenant group code (optional). +# DOC_KIND: scheme (default) or advisory. PIPELINE_INSTANCE: tenant instance code (optional). set -euo pipefail DOCS_DIR="${1:-$HOME/test-docs}" API="${PIPELINE_API:-http://127.0.0.1:8001}" DOC_KIND="${DOC_KIND:-scheme}" QUERY="auto_approve=true&kind=${DOC_KIND}" -if [[ -n "${PIPELINE_GROUP:-}" ]]; then - QUERY="${QUERY}&group=${PIPELINE_GROUP}" +if [[ -n "${PIPELINE_INSTANCE:-}" ]]; then + QUERY="${QUERY}&instance=${PIPELINE_INSTANCE}" fi if [[ ! -d "$DOCS_DIR" ]]; then diff --git a/scripts/migrate_sqlite_to_postgres.py b/scripts/migrate_sqlite_to_postgres.py index a37214b..366e54b 100644 --- a/scripts/migrate_sqlite_to_postgres.py +++ b/scripts/migrate_sqlite_to_postgres.py @@ -15,8 +15,8 @@ Mapping (Schema v2): one legacy ``documents`` row becomes one document plus its main workflow (``temporal_workflow_id`` = the legacy ``workflow_id``); each ``document_jobs`` row becomes a workflow. The legacy ``instance`` becomes a -``groups`` row (``bv`` is ``/global``, others ``/states/``, Keycloak id -filled in by the first login sync) and the group's ``catalogs`` row for the +``instances`` row (``bv`` is ``/global``, others ``/states/``, Keycloak id +filled in by the first login sync) and the instance's ``catalogs`` row for the document's kind. ``scheme_code`` / ``scheme_name`` / ``scheme_aliases_json`` (or the ``scheme_catalog_entries`` registry entry for the code) become the document's ``code`` / ``name`` / ``aliases``; the validity period, uploader and @@ -289,29 +289,29 @@ def _instance_of(doc: sqlite3.Row) -> str: return (_get(doc, "instance") or _default_instance()).strip().lower() or _default_instance() -def _group_for(conn, instance: str) -> str: - """The ``groups`` row of a tenant code, created with a deterministic id.""" +def _instance_for(conn, instance: str) -> str: + """The ``instances`` row of a tenant code, created with a deterministic id.""" path = db._tenant_path(instance) row = conn.execute( """ - INSERT INTO groups (id, name, path) VALUES (%s::uuid, %s, %s) - ON CONFLICT (path) DO UPDATE SET name = groups.name RETURNING id + INSERT INTO instances (id, name, path) VALUES (%s::uuid, %s, %s) + ON CONFLICT (path) DO UPDATE SET name = instances.name RETURNING id """, - (det_id("group", path), db._tenant_code_from_path(path), path), + (det_id("instance", path), db._tenant_code_from_path(path), path), ).fetchone() return str(row["id"]) -def _catalog_for(conn, group_id: str, kind: str) -> str: - """The ``(group, kind)`` catalog, created with a deterministic id.""" +def _catalog_for(conn, instance_id: str, kind: str) -> str: + """The ``(instance, kind)`` catalog, created with a deterministic id.""" catalog_type = catalog_type_for_kind(kind) row = conn.execute( """ - INSERT INTO catalogs (id, group_id, type, name) - SELECT %s::uuid, g.id, %s, g.name || ' ' || %s FROM groups g WHERE g.id = %s::uuid - ON CONFLICT (group_id, type) DO UPDATE SET type = EXCLUDED.type RETURNING id + INSERT INTO catalogs (id, instance_id, type, name) + SELECT %s::uuid, g.id, %s, g.name || ' ' || %s FROM instances g WHERE g.id = %s::uuid + ON CONFLICT (instance_id, type) DO UPDATE SET type = EXCLUDED.type RETURNING id """, - (det_id("catalog", f"{group_id}:{catalog_type}"), catalog_type, kind, group_id), + (det_id("catalog", f"{instance_id}:{catalog_type}"), catalog_type, kind, instance_id), ).fetchone() return str(row["id"]) @@ -442,7 +442,7 @@ def _build_workflows( def _skip_document(sq, conn, doc: sqlite3.Row, report: Report, audit_by_workflow: dict[str, list], reason: str) -> None: """Leave a document behind. Its history rows are counted as skipped; its audit - entries are kept, tied to the tenant's group but to no document.""" + entries are kept, tied to the tenant's instance but to no document.""" workflow_id = doc["workflow_id"] report.skip("documents", workflow_id, reason) for table in ("document_jobs", "document_artifacts", "pages", "chunks"): @@ -532,7 +532,7 @@ def migrate_document( canonical_input_type=_get(doc, "canonical_input_type"), ).model_dump(exclude_none=True) - catalog_id = _catalog_for(conn, _group_for(conn, _instance_of(doc)), kind) + catalog_id = _catalog_for(conn, _instance_for(conn, _instance_of(doc)), kind) code = (_get(doc, "scheme_code") or "").strip().lower() entry = registry.get(code) if kind == "scheme" and code else None if entry is not None: @@ -709,7 +709,7 @@ def insert_audit(conn, audit: sqlite3.Row, doc_uuid: Optional[str], source_insta ) inserted = conn.execute( """ - INSERT INTO audit_logs (id, document_id, workflow_id, group_id, action_type, entity_type, + INSERT INTO audit_logs (id, document_id, workflow_id, instance_id, action_type, entity_type, entity_number, field_name, old_value, new_value, metadata, actor_user_id, actor, created_at) VALUES (%s::uuid, %s::uuid, %s, %s::uuid, %s, %s, %s, %s, %s, %s, %s, @@ -718,7 +718,7 @@ def insert_audit(conn, audit: sqlite3.Row, doc_uuid: Optional[str], source_insta """, ( det_id("audit", str(audit["id"])), doc_uuid, workflow_id, - _group_for(conn, source_instance.strip().lower()) if (source_instance or "").strip() else None, + _instance_for(conn, source_instance.strip().lower()) if (source_instance or "").strip() else None, audit["action_type"], entity, audit["entity_id"], _legacy_audit_field(audit["field_name"], entity), audit["old_value"], audit["new_value"], Jsonb(metadata), actor_uuid, diff --git a/scripts/upload_test_docs.ps1 b/scripts/upload_test_docs.ps1 index ad80769..788fb77 100644 --- a/scripts/upload_test_docs.ps1 +++ b/scripts/upload_test_docs.ps1 @@ -2,10 +2,10 @@ # Prerequisite: ssh -L 18001:127.0.0.1:8001 -N docs-pipeline-host $ErrorActionPreference = "Stop" $Api = if ($env:PIPELINE_API) { $env:PIPELINE_API } else { "http://127.0.0.1:18001" } -# DOC_KIND: scheme (default) or advisory. PIPELINE_GROUP: tenant group code (optional). +# DOC_KIND: scheme (default) or advisory. PIPELINE_INSTANCE: tenant instance code (optional). $Kind = if ($env:DOC_KIND) { $env:DOC_KIND } else { "scheme" } $Query = "auto_approve=true&kind=$Kind" -if ($env:PIPELINE_GROUP) { $Query = "$Query&group=$($env:PIPELINE_GROUP)" } +if ($env:PIPELINE_INSTANCE) { $Query = "$Query&instance=$($env:PIPELINE_INSTANCE)" } $DocsDir = Join-Path $PSScriptRoot ".." "test docs" | Resolve-Path Write-Host "API: $Api" diff --git a/scripts/verify_document_validity_e2e.py b/scripts/verify_document_validity_e2e.py index 3b5d839..79ab4d4 100644 --- a/scripts/verify_document_validity_e2e.py +++ b/scripts/verify_document_validity_e2e.py @@ -75,7 +75,7 @@ def make_document(workflow_id, text, valid_from=None, valid_to=None, legacy=Fals stage="chunk_review", valid_from=valid_from, valid_to=valid_to, - group="bv", + instance="bv", kind="scheme", code=workflow_id, ) @@ -90,7 +90,7 @@ def make_document(workflow_id, text, valid_from=None, valid_to=None, legacy=Fals filename=f"{workflow_id}.pdf", chunks=chunks, workflow_id=workflow_id, - group="bv", + instance="bv", **kwargs, ) @@ -231,7 +231,7 @@ def search_docs(**kwargs): filename="wf-active.pdf", chunks=[{"chunk_number": 1, "original_text": "Kisan credit card subsidy for active season", "token_count": 8}], workflow_id="wf-active", - group="bv", + instance="bv", **_validity_fields_from_doc(doc), ) store.upsert(COLLECTION, again, batch_size=8) diff --git a/tests/conftest.py b/tests/conftest.py index fa276dd..4acf436 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -23,7 +23,7 @@ _TRUNCATE_SQL = """ - TRUNCATE documents, users, groups, catalogs, audit_logs, email_otps, + TRUNCATE documents, users, instances, catalogs, audit_logs, email_otps, webhook_records, search_settings CASCADE """ diff --git a/tests/test_activities.py b/tests/test_activities.py index 09d590b..89a561c 100644 --- a/tests/test_activities.py +++ b/tests/test_activities.py @@ -167,7 +167,7 @@ def test_prepare_records_carries_instance_name(self): document_id="test-doc", filename="test.pdf", chunks=chunks, - group=code, + instance=code, ) assert records[0]["instance"] == code assert records[0]["instance_name"] == expected @@ -213,7 +213,7 @@ def test_prepare_records_fills_scheme_aliases(self): document_id="test-doc", filename="NMEO-OPGUIEDELINES.pdf", chunks=chunks, - group="bv", + instance="bv", kind="scheme", code="nmeoop", name="National Mission on Edible Oils - Oil Palm", @@ -239,7 +239,7 @@ def test_prepare_records_non_scheme_has_no_aliases(self): document_id="test-doc", filename="test.pdf", chunks=chunks, - group="mh", + instance="mh", kind="document", ) @@ -859,7 +859,7 @@ async def test_passes_the_documents_group(self, monkeypatch, instance, group): seen = {} self._wire( monkeypatch, - {"kind": "advisory", "group": instance}, + {"kind": "advisory", "instance": instance}, self._PUBLISHED, seen, ) @@ -883,7 +883,7 @@ async def test_unmapped_kind_records_a_skip(self, monkeypatch): } recorded = self._wire( monkeypatch, - {"kind": "document", "group": "mh"}, + {"kind": "document", "instance": "mh"}, skipped, {}, ) @@ -908,7 +908,7 @@ def publish(self, transaction_id, document_kind, workflow_id=None, group=None): raise InvalidProviderIdError("Provider id 'advisory.ladakh_ut.provider.oan.dev' is invalid") monkeypatch.setattr(activities, "DiscoveryPublishService", FakeService) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory", "group": "ladakh_ut"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "advisory", "instance": "ladakh_ut"}) recorded = {} monkeypatch.setattr(db, "add_document_artifact", lambda **kwargs: recorded.update(kwargs)) @@ -1020,7 +1020,7 @@ async def test_a_key_another_live_document_is_indexed_under_is_kept_and_remember # A second document is indexed under the very key the first one replaced. db_connection.upsert_document( workflow_id="wf-twin", document_id="a" * 32, filename="a.pdf", - filepath="minio://documents/a.pdf", group="gj", kind="scheme", code="twin", + filepath="minio://documents/a.pdf", instance="gj", kind="scheme", code="twin", ) db_connection.upsert_document_index_status( "wf-twin", "documents-index", doc_id="a" * 32, status="indexed" diff --git a/tests/test_api.py b/tests/test_api.py index ef0bda9..1c9064e 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -1137,7 +1137,7 @@ def test_upload_creates_the_catalog_and_derives_the_code_from_the_filename( with db_connection.get_connection() as conn: catalog = conn.execute( "SELECT c.type, g.name, g.path FROM documents d JOIN catalogs c ON c.id = d.catalog_id " - "JOIN groups g ON g.id = c.group_id WHERE d.code = %s", + "JOIN instances g ON g.id = c.instance_id WHERE d.code = %s", (doc["code"],), ).fetchall() assert [(r["type"], r["name"], r["path"]) for r in catalog] == [ @@ -1261,7 +1261,7 @@ def _mh_doc(db, **extra): document_id="d-mh", filename="a.pdf", filepath="/tmp/a.pdf", - group="mh", + instance="mh", kind="scheme", code="pm-kisan", **extra, @@ -1273,7 +1273,7 @@ def _mh_doc(db, **extra): @pytest.mark.api @pytest.mark.unit - def test_replacing_a_document_needs_rights_in_its_group(self, db_connection): + def test_replacing_a_document_needs_rights_in_its_instance(self, db_connection): from fastapi import HTTPException from pipeline.api import api @@ -1296,7 +1296,7 @@ def test_replacing_a_document_needs_rights_in_its_group(self, db_connection): @pytest.mark.api @pytest.mark.unit - def test_upload_into_a_group_needs_upload_rights_there(self, db_connection): + def test_upload_into_a_instance_needs_upload_rights_there(self, db_connection): from fastapi import HTTPException from pipeline.api import api @@ -1376,15 +1376,15 @@ def test_disable_never_deletes_a_key_another_live_document_is_indexed_under(self db = db_connection uuid_a = db.upsert_document( workflow_id="wf-k-a", document_id="a" * 32, filename="a.pdf", - filepath="minio://documents/a.pdf", group="mh", kind="scheme", code="k-a", + filepath="minio://documents/a.pdf", instance="mh", kind="scheme", code="k-a", ) db.upsert_document( workflow_id="wf-k-b", document_id="a" * 32, filename="a.pdf", - filepath="minio://documents/a.pdf", group="gj", kind="scheme", code="k-b", + filepath="minio://documents/a.pdf", instance="gj", kind="scheme", code="k-b", ) db.upsert_document( workflow_id="wf-k-c", document_id="b" * 32, filename="c.pdf", - filepath="minio://documents/c.pdf", group="mh", kind="scheme", code="k-c", + filepath="minio://documents/c.pdf", instance="mh", kind="scheme", code="k-c", ) # The twin was only uploaded, never indexed, so nothing of it is at risk. diff --git a/tests/test_auth.py b/tests/test_auth.py index 5f7fdb0..80483d2 100644 --- a/tests/test_auth.py +++ b/tests/test_auth.py @@ -85,7 +85,7 @@ def test_claims_to_user_maps_keycloak_shape(): assert Permission.UPLOAD in user.permissions assert user.instances == ["tenant-a", "tenant-b"] assert user.envs == ["dev"] - assert user.has_group("Tenant-A") + assert user.has_instance("Tenant-A") assert not user.has_env("prod") diff --git a/tests/test_db.py b/tests/test_db.py index d98a0e0..01820a6 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -617,7 +617,7 @@ class TestSchemaConstraints: @staticmethod def _catalog_id(db): with db.get_connection() as conn: - return db.get_or_create_catalog(db.get_or_create_group("mh", conn=conn), "scheme", conn=conn)["id"] + return db.get_or_create_catalog(db.get_or_create_instance("mh", conn=conn), "scheme", conn=conn)["id"] @pytest.mark.db @pytest.mark.unit @@ -903,14 +903,14 @@ def test_source_spans_round_trip_and_derive_page_numbers(self, db_connection): @pytest.mark.db @pytest.mark.unit def test_search_chunks_matches_edited_text_first(self, db_connection): - _new_doc(db_connection, "wf-search", group="mh") + _new_doc(db_connection, "wf-search", instance="mh") db_connection.save_chunks("wf-search", [ {"chunk_number": 1, "original_text": "alpha beta", "edited_text": "gamma delta"}, ]) hits, total = db_connection.search_chunks(query="gamma") assert total == 1 and hits[0]["workflow_id"] == "wf-search" assert db_connection.search_chunks(query="alpha")[1] == 0 - assert db_connection.search_chunks(query="gamma", groups=["up"])[1] == 0 + assert db_connection.search_chunks(query="gamma", instances=["up"])[1] == 0 class TestIndexStatus: @@ -966,7 +966,7 @@ class TestAuditTrail: @pytest.mark.db @pytest.mark.unit def test_audit_outlives_the_document_and_keeps_tenant_scope(self, db_connection): - _new_doc(db_connection, "wf-aud", group="mh") + _new_doc(db_connection, "wf-aud", instance="mh") db_connection.log_audit( workflow_id="wf-aud", document_id="x", action_type="purge_document", entity_type="document", actor_user_id="kc-9", actor_username="boss", @@ -978,8 +978,8 @@ def test_audit_outlives_the_document_and_keeps_tenant_scope(self, db_connection) assert [entry["action_type"] for entry in logs] == ["purge_document"] assert logs[0]["actor"] == "b@x.in (superadmin)" assert logs[0]["actor_user_id"] == "kc-9" - assert db_connection.get_all_audit_log_count(groups=["mh"]) == 1 - assert db_connection.get_all_audit_log_count(groups=["up"]) == 0 + assert db_connection.get_all_audit_log_count(instances=["mh"]) == 1 + assert db_connection.get_all_audit_log_count(instances=["up"]) == 0 @pytest.mark.db @pytest.mark.unit @@ -993,7 +993,7 @@ def test_system_entries_need_no_document(self, db_connection): @pytest.mark.db @pytest.mark.unit - def test_settings_changes_are_logged_per_key_with_group_names(self, db_connection): + def test_settings_changes_are_logged_per_key_with_instance_names(self, db_connection): db_connection.update_search_settings({"alpha": 0.2, "limit": 3}) fields = {entry["field_name"] for entry in db_connection.get_settings_audit_logs()} assert fields == {"retrieval.alpha", "results.limit"} @@ -1022,7 +1022,7 @@ def test_values_are_validated(self, db_connection): @pytest.mark.db @pytest.mark.unit - def test_a_partial_update_keeps_the_rest_of_its_group(self, db_connection): + def test_a_partial_update_keeps_the_rest_of_its_instance(self, db_connection): db_connection.update_search_settings({"alpha": 0.3}) db_connection.update_search_settings({"hybridRrfK": 10}) settings = db_connection.get_search_settings() @@ -1063,8 +1063,8 @@ def test_status_rules(self, db_connection): def test_computed_columns(self, db_connection, monkeypatch): monkeypatch.setenv("PROD_VECTOR_DB_COLLECTION_NAME", "prod-docs") db = db_connection - self._scheme_doc(db, "wf-a", "scheme-a", "completed", group="mh") - self._scheme_doc(db, "wf-b", "scheme-b", "completed", group="up") + self._scheme_doc(db, "wf-a", "scheme-a", "completed", instance="mh") + self._scheme_doc(db, "wf-b", "scheme-b", "completed", instance="up") db.save_chunks("wf-a", [{"chunk_number": 1, "original_text": "x"}]) db.save_chunks("wf-b", [{"chunk_number": 1, "original_text": "y"}, {"chunk_number": 2, "original_text": "z"}]) @@ -1074,8 +1074,8 @@ def test_computed_columns(self, db_connection, monkeypatch): a = db.get_catalog_entry("scheme-a") b = db.get_catalog_entry("scheme-b") - assert (a["groups"], a["workflow_ids"], a["chunk_count"]) == (["mh"], ["wf-a"], 1) - assert (b["groups"], b["workflow_ids"], b["chunk_count"]) == (["up"], ["wf-b"], 2) + assert (a["instances"], a["workflow_ids"], a["chunk_count"]) == (["mh"], ["wf-a"], 1) + assert (b["instances"], b["workflow_ids"], b["chunk_count"]) == (["up"], ["wf-b"], 2) assert a["network_visible"] is True and b["network_visible"] is False assert a["promoted_at"].startswith("2026-05-01T00:00:00") assert b["promoted_at"] is None @@ -1085,7 +1085,7 @@ def test_computed_columns(self, db_connection, monkeypatch): def test_catalogs_without_documents_are_not_listed(self, db_connection): db = db_connection with db.get_connection() as conn: - db.get_or_create_catalog(db.get_or_create_group("mh", conn=conn), "scheme", conn=conn) + db.get_or_create_catalog(db.get_or_create_instance("mh", conn=conn), "scheme", conn=conn) assert db.list_catalog_entries() == [] assert db.get_catalog_entry("anything") is None @@ -1174,24 +1174,24 @@ class TestOperationsViews: @pytest.mark.db @pytest.mark.unit def test_summary_counts_and_queue_use_the_current_workflow(self, db_connection): - _new_doc(db_connection, "wf-q1", stage="ocr_review", group="mh") - _new_doc(db_connection, "wf-q2", stage="completed", group="up") + _new_doc(db_connection, "wf-q1", stage="ocr_review", instance="mh") + _new_doc(db_connection, "wf-q2", stage="completed", instance="up") db_connection.create_document_job("wf-q2", "reingest", current_stage="ingesting") counts = db_connection.get_document_summary_counts() assert counts["total_documents"] == 2 assert counts["ocr_review_documents"] == 1 assert counts["completed_documents"] == 0 assert counts["running_jobs"] == 3 - assert db_connection.get_document_summary_counts(groups=[])["total_documents"] == 0 - by_group = {r["group"]: r for r in db_connection.get_document_counts_by_group()} - assert by_group["mh"]["dev_approval"] == 1 + assert db_connection.get_document_summary_counts(instances=[])["total_documents"] == 0 + by_instance = {r["instance"]: r for r in db_connection.get_document_counts_by_instance()} + assert by_instance["mh"]["dev_approval"] == 1 rows, total = db_connection.list_operations_queue() assert total == 2 assert {r["workflow_id"]: r["job_type"] for r in rows} == { "wf-q1": "pipeline", "wf-q2": "reingest", } - assert db_connection.list_operations_queue(groups=["up"])[1] == 1 + assert db_connection.list_operations_queue(instances=["up"])[1] == 1 @pytest.mark.db @pytest.mark.unit diff --git a/tests/test_discovery_publish_service.py b/tests/test_discovery_publish_service.py index 06f4522..3539abf 100644 --- a/tests/test_discovery_publish_service.py +++ b/tests/test_discovery_publish_service.py @@ -98,7 +98,7 @@ def test_publish_success_envelope_shape(self, monkeypatch): mock_client.post.return_value = mock_response with patch("pipeline.discovery_publish_service.httpx.Client", return_value=mock_client): - result = service.publish(transaction_id="txn-123", kind="advisory") + result = service.publish(transaction_id="txn-123", document_kind="advisory") mock_client.post.assert_called_once() call_args = mock_client.post.call_args @@ -153,8 +153,8 @@ def test_publish_generates_fresh_message_id_per_call(self, monkeypatch): mock_client.post.return_value = mock_response with patch("pipeline.discovery_publish_service.httpx.Client", return_value=mock_client): - first = service.publish(transaction_id="txn-shared", kind="advisory") - second = service.publish(transaction_id="txn-shared", kind="advisory") + first = service.publish(transaction_id="txn-shared", document_kind="advisory") + second = service.publish(transaction_id="txn-shared", document_kind="advisory") assert first["envelope"]["context"]["transactionId"] == "txn-shared" assert second["envelope"]["context"]["transactionId"] == "txn-shared" @@ -180,7 +180,7 @@ def test_publish_raises_on_http_error(self, monkeypatch): with patch("pipeline.discovery_publish_service.httpx.Client", return_value=mock_client): with pytest.raises(httpx.HTTPStatusError): - service.publish(transaction_id="txn-123", kind="advisory") + service.publish(transaction_id="txn-123", document_kind="advisory") @pytest.mark.unit def test_publish_raises_on_connection_error(self, monkeypatch): @@ -197,7 +197,7 @@ def test_publish_raises_on_connection_error(self, monkeypatch): with patch("pipeline.discovery_publish_service.httpx.Client", return_value=mock_client): with pytest.raises(httpx.ConnectError): - service.publish(transaction_id="txn-123", kind="advisory") + service.publish(transaction_id="txn-123", document_kind="advisory") class TestDiscoveryPublishServiceSchemeCatalog: @@ -220,7 +220,7 @@ def test_scheme_kind_publishes_the_schemes_catalog(self, monkeypatch): mock_client.post.return_value = mock_response with patch("pipeline.discovery_publish_service.httpx.Client", return_value=mock_client): - result = service.publish(transaction_id="txn-9", kind="scheme") + result = service.publish(transaction_id="txn-9", document_kind="scheme") envelope = mock_client.post.call_args.kwargs["json"] catalog = envelope["message"]["catalogs"][0] @@ -244,7 +244,7 @@ def test_no_http_call_is_made(self, monkeypatch, kind): mock_client = MagicMock() with patch("pipeline.discovery_publish_service.httpx.Client", return_value=mock_client): - result = service.publish(transaction_id="txn-skip", kind=kind) + result = service.publish(transaction_id="txn-skip", document_kind=kind) mock_client.post.assert_not_called() assert result["skipped"] is True @@ -265,7 +265,7 @@ def test_reports_the_result_status_for_our_catalog(self, monkeypatch, status): "pipeline.discovery_publish_service.httpx.Client", return_value=_client_returning(response), ): - result = service.publish(transaction_id="txn-v", kind="advisory") + result = service.publish(transaction_id="txn-v", document_kind="advisory") assert result["result_status"] == status @@ -281,7 +281,7 @@ def test_carries_the_errors_array_through(self, monkeypatch): "pipeline.discovery_publish_service.httpx.Client", return_value=_client_returning(response), ): - result = service.publish(transaction_id="txn-v", kind="advisory") + result = service.publish(transaction_id="txn-v", document_kind="advisory") assert result["errors"] == errors @@ -306,7 +306,7 @@ def test_picks_our_catalog_out_of_several_results(self, monkeypatch): "pipeline.discovery_publish_service.httpx.Client", return_value=_client_returning(response), ): - result = service.publish(transaction_id="txn-v", kind="advisory") + result = service.publish(transaction_id="txn-v", document_kind="advisory") assert result["result_status"] == "ACCEPTED" @@ -333,7 +333,7 @@ def test_unreadable_result_is_unknown_not_a_failure(self, monkeypatch, body): "pipeline.discovery_publish_service.httpx.Client", return_value=_client_returning(response), ): - result = service.publish(transaction_id="txn-v", kind="advisory") + result = service.publish(transaction_id="txn-v", document_kind="advisory") assert result["result_status"] is None assert result["errors"] == [] @@ -351,7 +351,7 @@ def test_non_json_body_is_unknown_not_a_failure(self, monkeypatch): "pipeline.discovery_publish_service.httpx.Client", return_value=_client_returning(response), ): - result = service.publish(transaction_id="txn-v", kind="advisory") + result = service.publish(transaction_id="txn-v", document_kind="advisory") assert result["result_status"] is None assert result["errors"] == [] @@ -361,7 +361,7 @@ def test_skipped_publish_has_no_result_status(self, monkeypatch): service = _service(monkeypatch) with patch("pipeline.discovery_publish_service.httpx.Client", return_value=MagicMock()): - result = service.publish(transaction_id="txn-v", kind="video") + result = service.publish(transaction_id="txn-v", document_kind="video") assert result["skipped"] is True assert result["result_status"] is None @@ -384,7 +384,7 @@ def test_logs_the_request_and_the_response_status(self, monkeypatch, caplog): ) with patch("pipeline.discovery_publish_service.httpx.Client", return_value=ok): service.publish( - transaction_id="txn-log", kind="advisory", workflow_id="wf-log" + transaction_id="txn-log", document_kind="advisory", workflow_id="wf-log" ) messages = [r.getMessage() for r in caplog.records] @@ -410,7 +410,7 @@ def test_logs_an_error_when_the_call_fails(self, monkeypatch, caplog): with patch("pipeline.discovery_publish_service.httpx.Client", return_value=client): with pytest.raises(httpx.ConnectError): service.publish( - transaction_id="txn-log", kind="advisory", workflow_id="wf-log" + transaction_id="txn-log", document_kind="advisory", workflow_id="wf-log" ) errors = [r.getMessage() for r in caplog.records if r.levelno == logging.ERROR] @@ -430,7 +430,7 @@ def test_skipped_publish_logs_no_request(self, monkeypatch, caplog): "pipeline.discovery_publish_service.httpx.Client", return_value=MagicMock() ): service.publish( - transaction_id="txn-log", kind="video", workflow_id="wf-log" + transaction_id="txn-log", document_kind="video", workflow_id="wf-log" ) messages = [r.getMessage() for r in caplog.records] @@ -465,7 +465,7 @@ def _publish(monkeypatch, kind, group, sender_id="provider.oan.dev"): service = DiscoveryPublishService() client = _client_returning(_on_publish_response("any", "ACCEPTED")) with patch("pipeline.discovery_publish_service.httpx.Client", return_value=client): - result = service.publish(transaction_id="txn", kind=kind, group=group) + result = service.publish(transaction_id="txn", document_kind=kind, group=group) return client.post.call_args.kwargs["json"], result @@ -550,7 +550,7 @@ def test_unmapped_kind_is_skipped_for_a_group(self, monkeypatch): monkeypatch.setenv("NETWORK_SENDER_ID", "provider.oan.dev") service = DiscoveryPublishService() - result = service.publish(transaction_id="txn-doc", kind="document", group="ka") + result = service.publish(transaction_id="txn-doc", document_kind="document", group="ka") assert result["skipped"] is True assert result["envelope"] is None diff --git a/tests/test_display_name.py b/tests/test_display_name.py index 14f6c20..d32fddf 100644 --- a/tests/test_display_name.py +++ b/tests/test_display_name.py @@ -127,7 +127,7 @@ def capture(*args, **kwargs): monkeypatch.setattr(activities, "_prepare_records", capture) monkeypatch.setattr(db, "get_chunks", lambda *a, **k: [CHUNK]) monkeypatch.setattr( - db, "get_document", lambda workflow_id: {"name": "PM Kisan Guidelines", "group": "mh"} + db, "get_document", lambda workflow_id: {"name": "PM Kisan Guidelines", "instance": "mh"} ) return seen diff --git a/tests/test_document_repository.py b/tests/test_document_repository.py index 1d9e301..3335332 100644 --- a/tests/test_document_repository.py +++ b/tests/test_document_repository.py @@ -112,7 +112,7 @@ def _default_no_group_instances(self, monkeypatch): [("mh", "mh"), ("MH", "mh"), ("bv", None), ("default", None), (None, None)], ) def test_group_comes_from_the_documents_instance(self, instance, group): - repo = DocumentRepository(FakeDb(document={"group": instance})) + repo = DocumentRepository(FakeDb(document={"instance": instance})) assert repo.get_document_group("wf-1") == group diff --git a/tests/test_groups.py b/tests/test_groups.py index f90d1f7..96bcb23 100644 --- a/tests/test_groups.py +++ b/tests/test_groups.py @@ -9,11 +9,11 @@ ROLE_STATE_VIEW, ROLE_SUPER_ADMIN, parse_group_paths, - role_for_group, + role_for_instance, ) from pipeline.auth.jwt import claims_to_user from pipeline.auth.permissions import Permission, UserRole, permissions_for_roles -from pipeline.auth.tenancy import allowed_groups, user_can_access_group +from pipeline.auth.tenancy import allowed_instances, user_can_access_instance def test_parse_global_super_admin(): @@ -33,9 +33,9 @@ def test_parse_multi_state_roles(): assert access.is_super_admin is False assert access.state_roles == {"mh": ROLE_STATE_ADMIN, "up": ROLE_STATE_VIEW} assert set(access.instances) == {"mh", "up"} - assert role_for_group(access, "MH") == ROLE_STATE_ADMIN - assert role_for_group(access, "up") == ROLE_STATE_VIEW - assert role_for_group(access, "bh") is None + assert role_for_instance(access, "MH") == ROLE_STATE_ADMIN + assert role_for_instance(access, "up") == ROLE_STATE_VIEW + assert role_for_instance(access, "bh") is None def test_approver_and_contributor_paths(): @@ -61,17 +61,17 @@ def test_contributor_approves_stages_but_cannot_publish_to_dev(): # OCR / translation / chunk approvals all ride on REVIEW. assert user.has_permission(Permission.REVIEW) - assert user.has_permission_for_group(Permission.REVIEW, "up") + assert user.has_permission_for_instance(Permission.REVIEW, "up") # …but publishing to DEV is its own permission. assert not user.has_permission(Permission.APPROVE_INGESTION) - assert not user.has_permission_for_group(Permission.APPROVE_INGESTION, "up") + assert not user.has_permission_for_instance(Permission.APPROVE_INGESTION, "up") assert not user.has_permission(Permission.DELETE_OWN) # Approver and admin both clear the DEV publish gate. for group, role in (("approver", "state_approver"), ("admin", "state_admin")): stronger = claims_to_user({"sub": role, "groups": [f"/states/UP/{group}"]}) assert stronger.has_permission(Permission.APPROVE_INGESTION), role - assert stronger.has_permission_for_group(Permission.APPROVE_INGESTION, "up"), role + assert stronger.has_permission_for_instance(Permission.APPROVE_INGESTION, "up"), role # Only the admin may delete. approver = claims_to_user({"sub": "a", "groups": ["/states/UP/approver"]}) @@ -109,14 +109,14 @@ def test_bh_viewer_reads_every_state_without_write(): assert access.is_super_admin is False assert access.state_roles == {} # No per-state group, but any state resolves to view. - assert role_for_group(access, "mh") == ROLE_STATE_VIEW - assert role_for_group(access, "wb") == ROLE_STATE_VIEW + assert role_for_instance(access, "mh") == ROLE_STATE_VIEW + assert role_for_instance(access, "wb") == ROLE_STATE_VIEW def test_bh_viewer_keeps_stronger_state_role(): access = parse_group_paths(["/global/bh-viewer", "/states/MH/admin"]) - assert role_for_group(access, "mh") == ROLE_STATE_ADMIN - assert role_for_group(access, "up") == ROLE_STATE_VIEW + assert role_for_instance(access, "mh") == ROLE_STATE_ADMIN + assert role_for_instance(access, "up") == ROLE_STATE_VIEW def test_higher_role_wins_same_state(): @@ -172,17 +172,17 @@ def test_claims_to_user_from_groups_only(): assert user.state_roles["mh"] == "state_admin" assert user.state_roles["up"] == "state_view" assert user.is_superadmin is False - assert allowed_groups(user) == {"mh", "up"} - assert user_can_access_group(user, "mh") - assert not user_can_access_group(user, "bh") + assert allowed_instances(user) == {"mh", "up"} + assert user_can_access_instance(user, "mh") + assert not user_can_access_instance(user, "bh") # Union of state_admin + state_view permissions assert Permission.UPLOAD in user.permissions assert Permission.SEARCH in user.permissions assert Permission.MANAGE_USERS not in user.permissions - assert user.has_permission_for_group(Permission.UPLOAD, "mh") is True - assert user.has_permission_for_group(Permission.UPLOAD, "up") is False - assert user.has_permission_for_group(Permission.SEARCH, "up") is True - assert user.has_permission_for_group(Permission.REVIEW, "up") is False + assert user.has_permission_for_instance(Permission.UPLOAD, "mh") is True + assert user.has_permission_for_instance(Permission.UPLOAD, "up") is False + assert user.has_permission_for_instance(Permission.SEARCH, "up") is True + assert user.has_permission_for_instance(Permission.REVIEW, "up") is False def test_claims_super_admin_unrestricted(): @@ -194,8 +194,8 @@ def test_claims_super_admin_unrestricted(): } ) assert user.is_superadmin is True - assert allowed_groups(user) is None - assert user_can_access_group(user, "mh") + assert allowed_instances(user) is None + assert user_can_access_instance(user, "mh") assert Permission.MANAGE_USERS in user.permissions @@ -210,5 +210,5 @@ def test_legacy_instances_claim_still_works(): ) assert user.instances == ["tenant-a"] assert Permission.UPLOAD in user.permissions - assert user_can_access_group(user, "tenant-a") - assert not user_can_access_group(user, "mh") + assert user_can_access_instance(user, "tenant-a") + assert not user_can_access_instance(user, "mh") diff --git a/tests/test_group_names.py b/tests/test_instances.py similarity index 81% rename from tests/test_group_names.py rename to tests/test_instances.py index 857ca7a..e37c3c7 100644 --- a/tests/test_group_names.py +++ b/tests/test_instances.py @@ -1,4 +1,4 @@ -"""Tests for group (tenant) code → state name resolution.""" +"""Tests for instance (tenant) code → state name resolution.""" import pytest @@ -22,30 +22,30 @@ class TestGroupDisplayName: ], ) def test_known_codes(self, code, expected): - from pipeline.group_names import group_display_name + from pipeline.instances import instance_display_name - assert group_display_name(code) == expected + assert instance_display_name(code) == expected @pytest.mark.unit @pytest.mark.parametrize("code,expected", [("or", "Odisha"), ("od", "Odisha"), ("ts", "Telangana"), ("tg", "Telangana")]) def test_legacy_code_aliases(self, code, expected): - from pipeline.group_names import group_display_name + from pipeline.instances import instance_display_name - assert group_display_name(code) == expected + assert instance_display_name(code) == expected @pytest.mark.unit def test_blank_falls_back_to_default(self): - from pipeline.group_names import group_display_name + from pipeline.instances import instance_display_name - assert group_display_name("") == "Default" - assert group_display_name(None) == "Default" + assert instance_display_name("") == "Default" + assert instance_display_name(None) == "Default" @pytest.mark.unit def test_unknown_code_is_made_presentable(self): """A new state must degrade to a readable label, never break ingestion.""" - from pipeline.group_names import group_display_name + from pipeline.instances import instance_display_name - assert group_display_name("new-state") == "New State" + assert instance_display_name("new-state") == "New State" class TestProdStageDisabled: @@ -54,7 +54,7 @@ class TestProdStageDisabled: @pytest.mark.unit @pytest.mark.parametrize("value", ["true", "TRUE", "True", "1", "yes", "YES"]) def test_truthy_values_disable_prod(self, monkeypatch, value): - from pipeline.group_names import prod_stage_disabled + from pipeline.instances import prod_stage_disabled monkeypatch.setenv("DISABLE_PROD_SETTING", value) assert prod_stage_disabled() is True @@ -62,7 +62,7 @@ def test_truthy_values_disable_prod(self, monkeypatch, value): @pytest.mark.unit @pytest.mark.parametrize("value", ["false", "FALSE", "0", "no", "", "anything-else"]) def test_other_values_keep_prod_enabled(self, monkeypatch, value): - from pipeline.group_names import prod_stage_disabled + from pipeline.instances import prod_stage_disabled monkeypatch.setenv("DISABLE_PROD_SETTING", value) assert prod_stage_disabled() is False @@ -70,14 +70,14 @@ def test_other_values_keep_prod_enabled(self, monkeypatch, value): @pytest.mark.unit def test_defaults_to_enabled_when_unset(self, monkeypatch): """Absent flag must keep the full pipeline — never silently skip PROD.""" - from pipeline.group_names import prod_stage_disabled + from pipeline.instances import prod_stage_disabled monkeypatch.delenv("DISABLE_PROD_SETTING", raising=False) assert prod_stage_disabled() is False @pytest.mark.unit def test_whitespace_is_tolerated(self, monkeypatch): - from pipeline.group_names import prod_stage_disabled + from pipeline.instances import prod_stage_disabled monkeypatch.setenv("DISABLE_PROD_SETTING", " true ") assert prod_stage_disabled() is True diff --git a/tests/test_migrate_sqlite_to_postgres.py b/tests/test_migrate_sqlite_to_postgres.py index ab56a1b..5af95ae 100644 --- a/tests/test_migrate_sqlite_to_postgres.py +++ b/tests/test_migrate_sqlite_to_postgres.py @@ -254,7 +254,7 @@ def _counts(db): table: conn.execute(f"SELECT count(*) AS n FROM {table}").fetchone()["n"] for table in ( "documents", "workflows", "workflow_artifacts", "pages", "chunks", "audit_logs", - "users", "groups", "catalogs", "email_otps", "webhook_records", + "users", "instances", "catalogs", "email_otps", "webhook_records", ) } @@ -294,7 +294,7 @@ def test_a_full_run_maps_every_table(self, db_connection, postgres_url, legacy_d doc = db.get_document(WF1) assert doc["stage"] == "completed" - assert doc["group"] == "mh" + assert doc["instance"] == "mh" assert doc["document_id"] == FP1 assert doc["name"] == "Registry name" # one name per document in v2 assert (doc["valid_from"], doc["valid_to"]) == ("2026-03-01", "2027-03-01") @@ -344,9 +344,9 @@ def test_a_full_run_maps_every_table(self, db_connection, postgres_url, legacy_d assert db.get_catalog_meta()["version"] == 6 with db.get_connection() as conn: - groups = {g["path"]: g for g in conn.execute("SELECT * FROM groups").fetchall()} + groups = {g["path"]: g for g in conn.execute("SELECT * FROM instances").fetchall()} catalogs = conn.execute( - "SELECT g.path, c.type FROM catalogs c JOIN groups g ON g.id = c.group_id ORDER BY 1, 2" + "SELECT g.path, c.type FROM catalogs c JOIN instances g ON g.id = c.instance_id ORDER BY 1, 2" ).fetchall() meta = conn.execute( "SELECT metadata FROM documents WHERE id = %s::uuid", (doc["id"],) @@ -400,7 +400,7 @@ def test_in_flight_documents_keep_their_stage_and_prod_request(self, db_connecti assert doc["kind"] == "advisory" and doc["code"] == "inflight" assert doc["prod_ready_requested_at"] == "2026-04-02T09:00:00" assert doc["prod_ready_requested_by_username"] == "rita" - assert doc["group"] == "up" + assert doc["instance"] == "up" assert doc["ocr_completed_at"] == "2026-04-01T11:00:00" types = sorted(j["job_type"] for j in db.list_document_jobs(WF2)) assert types == ["pipeline", "promote_prod"] @@ -415,7 +415,7 @@ def test_unusable_values_are_repaired_and_noted(self, db_connection, postgres_ur doc = db.get_document(WF3) assert doc["stage"] == "failed" and doc["error_message"] == "OCR blew up" assert doc["document_id"] == hashlib.md5(b"not-a-hash").hexdigest() - assert doc["group"] == "default" + assert doc["instance"] == "default" assert doc["kind"] == "advisory" and doc["code"] == "broken" assert doc["valid_from"] == "2026-06-01" and doc["valid_to"] is None notes = " ".join(report.notes) diff --git a/tests/test_network_group.py b/tests/test_network_group.py index 14f17cd..9c0d13e 100644 --- a/tests/test_network_group.py +++ b/tests/test_network_group.py @@ -2,7 +2,7 @@ import pytest -from pipeline.group_names import GROUP_NAMES +from pipeline.instances import INSTANCE_NAMES from pipeline.network_group import ( DEFAULT_PROVIDER_CONFIG_OVERRIDE_PATH, PROVIDER_CONFIG_OVERRIDE_PATH_ENV, @@ -212,7 +212,7 @@ def test_id_template_must_render_a_valid_id(self, monkeypatch, template): @pytest.mark.unit def test_every_known_instance_is_checked_at_boot(self, monkeypatch): - monkeypatch.setitem(GROUP_NAMES, "ladakh_ut", "Ladakh") + monkeypatch.setitem(INSTANCE_NAMES, "ladakh_ut", "Ladakh") with pytest.raises(NetworkProviderConfigError, match="provider id"): load_provider_naming() diff --git a/tests/test_provider_id_on_chunks.py b/tests/test_provider_id_on_chunks.py index 2f31fee..f9e926c 100644 --- a/tests/test_provider_id_on_chunks.py +++ b/tests/test_provider_id_on_chunks.py @@ -44,14 +44,14 @@ class TestProviderIdFromDoc: def test_published_kinds_get_the_provider_id(self, kind, instance, expected): from pipeline.activities import _provider_id_from_doc - assert _provider_id_from_doc({"kind": kind, "group": instance}) == expected + assert _provider_id_from_doc({"kind": kind, "instance": instance}) == expected @pytest.mark.unit @pytest.mark.parametrize("kind", ["document", "video", "how_to_faq", None]) def test_kinds_that_never_publish_get_none(self, kind): from pipeline.activities import _provider_id_from_doc - assert _provider_id_from_doc({"kind": kind, "group": "mh"}) is None + assert _provider_id_from_doc({"kind": kind, "instance": "mh"}) is None @pytest.mark.unit def test_follows_the_overrides_file(self, monkeypatch, tmp_path): @@ -59,9 +59,9 @@ def test_follows_the_overrides_file(self, monkeypatch, tmp_path): _use_overrides(monkeypatch, tmp_path, '{"ka": "ka.example.com"}') - assert _provider_id_from_doc({"kind": "advisory", "group": "ka"}) == "ka.example.com" - assert _provider_id_from_doc({"kind": "scheme", "group": "ka"}) == "ka.example.com" - assert _provider_id_from_doc({"kind": "scheme", "group": "mh"}) == "mh.provider.oan.dev" + assert _provider_id_from_doc({"kind": "advisory", "instance": "ka"}) == "ka.example.com" + assert _provider_id_from_doc({"kind": "scheme", "instance": "ka"}) == "ka.example.com" + assert _provider_id_from_doc({"kind": "scheme", "instance": "mh"}) == "mh.provider.oan.dev" @pytest.mark.unit def test_no_network_id_means_no_provider_id(self, monkeypatch): @@ -69,14 +69,14 @@ def test_no_network_id_means_no_provider_id(self, monkeypatch): monkeypatch.delenv("NETWORK_SENDER_ID", raising=False) - assert _provider_id_from_doc({"kind": "advisory", "group": "mh"}) is None + assert _provider_id_from_doc({"kind": "advisory", "instance": "mh"}) is None @pytest.mark.unit def test_an_invalid_provider_id_fails_without_retry(self): from pipeline.activities import _provider_id_from_doc with pytest.raises(ApplicationError, match="ladakh_ut") as excinfo: - _provider_id_from_doc({"kind": "advisory", "group": "ladakh_ut"}) + _provider_id_from_doc({"kind": "advisory", "instance": "ladakh_ut"}) assert excinfo.value.non_retryable is True @@ -101,7 +101,7 @@ def test_matches_the_provider_id_sent_in_publish(self, monkeypatch, tmp_path, ki DiscoveryPublishService().publish("txn", kind, "wf", group_for_instance(instance)) published = client.post.call_args.kwargs["json"]["message"]["catalogs"][0]["provider"]["id"] - assert _provider_id_from_doc({"kind": kind, "group": instance}) == published + assert _provider_id_from_doc({"kind": kind, "instance": instance}) == published class TestProviderIdOnRecords: @@ -147,7 +147,7 @@ def capture(*args, **kwargs): monkeypatch.setattr(activities, "_prepare_records", capture) monkeypatch.setattr(db, "get_chunks", lambda *a, **k: [CHUNK]) - monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "scheme", "group": "mh"}) + monkeypatch.setattr(db, "get_document", lambda workflow_id: {"kind": "scheme", "instance": "mh"}) return seen @pytest.mark.unit diff --git a/tests/test_schema_v2.py b/tests/test_schema_v2.py index fa6b053..bfb8c77 100644 --- a/tests/test_schema_v2.py +++ b/tests/test_schema_v2.py @@ -26,88 +26,88 @@ def _doc(db, workflow_id, **kwargs): return db.upsert_document(workflow_id=workflow_id, **kwargs) -def _group(db, group="mh"): - return db.get_or_create_group(group) +def _instance(db, instance="mh"): + return db.get_or_create_instance(instance) class TestGroups: @pytest.mark.db @pytest.mark.unit - def test_get_or_create_group_is_idempotent_and_maps_tenant_codes_to_paths(self, db_connection): + def test_get_or_create_instance_is_idempotent_and_maps_tenant_codes_to_paths(self, db_connection): db = db_connection - mh = _group(db, "mh") - assert _group(db, "MH") == mh - assert _group(db, "up") != mh - bv = _group(db, "bv") + mh = _instance(db, "mh") + assert _instance(db, "MH") == mh + assert _instance(db, "up") != mh + bv = _instance(db, "bv") with db.get_connection() as conn: - rows = {r["name"]: r["path"] for r in conn.execute("SELECT name, path FROM groups").fetchall()} + rows = {r["name"]: r["path"] for r in conn.execute("SELECT name, path FROM instances").fetchall()} assert rows == {"mh": "/states/MH", "up": "/states/UP", "bv": "/global"} - assert _group(db, "bv") == bv + assert _instance(db, "bv") == bv @pytest.mark.db @pytest.mark.unit - def test_upsert_group_fills_the_keycloak_id_of_an_existing_path(self, db_connection): + def test_upsert_instance_fills_the_keycloak_id_of_an_existing_path(self, db_connection): db = db_connection - first = db.upsert_group("/states/MH") # as created by the SQLite move: no Keycloak id yet - again = db.upsert_group("/states/MH", keycloak_group_id="kc-group-1") + first = db.upsert_instance("/states/MH") # as created by the SQLite move: no Keycloak id yet + again = db.upsert_instance("/states/MH", keycloak_group_id="kc-group-1") assert again == first with db.get_connection() as conn: - row = conn.execute("SELECT keycloak_group_id, name FROM groups WHERE id = %s::uuid", (first,)).fetchone() + row = conn.execute("SELECT keycloak_group_id, name FROM instances WHERE id = %s::uuid", (first,)).fetchone() assert (row["keycloak_group_id"], row["name"]) == ("kc-group-1", "mh") # A later sync that does not know the id must not erase it. - db.upsert_group("/states/MH") + db.upsert_instance("/states/MH") with db.get_connection() as conn: - row = conn.execute("SELECT keycloak_group_id FROM groups WHERE id = %s::uuid", (first,)).fetchone() + row = conn.execute("SELECT keycloak_group_id FROM instances WHERE id = %s::uuid", (first,)).fetchone() assert row["keycloak_group_id"] == "kc-group-1" @pytest.mark.db @pytest.mark.unit def test_keycloak_group_id_is_unique_but_nullable(self, db_connection): with db_connection.get_connection() as conn: - conn.execute("INSERT INTO groups (name, path) VALUES ('a', '/states/A')") - conn.execute("INSERT INTO groups (name, path) VALUES ('b', '/states/B')") # two NULLs are fine - conn.execute("INSERT INTO groups (keycloak_group_id, name, path) VALUES ('k', 'c', '/states/C')") + conn.execute("INSERT INTO instances (name, path) VALUES ('a', '/states/A')") + conn.execute("INSERT INTO instances (name, path) VALUES ('b', '/states/B')") # two NULLs are fine + conn.execute("INSERT INTO instances (keycloak_group_id, name, path) VALUES ('k', 'c', '/states/C')") with pytest.raises(psycopg.errors.UniqueViolation): - conn.execute("INSERT INTO groups (keycloak_group_id, name, path) VALUES ('k', 'd', '/states/D')") + conn.execute("INSERT INTO instances (keycloak_group_id, name, path) VALUES ('k', 'd', '/states/D')") @pytest.mark.db @pytest.mark.unit - def test_a_group_with_catalogs_cannot_be_deleted(self, db_connection): + def test_a_instance_with_catalogs_cannot_be_deleted(self, db_connection): db = db_connection - group_id = _group(db) - db.get_or_create_catalog(group_id, "scheme") + instance_id = _instance(db) + db.get_or_create_catalog(instance_id, "scheme") with db.get_connection() as conn: with pytest.raises(psycopg.errors.ForeignKeyViolation): - conn.execute("DELETE FROM groups WHERE id = %s::uuid", (group_id,)) + conn.execute("DELETE FROM instances WHERE id = %s::uuid", (instance_id,)) @pytest.mark.db @pytest.mark.unit - def test_deleting_a_group_clears_users_and_audit_snapshots(self, db_connection): + def test_deleting_a_instance_clears_users_and_audit_snapshots(self, db_connection): db = db_connection - group_id = _group(db, "zz") + instance_id = _instance(db, "zz") db.upsert_user("kc-zz", "zed", None, [], tenants=["zz"]) with db.get_connection() as conn: conn.execute( - "INSERT INTO audit_logs (action_type, entity_type, group_id) VALUES ('x', 'setting', %s::uuid)", - (group_id,), + "INSERT INTO audit_logs (action_type, entity_type, instance_id) VALUES ('x', 'setting', %s::uuid)", + (instance_id,), ) - conn.execute("DELETE FROM catalogs WHERE group_id = %s::uuid", (group_id,)) - conn.execute("DELETE FROM groups WHERE id = %s::uuid", (group_id,)) - assert conn.execute("SELECT group_id FROM audit_logs WHERE action_type = 'x'").fetchone()["group_id"] is None - assert db.get_user_by_keycloak_id("kc-zz")["group_id"] is None + conn.execute("DELETE FROM catalogs WHERE instance_id = %s::uuid", (instance_id,)) + conn.execute("DELETE FROM instances WHERE id = %s::uuid", (instance_id,)) + assert conn.execute("SELECT instance_id FROM audit_logs WHERE action_type = 'x'").fetchone()["instance_id"] is None + assert db.get_user_by_keycloak_id("kc-zz")["instance_id"] is None class TestCatalogs: @pytest.mark.db @pytest.mark.unit - def test_a_new_group_gets_one_catalog_per_supported_type(self, db_connection): + def test_a_new_instance_gets_one_catalog_per_supported_type(self, db_connection): from pipeline.network_constants import KIND_TO_CATALOG_TYPE db = db_connection - mh = _group(db, "mh") + mh = _instance(db, "mh") with db.get_connection() as conn: rows = conn.execute( - "SELECT type, name FROM catalogs WHERE group_id = %s::uuid ORDER BY type", (mh,) + "SELECT type, name FROM catalogs WHERE instance_id = %s::uuid ORDER BY type", (mh,) ).fetchall() assert [r["type"] for r in rows] == sorted(KIND_TO_CATALOG_TYPE.values()) assert {r["name"] for r in rows} == { @@ -118,14 +118,14 @@ def test_a_new_group_gets_one_catalog_per_supported_type(self, db_connection): @pytest.mark.unit def test_upload_uses_the_seeded_catalog_and_creates_none(self, db_connection): db = db_connection - mh = _group(db, "mh") + mh = _instance(db, "mh") with db.get_connection() as conn: seeded = conn.execute( - "SELECT id FROM catalogs WHERE group_id = %s::uuid AND type = 'openagrinet:AgricultureResource'", + "SELECT id FROM catalogs WHERE instance_id = %s::uuid AND type = 'openagrinet:AgricultureResource'", (mh,), ).fetchone()["id"] before = conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] - _doc(db, "wf-seeded", group="mh", code="seeded") + _doc(db, "wf-seeded", instance="mh", code="seeded") assert db.get_document("wf-seeded")["catalog_id"] == str(seeded) with db.get_connection() as conn: assert conn.execute("SELECT count(*) AS n FROM catalogs").fetchone()["n"] == before @@ -134,39 +134,39 @@ def test_upload_uses_the_seeded_catalog_and_creates_none(self, db_connection): @pytest.mark.unit def test_a_missing_catalog_is_created_on_demand(self, db_connection): db = db_connection - mh = _group(db, "mh") + mh = _instance(db, "mh") with db.get_connection() as conn: conn.execute( - "DELETE FROM catalogs WHERE group_id = %s::uuid AND type = 'openagrinet:KnowledgeAdvisory'", + "DELETE FROM catalogs WHERE instance_id = %s::uuid AND type = 'openagrinet:KnowledgeAdvisory'", (mh,), ) - _doc(db, "wf-fallback", group="mh", code="fallback", kind="advisory") + _doc(db, "wf-fallback", instance="mh", code="fallback", kind="advisory") assert db.get_document("wf-fallback")["kind"] == "advisory" @pytest.mark.db @pytest.mark.unit - def test_startup_sync_fills_missing_catalogs_for_every_group_and_is_idempotent(self, db_connection): + def test_startup_sync_fills_missing_catalogs_for_every_instance_and_is_idempotent(self, db_connection): db = db_connection - mh, up = _group(db, "mh"), _group(db, "up") + mh, up = _instance(db, "mh"), _instance(db, "up") with db.get_connection() as conn: conn.execute("DELETE FROM catalogs WHERE type = 'openagrinet:VideoResource'") conn.execute( - "DELETE FROM catalogs WHERE group_id = %s::uuid AND type = 'openagrinet:KnowledgeAdvisory'", (up,) + "DELETE FROM catalogs WHERE instance_id = %s::uuid AND type = 'openagrinet:KnowledgeAdvisory'", (up,) ) - assert db.sync_catalogs_for_all_groups() == 3 - assert db.sync_catalogs_for_all_groups() == 0 + assert db.sync_catalogs_for_all_instances() == 3 + assert db.sync_catalogs_for_all_instances() == 0 with db.get_connection() as conn: counts = { - str(r["group_id"]): r["n"] - for r in conn.execute("SELECT group_id, count(*) AS n FROM catalogs GROUP BY group_id") + str(r["instance_id"]): r["n"] + for r in conn.execute("SELECT instance_id, count(*) AS n FROM catalogs GROUP BY instance_id") } assert counts == {mh: 3, up: 3} @pytest.mark.db @pytest.mark.unit - def test_get_or_create_catalog_is_idempotent_per_group_and_kind(self, db_connection): + def test_get_or_create_catalog_is_idempotent_per_instance_and_kind(self, db_connection): db = db_connection - mh = _group(db, "mh") + mh = _instance(db, "mh") scheme = db.get_or_create_catalog(mh, "scheme") assert db.get_or_create_catalog(mh, "scheme")["id"] == scheme["id"] assert scheme["type"] == "openagrinet:AgricultureResource" @@ -174,52 +174,52 @@ def test_get_or_create_catalog_is_idempotent_per_group_and_kind(self, db_connect advisory = db.get_or_create_catalog(mh, "advisory") assert advisory["id"] != scheme["id"] assert advisory["type"] == "openagrinet:KnowledgeAdvisory" - assert db.get_or_create_catalog(_group(db, "up"), "scheme")["id"] != scheme["id"] + assert db.get_or_create_catalog(_instance(db, "up"), "scheme")["id"] != scheme["id"] @pytest.mark.db @pytest.mark.unit def test_a_kind_without_a_catalog_type_is_rejected(self, db_connection): with pytest.raises(ValueError): - db_connection.get_or_create_catalog(_group(db_connection), "podcast") + db_connection.get_or_create_catalog(_instance(db_connection), "podcast") @pytest.mark.db @pytest.mark.unit - def test_one_catalog_per_group_and_type(self, db_connection): + def test_one_catalog_per_instance_and_type(self, db_connection): db = db_connection - group_id = _group(db) + instance_id = _instance(db) with db.get_connection() as conn: conn.execute( - "INSERT INTO catalogs (group_id, type, name) VALUES (%s::uuid, 'openagrinet:MandiPrice', 'A')", - (group_id,), + "INSERT INTO catalogs (instance_id, type, name) VALUES (%s::uuid, 'openagrinet:MandiPrice', 'A')", + (instance_id,), ) with pytest.raises(psycopg.errors.UniqueViolation): conn.execute( - "INSERT INTO catalogs (group_id, type, name) VALUES (%s::uuid, 'openagrinet:MandiPrice', 'B')", - (group_id,), + "INSERT INTO catalogs (instance_id, type, name) VALUES (%s::uuid, 'openagrinet:MandiPrice', 'B')", + (instance_id,), ) @pytest.mark.db @pytest.mark.unit @pytest.mark.parametrize("bad_type", ["scheme", "openagrinet:", ":Thing", "open agrinet:X", "a:b:c", ""]) def test_type_must_look_like_a_schema_name(self, db_connection, bad_type): - group_id = _group(db_connection) + instance_id = _instance(db_connection) with db_connection.get_connection() as conn: with pytest.raises(psycopg.errors.CheckViolation): conn.execute( - "INSERT INTO catalogs (group_id, type, name) VALUES (%s::uuid, %s, 'N')", - (group_id, bad_type), + "INSERT INTO catalogs (instance_id, type, name) VALUES (%s::uuid, %s, 'N')", + (instance_id, bad_type), ) @pytest.mark.db @pytest.mark.unit def test_catalog_metadata_must_be_an_object(self, db_connection): - group_id = _group(db_connection) + instance_id = _instance(db_connection) with db_connection.get_connection() as conn: with pytest.raises(psycopg.errors.CheckViolation): conn.execute( - "INSERT INTO catalogs (group_id, type, name, metadata) " + "INSERT INTO catalogs (instance_id, type, name, metadata) " "VALUES (%s::uuid, 'openagrinet:MandiPrice', 'N', '[]'::jsonb)", - (group_id,), + (instance_id,), ) @pytest.mark.db @@ -248,12 +248,12 @@ def test_a_new_document_needs_a_kind_and_a_code(self, db_connection): @pytest.mark.unit def test_a_new_document_lands_in_its_tenants_catalog_of_that_kind(self, db_connection): db = db_connection - _doc(db, "wf-land", group="mh", kind="advisory", code="kharif", name="Kharif advice") + _doc(db, "wf-land", instance="mh", kind="advisory", code="kharif", name="Kharif advice") doc = db.get_document("wf-land") - assert (doc["group"], doc["kind"], doc["code"], doc["name"]) == ( + assert (doc["instance"], doc["kind"], doc["code"], doc["name"]) == ( "mh", "advisory", "kharif", "Kharif advice", ) - expected = db.get_or_create_catalog(_group(db, "mh"), "advisory")["id"] + expected = db.get_or_create_catalog(_instance(db, "mh"), "advisory")["id"] assert doc["catalog_id"] == expected @pytest.mark.db @@ -266,18 +266,18 @@ def test_the_name_defaults_to_the_code(self, db_connection): @pytest.mark.unit def test_a_code_is_unique_within_its_catalog(self, db_connection): db = db_connection - _doc(db, "wf-one", group="mh", code="pm-kisan") + _doc(db, "wf-one", instance="mh", code="pm-kisan") with pytest.raises(db.DuplicateDocumentCode): - _doc(db, "wf-two", group="mh", code="pm-kisan") + _doc(db, "wf-two", instance="mh", code="pm-kisan") assert db.get_document("wf-two") is None @pytest.mark.db @pytest.mark.unit - def test_the_same_code_is_fine_in_another_group_or_kind(self, db_connection): + def test_the_same_code_is_fine_in_another_instance_or_kind(self, db_connection): db = db_connection - _doc(db, "wf-mh", group="mh", code="pm-kisan") - _doc(db, "wf-up", group="up", code="pm-kisan") - _doc(db, "wf-adv", group="mh", code="pm-kisan", kind="advisory") + _doc(db, "wf-mh", instance="mh", code="pm-kisan") + _doc(db, "wf-up", instance="up", code="pm-kisan") + _doc(db, "wf-adv", instance="mh", code="pm-kisan", kind="advisory") assert {d["workflow_id"] for d in db.list_documents()} == {"wf-mh", "wf-up", "wf-adv"} @pytest.mark.db @@ -295,7 +295,7 @@ def test_a_duplicate_code_leaves_no_half_written_rows(self, db_connection): @pytest.mark.unit def test_find_document_by_code(self, db_connection): db = db_connection - _doc(db, "wf-find", group="mh", code="pm-kisan") + _doc(db, "wf-find", instance="mh", code="pm-kisan") found = db.find_document_by_code("mh", "scheme", "pm-kisan") assert found and found["workflow_id"] == "wf-find" assert db.find_document_by_code("MH", "scheme", "pm-kisan")["id"] == found["id"] @@ -307,7 +307,7 @@ def test_find_document_by_code(self, db_connection): @pytest.mark.unit def test_the_kind_does_not_change_after_the_document_exists(self, db_connection): db = db_connection - _doc(db, "wf-kind", group="mh", code="fixed") + _doc(db, "wf-kind", instance="mh", code="fixed") before = db.get_document("wf-kind")["catalog_id"] db.update_document_fields("wf-kind", kind="advisory") doc = db.get_document("wf-kind") @@ -376,11 +376,11 @@ def test_a_fingerprint_must_be_hex(self, db_connection): class TestUserGroup: @pytest.mark.db @pytest.mark.unit - def test_a_single_tenant_sets_the_group_without_a_warning(self, db_connection, caplog): + def test_a_single_tenant_sets_the_instance_without_a_warning(self, db_connection, caplog): db = db_connection with caplog.at_level(logging.WARNING, logger="pipeline.db"): db.upsert_user("kc-one", "one", None, [], tenants=["MH"]) - assert db.get_user_by_keycloak_id("kc-one")["group_id"] == _group(db, "mh") + assert db.get_user_by_keycloak_id("kc-one")["instance_id"] == _instance(db, "mh") assert not [r for r in caplog.records if "tenants" in r.getMessage()] @pytest.mark.db @@ -389,18 +389,18 @@ def test_several_tenants_use_the_first_sorted_one_and_warn(self, db_connection, db = db_connection with caplog.at_level(logging.WARNING, logger="pipeline.db"): db.upsert_user("kc-many", "many", None, [], tenants=["up", "mh", "up"]) - assert db.get_user_by_keycloak_id("kc-many")["group_id"] == _group(db, "mh") + assert db.get_user_by_keycloak_id("kc-many")["instance_id"] == _instance(db, "mh") warnings = [r.getMessage() for r in caplog.records if r.levelno == logging.WARNING] assert any("kc-many" in w and "mh" in w for w in warnings) @pytest.mark.db @pytest.mark.unit - def test_a_login_without_tenants_keeps_the_stored_group(self, db_connection): + def test_a_login_without_tenants_keeps_the_stored_instance(self, db_connection): db = db_connection db.upsert_user("kc-keep", "keep", None, [], tenants=["up"]) db.upsert_user("kc-keep", "keep", None, []) db.upsert_user("kc-keep", "keep", None, [], tenants=[]) - assert db.get_user_by_keycloak_id("kc-keep")["group_id"] == _group(db, "up") + assert db.get_user_by_keycloak_id("kc-keep")["instance_id"] == _instance(db, "up") @pytest.mark.db @pytest.mark.unit @@ -408,23 +408,23 @@ def test_a_new_login_moves_the_user_to_the_new_first_tenant(self, db_connection) db = db_connection db.upsert_user("kc-move", "move", None, [], tenants=["up"]) db.upsert_user("kc-move", "move", None, [], tenants=["mh"]) - assert db.get_user_by_keycloak_id("kc-move")["group_id"] == _group(db, "mh") + assert db.get_user_by_keycloak_id("kc-move")["instance_id"] == _instance(db, "mh") @pytest.mark.db @pytest.mark.unit - def test_users_without_tenants_have_no_group(self, db_connection): + def test_users_without_tenants_have_no_instance(self, db_connection): db_connection.upsert_user("kc-none", "none", None, []) - assert db_connection.get_user_by_keycloak_id("kc-none")["group_id"] is None + assert db_connection.get_user_by_keycloak_id("kc-none")["instance_id"] is None @pytest.mark.db @pytest.mark.unit - def test_the_uploader_lands_in_the_documents_metadata_and_gets_the_tenant_group(self, db_connection): + def test_the_uploader_lands_in_the_documents_metadata_and_gets_the_tenant_instance(self, db_connection): db = db_connection - _doc(db, "wf-up", group="mh", uploaded_by_user_id="kc-up", uploaded_by_username="uma") + _doc(db, "wf-up", instance="mh", uploaded_by_user_id="kc-up", uploaded_by_username="uma") doc = db.get_document("wf-up") user = db.get_user_by_keycloak_id("kc-up") assert doc["uploaded_by_user_id"] == "kc-up" - assert user["group_id"] == _group(db, "mh") + assert user["instance_id"] == _instance(db, "mh") with db.get_connection() as conn: meta = conn.execute("SELECT metadata FROM documents").fetchone()["metadata"] assert meta["uploaded_by_user_id"] == user["id"] @@ -433,20 +433,20 @@ def test_the_uploader_lands_in_the_documents_metadata_and_gets_the_tenant_group( class TestAuditGroupSnapshot: @pytest.mark.db @pytest.mark.unit - def test_an_entry_snapshots_the_documents_group(self, db_connection): + def test_an_entry_snapshots_the_documents_instance(self, db_connection): db = db_connection - _doc(db, "wf-aud", group="mh") + _doc(db, "wf-aud", instance="mh") db.log_audit(workflow_id="wf-aud", document_id="x", action_type="page_edit", entity_type="page") with db.get_connection() as conn: - row = conn.execute("SELECT group_id FROM audit_logs WHERE action_type = 'page_edit'").fetchone() - assert str(row["group_id"]) == _group(db, "mh") + row = conn.execute("SELECT instance_id FROM audit_logs WHERE action_type = 'page_edit'").fetchone() + assert str(row["instance_id"]) == _instance(db, "mh") @pytest.mark.db @pytest.mark.unit def test_the_snapshot_survives_a_purge_and_keeps_tenant_scope(self, db_connection): db = db_connection - _doc(db, "wf-gone", group="mh") - _doc(db, "wf-other", group="up", code="other") + _doc(db, "wf-gone", instance="mh") + _doc(db, "wf-other", instance="up", code="other") for wf in ("wf-gone", "wf-other"): db.log_audit(workflow_id=wf, document_id="x", action_type="purge_document", entity_type="document") db.purge_document("wf-gone") @@ -454,26 +454,26 @@ def test_the_snapshot_survives_a_purge_and_keeps_tenant_scope(self, db_connectio with db.get_connection() as conn: rows = conn.execute( - "SELECT document_id, group_id, workflow_id FROM audit_logs WHERE action_type = 'purge_document'" + "SELECT document_id, instance_id, workflow_id FROM audit_logs WHERE action_type = 'purge_document'" ).fetchall() assert {r["workflow_id"] for r in rows} == {"wf-gone", "wf-other"} assert all(r["document_id"] is None for r in rows) - assert {str(r["group_id"]) for r in rows} == {_group(db, "mh"), _group(db, "up")} - assert db.get_all_audit_log_count(groups=["mh"]) == 1 - assert db.get_all_audit_log_count(groups=["up"]) == 1 - assert db.get_all_audit_log_count(groups=["gj"]) == 0 - assert [e["workflow_id"] for e in db.get_all_audit_logs(groups=["mh"])] == ["wf-gone"] - assert db.get_all_audit_logs(groups=["mh"])[0]["instance"] == "mh" + assert {str(r["instance_id"]) for r in rows} == {_instance(db, "mh"), _instance(db, "up")} + assert db.get_all_audit_log_count(instances=["mh"]) == 1 + assert db.get_all_audit_log_count(instances=["up"]) == 1 + assert db.get_all_audit_log_count(instances=["gj"]) == 0 + assert [e["workflow_id"] for e in db.get_all_audit_logs(instances=["mh"])] == ["wf-gone"] + assert db.get_all_audit_logs(instances=["mh"])[0]["instance"] == "mh" @pytest.mark.db @pytest.mark.unit - def test_system_entries_have_no_group(self, db_connection): + def test_system_entries_have_no_instance(self, db_connection): db = db_connection db.log_audit(workflow_id="__catalog__", document_id="__catalog__", action_type="catalog_rebuild", entity_type="catalog") with db.get_connection() as conn: - row = conn.execute("SELECT group_id, document_id FROM audit_logs WHERE action_type = 'catalog_rebuild'").fetchone() - assert row["group_id"] is None and row["document_id"] is None + row = conn.execute("SELECT instance_id, document_id FROM audit_logs WHERE action_type = 'catalog_rebuild'").fetchone() + assert row["instance_id"] is None and row["document_id"] is None class TestCatalogStatusView: @@ -484,16 +484,16 @@ def _rows(db): @pytest.mark.db @pytest.mark.unit - def test_one_row_per_document_with_catalog_group_and_type(self, db_connection): + def test_one_row_per_document_with_catalog_instance_and_type(self, db_connection): db = db_connection - _doc(db, "wf-1", group="mh", code="alpha", stage="completed") - _doc(db, "wf-2", group="mh", code="beta", stage="completed") - _doc(db, "wf-3", group="up", code="gamma", stage="completed", kind="advisory") + _doc(db, "wf-1", instance="mh", code="alpha", stage="completed") + _doc(db, "wf-2", instance="mh", code="beta", stage="completed") + _doc(db, "wf-3", instance="up", code="gamma", stage="completed", kind="advisory") rows = self._rows(db) assert set(rows) == {"alpha", "beta", "gamma"} assert rows["alpha"]["catalog_id"] == rows["beta"]["catalog_id"] != rows["gamma"]["catalog_id"] - assert str(rows["alpha"]["group_id"]) == _group(db, "mh") - assert (rows["alpha"]["group_name"], rows["gamma"]["group_name"]) == ("mh", "up") + assert str(rows["alpha"]["instance_id"]) == _instance(db, "mh") + assert (rows["alpha"]["instance_name"], rows["gamma"]["instance_name"]) == ("mh", "up") assert rows["alpha"]["type"] == "openagrinet:AgricultureResource" assert rows["gamma"]["type"] == "openagrinet:KnowledgeAdvisory" assert rows["alpha"]["workflow_id"] == "wf-1" @@ -568,7 +568,7 @@ def test_promoted_at_comes_from_the_prod_index_of_live_documents_only(self, db_c @pytest.mark.unit def test_a_catalog_without_documents_has_no_rows(self, db_connection): db = db_connection - db.get_or_create_catalog(_group(db), "scheme") + db.get_or_create_catalog(_instance(db), "scheme") assert self._rows(db) == {} @@ -725,7 +725,7 @@ class TestSharedIndexKeys: @staticmethod def _indexed_twin(db, workflow_id, key, env="dev", **kwargs): - _doc(db, workflow_id, document_id=key, group="gj", **kwargs) + _doc(db, workflow_id, document_id=key, instance="gj", **kwargs) index = "prod-index" if env == "prod" else "documents-index" db.upsert_document_index_status(workflow_id, index, doc_id=key, status="indexed") @@ -749,7 +749,7 @@ def test_a_key_is_in_use_while_another_live_document_is_indexed_under_it( def test_an_uploaded_but_unindexed_twin_does_not_block_deletion(self, db_connection): db = db_connection first = _doc(db, "wf-up-a", document_id="e" * 32) - _doc(db, "wf-up-b", document_id="e" * 32, group="gj") # same file, never indexed + _doc(db, "wf-up-b", document_id="e" * 32, instance="gj") # same file, never indexed assert db.doc_id_used_by_other_document("e" * 32, first, "dev") is False diff --git a/tests/test_scheme_catalog.py b/tests/test_scheme_catalog.py index 9ecda68..4e4cab7 100644 --- a/tests/test_scheme_catalog.py +++ b/tests/test_scheme_catalog.py @@ -13,7 +13,7 @@ def catalog_db(db_connection): def _doc(db, workflow_id, *, code, kind="scheme", name=None, stage="completed", - group="default", filename=None, **extra): + instance="default", filename=None, **extra): """Register a document the v2 way: it needs a kind (catalog) and a code.""" return db.upsert_document( workflow_id=workflow_id, @@ -21,7 +21,7 @@ def _doc(db, workflow_id, *, code, kind="scheme", name=None, stage="completed", filename=filename or f"{workflow_id}.pdf", filepath=f"/tmp/{workflow_id}.pdf", stage=stage, - group=group, + instance=instance, kind=kind, code=code, name=name, @@ -47,8 +47,8 @@ def test_catalog_has_no_bootstrap_entries(catalog_db): def test_snapshot_has_one_entry_per_live_scheme_document(catalog_db): db, scheme_catalog = catalog_db - _doc(db, "wf-a", code="pkvy", name="Paramparagat Krishi Vikas Yojana", group="mh") - _doc(db, "wf-b", code="mif", name="Micro Irrigation Fund", group="gj") + _doc(db, "wf-a", code="pkvy", name="Paramparagat Krishi Vikas Yojana", instance="mh") + _doc(db, "wf-b", code="mif", name="Micro Irrigation Fund", instance="gj") _doc(db, "wf-adv", code="rain-advice", kind="advisory", name="Rain advisory") _doc(db, "wf-draft", code="draft", stage="chunk_review") snap = scheme_catalog.build_snapshot() @@ -132,10 +132,10 @@ def test_derive_scheme_code_collision_appends_suffix(catalog_db): def test_collision_domain_spans_tenants(catalog_db): """Codes are unique per catalog in SQL, but derivation avoids every tenant's codes.""" db, scheme_catalog = catalog_db - _doc(db, "wf-mh", code="nmnf", group="mh") + _doc(db, "wf-mh", code="nmnf", instance="mh") assert scheme_catalog.derive_scheme_code("National Mission on Natural Farming") == "nmnf-2" # The same explicit code in another tenant's catalog is still allowed by the schema. - _doc(db, "wf-gj", code="nmnf", group="gj") + _doc(db, "wf-gj", code="nmnf", instance="gj") def test_apply_scheme_metadata_keeps_the_code_when_only_the_name_changes(catalog_db): @@ -179,7 +179,7 @@ def test_prepare_records_scheme_payload(): "pkvy.pdf", chunks, workflow_id="wf1", - group="default", + instance="default", kind="scheme", code="pkvy", name="Paramparagat Krishi Vikas Yojana", diff --git a/tests/test_tenancy.py b/tests/test_tenancy.py index f93354d..bf78d09 100644 --- a/tests/test_tenancy.py +++ b/tests/test_tenancy.py @@ -11,49 +11,49 @@ from pipeline.auth.jwt import claims_to_user from pipeline.auth.models import local_bypass_user from pipeline.auth.tenancy import ( - PORTAL_GROUP, - allowed_groups, - assert_document_group_access, - assert_group_access, + PORTAL_INSTANCE, + allowed_instances, + assert_document_instance_access, + assert_instance_access, group_for_instance, - resolve_create_group, - user_can_access_group, + resolve_create_instance, + user_can_access_instance, ) def test_bypass_user_is_unrestricted(): user = local_bypass_user() - assert allowed_groups(user) is None - assert user_can_access_group(user, "tenant-a") - assert user_can_access_group(user, "tenant-b") + assert allowed_instances(user) is None + assert user_can_access_instance(user, "tenant-a") + assert user_can_access_instance(user, "tenant-b") -def test_resolve_create_group_state_user_defaults_to_only_state(): +def test_resolve_create_instance_state_user_defaults_to_only_state(): user = claims_to_user( { "sub": "u1", "groups": ["/states/MH/contributor"], } ) - assert resolve_create_group(user, None) == "mh" - assert resolve_create_group(user, "MH") == "mh" + assert resolve_create_instance(user, None) == "mh" + assert resolve_create_instance(user, "MH") == "mh" with pytest.raises(HTTPException) as exc: - resolve_create_group(user, "up") + resolve_create_instance(user, "up") assert exc.value.status_code == 403 -def test_resolve_create_group_superadmin_defaults_to_portal(): +def test_resolve_create_instance_superadmin_defaults_to_portal(): user = claims_to_user( { "sub": "sa", "groups": ["/global/super-admin"], } ) - assert resolve_create_group(user, None) == PORTAL_GROUP - assert resolve_create_group(user, "mh") == "mh" + assert resolve_create_instance(user, None) == PORTAL_INSTANCE + assert resolve_create_instance(user, "mh") == "mh" -def test_resolve_create_group_multi_state_requires_choice(): +def test_resolve_create_instance_multi_state_requires_choice(): user = claims_to_user( { "sub": "u2", @@ -61,9 +61,9 @@ def test_resolve_create_group_multi_state_requires_choice(): } ) with pytest.raises(HTTPException) as exc: - resolve_create_group(user, None) + resolve_create_instance(user, None) assert exc.value.status_code == 400 - assert resolve_create_group(user, "up") == "up" + assert resolve_create_instance(user, "up") == "up" def test_user_instances_are_enforced(): @@ -74,11 +74,11 @@ def test_user_instances_are_enforced(): "instances": ["Tenant-A", "tenant-b"], } ) - assert allowed_groups(user) == {"tenant-a", "tenant-b"} - assert user_can_access_group(user, "tenant-a") - assert not user_can_access_group(user, "mh") + assert allowed_instances(user) == {"tenant-a", "tenant-b"} + assert user_can_access_instance(user, "tenant-a") + assert not user_can_access_instance(user, "mh") with pytest.raises(HTTPException) as exc: - assert_group_access(user, "mh") + assert_instance_access(user, "mh") assert exc.value.status_code == 403 @@ -91,11 +91,11 @@ def test_document_access_hides_other_instances(): } ) with pytest.raises(HTTPException) as exc: - assert_document_group_access(user, {"workflow_id": "wf", "group": "tenant-b"}) + assert_document_instance_access(user, {"workflow_id": "wf", "instance": "tenant-b"}) assert exc.value.status_code == 404 -def test_superadmin_token_is_group_unrestricted_despite_scoped_claim(): +def test_superadmin_token_is_instance_unrestricted_despite_scoped_claim(): """Platform superadmin stays unrestricted even with a narrow instances claim.""" for role in ("superadmin", "super_admin", "super-admin"): user = claims_to_user( @@ -107,14 +107,14 @@ def test_superadmin_token_is_group_unrestricted_despite_scoped_claim(): ) assert user.is_superadmin is True assert user.is_admin is True # alias - assert user.is_group_unrestricted() is True - # Unrestricted -> allowed_groups is None (every group visible). - assert allowed_groups(user) is None - assert user_can_access_group(user, "tenant-b") - doc = assert_document_group_access( - user, {"workflow_id": "wf", "group": "tenant-b"} + assert user.is_instance_unrestricted() is True + # Unrestricted -> allowed_instances is None (every group visible). + assert allowed_instances(user) is None + assert user_can_access_instance(user, "tenant-b") + doc = assert_document_instance_access( + user, {"workflow_id": "wf", "instance": "tenant-b"} ) - assert doc["group"] == "tenant-b" + assert doc["instance"] == "tenant-b" def test_state_admin_is_scoped_to_instances_claim(): @@ -129,15 +129,15 @@ def test_state_admin_is_scoped_to_instances_claim(): } ) assert user.is_superadmin is False - assert user.is_group_unrestricted() is False - assert allowed_groups(user) == {"tenant-a"} - assert user_can_access_group(user, "tenant-a") - assert not user_can_access_group(user, "tenant-b") + assert user.is_instance_unrestricted() is False + assert allowed_instances(user) == {"tenant-a"} + assert user_can_access_instance(user, "tenant-a") + assert not user_can_access_instance(user, "tenant-b") assert Permission.SEARCH in user.permissions assert Permission.UPLOAD in user.permissions assert Permission.ADMIN not in user.permissions with pytest.raises(HTTPException) as exc: - assert_document_group_access(user, {"workflow_id": "wf", "group": "tenant-b"}) + assert_document_instance_access(user, {"workflow_id": "wf", "instance": "tenant-b"}) assert exc.value.status_code == 404 @@ -151,12 +151,12 @@ def test_content_curator_with_scoped_claim_cannot_cross_tenants(): } ) assert user.is_admin is False - assert user.is_group_unrestricted() is False - assert allowed_groups(user) == {"tenant-a"} - assert user_can_access_group(user, "tenant-a") - assert not user_can_access_group(user, "tenant-b") + assert user.is_instance_unrestricted() is False + assert allowed_instances(user) == {"tenant-a"} + assert user_can_access_instance(user, "tenant-a") + assert not user_can_access_instance(user, "tenant-b") with pytest.raises(HTTPException) as exc: - assert_document_group_access(user, {"workflow_id": "wf", "group": "tenant-b"}) + assert_document_instance_access(user, {"workflow_id": "wf", "instance": "tenant-b"}) assert exc.value.status_code == 404 @@ -171,7 +171,7 @@ def _register(db, **kwargs): return db.upsert_document(**kwargs) -def test_list_documents_filters_by_group(db_connection): +def test_list_documents_filters_by_instance(db_connection): db = db_connection _register(db, workflow_id="wf-tenant-a", @@ -179,7 +179,7 @@ def test_list_documents_filters_by_group(db_connection): filename="a.pdf", filepath="/tmp/a.pdf", stage="completed", - group="tenant-a", + instance="tenant-a", ) _register(db, workflow_id="wf-tenant-b", @@ -187,20 +187,20 @@ def test_list_documents_filters_by_group(db_connection): filename="b.pdf", filepath="/tmp/b.pdf", stage="completed", - group="tenant-b", + instance="tenant-b", ) - tenant_a_only = db.list_documents(groups=["tenant-a"]) + tenant_a_only = db.list_documents(instances=["tenant-a"]) assert {d["workflow_id"] for d in tenant_a_only} == {"wf-tenant-a"} - both = db.list_documents(groups=["tenant-a", "tenant-b"]) + both = db.list_documents(instances=["tenant-a", "tenant-b"]) assert {d["workflow_id"] for d in both} == {"wf-tenant-a", "wf-tenant-b"} - none = db.list_documents(groups=[]) + none = db.list_documents(instances=[]) assert none == [] -def test_summary_counts_honor_group_filter(db_connection): +def test_summary_counts_honor_instance_filter(db_connection): db = db_connection _register(db, workflow_id="wf-tenant-a-2", @@ -208,7 +208,7 @@ def test_summary_counts_honor_group_filter(db_connection): filename="c.pdf", filepath="/tmp/c.pdf", stage="completed", - group="tenant-a", + instance="tenant-a", ) _register(db, workflow_id="wf-tenant-b-2", @@ -216,15 +216,15 @@ def test_summary_counts_honor_group_filter(db_connection): filename="d.pdf", filepath="/tmp/d.pdf", stage="failed", - group="tenant-b", + instance="tenant-b", ) - summary = db.get_document_summary_counts(groups=["tenant-a"]) + summary = db.get_document_summary_counts(instances=["tenant-a"]) assert summary["total_documents"] == 1 assert summary["completed_documents"] == 1 assert summary["failed_documents"] == 0 -def test_list_runs_filters_by_group(db_connection): +def test_list_runs_filters_by_instance(db_connection): """Runs must honor the same tenant scope as documents (role-based states).""" db = db_connection doc_a = _register(db, @@ -234,7 +234,7 @@ def test_list_runs_filters_by_group(db_connection): filename="a.pdf", filepath="/tmp/a.pdf", stage="completed", - group="tenant-a", + instance="tenant-a", ) doc_b = _register(db, with_workflow=False, @@ -243,28 +243,28 @@ def test_list_runs_filters_by_group(db_connection): filename="b.pdf", filepath="/tmp/b.pdf", stage="completed", - group="tenant-b", + instance="tenant-b", ) job_a = db.create_document_job(doc_a, "ocr_only", status="completed") job_b = db.create_document_job(doc_b, "ocr_only", status="running") - tenant_a_only = db.list_runs(groups=["tenant-a"]) + tenant_a_only = db.list_runs(instances=["tenant-a"]) assert {r["id"] for r in tenant_a_only} == {job_a} - assert all(r.get("group") == "tenant-a" for r in tenant_a_only) + assert all(r.get("instance") == "tenant-a" for r in tenant_a_only) - both = db.list_runs(groups=["tenant-a", "tenant-b"]) + both = db.list_runs(instances=["tenant-a", "tenant-b"]) assert {r["id"] for r in both} == {job_a, job_b} - none = db.list_runs(groups=[]) + none = db.list_runs(instances=[]) assert none == [] - unrestricted_rows = db.list_runs(groups=None) + unrestricted_rows = db.list_runs(instances=None) assert {r["id"] for r in unrestricted_rows} >= {job_a, job_b} # Status filter still works under group scope. - running_a = db.list_runs(status="running", groups=["tenant-a"]) + running_a = db.list_runs(status="running", instances=["tenant-a"]) assert running_a == [] - running_b = db.list_runs(status="running", groups=["tenant-b"]) + running_b = db.list_runs(status="running", instances=["tenant-b"]) assert {r["id"] for r in running_b} == {job_b} @@ -277,7 +277,7 @@ def test_state_user_cannot_see_bv_portal_documents(db_connection): filename="portal.pdf", filepath="/tmp/portal.pdf", stage="ocr_review", - group=PORTAL_GROUP, + instance=PORTAL_INSTANCE, ) _register(db, workflow_id="wf-mh", @@ -285,7 +285,7 @@ def test_state_user_cannot_see_bv_portal_documents(db_connection): filename="mh.pdf", filepath="/tmp/mh.pdf", stage="ocr_review", - group="mh", + instance="mh", ) db.create_document_job("wf-bv", "ocr_only", status="running") db.create_document_job("wf-mh", "ocr_only", status="running") @@ -299,38 +299,38 @@ def test_state_user_cannot_see_bv_portal_documents(db_connection): } ) assert mh_user.is_superadmin is False - assert allowed_groups(mh_user) == {"mh"} - assert not user_can_access_group(mh_user, PORTAL_GROUP) - assert not user_can_access_group(mh_user, "bv") + assert allowed_instances(mh_user) == {"mh"} + assert not user_can_access_instance(mh_user, PORTAL_INSTANCE) + assert not user_can_access_instance(mh_user, "bv") - mh_docs = db.list_documents(groups=sorted(allowed_groups(mh_user) or [])) + mh_docs = db.list_documents(instances=sorted(allowed_instances(mh_user) or [])) assert {d["workflow_id"] for d in mh_docs} == {"wf-mh"} - mh_runs = db.list_runs(groups=sorted(allowed_groups(mh_user) or [])) - assert all(r.get("group") == "mh" for r in mh_runs) + mh_runs = db.list_runs(instances=sorted(allowed_instances(mh_user) or [])) + assert all(r.get("instance") == "mh" for r in mh_runs) assert {r["workflow_id"] for r in mh_runs} == {"wf-mh"} mh_queue, mh_total = db.list_operations_queue( - groups=sorted(allowed_groups(mh_user) or []) + instances=sorted(allowed_instances(mh_user) or []) ) assert mh_total == 1 assert {r["workflow_id"] for r in mh_queue} == {"wf-mh"} with pytest.raises(HTTPException) as exc: - assert_document_group_access( - mh_user, {"workflow_id": "wf-bv", "group": PORTAL_GROUP} + assert_document_instance_access( + mh_user, {"workflow_id": "wf-bv", "instance": PORTAL_INSTANCE} ) assert exc.value.status_code == 404 # Super-admin sees both sa = claims_to_user({"sub": "sa", "groups": ["/global/super-admin"]}) - assert allowed_groups(sa) is None - assert user_can_access_group(sa, PORTAL_GROUP) - all_docs = db.list_documents(groups=None) + assert allowed_instances(sa) is None + assert user_can_access_instance(sa, PORTAL_INSTANCE) + all_docs = db.list_documents(instances=None) assert {d["workflow_id"] for d in all_docs} >= {"wf-bv", "wf-mh"} -def test_upsert_does_not_reassign_group(db_connection): +def test_upsert_does_not_reassign_instance(db_connection): db = db_connection _register(db, workflow_id="wf-owned", @@ -338,7 +338,7 @@ def test_upsert_does_not_reassign_group(db_connection): filename="a.pdf", filepath="/tmp/a.pdf", stage="registered", - group="tenant-a", + instance="tenant-a", ) _register(db, workflow_id="wf-owned", @@ -346,10 +346,10 @@ def test_upsert_does_not_reassign_group(db_connection): filename="a.pdf", filepath="/tmp/a.pdf", stage="ocr_review", - group="tenant-b", # must be ignored on update + instance="tenant-b", # must be ignored on update ) doc = db.get_document("wf-owned") - assert doc["group"] == "tenant-a" + assert doc["instance"] == "tenant-a" assert doc["stage"] == "ocr_review" @@ -365,7 +365,7 @@ def test_api_helpers_hide_cross_tenant_mutations(db_connection, monkeypatch): filename="b.pdf", filepath="/tmp/b.pdf", stage="ocr_review", - group="tenant-b", + instance="tenant-b", ) user = claims_to_user( { @@ -418,77 +418,77 @@ def test_follows_a_configured_default_instance(self, monkeypatch): assert group_for_instance("mh") == "mh" -def test_group_is_reached_through_the_documents_catalog(db_connection): +def test_instance_is_reached_through_the_documents_catalog(db_connection): """No documents.instance column: a document's group is a groups row reached through its catalog.""" db = db_connection _register(db, workflow_id="wf-g-a", document_id="g1", filename="a.pdf", - filepath="/tmp/a.pdf", group="tenant-a") + filepath="/tmp/a.pdf", instance="tenant-a") _register(db, workflow_id="wf-g-bv", document_id="g2", filename="b.pdf", - filepath="/tmp/b.pdf", group=PORTAL_GROUP) + filepath="/tmp/b.pdf", instance=PORTAL_INSTANCE) _register(db, workflow_id="wf-g-adv", document_id="g3", filename="c.pdf", - filepath="/tmp/c.pdf", group="tenant-a", kind="advisory") + filepath="/tmp/c.pdf", instance="tenant-a", kind="advisory") with db.get_connection() as conn: - groups = {r["name"]: r["path"] for r in conn.execute("SELECT name, path FROM groups")} + groups = {r["name"]: r["path"] for r in conn.execute("SELECT name, path FROM instances")} catalogs = conn.execute( - "SELECT g.name AS group_name, c.type FROM catalogs c JOIN groups g ON g.id = c.group_id" + "SELECT g.name AS instance_name, c.type FROM catalogs c JOIN instances g ON g.id = c.instance_id" ).fetchall() assert groups == { "tenant-a": "/states/TENANT-A", - PORTAL_GROUP: "/global" if PORTAL_GROUP == "bv" else f"/states/{PORTAL_GROUP.upper()}", + PORTAL_INSTANCE: "/global" if PORTAL_INSTANCE == "bv" else f"/states/{PORTAL_INSTANCE.upper()}", } # Every group is seeded with one catalog per supported type. from pipeline.network_constants import KIND_TO_CATALOG_TYPE - assert sorted((r["group_name"], r["type"]) for r in catalogs) == sorted( + assert sorted((r["instance_name"], r["type"]) for r in catalogs) == sorted( (group, catalog_type) - for group in ("tenant-a", PORTAL_GROUP) + for group in ("tenant-a", PORTAL_INSTANCE) for catalog_type in KIND_TO_CATALOG_TYPE.values() ) - assert db.get_document("wf-g-a")["group"] == "tenant-a" + assert db.get_document("wf-g-a")["instance"] == "tenant-a" assert db.get_document("wf-g-adv")["kind"] == "advisory" - assert {d["workflow_id"] for d in db.list_documents(groups=["tenant-a"])} == {"wf-g-a", "wf-g-adv"} + assert {d["workflow_id"] for d in db.list_documents(instances=["tenant-a"])} == {"wf-g-a", "wf-g-adv"} def test_same_code_is_allowed_in_another_tenants_catalog(db_connection): db = db_connection _register(db, workflow_id="wf-c-a", document_id="c1", filename="a.pdf", - filepath="/tmp/a.pdf", group="tenant-a", code="pm-kisan") + filepath="/tmp/a.pdf", instance="tenant-a", code="pm-kisan") _register(db, workflow_id="wf-c-b", document_id="c2", filename="b.pdf", - filepath="/tmp/b.pdf", group="tenant-b", code="pm-kisan") + filepath="/tmp/b.pdf", instance="tenant-b", code="pm-kisan") assert db.get_document("wf-c-a")["code"] == db.get_document("wf-c-b")["code"] == "pm-kisan" # ...but not twice inside one tenant's catalog of that kind. with pytest.raises(db.DuplicateDocumentCode): _register(db, workflow_id="wf-c-a2", document_id="c3", filename="a2.pdf", - filepath="/tmp/a2.pdf", group="tenant-a", code="pm-kisan") + filepath="/tmp/a2.pdf", instance="tenant-a", code="pm-kisan") def test_audit_keeps_the_tenant_scope_after_the_document_is_purged(db_connection): """audit_logs snapshots the group, so a purged document's history stays tenant-scoped.""" db = db_connection _register(db, workflow_id="wf-aud-a", document_id="a1", filename="a.pdf", - filepath="/tmp/a.pdf", group="tenant-a") + filepath="/tmp/a.pdf", instance="tenant-a") _register(db, workflow_id="wf-aud-b", document_id="a2", filename="b.pdf", - filepath="/tmp/b.pdf", group="tenant-b") + filepath="/tmp/b.pdf", instance="tenant-b") db.log_audit(workflow_id="wf-aud-a", document_id="", action_type="page_edit", entity_type="document") db.log_audit(workflow_id="wf-aud-b", document_id="", action_type="page_edit", entity_type="document") db.purge_document("wf-aud-a", keep_audit=True) - a_rows = db.get_all_audit_logs(groups=["tenant-a"]) + a_rows = db.get_all_audit_logs(instances=["tenant-a"]) assert [r["workflow_id"] for r in a_rows if r["action_type"] == "page_edit"] == ["wf-aud-a"] - b_rows = db.get_all_audit_logs(groups=["tenant-b"]) + b_rows = db.get_all_audit_logs(instances=["tenant-b"]) assert [r["workflow_id"] for r in b_rows if r["action_type"] == "page_edit"] == ["wf-aud-b"] - assert db.get_all_audit_log_count(groups=["tenant-a"]) == len(a_rows) - assert db.get_all_audit_logs(groups=[]) == [] + assert db.get_all_audit_log_count(instances=["tenant-a"]) == len(a_rows) + assert db.get_all_audit_logs(instances=[]) == [] -def test_user_group_follows_the_first_tenant_in_the_token(db_connection, caplog): +def test_user_instance_follows_the_first_tenant_in_the_token(db_connection, caplog): db = db_connection def group_of(keycloak_id): with db.get_connection() as conn: row = conn.execute( - "SELECT g.name FROM users u LEFT JOIN groups g ON g.id = u.group_id " + "SELECT g.name FROM users u LEFT JOIN instances g ON g.id = u.instance_id " "WHERE u.keycloak_user_id = %s", (keycloak_id,)).fetchone() return row["name"] @@ -507,20 +507,20 @@ def group_of(keycloak_id): assert group_of("kc-none") is None -def test_group_path_fills_the_keycloak_id_on_sync(db_connection): +def test_instance_path_fills_the_keycloak_id_on_sync(db_connection): """The SQLite move creates groups without a Keycloak id; a later sync matches by path.""" db = db_connection - first = db.upsert_group("/states/MH") + first = db.upsert_instance("/states/MH") with db.get_connection() as conn: - assert conn.execute("SELECT keycloak_group_id FROM groups WHERE id = %s::uuid", + assert conn.execute("SELECT keycloak_group_id FROM instances WHERE id = %s::uuid", (first,)).fetchone()["keycloak_group_id"] is None - again = db.upsert_group("/states/MH", keycloak_group_id="kc-group-123") + again = db.upsert_instance("/states/MH", keycloak_group_id="kc-group-123") assert again == first with db.get_connection() as conn: - row = conn.execute("SELECT name, keycloak_group_id FROM groups WHERE id = %s::uuid", (first,)).fetchone() + row = conn.execute("SELECT name, keycloak_group_id FROM instances WHERE id = %s::uuid", (first,)).fetchone() assert (row["name"], row["keycloak_group_id"]) == ("mh", "kc-group-123") # A sync without an id never erases one already stored. - assert db.upsert_group("/states/MH") == first + assert db.upsert_instance("/states/MH") == first with db.get_connection() as conn: - assert conn.execute("SELECT keycloak_group_id FROM groups WHERE id = %s::uuid", + assert conn.execute("SELECT keycloak_group_id FROM instances WHERE id = %s::uuid", (first,)).fetchone()["keycloak_group_id"] == "kc-group-123" diff --git a/ui/src/auth/AuthProvider.jsx b/ui/src/auth/AuthProvider.jsx index 998f86c..fdf8e27 100644 --- a/ui/src/auth/AuthProvider.jsx +++ b/ui/src/auth/AuthProvider.jsx @@ -366,11 +366,11 @@ export function AuthProvider({ children }) { ) /** Role for a state tenant (mh, up, …). Super-admin always returns super_admin. */ - const roleForGroup = useCallback( - (group) => { + const roleForInstance = useCallback( + (instance) => { if (!AUTH_ENABLED || isSuperAdmin) return 'super_admin' - if (!group) return null - const key = String(group).trim().toLowerCase() + if (!instance) return null + const key = String(instance).trim().toLowerCase() // A BH viewer holds no per-state group but reads every state. return stateRoles[key] || (isBhViewer ? 'state_view' : null) }, @@ -378,11 +378,11 @@ export function AuthProvider({ children }) { ) /** Whether the user may see a tenant (state). Super-admin → all. */ - const canAccessGroup = useCallback( - (group) => { + const canAccessInstance = useCallback( + (instance) => { if (!AUTH_ENABLED || isSuperAdmin || isBhViewer) return true - if (!group) return false - const key = String(group).trim().toLowerCase() + if (!instance) return false + const key = String(instance).trim().toLowerCase() if (instances.length === 0) return false return instances.map((i) => String(i).toLowerCase()).includes(key) }, @@ -397,17 +397,17 @@ export function AuthProvider({ children }) { * may approve OCR / translation / chunking) but NOT 'approve_ingestion' * (it may not publish to DEV). */ - const hasPermissionForGroup = useCallback( - (perm, group) => { + const hasPermissionForInstance = useCallback( + (perm, instance) => { if (!AUTH_ENABLED) return true if (isSuperAdmin) return hasPermission(perm) - const role = normalizeProductRole(roleForGroup(group)) + const role = normalizeProductRole(roleForInstance(instance)) if (!role) return false // Platform-level capabilities are never granted by a state role. if (perm === 'admin' || perm === 'manage_users') return false return roleGrants(role, perm) }, - [hasPermission, isSuperAdmin, roleForGroup], + [hasPermission, isSuperAdmin, roleForInstance], ) const value = useMemo( @@ -432,10 +432,10 @@ export function AuthProvider({ children }) { isSuperAdmin, isBhViewer, hasPermission, - hasPermissionForGroup, + hasPermissionForInstance, hasRole, - roleForGroup, - canAccessGroup, + roleForInstance, + canAccessInstance, loginWithSso, loginWithOtpTokens, logout, @@ -460,10 +460,10 @@ export function AuthProvider({ children }) { isSuperAdmin, isBhViewer, hasPermission, - hasPermissionForGroup, + hasPermissionForInstance, hasRole, - roleForGroup, - canAccessGroup, + roleForInstance, + canAccessInstance, loginWithSso, loginWithOtpTokens, logout, diff --git a/ui/src/components/DocumentHeaderSummary.jsx b/ui/src/components/DocumentHeaderSummary.jsx index 3af75e9..5098450 100644 --- a/ui/src/components/DocumentHeaderSummary.jsx +++ b/ui/src/components/DocumentHeaderSummary.jsx @@ -1,6 +1,6 @@ import { useMemo, useState } from 'react' import { ArrowLeft, Info } from 'lucide-react' -import { formatGroupLabel, groupBadgeTitle } from '../lib/groupLabels' +import { formatInstanceLabel, instanceBadgeTitle } from '../lib/instanceLabels' import { formatCompactDateTime, getDocumentFileLabel, @@ -74,7 +74,7 @@ export function DocumentHeaderSummary({ const pages = pageCount ?? doc?.page_count ?? 0 const chunks = chunkCount ?? doc?.chunk_count ?? 0 const uploader = uploaderName(doc) - const stateLabel = formatGroupLabel(doc?.instance) + const stateLabel = formatInstanceLabel(doc?.instance) const roleLabels = useMemo( () => productRoleLabels(doc?.uploaded_by_roles || []), [doc?.uploaded_by_roles], @@ -126,7 +126,7 @@ export function DocumentHeaderSummary({ {stateLabel ? ( - {stateLabel} + {stateLabel} ) : ( '—' )} diff --git a/ui/src/components/GroupBadge.jsx b/ui/src/components/GroupBadge.jsx deleted file mode 100644 index edb2c2e..0000000 --- a/ui/src/components/GroupBadge.jsx +++ /dev/null @@ -1,21 +0,0 @@ -import { formatGroupLabel, groupBadgeTitle } from '../lib/groupLabels' -import { Badge } from './ui/badge' -import { cn } from '../lib/utils' - -/** Plan 2: show document state / BV portal tag from ``group``. */ -export function GroupBadge({ group, className }) { - const label = formatGroupLabel(group) - if (!label) return null - const isPortal = label === 'BV' - return ( - - {label} - - ) -} - -export default GroupBadge diff --git a/ui/src/components/InstanceBadge.jsx b/ui/src/components/InstanceBadge.jsx new file mode 100644 index 0000000..87d4b4a --- /dev/null +++ b/ui/src/components/InstanceBadge.jsx @@ -0,0 +1,21 @@ +import { formatInstanceLabel, instanceBadgeTitle } from '../lib/instanceLabels' +import { Badge } from './ui/badge' +import { cn } from '../lib/utils' + +/** Plan 2: show document state / BV portal tag from ``instance``. */ +export function InstanceBadge({ instance, className }) { + const label = formatInstanceLabel(instance) + if (!label) return null + const isPortal = label === 'BV' + return ( + + {label} + + ) +} + +export default InstanceBadge diff --git a/ui/src/lib/groupLabels.js b/ui/src/lib/instanceLabels.js similarity index 63% rename from ui/src/lib/groupLabels.js rename to ui/src/lib/instanceLabels.js index 95e7840..b174b45 100644 --- a/ui/src/lib/groupLabels.js +++ b/ui/src/lib/instanceLabels.js @@ -1,13 +1,13 @@ /** - * Plan 2: display labels for the document group (state / portal tag). + * Plan 2: display labels for the document instance (state / portal tag). * Authorization still comes from Keycloak JWT — this is UI only. */ -export const PORTAL_GROUP = 'bv' +export const PORTAL_INSTANCE = 'bv' /** Short badge label: mh → MH, bv → BV */ -export function formatGroupLabel(group) { - const code = String(group || '') +export function formatInstanceLabel(instance) { + const code = String(instance || '') .trim() .toLowerCase() if (!code || code === 'default') return null @@ -17,8 +17,8 @@ export function formatGroupLabel(group) { return code.toUpperCase() } -export function groupBadgeTitle(group) { - const code = String(group || '') +export function instanceBadgeTitle(instance) { + const code = String(instance || '') .trim() .toLowerCase() if (!code || code === 'default') return 'No state tag' @@ -29,11 +29,11 @@ export function groupBadgeTitle(group) { } /** - * Default group for upload form. + * Default instance for upload form. * Super admin → portal (bv); single state → that state; multi → first (user can change). */ -export function defaultUploadGroup({ isSuperAdmin, instances = [], portalGroup = PORTAL_GROUP }) { - if (isSuperAdmin) return portalGroup || PORTAL_GROUP +export function defaultUploadInstance({ isSuperAdmin, instances = [], portalInstance = PORTAL_INSTANCE }) { + if (isSuperAdmin) return portalInstance || PORTAL_INSTANCE const list = (instances || []).map((i) => String(i).trim().toLowerCase()).filter(Boolean) if (list.length >= 1) return list[0] return '' diff --git a/ui/src/views/DashboardView.jsx b/ui/src/views/DashboardView.jsx index 2cfd9d4..becf4a6 100644 --- a/ui/src/views/DashboardView.jsx +++ b/ui/src/views/DashboardView.jsx @@ -5,7 +5,7 @@ import { Skeleton } from '../components/ui/skeleton' import { StatCard } from '../components/StatCard' import { StageBadge } from '../components/StageBadge' import { fetchJson, formatCount, getDocumentListLabel, summarizeQueueReason } from '../lib/pipelineUi' -import { formatGroupLabel } from '../lib/groupLabels' +import { formatInstanceLabel } from '../lib/instanceLabels' import { useAuth } from '../auth/AuthProvider' import { AlertTriangle, CheckCircle, FileText, Globe2, ListTodo } from 'lucide-react' @@ -66,8 +66,8 @@ export default function DashboardView() { const reindexCount = summary?.needs_reindex || 0 const displayedQueueItems = queue.slice(0, QUEUE_ROWS_SHOWN) - const byGroup = useMemo(() => (Array.isArray(summary?.by_instance) ? summary.by_instance : []), [summary]) - const stateRows = byGroup + const byInstance = useMemo(() => (Array.isArray(summary?.by_instance) ? summary.by_instance : []), [summary]) + const stateRows = byInstance const maxStateCount = useMemo( () => stateRows.reduce((max, item) => Math.max(max, item.count || 0), 0), @@ -113,7 +113,7 @@ export default function DashboardView() { /> } /> { const label = row.otherStateCount ? `Others (${row.otherStateCount} states)` - : formatGroupLabel(row.instance) || 'No state tag' + : formatInstanceLabel(row.instance) || 'No state tag' const pct = (value) => (maxStateCount ? (value / maxStateCount) * 100 : 0) return (
diff --git a/ui/src/views/DocumentsView.jsx b/ui/src/views/DocumentsView.jsx index f164edb..5155a66 100644 --- a/ui/src/views/DocumentsView.jsx +++ b/ui/src/views/DocumentsView.jsx @@ -6,7 +6,7 @@ import { Input } from '../components/ui/input' import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '../components/ui/select' import { Skeleton } from '../components/ui/skeleton' import { StageBadge } from '../components/StageBadge' -import { GroupBadge } from '../components/GroupBadge' +import { InstanceBadge } from '../components/InstanceBadge' import { fetchAllDocuments, formatCompactDateTime, getDocumentListLabel, getDocumentMetaLabel, getStageLabel } from '../lib/pipelineUi' import { useAuth } from '../auth/AuthProvider' import { Search, FileText, AlertTriangle, RefreshCw, ChevronLeft, ChevronRight, Loader2 } from 'lucide-react' @@ -21,7 +21,7 @@ export default function DocumentsView() { const [query, setQuery] = useState('') const [stageFilter, setStageFilter] = useState('all') const [authFilter, setAuthFilter] = useState('all') - const [groupFilter, setGroupFilter] = useState('all') + const [instanceFilter, setInstanceFilter] = useState('all') const [showFailed, setShowFailed] = useState(false) const [showReindex, setShowReindex] = useState(false) const [loading, setLoading] = useState(true) @@ -47,7 +47,7 @@ export default function DocumentsView() { } const stageOptions = useMemo(() => ['all', ...new Set(documents.map(doc => doc.stage))], [documents]) - const groupOptions = useMemo(() => { + const instanceOptions = useMemo(() => { const fromDocs = documents.map((d) => (d.instance || '').trim().toLowerCase()).filter(Boolean) return ['all', ...new Set(fromDocs)] }, [documents]) @@ -59,26 +59,26 @@ export default function DocumentsView() { if (stageFilter !== 'all' && doc.stage !== stageFilter) return false if (authFilter === 'authoritative' && !doc.authoritative) return false if (authFilter === 'legacy' && doc.authoritative) return false - if (groupFilter !== 'all') { + if (instanceFilter !== 'all') { const code = (doc.instance || 'default').trim().toLowerCase() - if (code !== groupFilter) return false + if (code !== instanceFilter) return false } if (showFailed && !doc.failed && doc.stage !== 'failed') return false if (showReindex && !doc.reindex_required) return false return true }) - }, [documents, query, stageFilter, authFilter, groupFilter, showFailed, showReindex]) + }, [documents, query, stageFilter, authFilter, instanceFilter, showFailed, showReindex]) useEffect(() => { setPage(1) - }, [query, stageFilter, authFilter, groupFilter, showFailed, showReindex]) + }, [query, stageFilter, authFilter, instanceFilter, showFailed, showReindex]) const totalPages = Math.max(1, Math.ceil(filtered.length / PAGE_SIZE)) const paginated = filtered.slice((page - 1) * PAGE_SIZE, page * PAGE_SIZE) const activeFilters = [ stageFilter !== 'all' && `Stage: ${getStageLabel(stageFilter, { compact: true })}`, authFilter !== 'all' && `Type: ${authFilter}`, - groupFilter !== 'all' && `State: ${groupFilter.toUpperCase()}`, + instanceFilter !== 'all' && `State: ${instanceFilter.toUpperCase()}`, showFailed && 'Failed only', showReindex && 'Reindex required', ].filter(Boolean) @@ -142,14 +142,14 @@ export default function DocumentsView() { Legacy - {isSuperAdmin && groupOptions.length > 1 ? ( - All states - {groupOptions + {instanceOptions .filter((s) => s !== 'all') .map((code) => ( @@ -178,7 +178,7 @@ export default function DocumentsView() { onClick={() => { setStageFilter('all') setAuthFilter('all') - setGroupFilter('all') + setInstanceFilter('all') setShowFailed(false) setShowReindex(false) setQuery('') @@ -233,7 +233,7 @@ export default function DocumentsView() {
{getDocumentMetaLabel(doc)}
 - + diff --git a/ui/src/views/NewDocumentView.jsx b/ui/src/views/NewDocumentView.jsx index f12be02..93c5088 100644 --- a/ui/src/views/NewDocumentView.jsx +++ b/ui/src/views/NewDocumentView.jsx @@ -24,14 +24,14 @@ import { import { API_BASE } from '../config' import { apiFetch } from '../auth/keycloak' import { useAuth } from '../auth/AuthProvider' -import { defaultUploadGroup, PORTAL_GROUP } from '../lib/groupLabels' +import { defaultUploadInstance, PORTAL_INSTANCE } from '../lib/instanceLabels' import { fetchJson, formatCompactDateTime, getDocumentListLabel, summarizeIngestStatus } from '../lib/pipelineUi' import { displayNameError, filenameStem } from '../lib/displayName' const SUPPORTED_TYPES = ['.pdf', '.doc', '.docx'] const MAX_SIZE_MB = 100 const FILE_TYPE_CHIPS = ['PDF', 'Word'] -// The catalog a document joins is (your group, type), so the type is chosen up front. +// The catalog a document joins is (your instance, type), so the type is chosen up front. const DOCUMENT_TYPE_OPTIONS = [ { value: 'scheme', label: 'Scheme' }, { value: 'advisory', label: 'Advisory' }, @@ -62,11 +62,11 @@ export default function NewDocumentView() { const navigate = useNavigate() const { hasPermission, instances, isSuperAdmin, user } = useAuth() const canUpload = hasPermission('upload') - // Group from login (Keycloak) — not user-selected on this form. - const documentGroup = defaultUploadGroup({ + // Instance from login (Keycloak) — not user-selected on this form. + const documentInstance = defaultUploadInstance({ isSuperAdmin, instances, - portalGroup: user?.portal_instance || PORTAL_GROUP, + portalInstance: user?.portal_instance || PORTAL_INSTANCE, }) const [file, setFile] = useState(null) const [displayName, setDisplayName] = useState('') @@ -135,7 +135,7 @@ export default function NewDocumentView() { async function handleSubmit() { if (!file || !documentKind || !canUpload || nameError) return - if (!documentGroup && !isSuperAdmin) { + if (!documentInstance && !isSuperAdmin) { setUploadError( 'No state on your account. Ask an admin to assign a Keycloak group (e.g. /states/MH/admin).', ) @@ -148,8 +148,8 @@ export default function NewDocumentView() { formData.append('file', file) formData.append('display_name', displayName.trim()) // State users: Keycloak group state only. Super-admin: portal BV when unset. - const uploadGroup = documentGroup || (isSuperAdmin ? PORTAL_GROUP : '') - if (!uploadGroup) { + const uploadInstance = documentInstance || (isSuperAdmin ? PORTAL_INSTANCE : '') + if (!uploadInstance) { setUploadError( 'No state on your account. Ask an admin to assign a Keycloak group (e.g. /states/MH/admin).', ) @@ -157,7 +157,7 @@ export default function NewDocumentView() { } const params = new URLSearchParams({ auto_approve: 'false', - instance: uploadGroup, + instance: uploadInstance, kind: documentKind, }) const response = await apiFetch(`${API_BASE}/upload?${params}`, { method: 'POST', body: formData }) diff --git a/ui/src/views/RunsView.jsx b/ui/src/views/RunsView.jsx index 727f2df..c3a9bfb 100644 --- a/ui/src/views/RunsView.jsx +++ b/ui/src/views/RunsView.jsx @@ -4,7 +4,7 @@ import { Badge } from '../components/ui/badge' import { Button } from '../components/ui/button' import { Skeleton } from '../components/ui/skeleton' import { StageBadge } from '../components/StageBadge' -import { GroupBadge } from '../components/GroupBadge' +import { InstanceBadge } from '../components/InstanceBadge' import PipelineStepper from '../components/PipelineStepper' import { fetchJson, @@ -356,7 +356,7 @@ export default function RunsView() { {title}
- + #{runId} {fileLabel && fileLabel !== title ? ` · ${fileLabel}` : ''} From 6ce362e2e4b9b9d1d29603e86e792a562da12ea6 Mon Sep 17 00:00:00 2001 From: Ravi Mula Date: Wed, 11 Nov 2026 14:23:07 +0530 Subject: [PATCH 53/53] fix: preserve the existing search group filter --- README.md | 2 +- docs/openapi-search.yaml | 28 ++++++------- pipeline/api/search.py | 16 ++++---- tests/test_api.py | 48 ++++++++++++----------- tests/test_search_group_integration.py | 26 ++++++------ tests/test_search_provider_integration.py | 4 +- 6 files changed, 64 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 9eb9b96..5c79f6d 100644 --- a/README.md +++ b/README.md @@ -530,7 +530,7 @@ Translation data is surfaced through the page model and review endpoints. - `GET /documents/{workflow_id}/qdrant` - `GET /documents/{workflow_id}/qdrant/chunks` - `POST /documents/{workflow_id}/reingest` -- `POST /search` — only answers from documents whose validity period covers today; `valid_on` asks about another day and `include_expired` drops the filter. Optional `instance` (e.g. `mh`) limits results to one instance's documents; omitted means every instance, and the response's `effective_config.instance` echoes what was applied (see `docs/openapi-search.yaml`) +- `POST /search` — only answers from documents whose validity period covers today; `valid_on` asks about another day and `include_expired` drops the filter. Optional `group` (e.g. `mh`) limits results to one group's documents; omitted means every group, and the response's `effective_config.group` echoes what was applied (see `docs/openapi-search.yaml`) - `GET /indexes/summary` - `GET /indexes/{index_name}/settings` - `GET /indexes/{index_name}/stats` diff --git a/docs/openapi-search.yaml b/docs/openapi-search.yaml index 220f079..5115ef8 100644 --- a/docs/openapi-search.yaml +++ b/docs/openapi-search.yaml @@ -52,11 +52,11 @@ paths: top_k: 10 hybrid_alpha: 0.6 rerank_mode: "none" - one_instance: - summary: Only one instance's documents (e.g. a state's DSS) + one_group: + summary: Only one group's documents (e.g. a state's DSS) value: query: "cotton pest advisory" - instance: "mh" + group: "mh" one_provider: summary: Only the chunks one network provider publishes value: @@ -70,7 +70,7 @@ paths: schema: $ref: '#/components/schemas/SearchResponse' '400': - description: Missing/empty query, malformed valid_on, non-string instance, non-string provider_id, or the underlying vector-store search failed + description: Missing/empty query, malformed valid_on, non-string group, non-string provider_id, or the underlying vector-store search failed content: application/json: schema: @@ -263,21 +263,21 @@ components: what search would have returned, or will return, on that date. A malformed value returns 400. Ignored when include_expired is true. default: "(today)" - instance: + group: type: string nullable: true description: | - Limit results to chunks tagged with this instance code (for example + Limit results to chunks tagged with this group code (for example `mh` for Maharashtra, `bv` for Bharat Vistaar). Case-insensitive and trimmed: `MH` and ` mh ` both mean `mh`. One value only — - a caller belongs to one instance; fallback to another instance is the - caller's second request. Omitted, null or empty means no instance - filter: chunks from every instance are searched as one pool. An - unknown instance is not an error: + a caller belongs to one group; fallback to another group is the + caller's second request. Omitted, null or empty means no group + filter: chunks from every group are searched as one pool. An + unknown group is not an error: the response is a success with zero hits. A non-string value returns 400. Applies in every search mode. A filter, not an access check: this endpoint applies no tenant scoping, so any - search-capable caller may ask for any instance. + search-capable caller may ask for any group. example: "mh" provider_id: type: string @@ -289,7 +289,7 @@ components: empty means no provider filter; an unknown id returns zero hits; a non-string value returns 400. Only advisory and scheme chunks carry a provider id, so a provider filter leaves other kinds out. - Combines with `instance` and validity. Any other type returns 400. + Combines with `group` and validity. Any other type returns 400. A filter, not an access check. example: "mh.provider.oan.dev" @@ -424,10 +424,10 @@ components: type: boolean description: True if the query-expansion profile actually rewrote the query. rerank_mode: { type: string } - instance: + group: type: string nullable: true - description: The normalised instance filter that was applied (e.g. `mh`), or null when the search spanned every instance. + description: The normalised group filter that was applied (e.g. `mh`), or null when the search spanned every group. provider_id: type: string nullable: true diff --git a/pipeline/api/search.py b/pipeline/api/search.py index a621082..70beb7c 100644 --- a/pipeline/api/search.py +++ b/pipeline/api/search.py @@ -170,10 +170,10 @@ def _rerank_hits(query: str, hits: list[dict], rerank_mode: str) -> list[dict]: -def _requested_instance(payload: dict) -> str | None: - raw = payload.get("instance") +def _requested_group(payload: dict) -> str | None: + raw = payload.get("group") if raw is not None and not isinstance(raw, str): - raise HTTPException(400, "instance must be a string") + raise HTTPException(400, "group must be a string") return (raw or "").strip().lower() or None @@ -188,9 +188,9 @@ def _requested_provider_id(payload: dict) -> str | None: async def run_search(payload: dict, user: RequireSearch): """Search the vector index. - `instance` is a filter, not an access check: this endpoint applies no tenant + `group` is a filter, not an access check: this endpoint applies no tenant scoping to its results (unlike `GET /chunks/search`), so any - search-capable caller may ask for any instance. + search-capable caller may ask for any group. """ from ..vector_store import get_default_index_name, get_vector_store @@ -247,7 +247,7 @@ async def run_search(payload: dict, user: RequireSearch): valid_on = document_validity.parse_day(valid_on, "valid_on") except document_validity.DocumentValidityError as exc: raise HTTPException(400, str(exc)) from None - instance = _requested_instance(payload) + group = _requested_group(payload) provider_id = _requested_provider_id(payload) expanded_query = _expand_query(query, query_expansion_profile) # Qdrant embeddings apply E5 prefixes internally; don't pre-prefix here. @@ -270,7 +270,7 @@ async def run_search(payload: dict, user: RequireSearch): ef_search=ef_search, apply_validity=not include_expired, valid_on=valid_on, - instance=instance, + instance=group, provider_id=provider_id, ) except Exception as error: @@ -320,7 +320,7 @@ async def run_search(payload: dict, user: RequireSearch): "rerank_mode": rerank_mode, "apply_validity": not include_expired, "valid_on": result.get("valid_on"), - "instance": instance, + "group": group, "provider_id": provider_id, "filter_string": None, }, diff --git a/tests/test_api.py b/tests/test_api.py index 1c9064e..0cf8040 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -615,68 +615,72 @@ def test_rejects_a_malformed_valid_on(self, test_client): assert "YYYY-MM-DD" in response.json()["detail"] -class TestSearchInstance: - """`instance` narrows a search to one instance's chunks.""" +class TestSearchGroup: + """`group` narrows a search to one group's chunks (story #202). + + The API says `group`; the store and the chunk payload say `instance`. + The endpoint is the only place that maps one name to the other. + """ @pytest.mark.api @pytest.mark.unit - def test_passes_the_instance_to_the_store(self, test_client, capturing_store): - response = test_client.post("/search", json={"query": "kisan", "instance": "mh"}) + def test_passes_the_group_to_the_store_as_instance(self, test_client, capturing_store): + response = test_client.post("/search", json={"query": "kisan", "group": "mh"}) assert response.status_code == 200 assert capturing_store.captured["instance"] == "mh" - assert response.json()["effective_config"]["instance"] == "mh" + assert response.json()["effective_config"]["group"] == "mh" @pytest.mark.api @pytest.mark.unit @pytest.mark.parametrize("raw", ["MH", " mh ", " Mh\n"]) def test_normalises_case_and_whitespace(self, test_client, capturing_store, raw): - response = test_client.post("/search", json={"query": "kisan", "instance": raw}) + response = test_client.post("/search", json={"query": "kisan", "group": raw}) assert capturing_store.captured["instance"] == "mh" - assert response.json()["effective_config"]["instance"] == "mh" + assert response.json()["effective_config"]["group"] == "mh" @pytest.mark.api @pytest.mark.unit @pytest.mark.parametrize("body", [ {"query": "kisan"}, - {"query": "kisan", "instance": ""}, - {"query": "kisan", "instance": " "}, - {"query": "kisan", "instance": None}, + {"query": "kisan", "group": ""}, + {"query": "kisan", "group": " "}, + {"query": "kisan", "group": None}, ]) - def test_absent_or_empty_instance_means_no_filter(self, test_client, capturing_store, body): + def test_absent_or_empty_group_means_no_filter(self, test_client, capturing_store, body): response = test_client.post("/search", json=body) assert response.status_code == 200 assert capturing_store.captured["instance"] is None - assert response.json()["effective_config"]["instance"] is None + assert response.json()["effective_config"]["group"] is None @pytest.mark.api @pytest.mark.unit @pytest.mark.parametrize("bad", [["mh"], 7, {"code": "mh"}]) - def test_rejects_a_non_string_instance(self, test_client, capturing_store, bad): - response = test_client.post("/search", json={"query": "kisan", "instance": bad}) + def test_rejects_a_non_string_group(self, test_client, capturing_store, bad): + response = test_client.post("/search", json={"query": "kisan", "group": bad}) assert response.status_code == 400 - assert response.json()["detail"] == "instance must be a string" + assert response.json()["detail"] == "group must be a string" assert capturing_store.captured == {}, "the store must not be called for a bad request" @pytest.mark.api @pytest.mark.unit - def test_unknown_instance_is_an_empty_success(self, test_client, capturing_store): - response = test_client.post("/search", json={"query": "kisan", "instance": "xx"}) + def test_unknown_group_is_an_empty_success(self, test_client, capturing_store): + response = test_client.post("/search", json={"query": "kisan", "group": "xx"}) assert response.status_code == 200 body = response.json() assert body["hits"] == [] assert body["final_count"] == 0 - assert body["effective_config"]["instance"] == "xx" + assert body["effective_config"]["group"] == "xx" @pytest.mark.api @pytest.mark.unit - def test_instance_composes_with_include_expired(self, test_client, capturing_store): + def test_group_composes_with_include_expired(self, test_client, capturing_store): test_client.post( - "/search", json={"query": "kisan", "instance": "mh", "include_expired": True} + "/search", json={"query": "kisan", "group": "mh", "include_expired": True} ) assert capturing_store.captured["instance"] == "mh" @@ -726,8 +730,8 @@ def test_rejects_anything_but_a_string(self, test_client, capturing_store, bad): @pytest.mark.api @pytest.mark.unit - def test_composes_with_instance(self, test_client, capturing_store): - test_client.post("/search", json={"query": "kisan", "instance": "mh", "provider_id": "mh.provider.oan.dev"}) + def test_composes_with_group(self, test_client, capturing_store): + test_client.post("/search", json={"query": "kisan", "group": "mh", "provider_id": "mh.provider.oan.dev"}) assert capturing_store.captured["instance"] == "mh" assert capturing_store.captured["provider_id"] == "mh.provider.oan.dev" diff --git a/tests/test_search_group_integration.py b/tests/test_search_group_integration.py index 93da320..505de70 100644 --- a/tests/test_search_group_integration.py +++ b/tests/test_search_group_integration.py @@ -1,9 +1,9 @@ -"""Integration: ``POST /search`` with ``instance`` through the real Qdrant engine. +"""Integration: ``POST /search`` with ``group`` through the real Qdrant engine. qdrant-client ships an in-process mode (``QdrantClient(":memory:")``) that runs the real filtering and vector engine, so this exercises the route, the store and the engine together without Docker. Only the embedding model is -faked: search quality is not under test here, the instance filter is. +faked: search quality is not under test here, the group filter is. """ import zlib @@ -14,7 +14,7 @@ from pipeline.vector_store import qdrant_store from tests.test_qdrant_validity import TODAY, clock_at -INDEX = "instance-integration" +INDEX = "group-integration" VECTOR_SIZE = 8 CHUNKS = [ @@ -67,27 +67,27 @@ def test_every_chunk_is_indexed(local_store): @pytest.mark.integration @pytest.mark.api @pytest.mark.parametrize("mode", ["HYBRID", "TENSOR", "LEXICAL"]) -def test_instance_limits_hits_to_that_group_in_every_mode(local_store, test_client, mode): - body = search(test_client, instance="mh", search_mode=mode) +def test_group_limits_hits_to_that_group_in_every_mode(local_store, test_client, mode): + body = search(test_client, group="mh", search_mode=mode) - assert body["effective_config"]["instance"] == "mh" + assert body["effective_config"]["group"] == "mh" assert body["hits"], "the mh chunk must be found" assert {hit["instance"] for hit in body["hits"]} == {"mh"} @pytest.mark.integration @pytest.mark.api -def test_no_instance_searches_every_group(local_store, test_client): +def test_no_group_searches_every_group(local_store, test_client): body = search(test_client) - assert body["effective_config"]["instance"] is None + assert body["effective_config"]["group"] is None assert {hit["instance"] for hit in body["hits"]} == {"mh", "ka", "bv"} @pytest.mark.integration @pytest.mark.api -def test_unknown_instance_is_an_empty_success(local_store, test_client): - body = search(test_client, instance="xx") +def test_unknown_group_is_an_empty_success(local_store, test_client): + body = search(test_client, group="xx") assert body["hits"] == [] assert body["final_count"] == 0 @@ -95,9 +95,9 @@ def test_unknown_instance_is_an_empty_success(local_store, test_client): @pytest.mark.integration @pytest.mark.api -def test_instance_and_validity_compose_in_the_engine(local_store, test_client): - current_only = search(test_client, instance="mh") - with_expired = search(test_client, instance="mh", include_expired=True) +def test_group_and_validity_compose_in_the_engine(local_store, test_client): + current_only = search(test_client, group="mh") + with_expired = search(test_client, group="mh", include_expired=True) assert {hit["doc_id"] for hit in current_only["hits"]} == {"mh-current"} assert {hit["doc_id"] for hit in with_expired["hits"]} == {"mh-current", "mh-expired"} diff --git a/tests/test_search_provider_integration.py b/tests/test_search_provider_integration.py index 9589206..bd28f3f 100644 --- a/tests/test_search_provider_integration.py +++ b/tests/test_search_provider_integration.py @@ -60,7 +60,7 @@ def test_no_provider_searches_every_chunk(local_store, test_client): @pytest.mark.integration @pytest.mark.api -def test_provider_composes_with_instance(local_store, test_client): - body = search(test_client, provider_id="bv.provider.oan.dev", instance="mh") +def test_provider_composes_with_group(local_store, test_client): + body = search(test_client, provider_id="bv.provider.oan.dev", group="mh") assert body["hits"] == []