From 8ce8bfdc8e03d101ff6321c637e9f245db368b8f Mon Sep 17 00:00:00 2001 From: Ravi Mula Date: Wed, 11 Nov 2026 10:20:32 +0530 Subject: [PATCH] fix: use instance for knowledge provider ownership --- README.md | 2 +- docs/ADR/0006-groups-and-catalogs-model.md | 2 +- docs/openapi-search.yaml | 28 +++---- pipeline/api/api.py | 22 ++++-- pipeline/api/documents.py | 16 ++-- pipeline/api/search.py | 16 ++-- pipeline/api/upload.py | 10 +-- pipeline/auth/keycloak_admin.py | 4 +- pipeline/db.py | 2 +- pipeline/models.py | 10 +-- scripts/start_ocr_only_batch.py | 10 +-- scripts/test_pipeline_e2e.py | 10 +-- tests/test_api.py | 86 +++++++++++++-------- tests/test_migrate_sqlite_to_postgres.py | 2 +- tests/test_schema_v2.py | 2 +- tests/test_search_group_integration.py | 26 +++---- tests/test_search_provider_integration.py | 4 +- ui/src/components/DocumentHeaderSummary.jsx | 4 +- ui/src/views/DashboardView.jsx | 6 +- ui/src/views/DocumentsView.jsx | 6 +- ui/src/views/GlobalAuditView.jsx | 4 +- ui/src/views/NewDocumentView.jsx | 4 +- ui/src/views/RunsView.jsx | 2 +- 23 files changed, 152 insertions(+), 126 deletions(-) diff --git a/README.md b/README.md index 81d055f..7a78c7b 100644 --- a/README.md +++ b/README.md @@ -518,7 +518,7 @@ Translation data is surfaced through the page model and review endpoints. - `GET /documents/{workflow_id}/qdrant` - `GET /documents/{workflow_id}/qdrant/chunks` - `POST /documents/{workflow_id}/reingest` -- `POST /search` — only answers from documents whose validity period covers today; `valid_on` asks about another day and `include_expired` drops the filter. Optional `group` (e.g. `mh`) limits results to one group's documents; omitted means every group, and the response's `effective_config.group` echoes what was applied (see `docs/openapi-search.yaml`) +- `POST /search` — only answers from documents whose validity period covers today; `valid_on` asks about another day and `include_expired` drops the filter. Optional `instance` (e.g. `mh`) limits results to one instance's documents; omitted means every instance, and the response's `effective_config.instance` echoes what was applied (see `docs/openapi-search.yaml`) - `GET /indexes/summary` - `GET /indexes/{index_name}/settings` - `GET /indexes/{index_name}/stats` diff --git a/docs/ADR/0006-groups-and-catalogs-model.md b/docs/ADR/0006-groups-and-catalogs-model.md index 474b8cd..74e1f53 100644 --- a/docs/ADR/0006-groups-and-catalogs-model.md +++ b/docs/ADR/0006-groups-and-catalogs-model.md @@ -5,7 +5,7 @@ Context: the first Postgres schema modelled a tenant as a free-text `documents.i Decision: - A **group** is a provider: one `groups` row per Keycloak tenant node (`/states/MH`, `/global`). Role leaves are not mirrored; permissions still come from the token. `users.group_id` is a single nullable FK for now (first tenant in the token, sorted, with a warning when there are several); a join table can replace it if multi-group membership is needed. - A **catalog** is `(group, type)`: `catalogs.group_id` plus `catalogs.type`, unique together. `type` holds the network schema name. The kind-to-schema map lives in `pipeline/network_constants.py` and is a placeholder: `scheme` is `openagrinet:AgricultureResource`, `advisory` is `openagrinet:KnowledgeAdvisory`, `video` is `openagrinet:VideoResource`. Code branches through that map, never on raw `openagrinet:*` strings. Catalog Beckn information (`publish_directives`, `visible_networks`, `criteria`, `network`) sits in `catalogs.metadata`. -- A **document** belongs to one catalog and is identified there by `code`: `documents.catalog_id` NOT NULL, `UNIQUE (catalog_id, code)`, with `name` and `aliases` on the document. The API fields `code`, `name` and `aliases` are those columns. `documents.instance`, `document_kind` and `is_demo` are gone; the API returns the derived `group` (the group name) and `kind` (from the type map). The `/catalog/v1/*` snapshot keys, the Qdrant payload keys and Temporal arguments keep their earlier names (`scheme_code`, `document_kind`, `instance`, ...) because other services and in-flight workflows depend on them. +- A **document** belongs to one catalog and is identified there by `code`: `documents.catalog_id` NOT NULL, `UNIQUE (catalog_id, code)`, with `name` and `aliases` on the document. The API fields `code`, `name` and `aliases` are those columns. `documents.instance`, `document_kind` and `is_demo` are gone; the API returns the derived `instance` (the group name) and `kind` (from the type map). The `/catalog/v1/*` snapshot keys, the Qdrant payload keys and Temporal arguments keep their earlier names (`scheme_code`, `document_kind`, `instance`, ...) because other services and in-flight workflows depend on them. - Re-uploading an existing code updates the same document (new workflow, `source` replaced, pages and chunks cleared, `reindex_required` set) rather than adding a second one. - Catalogs are created on upload for the tenant and kind. The upload therefore takes the document type up front. - Bootstrap schemes (the code-seeded entries with no documents) are dropped. `/catalog/v1/*` is served from the `catalog_status` view, one row per document, and a scheme appears once a document exists. diff --git a/docs/openapi-search.yaml b/docs/openapi-search.yaml index 5115ef8..220f079 100644 --- a/docs/openapi-search.yaml +++ b/docs/openapi-search.yaml @@ -52,11 +52,11 @@ paths: top_k: 10 hybrid_alpha: 0.6 rerank_mode: "none" - one_group: - summary: Only one group's documents (e.g. a state's DSS) + one_instance: + summary: Only one instance's documents (e.g. a state's DSS) value: query: "cotton pest advisory" - group: "mh" + instance: "mh" one_provider: summary: Only the chunks one network provider publishes value: @@ -70,7 +70,7 @@ paths: schema: $ref: '#/components/schemas/SearchResponse' '400': - description: Missing/empty query, malformed valid_on, non-string group, non-string provider_id, or the underlying vector-store search failed + description: Missing/empty query, malformed valid_on, non-string instance, non-string provider_id, or the underlying vector-store search failed content: application/json: schema: @@ -263,21 +263,21 @@ components: what search would have returned, or will return, on that date. A malformed value returns 400. Ignored when include_expired is true. default: "(today)" - group: + instance: type: string nullable: true description: | - Limit results to chunks tagged with this group code (for example + Limit results to chunks tagged with this instance code (for example `mh` for Maharashtra, `bv` for Bharat Vistaar). Case-insensitive and trimmed: `MH` and ` mh ` both mean `mh`. One value only — - a caller belongs to one group; fallback to another group is the - caller's second request. Omitted, null or empty means no group - filter: chunks from every group are searched as one pool. An - unknown group is not an error: + a caller belongs to one instance; fallback to another instance is the + caller's second request. Omitted, null or empty means no instance + filter: chunks from every instance are searched as one pool. An + unknown instance is not an error: the response is a success with zero hits. A non-string value returns 400. Applies in every search mode. A filter, not an access check: this endpoint applies no tenant scoping, so any - search-capable caller may ask for any group. + search-capable caller may ask for any instance. example: "mh" provider_id: type: string @@ -289,7 +289,7 @@ components: empty means no provider filter; an unknown id returns zero hits; a non-string value returns 400. Only advisory and scheme chunks carry a provider id, so a provider filter leaves other kinds out. - Combines with `group` and validity. Any other type returns 400. + Combines with `instance` and validity. Any other type returns 400. A filter, not an access check. example: "mh.provider.oan.dev" @@ -424,10 +424,10 @@ components: type: boolean description: True if the query-expansion profile actually rewrote the query. rerank_mode: { type: string } - group: + instance: type: string nullable: true - description: The normalised group filter that was applied (e.g. `mh`), or null when the search spanned every group. + description: The normalised instance filter that was applied (e.g. `mh`), or null when the search spanned every instance. provider_id: type: string nullable: true diff --git a/pipeline/api/api.py b/pipeline/api/api.py index d42b79b..d277584 100644 --- a/pipeline/api/api.py +++ b/pipeline/api/api.py @@ -418,13 +418,15 @@ def _i(key: str, default: int = 0) -> int: "ready_for_ingestion": _i("ready_for_ingestion_documents"), "failed": _i("failed_documents"), }, - "by_group": by_group or [], + "by_instance": [ + {**row, "instance": row["group"]} for row in (by_group or []) + ], } -def _resolve_create_group(user: AuthUser, requested: Optional[str] = None) -> str: - """Pick the document's group from Keycloak-allowed states / portal.""" - return resolve_create_group(user, requested) +def _resolve_upload_instance(user: AuthUser, instance: str) -> str: + """Resolve the upload owner using the existing Keycloak tenancy rules.""" + return resolve_create_group(user, instance) def _document_title(filename: str) -> str: @@ -558,7 +560,7 @@ def _document_summary_from_row(doc: dict, current_job: Optional[dict] = None) -> source_manifest_name=doc.get("source_manifest_name"), source_file_fingerprint=doc.get("source_file_fingerprint"), authoritative=bool(doc.get("source_manifest_name")), - group=normalize_group(doc.get("group")), + instance=normalize_group(doc.get("group")), stage=DocumentStage(doc["stage"]), page_count=doc.get("page_count") or 0, chunk_count=doc.get("chunk_count") or 0, @@ -804,7 +806,7 @@ async def auth_me(user: CurrentUser): "state_roles": dict(user.state_roles or {}), "is_super_admin": bool(user.is_superadmin), # Portal group code for BV / platform docs (not a state). - "portal_group": PORTAL_GROUP, + "portal_instance": PORTAL_GROUP, "auth_disabled": user.token_disabled_mode, } @@ -945,13 +947,17 @@ async def list_runs( Restricted users only see jobs whose document ``group`` is in their allowed states. Super-admin / bypass users see all tenants. """ - return db.list_runs( + runs = db.list_runs( limit=limit, offset=offset, status=status, groups=_group_scope_for_user(user), ) + for run in runs: + run["instance"] = run.pop("group") + return runs + @app.get("/runs/{job_id}") async def get_run(job_id: str, user: RequireSearch): @@ -1010,7 +1016,7 @@ def _build_document_detail(doc: dict) -> DocumentDetail: source_manifest_name=doc.get("source_manifest_name"), source_file_fingerprint=doc.get("source_file_fingerprint"), authoritative=bool(doc.get("source_manifest_name")), - group=normalize_group(doc.get("group")), + instance=normalize_group(doc.get("group")), filepath=doc["filepath"], stage=DocumentStage(doc["stage"]), page_count=doc.get("page_count", 0), diff --git a/pipeline/api/documents.py b/pipeline/api/documents.py index aae3ceb..0432ebf 100644 --- a/pipeline/api/documents.py +++ b/pipeline/api/documents.py @@ -36,7 +36,7 @@ async def start_document_workflow( min_tokens: int = 100, index_name: str = "documents-index", stop_after_ocr: bool = False, - group: str = "", + instance: str = "", kind: str = Query(..., description="scheme or advisory"), code: str = "", ): @@ -54,7 +54,7 @@ async def start_document_workflow( Rate limited to 10 requests/minute per IP. Requires permission: upload (no-op while AUTH_DISABLED=true). """ - create_group = _api._resolve_create_group(user, group) + create_group = _api._resolve_upload_instance(user, instance) # Validate file path to prevent path traversal attacks filepath = _api.validate_file_path(data.filepath) source_filename = _api.get_filename_from_path(filepath) @@ -85,7 +85,7 @@ async def start_document_workflow( source_filename=source_filename, source_file_fingerprint=source_file_fingerprint, authoritative=bool(existing_doc.get("source_manifest_name")) if existing_doc else False, - group=normalize_group(existing_doc.get("group")), + instance=normalize_group(existing_doc.get("group")), stage=DocumentStage(state.get("stage", "registered")), page_count=state.get("page_count", 0), chunk_count=state.get("chunk_count", 0), @@ -182,7 +182,7 @@ async def start_document_workflow( new_value=source_filename, metadata={ "source": "register_endpoint", - "group": create_group, + "instance": create_group, "kind": doc_kind, "code": doc_code, "replaced_existing": bool(replace_doc), @@ -198,7 +198,7 @@ async def start_document_workflow( source_filename=source_filename, source_file_fingerprint=source_file_fingerprint, authoritative=False, - group=create_group, + instance=create_group, stage=DocumentStage.REGISTERED, page_count=0, chunk_count=0, @@ -220,11 +220,11 @@ async def start_batch_workflows( chunk_overlap: int = 128, min_tokens: int = 100, stop_after_ocr: bool = False, - group: str = "", + instance: str = "", kind: str = Query(..., description="scheme or advisory"), ): """Start workflows for all supported documents in a directory.""" - create_group = _api._resolve_create_group(user, group) + create_group = _api._resolve_upload_instance(user, instance) directory = Path(data.directory) if not directory.exists(): raise HTTPException(404, f"Directory not found: {data.directory}") @@ -245,7 +245,7 @@ async def start_batch_workflows( chunk_overlap=chunk_overlap, min_tokens=min_tokens, stop_after_ocr=stop_after_ocr, - group=create_group, + instance=create_group, kind=kind, code="", ) diff --git a/pipeline/api/search.py b/pipeline/api/search.py index 70beb7c..a621082 100644 --- a/pipeline/api/search.py +++ b/pipeline/api/search.py @@ -170,10 +170,10 @@ def _rerank_hits(query: str, hits: list[dict], rerank_mode: str) -> list[dict]: -def _requested_group(payload: dict) -> str | None: - raw = payload.get("group") +def _requested_instance(payload: dict) -> str | None: + raw = payload.get("instance") if raw is not None and not isinstance(raw, str): - raise HTTPException(400, "group must be a string") + raise HTTPException(400, "instance must be a string") return (raw or "").strip().lower() or None @@ -188,9 +188,9 @@ def _requested_provider_id(payload: dict) -> str | None: async def run_search(payload: dict, user: RequireSearch): """Search the vector index. - `group` is a filter, not an access check: this endpoint applies no tenant + `instance` is a filter, not an access check: this endpoint applies no tenant scoping to its results (unlike `GET /chunks/search`), so any - search-capable caller may ask for any group. + search-capable caller may ask for any instance. """ from ..vector_store import get_default_index_name, get_vector_store @@ -247,7 +247,7 @@ async def run_search(payload: dict, user: RequireSearch): valid_on = document_validity.parse_day(valid_on, "valid_on") except document_validity.DocumentValidityError as exc: raise HTTPException(400, str(exc)) from None - group = _requested_group(payload) + instance = _requested_instance(payload) provider_id = _requested_provider_id(payload) expanded_query = _expand_query(query, query_expansion_profile) # Qdrant embeddings apply E5 prefixes internally; don't pre-prefix here. @@ -270,7 +270,7 @@ async def run_search(payload: dict, user: RequireSearch): ef_search=ef_search, apply_validity=not include_expired, valid_on=valid_on, - instance=group, + instance=instance, provider_id=provider_id, ) except Exception as error: @@ -320,7 +320,7 @@ async def run_search(payload: dict, user: RequireSearch): "rerank_mode": rerank_mode, "apply_validity": not include_expired, "valid_on": result.get("valid_on"), - "group": group, + "instance": instance, "provider_id": provider_id, "filter_string": None, }, diff --git a/pipeline/api/upload.py b/pipeline/api/upload.py index f4fb5c7..dd62f41 100644 --- a/pipeline/api/upload.py +++ b/pipeline/api/upload.py @@ -31,7 +31,7 @@ async def upload_and_process( min_tokens: int = 100, index_name: str = "documents-index", stop_after_ocr: bool = False, - group: str = "", + instance: str = "", kind: str = Query(..., description="scheme or advisory"), code: str = "", display_name: Optional[str] = Form(None), @@ -44,7 +44,7 @@ async def upload_and_process( Rate limited to 10 requests/minute per IP. Requires permission: upload (no-op while AUTH_DISABLED=true). """ - create_group = _api._resolve_create_group(user, group) + create_group = _api._resolve_upload_instance(user, instance) doc_kind, doc_code, replace_doc = _api._resolve_document_identity( user, create_group, kind, code, file.filename or "" ) @@ -104,7 +104,7 @@ async def upload_and_process( source_filename=file.filename, source_file_fingerprint=file_hash, authoritative=bool(existing_doc.get("source_manifest_name")) if existing_doc else False, - group=normalize_group(existing_doc.get("group")), + instance=normalize_group(existing_doc.get("group")), stage=DocumentStage(state.get("stage", "registered")), page_count=state.get("page_count", 0), chunk_count=state.get("chunk_count", 0), @@ -211,7 +211,7 @@ async def upload_and_process( new_value=file.filename, metadata={ "source": "upload_endpoint", - "group": create_group, + "instance": create_group, "kind": doc_kind, "code": doc_code, "replaced_existing": bool(replace_doc), @@ -237,7 +237,7 @@ async def upload_and_process( source_filename=file.filename, source_file_fingerprint=file_hash, authoritative=False, - group=create_group, + instance=create_group, stage=DocumentStage.REGISTERED, page_count=0, chunk_count=0, diff --git a/pipeline/auth/keycloak_admin.py b/pipeline/auth/keycloak_admin.py index f4b1a2c..d1bbdc8 100644 --- a/pipeline/auth/keycloak_admin.py +++ b/pipeline/auth/keycloak_admin.py @@ -506,7 +506,7 @@ def provision_user( "roles": [r.get("name") for r in (roles or []) if isinstance(r, dict)], "app_url": ui_url, "login_url": login_url, - "portal_group": PORTAL_GROUP, + "portal_instance": PORTAL_GROUP, "share_message": "\n".join(share_lines), } @@ -517,7 +517,7 @@ def list_access_options() -> dict[str, Any]: return { "keycloak_admin_configured": cfg.configured, "realm": cfg.realm, - "portal_group": PORTAL_GROUP, + "portal_instance": PORTAL_GROUP, "access_types": [ { "id": "super_admin", diff --git a/pipeline/db.py b/pipeline/db.py index 0dff900..f8d9b78 100644 --- a/pipeline/db.py +++ b/pipeline/db.py @@ -2612,7 +2612,7 @@ def _audit_row_to_legacy(row: dict) -> dict: "actor_roles": ",".join(roles) if roles else None, "filename": row["filename"], "name": row["document_name"], - "group": row["doc_group"] or row["audit_group"], + "instance": row["doc_group"] or row["audit_group"], "uploaded_by_username": row["uploaded_by_username"], "uploaded_by_email": row["uploaded_by_email"], } diff --git a/pipeline/models.py b/pipeline/models.py index 79ba659..011b101 100644 --- a/pipeline/models.py +++ b/pipeline/models.py @@ -218,7 +218,7 @@ class DocumentSummary(BaseModel): source_manifest_name: Optional[str] = None source_file_fingerprint: Optional[str] = None authoritative: bool = False - group: str = "default" # tenant code (groups.name) of the document's catalog + instance: str = "default" # tenant code of the document's catalog stage: DocumentStage page_count: int chunk_count: int @@ -352,8 +352,8 @@ class DocumentGraph(BaseModel): runtime: dict[str, Any] -class GroupDocumentCount(BaseModel): - group: str +class InstanceDocumentCount(BaseModel): + instance: str count: int success: int = 0 failed: int = 0 @@ -368,7 +368,7 @@ class DocumentCohortsResponse(BaseModel): review_queue: int failed_documents: int by_stage: dict[str, int] - by_group: list[GroupDocumentCount] = [] + by_instance: list[InstanceDocumentCount] = [] needs_reindex: int running_jobs: int = 0 @@ -445,7 +445,7 @@ class AuditLogEntry(BaseModel): # strips them from the response and the UI can only show a raw doc_id hash. filename: Optional[str] = None name: Optional[str] = None - group: Optional[str] = None + instance: Optional[str] = None uploaded_by_username: Optional[str] = None uploaded_by_email: Optional[str] = None diff --git a/scripts/start_ocr_only_batch.py b/scripts/start_ocr_only_batch.py index a48fb47..234fa61 100644 --- a/scripts/start_ocr_only_batch.py +++ b/scripts/start_ocr_only_batch.py @@ -45,10 +45,10 @@ def already_has_ocr(filepath: str, fingerprint: str) -> bool: return bool(db.get_saved_page_numbers(doc["workflow_id"])) -def post_document(api_base: str, filepath: str, kind: str, group: str = "") -> dict: +def post_document(api_base: str, filepath: str, kind: str, instance: str = "") -> dict: params = {"stop_after_ocr": "true", "kind": kind} - if group: - params["group"] = group + if instance: + params["instance"] = instance query = urllib.parse.urlencode(params) url = f"{api_base.rstrip('/')}/documents?{query}" payload = json.dumps({"filepath": filepath}).encode("utf-8") @@ -68,7 +68,7 @@ def main() -> int: parser.add_argument("--api-base", default="http://localhost:8001", help="Docs-pipeline API base URL") parser.add_argument("--limit", type=int, default=0, help="Optional max files to submit") parser.add_argument("--kind", choices=["scheme", "advisory"], default="scheme", help="Document kind (catalog)") - parser.add_argument("--group", default="", help="Tenant group code (optional; defaults to your group)") + parser.add_argument("--instance", default="", help="Tenant instance code (optional; defaults to your instance)") args = parser.parse_args() directory = Path(args.directory) @@ -94,7 +94,7 @@ def main() -> int: print(f"SKIP {path.name}: OCR already present") continue try: - result = post_document(args.api_base, str(path), args.kind, args.group) + result = post_document(args.api_base, str(path), args.kind, args.instance) submitted += 1 print(f"SUBMIT {path.name}: {result.get('workflow_id')}") except urllib.error.HTTPError as exc: diff --git a/scripts/test_pipeline_e2e.py b/scripts/test_pipeline_e2e.py index 66d8eeb..eda4ff7 100644 --- a/scripts/test_pipeline_e2e.py +++ b/scripts/test_pipeline_e2e.py @@ -38,7 +38,7 @@ def check_health(name: str, url: str) -> bool: def upload_pdf( - api: str, pdf_path: Path, *, auto_approve: bool, stop_after_ocr: bool, kind: str, group: str = "" + api: str, pdf_path: Path, *, auto_approve: bool, stop_after_ocr: bool, kind: str, instance: str = "" ) -> str: import subprocess @@ -46,8 +46,8 @@ def upload_pdf( f"auto_approve={'true' if auto_approve else 'false'}" f"&stop_after_ocr={'true' if stop_after_ocr else 'false'}&kind={kind}" ) - if group: - query += f"&group={group}" + if instance: + query += f"&instance={instance}" url = f"{api.rstrip('/')}/upload?{query}" result = subprocess.run( ["curl", "-sf", "-X", "POST", url, "-F", f"file=@{pdf_path}"], @@ -100,7 +100,7 @@ def main() -> int: parser.add_argument("--gemma-health", default=os.environ.get("TRANSLATION_VLLM_BASE_URL", "http://localhost:8020/v1")) parser.add_argument("--mode", choices=["ocr", "full"], default="ocr", help="ocr=stop after OCR; full=entire pipeline") parser.add_argument("--kind", choices=["scheme", "advisory"], default="scheme", help="Document kind (catalog)") - parser.add_argument("--group", default="", help="Tenant group code (optional; defaults to your group)") + parser.add_argument("--instance", default="", help="Tenant instance code (optional; defaults to your instance)") parser.add_argument("--timeout", type=int, default=1800, help="Max seconds to wait for workflow") parser.add_argument("--poll-interval", type=float, default=5.0) args = parser.parse_args() @@ -132,7 +132,7 @@ def main() -> int: print(f"==> Upload ({'OCR only' if stop_after_ocr else 'full pipeline'})") workflow_id = upload_pdf( args.api, args.pdf, auto_approve=True, stop_after_ocr=stop_after_ocr, - kind=args.kind, group=args.group, + kind=args.kind, instance=args.instance, ) print("==> Polling workflow") diff --git a/tests/test_api.py b/tests/test_api.py index ec690fc..ef0bda9 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -615,72 +615,68 @@ def test_rejects_a_malformed_valid_on(self, test_client): assert "YYYY-MM-DD" in response.json()["detail"] -class TestSearchGroup: - """`group` narrows a search to one group's chunks (story #202). - - The API says `group`; the store and the chunk payload say `instance`. - The endpoint is the only place that maps one name to the other. - """ +class TestSearchInstance: + """`instance` narrows a search to one instance's chunks.""" @pytest.mark.api @pytest.mark.unit - def test_passes_the_group_to_the_store_as_instance(self, test_client, capturing_store): - response = test_client.post("/search", json={"query": "kisan", "group": "mh"}) + def test_passes_the_instance_to_the_store(self, test_client, capturing_store): + response = test_client.post("/search", json={"query": "kisan", "instance": "mh"}) assert response.status_code == 200 assert capturing_store.captured["instance"] == "mh" - assert response.json()["effective_config"]["group"] == "mh" + assert response.json()["effective_config"]["instance"] == "mh" @pytest.mark.api @pytest.mark.unit @pytest.mark.parametrize("raw", ["MH", " mh ", " Mh\n"]) def test_normalises_case_and_whitespace(self, test_client, capturing_store, raw): - response = test_client.post("/search", json={"query": "kisan", "group": raw}) + response = test_client.post("/search", json={"query": "kisan", "instance": raw}) assert capturing_store.captured["instance"] == "mh" - assert response.json()["effective_config"]["group"] == "mh" + assert response.json()["effective_config"]["instance"] == "mh" @pytest.mark.api @pytest.mark.unit @pytest.mark.parametrize("body", [ {"query": "kisan"}, - {"query": "kisan", "group": ""}, - {"query": "kisan", "group": " "}, - {"query": "kisan", "group": None}, + {"query": "kisan", "instance": ""}, + {"query": "kisan", "instance": " "}, + {"query": "kisan", "instance": None}, ]) - def test_absent_or_empty_group_means_no_filter(self, test_client, capturing_store, body): + def test_absent_or_empty_instance_means_no_filter(self, test_client, capturing_store, body): response = test_client.post("/search", json=body) assert response.status_code == 200 assert capturing_store.captured["instance"] is None - assert response.json()["effective_config"]["group"] is None + assert response.json()["effective_config"]["instance"] is None @pytest.mark.api @pytest.mark.unit @pytest.mark.parametrize("bad", [["mh"], 7, {"code": "mh"}]) - def test_rejects_a_non_string_group(self, test_client, capturing_store, bad): - response = test_client.post("/search", json={"query": "kisan", "group": bad}) + def test_rejects_a_non_string_instance(self, test_client, capturing_store, bad): + response = test_client.post("/search", json={"query": "kisan", "instance": bad}) assert response.status_code == 400 - assert response.json()["detail"] == "group must be a string" + assert response.json()["detail"] == "instance must be a string" assert capturing_store.captured == {}, "the store must not be called for a bad request" @pytest.mark.api @pytest.mark.unit - def test_unknown_group_is_an_empty_success(self, test_client, capturing_store): - response = test_client.post("/search", json={"query": "kisan", "group": "xx"}) + def test_unknown_instance_is_an_empty_success(self, test_client, capturing_store): + response = test_client.post("/search", json={"query": "kisan", "instance": "xx"}) assert response.status_code == 200 body = response.json() assert body["hits"] == [] assert body["final_count"] == 0 - assert body["effective_config"]["group"] == "xx" + assert body["effective_config"]["instance"] == "xx" @pytest.mark.api @pytest.mark.unit - def test_group_composes_with_include_expired(self, test_client, capturing_store): + def test_instance_composes_with_include_expired(self, test_client, capturing_store): test_client.post( - "/search", json={"query": "kisan", "group": "mh", "include_expired": True} + "/search", json={"query": "kisan", "instance": "mh", "include_expired": True} ) assert capturing_store.captured["instance"] == "mh" @@ -730,8 +726,8 @@ def test_rejects_anything_but_a_string(self, test_client, capturing_store, bad): @pytest.mark.api @pytest.mark.unit - def test_composes_with_group(self, test_client, capturing_store): - test_client.post("/search", json={"query": "kisan", "group": "mh", "provider_id": "mh.provider.oan.dev"}) + def test_composes_with_instance(self, test_client, capturing_store): + test_client.post("/search", json={"query": "kisan", "instance": "mh", "provider_id": "mh.provider.oan.dev"}) assert capturing_store.captured["instance"] == "mh" assert capturing_store.captured["provider_id"] == "mh.provider.oan.dev" @@ -1126,14 +1122,16 @@ def test_upload_creates_the_catalog_and_derives_the_code_from_the_filename( ): response = test_client.post( "/upload", - params={"kind": "scheme", "group": "mh"}, + params={"kind": "scheme", "instance": "mh"}, files={"file": ("National Mission on Natural Farming.pdf", sample_pdf_content, "application/pdf")}, ) assert response.status_code == 200 doc = test_client.get(f"/documents/{response.json()['workflow_id']}").json() assert doc["kind"] == "scheme" - assert doc["group"] == "mh" + assert response.json()["instance"] == "mh" + assert doc["instance"] == "mh" + assert "group" not in doc assert doc["code"] == "nmnf" assert doc["name"] == "National Mission on Natural Farming" with db_connection.get_connection() as conn: @@ -1146,6 +1144,28 @@ def test_upload_creates_the_catalog_and_derives_the_code_from_the_filename( ("openagrinet:AgricultureResource", "mh", "/states/MH") ] + @pytest.mark.api + @pytest.mark.unit + def test_upload_rejects_an_unauthorized_instance( + self, test_client, db_connection, sample_pdf_content, monkeypatch + ): + from pipeline.api.api import app + from pipeline.auth.deps import get_current_user + + monkeypatch.setitem( + app.dependency_overrides, get_current_user, + lambda: self._user("/states/MH/contributor"), + ) + response = test_client.post( + "/upload", + params={"kind": "scheme", "instance": "gj"}, + files={"file": ("a.pdf", sample_pdf_content, "application/pdf")}, + ) + + assert response.status_code == 403 + with db_connection.get_connection() as conn: + assert conn.execute("SELECT count(*) AS n FROM documents").fetchone()["n"] == 0 + @pytest.mark.api @pytest.mark.unit def test_a_second_upload_reuses_the_catalog_and_gets_a_fresh_code( @@ -1153,12 +1173,12 @@ def test_a_second_upload_reuses_the_catalog_and_gets_a_fresh_code( ): first = test_client.post( "/upload", - params={"kind": "advisory", "group": "mh"}, + params={"kind": "advisory", "instance": "mh"}, files={"file": ("Crop Advisory.pdf", self._pdf(sample_pdf_content, "one"), "application/pdf")}, ) second = test_client.post( "/upload", - params={"kind": "advisory", "group": "mh"}, + params={"kind": "advisory", "instance": "mh"}, files={"file": ("Crop Advisory.pdf", self._pdf(sample_pdf_content, "two"), "application/pdf")}, ) @@ -1178,7 +1198,7 @@ def test_an_existing_code_updates_the_same_document( ): first = test_client.post( "/upload", - params={"kind": "scheme", "group": "mh", "code": "pm-kisan"}, + params={"kind": "scheme", "instance": "mh", "code": "pm-kisan"}, files={"file": ("pmkisan.pdf", self._pdf(sample_pdf_content, "v1"), "application/pdf")}, ) assert first.status_code == 200 @@ -1194,7 +1214,7 @@ def test_an_existing_code_updates_the_same_document( second = test_client.post( "/upload", - params={"kind": "scheme", "group": "mh", "code": "pm-kisan"}, + params={"kind": "scheme", "instance": "mh", "code": "pm-kisan"}, files={"file": ("pmkisan-2026.pdf", self._pdf(sample_pdf_content, "v2"), "application/pdf")}, ) @@ -1219,7 +1239,7 @@ def test_the_same_code_in_another_tenant_is_a_different_document( for group, tag in (("mh", "a"), ("gj", "b")): response = test_client.post( "/upload", - params={"kind": "scheme", "group": group, "code": "pm-kisan"}, + params={"kind": "scheme", "instance": group, "code": "pm-kisan"}, files={"file": ("p.pdf", self._pdf(sample_pdf_content, tag), "application/pdf")}, ) assert response.status_code == 200 diff --git a/tests/test_migrate_sqlite_to_postgres.py b/tests/test_migrate_sqlite_to_postgres.py index 99b2884..ab56a1b 100644 --- a/tests/test_migrate_sqlite_to_postgres.py +++ b/tests/test_migrate_sqlite_to_postgres.py @@ -375,7 +375,7 @@ def test_a_full_run_maps_every_table(self, db_connection, postgres_url, legacy_d assert [e["action_type"] for e in purged] == ["purge_document"] demo_trail = db.get_audit_logs(WF5) # kept for the skipped demo document, tied to its group assert [e["action_type"] for e in demo_trail] == ["stage_change"] - assert demo_trail[0]["group"] == "mh" + assert demo_trail[0]["instance"] == "mh" settings_log = db.get_settings_audit_logs() assert settings_log[0]["field_name"] == "retrieval.alpha" diff --git a/tests/test_schema_v2.py b/tests/test_schema_v2.py index d1ce3b2..fa6b053 100644 --- a/tests/test_schema_v2.py +++ b/tests/test_schema_v2.py @@ -463,7 +463,7 @@ def test_the_snapshot_survives_a_purge_and_keeps_tenant_scope(self, db_connectio assert db.get_all_audit_log_count(groups=["up"]) == 1 assert db.get_all_audit_log_count(groups=["gj"]) == 0 assert [e["workflow_id"] for e in db.get_all_audit_logs(groups=["mh"])] == ["wf-gone"] - assert db.get_all_audit_logs(groups=["mh"])[0]["group"] == "mh" + assert db.get_all_audit_logs(groups=["mh"])[0]["instance"] == "mh" @pytest.mark.db @pytest.mark.unit diff --git a/tests/test_search_group_integration.py b/tests/test_search_group_integration.py index 505de70..93da320 100644 --- a/tests/test_search_group_integration.py +++ b/tests/test_search_group_integration.py @@ -1,9 +1,9 @@ -"""Integration: ``POST /search`` with ``group`` through the real Qdrant engine. +"""Integration: ``POST /search`` with ``instance`` through the real Qdrant engine. qdrant-client ships an in-process mode (``QdrantClient(":memory:")``) that runs the real filtering and vector engine, so this exercises the route, the store and the engine together without Docker. Only the embedding model is -faked: search quality is not under test here, the group filter is. +faked: search quality is not under test here, the instance filter is. """ import zlib @@ -14,7 +14,7 @@ from pipeline.vector_store import qdrant_store from tests.test_qdrant_validity import TODAY, clock_at -INDEX = "group-integration" +INDEX = "instance-integration" VECTOR_SIZE = 8 CHUNKS = [ @@ -67,27 +67,27 @@ def test_every_chunk_is_indexed(local_store): @pytest.mark.integration @pytest.mark.api @pytest.mark.parametrize("mode", ["HYBRID", "TENSOR", "LEXICAL"]) -def test_group_limits_hits_to_that_group_in_every_mode(local_store, test_client, mode): - body = search(test_client, group="mh", search_mode=mode) +def test_instance_limits_hits_to_that_group_in_every_mode(local_store, test_client, mode): + body = search(test_client, instance="mh", search_mode=mode) - assert body["effective_config"]["group"] == "mh" + assert body["effective_config"]["instance"] == "mh" assert body["hits"], "the mh chunk must be found" assert {hit["instance"] for hit in body["hits"]} == {"mh"} @pytest.mark.integration @pytest.mark.api -def test_no_group_searches_every_group(local_store, test_client): +def test_no_instance_searches_every_group(local_store, test_client): body = search(test_client) - assert body["effective_config"]["group"] is None + assert body["effective_config"]["instance"] is None assert {hit["instance"] for hit in body["hits"]} == {"mh", "ka", "bv"} @pytest.mark.integration @pytest.mark.api -def test_unknown_group_is_an_empty_success(local_store, test_client): - body = search(test_client, group="xx") +def test_unknown_instance_is_an_empty_success(local_store, test_client): + body = search(test_client, instance="xx") assert body["hits"] == [] assert body["final_count"] == 0 @@ -95,9 +95,9 @@ def test_unknown_group_is_an_empty_success(local_store, test_client): @pytest.mark.integration @pytest.mark.api -def test_group_and_validity_compose_in_the_engine(local_store, test_client): - current_only = search(test_client, group="mh") - with_expired = search(test_client, group="mh", include_expired=True) +def test_instance_and_validity_compose_in_the_engine(local_store, test_client): + current_only = search(test_client, instance="mh") + with_expired = search(test_client, instance="mh", include_expired=True) assert {hit["doc_id"] for hit in current_only["hits"]} == {"mh-current"} assert {hit["doc_id"] for hit in with_expired["hits"]} == {"mh-current", "mh-expired"} diff --git a/tests/test_search_provider_integration.py b/tests/test_search_provider_integration.py index bd28f3f..9589206 100644 --- a/tests/test_search_provider_integration.py +++ b/tests/test_search_provider_integration.py @@ -60,7 +60,7 @@ def test_no_provider_searches_every_chunk(local_store, test_client): @pytest.mark.integration @pytest.mark.api -def test_provider_composes_with_group(local_store, test_client): - body = search(test_client, provider_id="bv.provider.oan.dev", group="mh") +def test_provider_composes_with_instance(local_store, test_client): + body = search(test_client, provider_id="bv.provider.oan.dev", instance="mh") assert body["hits"] == [] diff --git a/ui/src/components/DocumentHeaderSummary.jsx b/ui/src/components/DocumentHeaderSummary.jsx index 999c654..3af75e9 100644 --- a/ui/src/components/DocumentHeaderSummary.jsx +++ b/ui/src/components/DocumentHeaderSummary.jsx @@ -74,7 +74,7 @@ export function DocumentHeaderSummary({ const pages = pageCount ?? doc?.page_count ?? 0 const chunks = chunkCount ?? doc?.chunk_count ?? 0 const uploader = uploaderName(doc) - const stateLabel = formatGroupLabel(doc?.group) + const stateLabel = formatGroupLabel(doc?.instance) const roleLabels = useMemo( () => productRoleLabels(doc?.uploaded_by_roles || []), [doc?.uploaded_by_roles], @@ -126,7 +126,7 @@ export function DocumentHeaderSummary({ {stateLabel ? ( - {stateLabel} + {stateLabel} ) : ( '—' )} diff --git a/ui/src/views/DashboardView.jsx b/ui/src/views/DashboardView.jsx index 2538e72..2cfd9d4 100644 --- a/ui/src/views/DashboardView.jsx +++ b/ui/src/views/DashboardView.jsx @@ -66,7 +66,7 @@ export default function DashboardView() { const reindexCount = summary?.needs_reindex || 0 const displayedQueueItems = queue.slice(0, QUEUE_ROWS_SHOWN) - const byGroup = useMemo(() => (Array.isArray(summary?.by_group) ? summary.by_group : []), [summary]) + const byGroup = useMemo(() => (Array.isArray(summary?.by_instance) ? summary.by_instance : []), [summary]) const stateRows = byGroup const maxStateCount = useMemo( @@ -172,10 +172,10 @@ export default function DashboardView() { {stateRows.map((row) => { const label = row.otherStateCount ? `Others (${row.otherStateCount} states)` - : formatGroupLabel(row.group) || 'No state tag' + : formatGroupLabel(row.instance) || 'No state tag' const pct = (value) => (maxStateCount ? (value / maxStateCount) * 100 : 0) return ( -
+
{label} {formatCount(row.count)} diff --git a/ui/src/views/DocumentsView.jsx b/ui/src/views/DocumentsView.jsx index 47024c9..f164edb 100644 --- a/ui/src/views/DocumentsView.jsx +++ b/ui/src/views/DocumentsView.jsx @@ -48,7 +48,7 @@ export default function DocumentsView() { const stageOptions = useMemo(() => ['all', ...new Set(documents.map(doc => doc.stage))], [documents]) const groupOptions = useMemo(() => { - const fromDocs = documents.map((d) => (d.group || '').trim().toLowerCase()).filter(Boolean) + const fromDocs = documents.map((d) => (d.instance || '').trim().toLowerCase()).filter(Boolean) return ['all', ...new Set(fromDocs)] }, [documents]) @@ -60,7 +60,7 @@ export default function DocumentsView() { if (authFilter === 'authoritative' && !doc.authoritative) return false if (authFilter === 'legacy' && doc.authoritative) return false if (groupFilter !== 'all') { - const code = (doc.group || 'default').trim().toLowerCase() + const code = (doc.instance || 'default').trim().toLowerCase() if (code !== groupFilter) return false } if (showFailed && !doc.failed && doc.stage !== 'failed') return false @@ -233,7 +233,7 @@ export default function DocumentsView() {
{getDocumentMetaLabel(doc)}
- + diff --git a/ui/src/views/GlobalAuditView.jsx b/ui/src/views/GlobalAuditView.jsx index a3aed6c..c2aaa93 100644 --- a/ui/src/views/GlobalAuditView.jsx +++ b/ui/src/views/GlobalAuditView.jsx @@ -122,8 +122,8 @@ export default function GlobalAuditView() { > {entry.name || entry.filename || entry.document_id || entry.workflow_id} - {entry.group && ( - {entry.group} + {entry.instance && ( + {entry.instance} )}
{/* What actually changed, in words — the raw before/after stays diff --git a/ui/src/views/NewDocumentView.jsx b/ui/src/views/NewDocumentView.jsx index 41acaed..f12be02 100644 --- a/ui/src/views/NewDocumentView.jsx +++ b/ui/src/views/NewDocumentView.jsx @@ -66,7 +66,7 @@ export default function NewDocumentView() { const documentGroup = defaultUploadGroup({ isSuperAdmin, instances, - portalGroup: user?.portal_group || PORTAL_GROUP, + portalGroup: user?.portal_instance || PORTAL_GROUP, }) const [file, setFile] = useState(null) const [displayName, setDisplayName] = useState('') @@ -157,7 +157,7 @@ export default function NewDocumentView() { } const params = new URLSearchParams({ auto_approve: 'false', - group: uploadGroup, + instance: uploadGroup, kind: documentKind, }) const response = await apiFetch(`${API_BASE}/upload?${params}`, { method: 'POST', body: formData }) diff --git a/ui/src/views/RunsView.jsx b/ui/src/views/RunsView.jsx index d11a44d..727f2df 100644 --- a/ui/src/views/RunsView.jsx +++ b/ui/src/views/RunsView.jsx @@ -356,7 +356,7 @@ export default function RunsView() { {title}
- + #{runId} {fileLabel && fileLabel !== title ? ` · ${fileLabel}` : ''}