From c49234b7e1daec68d005774029c3121df960c61a Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Wed, 12 Aug 2026 13:10:35 +0530 Subject: [PATCH 1/8] feat(api): generate and commit the API deployment OpenAPI spec MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The published Python clients and their generated SDKs are built from a spec of the deployment execute/status endpoints, which until now was produced by a script living outside this repo — so a route or serializer change here could silently invalidate it. The schema annotation for DeploymentExecution now lives next to the view, and `manage.py generate_docstudio_spec` writes specs/docstudio-oss.json. A unit test regenerates and compares, so drift fails in this repo's existing CI tier rather than in a client repo, with no database or extra CI job needed. The generated spec is unchanged from what the external script produced, apart from a root `tags` array — clients had nowhere to read group descriptions from. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014f9oEEYspPH4fmPULTnLkJ --- backend/api_v2/api_deployment_views.py | 98 ++++ backend/api_v2/deployment_spec_urls.py | 13 + .../commands/generate_docstudio_spec.py | 60 +++ backend/api_v2/tests/test_docstudio_spec.py | 38 ++ backend/backend/settings/base.py | 21 + backend/pyproject.toml | 1 + backend/uv.lock | 19 + specs/docstudio-oss.json | 442 ++++++++++++++++++ 8 files changed, 692 insertions(+) create mode 100644 backend/api_v2/deployment_spec_urls.py create mode 100644 backend/api_v2/management/commands/generate_docstudio_spec.py create mode 100644 backend/api_v2/tests/test_docstudio_spec.py create mode 100644 specs/docstudio-oss.json diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.py index d5cfa800a1..6fb1e08234 100644 --- a/backend/api_v2/api_deployment_views.py +++ b/backend/api_v2/api_deployment_views.py @@ -5,6 +5,13 @@ from django.db.models import F, OuterRef, QuerySet, Subquery from django.http import HttpResponse +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import ( + OpenApiParameter, + extend_schema, + extend_schema_field, + extend_schema_view, +) from permissions.membership_views import OwnerManagementMixin from permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrg from permissions.resource_share_views import ResourceShareManagementMixin @@ -50,6 +57,97 @@ logger = logging.getLogger(__name__) +@extend_schema_field(OpenApiTypes.BINARY) +class UploadField(serializers.FileField): + """A bare ``FileField`` maps to ``format: uri`` — correct on output, wrong + for a multipart upload, and generators emit ``str`` for it. + """ + + +class ExecuteRequest(ExecutionRequestSerializer): + """Subclasses the real serializer so every backend param arrives free.""" + + # ``files`` arrives via ``request.FILES``, so no serializer declares it. + files = serializers.ListField(child=UploadField(), required=False) + + +class FileResult(serializers.Serializer): + file = serializers.CharField() + file_execution_id = serializers.CharField(required=False) + status = serializers.CharField(required=False) + result = serializers.JSONField(required=False) + metadata = serializers.JSONField(required=False) + metrics = serializers.JSONField(required=False) + error = serializers.CharField(required=False, allow_null=True) + + +class ExecutionMessage(serializers.Serializer): + execution_status = serializers.CharField() + execution_id = serializers.CharField(required=False) + workflow_id = serializers.CharField(required=False) + status_api = serializers.CharField(required=False, allow_null=True) + error = serializers.CharField(required=False, allow_null=True) + # The backend sends `result: null` while pending; without allow_null the + # generated deserialiser iterates None and crashes. + result = FileResult(many=True, required=False, allow_null=True) + + +class ExecuteResponse(serializers.Serializer): + message = ExecutionMessage() + + +class StatusResponse(serializers.Serializer): + status = serializers.CharField() + message = FileResult(many=True, required=False, allow_null=True) + + +class ErrorResponse(serializers.Serializer): + status = serializers.CharField(required=False) + message = serializers.JSONField(required=False, allow_null=True) + + +DEPLOYMENT_PATH_PARAMETERS = [ + OpenApiParameter( + "org_name", + str, + OpenApiParameter.PATH, + description="Organization identifier.", + ), + OpenApiParameter( + "api_name", str, OpenApiParameter.PATH, description="API deployment name." + ), +] + + +# The generated clients take their command names, module paths and request +# shapes from here, so this block is part of the public API surface. +@extend_schema_view( + post=extend_schema( + operation_id="execute", + tags=["deployment"], + parameters=DEPLOYMENT_PATH_PARAMETERS, + request={"multipart/form-data": ExecuteRequest}, + responses={ + 200: ExecuteResponse, + 422: ExecuteResponse, + 500: ErrorResponse, + }, + description="Execute an API deployment against one or more files.", + ), + get=extend_schema( + operation_id="status", + tags=["deployment"], + parameters=DEPLOYMENT_PATH_PARAMETERS + [ExecutionQuerySerializer], + # 406 means the result was already consumed — this GET is one-shot. + responses={ + 200: StatusResponse, + 406: ErrorResponse, + 422: StatusResponse, + 500: ErrorResponse, + }, + description="Poll the status of a previously started execution.", + ), +) class DeploymentExecution(views.APIView): def initialize_request(self, request: Request, *args: Any, **kwargs: Any) -> Request: """To remove csrf request for public API. diff --git a/backend/api_v2/deployment_spec_urls.py b/backend/api_v2/deployment_spec_urls.py new file mode 100644 index 0000000000..f61d714d2c --- /dev/null +++ b/backend/api_v2/deployment_spec_urls.py @@ -0,0 +1,13 @@ +"""URLconf used only to generate the API deployment OpenAPI spec. + +``api_v2.execution_urls`` is an included sub-urlconf, so generating against it +directly yields paths without the prefix it is mounted at — a spec describing +URLs the server does not serve. This mirrors the mount in ``base_urls``. +""" + +from django.conf import settings +from django.urls import include, path + +urlpatterns = [ + path(f"{settings.API_DEPLOYMENT_PATH_PREFIX}/", include("api_v2.execution_urls")) +] diff --git a/backend/api_v2/management/commands/generate_docstudio_spec.py b/backend/api_v2/management/commands/generate_docstudio_spec.py new file mode 100644 index 0000000000..601d59506c --- /dev/null +++ b/backend/api_v2/management/commands/generate_docstudio_spec.py @@ -0,0 +1,60 @@ +"""Regenerate the committed API deployment OpenAPI spec. + +The spec is the contract the published clients and their generated SDKs are +built from, so it is committed and CI fails on drift: change a route, a +serializer or the schema annotation, and regenerate in the same PR. + + python manage.py generate_docstudio_spec + python manage.py generate_docstudio_spec --check # CI: no write, drift is an error +""" + +import json +from pathlib import Path +from typing import Any + +from django.core.management.base import BaseCommand, CommandError +from drf_spectacular.generators import SchemaGenerator + +DEFAULT_OUT = Path(__file__).resolve().parents[4] / "specs" / "docstudio-oss.json" +URLCONF = "api_v2.deployment_spec_urls" + + +class Command(BaseCommand): + help = "Generate the API deployment OpenAPI spec." + + def add_arguments(self, parser: Any) -> None: + parser.add_argument("--out", type=Path, default=DEFAULT_OUT) + parser.add_argument( + "--check", + action="store_true", + help="Fail if the file on disk differs, instead of writing it.", + ) + + def handle(self, *args: Any, **options: Any) -> None: + schema = SchemaGenerator(urlconf=URLCONF).get_schema(request=None, public=True) + # Sorted keys are what make the committed artifact a usable drift signal. + rendered = json.dumps(schema, indent=2, sort_keys=True) + "\n" + + out: Path = options["out"] + if options["check"]: + current = out.read_text() if out.exists() else "" + if current != rendered: + raise CommandError( + f"{out} is out of date. Run `python manage.py " + f"generate_docstudio_spec` and commit the result." + ) + self.stdout.write(f"{out} is up to date") + return + + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(rendered) + operations = sum( + 1 + for methods in schema["paths"].values() + for method in methods + if method in {"get", "post", "put", "patch", "delete"} + ) + self.stdout.write( + f"{out}: {len(schema['paths'])} paths, {operations} operations, " + f"{len(schema.get('components', {}).get('schemas', {}))} schemas" + ) diff --git a/backend/api_v2/tests/test_docstudio_spec.py b/backend/api_v2/tests/test_docstudio_spec.py new file mode 100644 index 0000000000..602d8b5c9f --- /dev/null +++ b/backend/api_v2/tests/test_docstudio_spec.py @@ -0,0 +1,38 @@ +"""The committed spec is the contract the published clients are generated from. + +A route, serializer or schema-annotation change that is not regenerated ships a +spec describing an API the server no longer serves, so drift fails here rather +than in a client repo. +""" + +import json +from pathlib import Path + +from drf_spectacular.generators import SchemaGenerator + +from api_v2.management.commands.generate_docstudio_spec import DEFAULT_OUT, URLCONF + + +def _render() -> str: + schema = SchemaGenerator(urlconf=URLCONF).get_schema(request=None, public=True) + return json.dumps(schema, indent=2, sort_keys=True) + "\n" + + +def test_committed_spec_matches_the_code() -> None: + assert DEFAULT_OUT.exists(), f"{DEFAULT_OUT} is missing" + assert DEFAULT_OUT.read_text() == _render(), ( + f"{DEFAULT_OUT} is out of date. Run " + "`python manage.py generate_docstudio_spec` and commit the result." + ) + + +def test_spec_covers_the_deployment_routes() -> None: + """Guards the mount: generating against the included sub-urlconf silently + drops the prefix, leaving paths the server does not serve.""" + spec = json.loads(Path(DEFAULT_OUT).read_text()) + execute = "/deployment/api/{org_name}/{api_name}/" + + assert set(spec["paths"]) == {execute, f"{execute}mcp/"} + assert spec["paths"][execute]["post"]["operationId"] == "execute" + assert spec["paths"][execute]["get"]["operationId"] == "status" + assert [tag["name"] for tag in spec["tags"]] == ["deployment"] diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index d14f87b304..bbc6b440d4 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -653,6 +653,27 @@ def filter(self, record): "DEFAULT_VERSION": "v1", "ALLOWED_VERSIONS": ["v1"], "VERSION_PARAM": "version", + "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", +} + +# Read only while generating the API deployment OpenAPI spec +# (``manage.py generate_docstudio_spec``); no effect at request time. +SPECTACULAR_SETTINGS = { + "TITLE": "Unstract Document Studio", + "VERSION": "v1", + "PREPROCESSING_HOOKS": ["drf_spectacular.hooks.preprocess_exclude_path_format"], + "SERVE_INCLUDE_SCHEMA": False, + # Group descriptions clients show in their help; without this the spec has + # no root `tags` array and the text has nowhere to live. + "TAGS": [ + { + "name": "deployment", + "description": ( + "Run an API deployment against one or more documents and poll " + "the result." + ), + } + ], } # These paths will work without authentication diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 7eee250b9b..005906884b 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -27,6 +27,7 @@ dependencies = [ "django-redis==5.4.0", "django-tenants==3.5.0", "drf-standardized-errors>=0.12.6", + "drf-spectacular>=0.28.0", # For the generated API deployment OpenAPI spec "drf-yasg>=1.21.8", # For API docs "psycopg2-binary==2.9.9", "python-dotenv==1.2.2", diff --git a/backend/uv.lock b/backend/uv.lock index c80f0ecea6..0c94322123 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -900,6 +900,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/db/be/3032490fa33b36ddc8c4b1da3252c6f974e7133f1a50de00c6b85cca203a/docker-6.1.3-py3-none-any.whl", hash = "sha256:aecd2277b8bf8e506e484f6ab7aec39abe0038e29fa4a6d3ba86c3fe01844ed9", size = 148096, upload-time = "2023-06-01T14:24:47.769Z" }, ] +[[package]] +name = "drf-spectacular" +version = "0.30.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "django" }, + { name = "djangorestframework" }, + { name = "inflection" }, + { name = "jsonschema" }, + { name = "pyyaml" }, + { name = "uritemplate" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/43/41d25039a6a53545420ebc98eb9f877ec9fe30c7bd03fefabcaf9b953af7/drf_spectacular-0.30.0.tar.gz", hash = "sha256:53e79e7ba00e240441b63c32273754a5368e4c2ab44a19f2595277cc1cd559c9", size = 252311, upload-time = "2026-07-06T11:29:46.264Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/56/74dd7b45bbde6d24494220b98d6961cb1200b63a1800332b430daa2c4551/drf_spectacular-0.30.0-py3-none-any.whl", hash = "sha256:006cf5921ebe20a9bd24f7c846261ebbf78780be5961b0d6e87afaa82afd62ff", size = 111150, upload-time = "2026-07-06T11:29:45.12Z" }, +] + [[package]] name = "drf-standardized-errors" version = "0.15.0" @@ -3688,6 +3705,7 @@ dependencies = [ { name = "django-redis" }, { name = "django-tenants" }, { name = "djangorestframework" }, + { name = "drf-spectacular" }, { name = "drf-standardized-errors" }, { name = "drf-yasg" }, { name = "google-cloud-recaptcha-enterprise" }, @@ -3755,6 +3773,7 @@ requires-dist = [ { name = "django-redis", specifier = "==5.4.0" }, { name = "django-tenants", specifier = "==3.5.0" }, { name = "djangorestframework", specifier = "==3.17.1" }, + { name = "drf-spectacular", specifier = ">=0.28.0" }, { name = "drf-standardized-errors", specifier = ">=0.12.6" }, { name = "drf-yasg", specifier = ">=1.21.8" }, { name = "google-cloud-recaptcha-enterprise", specifier = ">=1.28.2" }, diff --git a/specs/docstudio-oss.json b/specs/docstudio-oss.json new file mode 100644 index 0000000000..424b30ff80 --- /dev/null +++ b/specs/docstudio-oss.json @@ -0,0 +1,442 @@ +{ + "components": { + "schemas": { + "ErrorResponse": { + "properties": { + "message": { + "nullable": true + }, + "status": { + "type": "string" + } + }, + "type": "object" + }, + "ExecuteRequest": { + "description": "Subclasses the real serializer so every backend param arrives free.", + "properties": { + "custom_data": { + "nullable": true + }, + "files": { + "items": { + "format": "binary", + "type": "string" + }, + "type": "array" + }, + "hitl_packet_id": { + "nullable": true, + "type": "string" + }, + "hitl_queue_name": { + "nullable": true, + "type": "string" + }, + "include_extracted_text": { + "default": false, + "type": "boolean" + }, + "include_metadata": { + "default": false, + "type": "boolean" + }, + "include_metrics": { + "default": false, + "type": "boolean" + }, + "llm_profile_id": { + "nullable": true, + "type": "string" + }, + "presigned_urls": { + "items": { + "format": "uri", + "type": "string" + }, + "type": "array" + }, + "tags": { + "default": "", + "description": "Comma-separated list of tag names (EX:'tag1,tag2-name,tag3_name')", + "type": "string" + }, + "timeout": { + "default": -1, + "maximum": 300, + "minimum": -1, + "type": "integer" + }, + "use_file_history": { + "default": false, + "type": "boolean" + } + }, + "type": "object" + }, + "ExecuteResponse": { + "properties": { + "message": { + "$ref": "#/components/schemas/ExecutionMessage" + } + }, + "required": [ + "message" + ], + "type": "object" + }, + "ExecutionMessage": { + "properties": { + "error": { + "nullable": true, + "type": "string" + }, + "execution_id": { + "type": "string" + }, + "execution_status": { + "type": "string" + }, + "result": { + "items": { + "$ref": "#/components/schemas/FileResult" + }, + "nullable": true, + "type": "array" + }, + "status_api": { + "nullable": true, + "type": "string" + }, + "workflow_id": { + "type": "string" + } + }, + "required": [ + "execution_status" + ], + "type": "object" + }, + "FileResult": { + "properties": { + "error": { + "nullable": true, + "type": "string" + }, + "file": { + "type": "string" + }, + "file_execution_id": { + "type": "string" + }, + "metadata": {}, + "metrics": {}, + "result": {}, + "status": { + "type": "string" + } + }, + "required": [ + "file" + ], + "type": "object" + }, + "StatusResponse": { + "properties": { + "message": { + "items": { + "$ref": "#/components/schemas/FileResult" + }, + "nullable": true, + "type": "array" + }, + "status": { + "type": "string" + } + }, + "required": [ + "status" + ], + "type": "object" + } + }, + "securitySchemes": { + "basicAuth": { + "scheme": "basic", + "type": "http" + }, + "cookieAuth": { + "in": "cookie", + "name": "sessionid", + "type": "apiKey" + } + } + }, + "info": { + "title": "Unstract Document Studio", + "version": "v1" + }, + "openapi": "3.0.3", + "paths": { + "/deployment/api/{org_name}/{api_name}/": { + "get": { + "description": "Poll the status of a previously started execution.", + "operationId": "status", + "parameters": [ + { + "description": "API deployment name.", + "in": "path", + "name": "api_name", + "required": true, + "schema": { + "type": "string" + } + }, + { + "in": "query", + "name": "execution_id", + "required": true, + "schema": { + "minLength": 1, + "type": "string" + } + }, + { + "in": "query", + "name": "include_extracted_text", + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "include_metadata", + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "in": "query", + "name": "include_metrics", + "schema": { + "default": false, + "type": "boolean" + } + }, + { + "description": "Organization identifier.", + "in": "path", + "name": "org_name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StatusResponse" + } + } + }, + "description": "" + }, + "406": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/StatusResponse" + } + } + }, + "description": "" + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "" + } + }, + "security": [ + { + "cookieAuth": [] + }, + { + "basicAuth": [] + } + ], + "tags": [ + "deployment" + ] + }, + "post": { + "description": "Execute an API deployment against one or more files.", + "operationId": "execute", + "parameters": [ + { + "description": "API deployment name.", + "in": "path", + "name": "api_name", + "required": true, + "schema": { + "type": "string" + } + }, + { + "description": "Organization identifier.", + "in": "path", + "name": "org_name", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "multipart/form-data": { + "schema": { + "$ref": "#/components/schemas/ExecuteRequest" + } + } + } + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExecuteResponse" + } + } + }, + "description": "" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExecuteResponse" + } + } + }, + "description": "" + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "" + } + }, + "security": [ + { + "cookieAuth": [] + }, + { + "basicAuth": [] + } + ], + "tags": [ + "deployment" + ] + } + }, + "/deployment/api/{org_name}/{api_name}/mcp/": { + "get": { + "description": "Refuse the SSE stream, but say who is here.\n\nUnder Streamable HTTP a client issues GET to open a server-to-client\nSSE stream, and a server that offers none must answer 405 (spec rev\n2025-06-18). Nothing here pushes messages \u2014 every tool call is\nrequest/response \u2014 so 405 is the honest answer, and returning\n``200 application/json`` instead would leave a conformant client\nparsing an identity document as an event stream.\n\nThe body is kept anyway: uptime checks and humans with curl probe this\npath, and a 405 may carry one. It stays deliberately free of tenant\ndetail \u2014 it reveals only that an MCP server is mounted here.\n\n``JsonResponse``, not DRF's ``Response``, for the same reason ``post``\nuses it: a DRF response runs content negotiation, so a client sending\n``Accept: text/html`` would be handed the browsable-API renderer.\n\nNo ``Allow`` header is set here. RFC 9110 asks for one on a 405, but a\nhandler cannot control it and pretending otherwise misleads a reader:\nDRF's ``finalize_response`` overwrites any handler-set value with\n``self.allowed_methods`` (``GET, POST, HEAD, OPTIONS``, since this view\ndefines both verbs), and ``RemoveAllowHeaderMiddleware`` \u2014 global in\n``MIDDLEWARE`` \u2014 then pops the header from every response before it\nleaves the process. So a client sees no ``Allow`` at all; a test driving\nthe view through ``APIRequestFactory`` bypasses that middleware and sees\nDRF's value.", + "operationId": "mcp_retrieve", + "parameters": [ + { + "in": "path", + "name": "api_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + }, + { + "in": "path", + "name": "org_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "No response body" + } + }, + "tags": [ + "mcp" + ] + }, + "post": { + "description": "Handle a single JSON-RPC request.", + "operationId": "mcp_create", + "parameters": [ + { + "in": "path", + "name": "api_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + }, + { + "in": "path", + "name": "org_name", + "required": true, + "schema": { + "pattern": "^[\\w-]+$", + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "No response body" + } + }, + "tags": [ + "mcp" + ] + } + } + }, + "tags": [ + { + "description": "Run an API deployment against one or more documents and poll the result.", + "name": "deployment" + } + ] +} From 482ca1b0d15b26f36b00ca50d0b4fc5debffabeb Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Wed, 12 Aug 2026 17:38:21 +0530 Subject: [PATCH 2/8] fix(api): publish the deployment contract the server actually implements The committed spec is what published clients are generated from, so the places where it disagreed with the server are places every SDK inherits. - Declare the bearer scheme the endpoints enforce. DRF's unset authentication default was being introspected as a decision and published session and basic auth, which these endpoints do not accept. - Declare the failures a caller has to handle (400/401/403/404/409/429) and describe the 406, so a generated client can branch on them. - Derive the response model from the serializer that builds the response, which drops `workflow_id` -- a field no code path produces. - Stop shadowing `files`: the real field carries the binary annotation, so a change to it now moves the spec. - Drop the MCP operations. MCP speaks JSON-RPC over one POST, so it had no REST shape to describe and was published with guessed responses, no security, and an internal docstring as its description. - Say in the shipped text that a status read is one-shot, and that documents may be supplied as files or presigned URLs. The gate had the same blind spots. It now resolves the real mount instead of comparing against a hand-written copy of it, fails when the generator reports a diagnostic instead of certifying its guess, and asserts the auth scheme and error statuses. Verified by mutation: moving the mount, adding a response field, changing the `files` constraint and dropping the auth annotation each redden the suite, and none of them did before. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014f9oEEYspPH4fmPULTnLkJ --- backend/api_v2/api_deployment_views.py | 109 ++++++--- backend/api_v2/deployment_spec_urls.py | 22 +- .../commands/generate_docstudio_spec.py | 51 +++- backend/api_v2/serializers.py | 11 +- backend/api_v2/tests/test_docstudio_spec.py | 83 +++++-- backend/backend/settings/base.py | 18 +- backend/mcp_server/views.py | 5 + backend/pyproject.toml | 4 +- backend/uv.lock | 2 +- specs/docstudio-oss.json | 219 ++++++++++-------- 10 files changed, 372 insertions(+), 152 deletions(-) diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.py index 6fb1e08234..4dce5967b2 100644 --- a/backend/api_v2/api_deployment_views.py +++ b/backend/api_v2/api_deployment_views.py @@ -5,11 +5,11 @@ from django.db.models import F, OuterRef, QuerySet, Subquery from django.http import HttpResponse -from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import ( OpenApiParameter, + OpenApiResponse, extend_schema, - extend_schema_field, + extend_schema_serializer, extend_schema_view, ) from permissions.membership_views import OwnerManagementMixin @@ -44,6 +44,7 @@ from api_v2.serializers import ( APIDeploymentListSerializer, APIDeploymentSerializer, + APIExecutionResponseSerializer, DeploymentResponseSerializer, ExecutionQuerySerializer, ExecutionRequestSerializer, @@ -57,18 +58,16 @@ logger = logging.getLogger(__name__) -@extend_schema_field(OpenApiTypes.BINARY) -class UploadField(serializers.FileField): - """A bare ``FileField`` maps to ``format: uri`` — correct on output, wrong - for a multipart upload, and generators emit ``str`` for it. - """ - - +# Declares no field of its own, so every backend parameter arrives free and a +# change to the real serializer moves the spec. It exists only to carry a +# caller-facing description in place of the implementation docstring, and to +# keep the published model name stable. +@extend_schema_serializer(component_name="ExecuteRequest") class ExecuteRequest(ExecutionRequestSerializer): - """Subclasses the real serializer so every backend param arrives free.""" + """The documents to run, and the options that shape the result. - # ``files`` arrives via ``request.FILES``, so no serializer declares it. - files = serializers.ListField(child=UploadField(), required=False) + Supply `files`, `presigned_urls`, or both. + """ class FileResult(serializers.Serializer): @@ -81,14 +80,19 @@ class FileResult(serializers.Serializer): error = serializers.CharField(required=False, allow_null=True) -class ExecutionMessage(serializers.Serializer): - execution_status = serializers.CharField() - execution_id = serializers.CharField(required=False) - workflow_id = serializers.CharField(required=False) - status_api = serializers.CharField(required=False, allow_null=True) - error = serializers.CharField(required=False, allow_null=True) - # The backend sends `result: null` while pending; without allow_null the - # generated deserialiser iterates None and crashes. +# Subclasses the serializer that builds the response, so a field added or +# removed there moves the spec. Docstrings on these annotation serializers are +# published as the client-facing model description, so they are written for +# the caller rather than the maintainer. +class ExecutionMessage(APIExecutionResponseSerializer): + """The execution's identity and, once it has finished, its per-file + results. + """ + + # The one field that has to be restated: the real declaration is an + # untyped JSONField, which gives generated clients nothing to work with. + # The backend also sends `result: null` while pending, and without + # allow_null the generated deserialiser iterates None and crashes. result = FileResult(many=True, required=False, allow_null=True) @@ -106,46 +110,95 @@ class ErrorResponse(serializers.Serializer): message = serializers.JSONField(required=False, allow_null=True) +# The pattern the route itself enforces, restated so a generated client can +# reject a mistyped identifier without a round trip. +PATH_SEGMENT = {"type": "string", "pattern": r"^[\w-]+$"} + DEPLOYMENT_PATH_PARAMETERS = [ OpenApiParameter( "org_name", - str, + PATH_SEGMENT, OpenApiParameter.PATH, description="Organization identifier.", ), OpenApiParameter( - "api_name", str, OpenApiParameter.PATH, description="API deployment name." + "api_name", + PATH_SEGMENT, + OpenApiParameter.PATH, + description="API deployment name.", ), ] +DEPLOYMENT_AUTH = [{"deploymentKey": []}] + +# Every failure a caller has to handle. Declared explicitly because a client +# generated without them treats an authentication or rate-limit response as an +# unknown status and has nothing to branch on. +DEPLOYMENT_ERRORS = { + 400: OpenApiResponse(ErrorResponse, description="The request failed validation."), + 401: OpenApiResponse(ErrorResponse, description="The API key is not valid."), + 403: OpenApiResponse(ErrorResponse, description="No API key was supplied."), + 404: OpenApiResponse(ErrorResponse, description="No such active deployment."), + 429: OpenApiResponse( + ErrorResponse, description="Too many concurrent executions; retry later." + ), + 500: ErrorResponse, +} + +EXECUTE_DESCRIPTION = ( + "Execute an API deployment against one or more documents.\n\n" + "Supply the documents either as `files` (multipart upload) or as " + "`presigned_urls` (HTTPS S3 URLs), or both — a request carrying neither is " + f"rejected, and the two together may not exceed " + f"{ExecutionRequestSerializer.MAX_FILES_ALLOWED} documents.\n\n" + "With the default `timeout` of -1 the call returns as soon as the " + "execution is queued; read the outcome from the status endpoint." +) + +STATUS_DESCRIPTION = ( + "Read the result of a previously started execution.\n\n" + "This read is one-shot: the first call that observes a completed execution " + "acknowledges it and the stored result is discarded, so every later call " + "for that execution answers 406. Poll while the execution is pending, and " + "keep the payload of the call that returns it — it cannot be fetched again." +) + + # The generated clients take their command names, module paths and request # shapes from here, so this block is part of the public API surface. @extend_schema_view( post=extend_schema( operation_id="execute", tags=["deployment"], + auth=DEPLOYMENT_AUTH, parameters=DEPLOYMENT_PATH_PARAMETERS, request={"multipart/form-data": ExecuteRequest}, responses={ 200: ExecuteResponse, + 409: OpenApiResponse( + ErrorResponse, description="The deployment has no active API key." + ), 422: ExecuteResponse, - 500: ErrorResponse, + **DEPLOYMENT_ERRORS, }, - description="Execute an API deployment against one or more files.", + description=EXECUTE_DESCRIPTION, ), get=extend_schema( operation_id="status", tags=["deployment"], + auth=DEPLOYMENT_AUTH, parameters=DEPLOYMENT_PATH_PARAMETERS + [ExecutionQuerySerializer], - # 406 means the result was already consumed — this GET is one-shot. responses={ 200: StatusResponse, - 406: ErrorResponse, + 406: OpenApiResponse( + ErrorResponse, + description="The result was already consumed by an earlier call.", + ), 422: StatusResponse, - 500: ErrorResponse, + **DEPLOYMENT_ERRORS, }, - description="Poll the status of a previously started execution.", + description=STATUS_DESCRIPTION, ), ) class DeploymentExecution(views.APIView): diff --git a/backend/api_v2/deployment_spec_urls.py b/backend/api_v2/deployment_spec_urls.py index f61d714d2c..0aa2d998bd 100644 --- a/backend/api_v2/deployment_spec_urls.py +++ b/backend/api_v2/deployment_spec_urls.py @@ -2,12 +2,26 @@ ``api_v2.execution_urls`` is an included sub-urlconf, so generating against it directly yields paths without the prefix it is mounted at — a spec describing -URLs the server does not serve. This mirrors the mount in ``base_urls``. +URLs the server does not serve. The mount is selected out of the served +urlconf rather than restated here, so a change to where the deployment API is +mounted moves the generated paths with it. """ -from django.conf import settings -from django.urls import include, path +from django.core.exceptions import ImproperlyConfigured + +from backend import base_urls + +DEPLOYMENT_URLCONF = "api_v2.execution_urls" urlpatterns = [ - path(f"{settings.API_DEPLOYMENT_PATH_PREFIX}/", include("api_v2.execution_urls")) + entry + for entry in base_urls.urlpatterns + if getattr(getattr(entry, "urlconf_name", None), "__name__", None) + == DEPLOYMENT_URLCONF ] + +if not urlpatterns: + raise ImproperlyConfigured( + f"{DEPLOYMENT_URLCONF} is not mounted in backend.base_urls; the API " + "deployment spec would be generated for no routes at all." + ) diff --git a/backend/api_v2/management/commands/generate_docstudio_spec.py b/backend/api_v2/management/commands/generate_docstudio_spec.py index 601d59506c..a091073a39 100644 --- a/backend/api_v2/management/commands/generate_docstudio_spec.py +++ b/backend/api_v2/management/commands/generate_docstudio_spec.py @@ -4,8 +4,12 @@ built from, so it is committed and CI fails on drift: change a route, a serializer or the schema annotation, and regenerate in the same PR. - python manage.py generate_docstudio_spec - python manage.py generate_docstudio_spec --check # CI: no write, drift is an error + uv run python manage.py generate_docstudio_spec # from backend/ + uv run python manage.py generate_docstudio_spec --check # no write, drift is an error + +The generated paths carry ``API_DEPLOYMENT_PATH_PREFIX``, so regenerate in an +environment that does not override it — the committed artifact describes the +deployment as it is served publicly, not as one installation mounts it. """ import json @@ -13,10 +17,44 @@ from typing import Any from django.core.management.base import BaseCommand, CommandError +from drf_spectacular.drainage import GENERATOR_STATS from drf_spectacular.generators import SchemaGenerator DEFAULT_OUT = Path(__file__).resolve().parents[4] / "specs" / "docstudio-oss.json" URLCONF = "api_v2.deployment_spec_urls" +REGENERATE = "uv run python manage.py generate_docstudio_spec" + + +class SpecGenerationFailed(CommandError): + """Raised when the generator had to guess.""" + + +def render_spec() -> str: + """The committed artifact, byte for byte. + + Shared with the drift test: two copies of this could disagree, and then + the gate rejects exactly the file the command it names produces. + """ + GENERATOR_STATS.reset() + schema = SchemaGenerator(urlconf=URLCONF).get_schema(request=None, public=True) + if GENERATOR_STATS: + # spectacular downgrades "unable to guess serializer" to a warning and + # writes a plausible, wrong operation. Nothing downstream can tell that + # apart from an annotation that is simply thin. + diagnostics = "\n".join( + f" {severity}: {message}" + for severity, cache in ( + ("error", GENERATOR_STATS._error_cache), + ("warning", GENERATOR_STATS._warn_cache), + ) + for message in cache + ) + raise SpecGenerationFailed( + f"The generator reported problems, so the spec would describe an " + f"API nobody implements:\n{diagnostics}" + ) + # Sorted keys are what make the committed artifact a usable drift signal. + return json.dumps(schema, indent=2, sort_keys=True) + "\n" class Command(BaseCommand): @@ -31,23 +69,22 @@ def add_arguments(self, parser: Any) -> None: ) def handle(self, *args: Any, **options: Any) -> None: - schema = SchemaGenerator(urlconf=URLCONF).get_schema(request=None, public=True) - # Sorted keys are what make the committed artifact a usable drift signal. - rendered = json.dumps(schema, indent=2, sort_keys=True) + "\n" + rendered = render_spec() out: Path = options["out"] if options["check"]: current = out.read_text() if out.exists() else "" if current != rendered: raise CommandError( - f"{out} is out of date. Run `python manage.py " - f"generate_docstudio_spec` and commit the result." + f"{out} is out of date. Run `{REGENERATE}` from `backend/` " + "and commit the result." ) self.stdout.write(f"{out} is up to date") return out.parent.mkdir(parents=True, exist_ok=True) out.write_text(rendered) + schema = json.loads(rendered) operations = sum( 1 for methods in schema["paths"].values() diff --git a/backend/api_v2/serializers.py b/backend/api_v2/serializers.py index 3db7f53db6..e376cd401b 100644 --- a/backend/api_v2/serializers.py +++ b/backend/api_v2/serializers.py @@ -6,6 +6,8 @@ from django.apps import apps from django.core.validators import RegexValidator +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import extend_schema_field from pipeline_v2.models import Pipeline from prompt_studio.prompt_profile_manager_v2.models import ProfileManager from rest_framework import serializers @@ -218,6 +220,13 @@ def to_representation(self, instance: APIKey) -> OrderedDict[str, Any]: return representation +@extend_schema_field(OpenApiTypes.BINARY) +class UploadField(FileField): + """A bare ``FileField`` maps to ``format: uri`` -- correct on output, wrong + for a multipart upload, and generators emit ``str`` for it. + """ + + class ExecutionRequestSerializer(TagParamsSerializer): """Execution request serializer. @@ -320,7 +329,7 @@ def validate_custom_data(self, value): return value files = ListField( - child=FileField(), + child=UploadField(), required=False, allow_empty=True, ) diff --git a/backend/api_v2/tests/test_docstudio_spec.py b/backend/api_v2/tests/test_docstudio_spec.py index 602d8b5c9f..b94f647450 100644 --- a/backend/api_v2/tests/test_docstudio_spec.py +++ b/backend/api_v2/tests/test_docstudio_spec.py @@ -6,33 +6,82 @@ """ import json -from pathlib import Path -from drf_spectacular.generators import SchemaGenerator +from django.urls import reverse +from drf_spectacular.drainage import GENERATOR_STATS -from api_v2.management.commands.generate_docstudio_spec import DEFAULT_OUT, URLCONF +from api_v2.management.commands.generate_docstudio_spec import ( + DEFAULT_OUT, + REGENERATE, + render_spec, +) -def _render() -> str: - schema = SchemaGenerator(urlconf=URLCONF).get_schema(request=None, public=True) - return json.dumps(schema, indent=2, sort_keys=True) + "\n" +def _committed() -> dict: + return json.loads(DEFAULT_OUT.read_text()) def test_committed_spec_matches_the_code() -> None: assert DEFAULT_OUT.exists(), f"{DEFAULT_OUT} is missing" - assert DEFAULT_OUT.read_text() == _render(), ( - f"{DEFAULT_OUT} is out of date. Run " - "`python manage.py generate_docstudio_spec` and commit the result." + assert DEFAULT_OUT.read_text() == render_spec(), ( + f"{DEFAULT_OUT} is out of date. Run `{REGENERATE}` from `backend/` and " + "commit the result." ) -def test_spec_covers_the_deployment_routes() -> None: - """Guards the mount: generating against the included sub-urlconf silently - drops the prefix, leaving paths the server does not serve.""" - spec = json.loads(Path(DEFAULT_OUT).read_text()) - execute = "/deployment/api/{org_name}/{api_name}/" +def test_generation_reports_no_diagnostics() -> None: + """A warned-about operation is published with guessed request and response + shapes, and the drift comparison certifies the guess.""" + render_spec() + assert not GENERATOR_STATS._error_cache + assert not GENERATOR_STATS._warn_cache - assert set(spec["paths"]) == {execute, f"{execute}mcp/"} - assert spec["paths"][execute]["post"]["operationId"] == "execute" - assert spec["paths"][execute]["get"]["operationId"] == "status" + +def test_spec_paths_are_the_urls_the_server_serves() -> None: + """Resolves the real mount rather than restating it: a spec generated for + URLs the server does not serve is the failure this file exists to catch.""" + served = reverse( + "api_deployment_execution", kwargs={"org_name": "ORG", "api_name": "API"} + ) + documented = [ + path.replace("{org_name}", "ORG").replace("{api_name}", "API").rstrip("/") + for path in _committed()["paths"] + ] + + assert documented == [served.rstrip("/")] + + +def test_spec_documents_the_deployment_operations() -> None: + spec = _committed() + (operations,) = spec["paths"].values() + + assert operations["post"]["operationId"] == "execute" + assert operations["get"]["operationId"] == "status" assert [tag["name"] for tag in spec["tags"]] == ["deployment"] + + +def test_operations_require_the_deployment_key() -> None: + """Without this the unset DRF authentication default is published as + though it were a decision, and no generated client can authenticate.""" + spec = _committed() + scheme = spec["components"]["securitySchemes"]["deploymentKey"] + + assert (scheme["type"], scheme["scheme"]) == ("http", "bearer") + for operation in spec["paths"].values(): + for method in ("get", "post"): + assert operation[method]["security"] == [{"deploymentKey": []}] + + +def test_clients_can_branch_on_every_failure_they_will_see() -> None: + for method in ("get", "post"): + (operation,) = (ops[method] for ops in _committed()["paths"].values()) + assert {"400", "401", "403", "404", "429", "500"} <= set(operation["responses"]) + + +def test_the_one_shot_read_is_documented_where_a_client_will_see_it() -> None: + """The semantics that a status read destroys the result must reach the + generated client, not live in a source comment.""" + (status_op,) = (ops["get"] for ops in _committed()["paths"].values()) + + assert "one-shot" in status_op["description"] + assert status_op["responses"]["406"]["description"].strip() diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index bbc6b440d4..a00c2fcc5c 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -656,13 +656,27 @@ def filter(self, record): "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", } -# Read only while generating the API deployment OpenAPI spec -# (``manage.py generate_docstudio_spec``); no effect at request time. +# Read while generating the API deployment OpenAPI spec +# (``manage.py generate_docstudio_spec``). ``DEFAULT_SCHEMA_CLASS`` above is a +# project-wide DRF default, but DRF dereferences it only when a schema is +# generated, so neither has an effect at request time. SPECTACULAR_SETTINGS = { "TITLE": "Unstract Document Studio", "VERSION": "v1", "PREPROCESSING_HOOKS": ["drf_spectacular.hooks.preprocess_exclude_path_format"], "SERVE_INCLUDE_SCHEMA": False, + # DRF's unset ``DEFAULT_AUTHENTICATION_CLASSES`` would otherwise be + # introspected as a decision and publish session and basic auth, which + # these endpoints do not accept. + "APPEND_COMPONENTS": { + "securitySchemes": { + "deploymentKey": { + "type": "http", + "scheme": "bearer", + "description": "The API deployment's own key.", + } + } + }, # Group descriptions clients show in their help; without this the spec has # no root `tags` array and the text has nowhere to live. "TAGS": [ diff --git a/backend/mcp_server/views.py b/backend/mcp_server/views.py index 53ef1f4781..7c85f5d237 100644 --- a/backend/mcp_server/views.py +++ b/backend/mcp_server/views.py @@ -14,6 +14,7 @@ from typing import Any from api_v2.deployment_helper import DeploymentHelper +from drf_spectacular.utils import extend_schema from rest_framework.request import Request from mcp_server.context import MCPContext @@ -23,6 +24,10 @@ logger = logging.getLogger(__name__) +# MCP speaks JSON-RPC over one POST, so it has no REST surface worth +# describing; leaving it in would publish guessed request and response shapes +# to every client generated from the spec. +@extend_schema(exclude=True) class MCPServerView(BaseMCPView): """MCP JSON-RPC endpoint for a single API deployment. diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 005906884b..1cd74ecdcb 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -27,7 +27,9 @@ dependencies = [ "django-redis==5.4.0", "django-tenants==3.5.0", "drf-standardized-errors>=0.12.6", - "drf-spectacular>=0.28.0", # For the generated API deployment OpenAPI spec + # Pinned: its rendering is the committed spec, so an upgrade rewrites the + # contract published clients are generated from. + "drf-spectacular==0.30.0", "drf-yasg>=1.21.8", # For API docs "psycopg2-binary==2.9.9", "python-dotenv==1.2.2", diff --git a/backend/uv.lock b/backend/uv.lock index 0c94322123..52bd298262 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -3773,7 +3773,7 @@ requires-dist = [ { name = "django-redis", specifier = "==5.4.0" }, { name = "django-tenants", specifier = "==3.5.0" }, { name = "djangorestframework", specifier = "==3.17.1" }, - { name = "drf-spectacular", specifier = ">=0.28.0" }, + { name = "drf-spectacular", specifier = "==0.30.0" }, { name = "drf-standardized-errors", specifier = ">=0.12.6" }, { name = "drf-yasg", specifier = ">=1.21.8" }, { name = "google-cloud-recaptcha-enterprise", specifier = ">=1.28.2" }, diff --git a/specs/docstudio-oss.json b/specs/docstudio-oss.json index 424b30ff80..265b7fac10 100644 --- a/specs/docstudio-oss.json +++ b/specs/docstudio-oss.json @@ -13,7 +13,7 @@ "type": "object" }, "ExecuteRequest": { - "description": "Subclasses the real serializer so every backend param arrives free.", + "description": "The documents to run, and the options that shape the result.\n\nSupply `files`, `presigned_urls`, or both.", "properties": { "custom_data": { "nullable": true @@ -86,9 +86,9 @@ "type": "object" }, "ExecutionMessage": { + "description": "The execution's identity and, once it has finished, its per-file\nresults.", "properties": { "error": { - "nullable": true, "type": "string" }, "execution_id": { @@ -105,15 +105,14 @@ "type": "array" }, "status_api": { - "nullable": true, - "type": "string" - }, - "workflow_id": { "type": "string" } }, "required": [ - "execution_status" + "error", + "execution_id", + "execution_status", + "status_api" ], "type": "object" }, @@ -161,14 +160,10 @@ } }, "securitySchemes": { - "basicAuth": { - "scheme": "basic", + "deploymentKey": { + "description": "The API deployment's own key.", + "scheme": "bearer", "type": "http" - }, - "cookieAuth": { - "in": "cookie", - "name": "sessionid", - "type": "apiKey" } } }, @@ -180,7 +175,7 @@ "paths": { "/deployment/api/{org_name}/{api_name}/": { "get": { - "description": "Poll the status of a previously started execution.", + "description": "Read the result of a previously started execution.\n\nThis read is one-shot: the first call that observes a completed execution acknowledges it and the stored result is discarded, so every later call for that execution answers 406. Poll while the execution is pending, and keep the payload of the call that returns it \u2014 it cannot be fetched again.", "operationId": "status", "parameters": [ { @@ -189,6 +184,7 @@ "name": "api_name", "required": true, "schema": { + "pattern": "^[\\w-]+$", "type": "string" } }, @@ -231,6 +227,7 @@ "name": "org_name", "required": true, "schema": { + "pattern": "^[\\w-]+$", "type": "string" } } @@ -246,6 +243,46 @@ }, "description": "" }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The request failed validation." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The API key is not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No API key was supplied." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No such active deployment." + }, "406": { "content": { "application/json": { @@ -254,7 +291,7 @@ } } }, - "description": "" + "description": "The result was already consumed by an earlier call." }, "422": { "content": { @@ -266,6 +303,16 @@ }, "description": "" }, + "429": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Too many concurrent executions; retry later." + }, "500": { "content": { "application/json": { @@ -279,10 +326,7 @@ }, "security": [ { - "cookieAuth": [] - }, - { - "basicAuth": [] + "deploymentKey": [] } ], "tags": [ @@ -290,7 +334,7 @@ ] }, "post": { - "description": "Execute an API deployment against one or more files.", + "description": "Execute an API deployment against one or more documents.\n\nSupply the documents either as `files` (multipart upload) or as `presigned_urls` (HTTPS S3 URLs), or both \u2014 a request carrying neither is rejected, and the two together may not exceed 32 documents.\n\nWith the default `timeout` of -1 the call returns as soon as the execution is queued; read the outcome from the status endpoint.", "operationId": "execute", "parameters": [ { @@ -299,6 +343,7 @@ "name": "api_name", "required": true, "schema": { + "pattern": "^[\\w-]+$", "type": "string" } }, @@ -308,6 +353,7 @@ "name": "org_name", "required": true, "schema": { + "pattern": "^[\\w-]+$", "type": "string" } } @@ -332,6 +378,56 @@ }, "description": "" }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The request failed validation." + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The API key is not valid." + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No API key was supplied." + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "No such active deployment." + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "The deployment has no active API key." + }, "422": { "content": { "application/json": { @@ -342,6 +438,16 @@ }, "description": "" }, + "429": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Too many concurrent executions; retry later." + }, "500": { "content": { "application/json": { @@ -355,82 +461,13 @@ }, "security": [ { - "cookieAuth": [] - }, - { - "basicAuth": [] + "deploymentKey": [] } ], "tags": [ "deployment" ] } - }, - "/deployment/api/{org_name}/{api_name}/mcp/": { - "get": { - "description": "Refuse the SSE stream, but say who is here.\n\nUnder Streamable HTTP a client issues GET to open a server-to-client\nSSE stream, and a server that offers none must answer 405 (spec rev\n2025-06-18). Nothing here pushes messages \u2014 every tool call is\nrequest/response \u2014 so 405 is the honest answer, and returning\n``200 application/json`` instead would leave a conformant client\nparsing an identity document as an event stream.\n\nThe body is kept anyway: uptime checks and humans with curl probe this\npath, and a 405 may carry one. It stays deliberately free of tenant\ndetail \u2014 it reveals only that an MCP server is mounted here.\n\n``JsonResponse``, not DRF's ``Response``, for the same reason ``post``\nuses it: a DRF response runs content negotiation, so a client sending\n``Accept: text/html`` would be handed the browsable-API renderer.\n\nNo ``Allow`` header is set here. RFC 9110 asks for one on a 405, but a\nhandler cannot control it and pretending otherwise misleads a reader:\nDRF's ``finalize_response`` overwrites any handler-set value with\n``self.allowed_methods`` (``GET, POST, HEAD, OPTIONS``, since this view\ndefines both verbs), and ``RemoveAllowHeaderMiddleware`` \u2014 global in\n``MIDDLEWARE`` \u2014 then pops the header from every response before it\nleaves the process. So a client sees no ``Allow`` at all; a test driving\nthe view through ``APIRequestFactory`` bypasses that middleware and sees\nDRF's value.", - "operationId": "mcp_retrieve", - "parameters": [ - { - "in": "path", - "name": "api_name", - "required": true, - "schema": { - "pattern": "^[\\w-]+$", - "type": "string" - } - }, - { - "in": "path", - "name": "org_name", - "required": true, - "schema": { - "pattern": "^[\\w-]+$", - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "No response body" - } - }, - "tags": [ - "mcp" - ] - }, - "post": { - "description": "Handle a single JSON-RPC request.", - "operationId": "mcp_create", - "parameters": [ - { - "in": "path", - "name": "api_name", - "required": true, - "schema": { - "pattern": "^[\\w-]+$", - "type": "string" - } - }, - { - "in": "path", - "name": "org_name", - "required": true, - "schema": { - "pattern": "^[\\w-]+$", - "type": "string" - } - } - ], - "responses": { - "200": { - "description": "No response body" - } - }, - "tags": [ - "mcp" - ] - } } }, "tags": [ From 86761cc3b3d228151efe4476f61e77e6901dfd54 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Wed, 12 Aug 2026 18:28:25 +0530 Subject: [PATCH 3/8] Name the spec after the API, not one product The deployment endpoints are the public API surface, and the generated clients carry this title into their own documentation. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014f9oEEYspPH4fmPULTnLkJ --- backend/backend/settings/base.py | 2 +- specs/docstudio-oss.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index a00c2fcc5c..cf21bd3e1e 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -661,7 +661,7 @@ def filter(self, record): # project-wide DRF default, but DRF dereferences it only when a schema is # generated, so neither has an effect at request time. SPECTACULAR_SETTINGS = { - "TITLE": "Unstract Document Studio", + "TITLE": "Unstract API", "VERSION": "v1", "PREPROCESSING_HOOKS": ["drf_spectacular.hooks.preprocess_exclude_path_format"], "SERVE_INCLUDE_SCHEMA": False, diff --git a/specs/docstudio-oss.json b/specs/docstudio-oss.json index 265b7fac10..edf3196660 100644 --- a/specs/docstudio-oss.json +++ b/specs/docstudio-oss.json @@ -168,7 +168,7 @@ } }, "info": { - "title": "Unstract Document Studio", + "title": "Unstract API", "version": "v1" }, "openapi": "3.0.3", From 05413c16f3536e2411fbdc50a8f910b3308179b8 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Thu, 13 Aug 2026 17:46:46 +0530 Subject: [PATCH 4/8] test: assert over every documented operation, not exactly one The spec describes one endpoint today, and five of these tests read it by unpacking a single item or by indexing get and post directly. The first endpoint added turns all five red for no reason, and a GET-only one raises KeyError. They now walk whatever the spec documents. The drift and diagnostics gates are unchanged. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014f9oEEYspPH4fmPULTnLkJ --- backend/api_v2/tests/test_docstudio_spec.py | 59 +++++++++++++++------ 1 file changed, 43 insertions(+), 16 deletions(-) diff --git a/backend/api_v2/tests/test_docstudio_spec.py b/backend/api_v2/tests/test_docstudio_spec.py index b94f647450..50b0e99cad 100644 --- a/backend/api_v2/tests/test_docstudio_spec.py +++ b/backend/api_v2/tests/test_docstudio_spec.py @@ -7,7 +7,7 @@ import json -from django.urls import reverse +from django.urls import resolve, reverse from drf_spectacular.drainage import GENERATOR_STATS from api_v2.management.commands.generate_docstudio_spec import ( @@ -17,10 +17,29 @@ ) +#: Keys under a path item that are operations. The rest -- `parameters`, +#: `summary`, vendor extensions -- describe the path, not a call. +_METHODS = ("get", "put", "post", "delete", "options", "head", "patch", "trace") + + def _committed() -> dict: return json.loads(DEFAULT_OUT.read_text()) +def _operations(spec: dict) -> list[tuple[str, str, dict]]: + """Every (path, method, operation) the spec documents. + + The spec grows an endpoint at a time, and a check written against exactly + one of them fails on the next addition without anything being wrong. + """ + return [ + (path, method, operation) + for path, path_item in spec["paths"].items() + for method, operation in path_item.items() + if method in _METHODS + ] + + def test_committed_spec_matches_the_code() -> None: assert DEFAULT_OUT.exists(), f"{DEFAULT_OUT} is missing" assert DEFAULT_OUT.read_text() == render_spec(), ( @@ -44,20 +63,22 @@ def test_spec_paths_are_the_urls_the_server_serves() -> None: "api_deployment_execution", kwargs={"org_name": "ORG", "api_name": "API"} ) documented = [ - path.replace("{org_name}", "ORG").replace("{api_name}", "API").rstrip("/") + path.replace("{org_name}", "ORG").replace("{api_name}", "API") for path in _committed()["paths"] ] - assert documented == [served.rstrip("/")] + assert served.rstrip("/") in [path.rstrip("/") for path in documented] + for path in documented: + # Raises Resolver404 if the spec documents a URL nothing answers. + resolve(path if path.endswith("/") else f"{path}/") def test_spec_documents_the_deployment_operations() -> None: spec = _committed() - (operations,) = spec["paths"].values() + documented = {operation["operationId"] for _, _, operation in _operations(spec)} - assert operations["post"]["operationId"] == "execute" - assert operations["get"]["operationId"] == "status" - assert [tag["name"] for tag in spec["tags"]] == ["deployment"] + assert {"execute", "status"} <= documented + assert "deployment" in [tag["name"] for tag in spec["tags"]] def test_operations_require_the_deployment_key() -> None: @@ -67,21 +88,27 @@ def test_operations_require_the_deployment_key() -> None: scheme = spec["components"]["securitySchemes"]["deploymentKey"] assert (scheme["type"], scheme["scheme"]) == ("http", "bearer") - for operation in spec["paths"].values(): - for method in ("get", "post"): - assert operation[method]["security"] == [{"deploymentKey": []}] + for path, method, operation in _operations(spec): + assert operation["security"] == [{"deploymentKey": []}], f"{method} {path}" def test_clients_can_branch_on_every_failure_they_will_see() -> None: - for method in ("get", "post"): - (operation,) = (ops[method] for ops in _committed()["paths"].values()) - assert {"400", "401", "403", "404", "429", "500"} <= set(operation["responses"]) + for path, method, operation in _operations(_committed()): + assert {"400", "401", "403", "404", "429", "500"} <= set( + operation["responses"] + ), f"{method} {path}" def test_the_one_shot_read_is_documented_where_a_client_will_see_it() -> None: """The semantics that a status read destroys the result must reach the generated client, not live in a source comment.""" - (status_op,) = (ops["get"] for ops in _committed()["paths"].values()) + reads = [ + operation + for _, _, operation in _operations(_committed()) + if operation["operationId"] == "status" + ] - assert "one-shot" in status_op["description"] - assert status_op["responses"]["406"]["description"].strip() + assert reads + for status_op in reads: + assert "one-shot" in status_op["description"] + assert status_op["responses"]["406"]["description"].strip() From 3ebfafbd03922de79e3382862d9c7213ed9ce397 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 18 Aug 2026 15:46:45 +0530 Subject: [PATCH 5/8] chore: drop drf-yasg now that spectacular generates the spec The `docs` app served a redoc UI over a schema drf-yasg introspected with `public=False`, so an anonymous caller saw nothing and no endpoint carried a `@swagger_auto_schema` annotation. Nothing generates or consumes it. Removes the dependency, the `docs` app and its two mounts. The `/doc/` route goes with it; drf-spectacular serves no UI, only the committed spec. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BFSunNN6RKRA1xo6kWkztx --- backend/backend/public_urls.py | 2 -- backend/backend/public_urls_v2.py | 2 -- backend/backend/settings/base.py | 7 ++----- backend/docs/__init__.py | 0 backend/docs/urls.py | 20 -------------------- backend/pyproject.toml | 1 - backend/uv.lock | 20 -------------------- 7 files changed, 2 insertions(+), 50 deletions(-) delete mode 100644 backend/docs/__init__.py delete mode 100644 backend/docs/urls.py diff --git a/backend/backend/public_urls.py b/backend/backend/public_urls.py index 2ca815442e..3d9a8130a2 100644 --- a/backend/backend/public_urls.py +++ b/backend/backend/public_urls.py @@ -28,8 +28,6 @@ path(f"{path_prefix}/", include("account.urls")), # Connector OAuth path(f"{path_prefix}/", include("connector_auth.urls")), - # Docs - path(f"{path_prefix}/", include("docs.urls")), # API deployment path(f"{api_path_prefix}/", include("api.urls")), path(f"{api_path_prefix}/pipeline/", include("pipeline.public_api_urls")), diff --git a/backend/backend/public_urls_v2.py b/backend/backend/public_urls_v2.py index 7034e86141..336ea9cdbc 100644 --- a/backend/backend/public_urls_v2.py +++ b/backend/backend/public_urls_v2.py @@ -25,8 +25,6 @@ path("", include("account_v2.urls")), # Connector OAuth path("", include("connector_auth_v2.urls")), - # Docs - path("", include("docs.urls")), # Feature flags path("flags/", include("feature_flag.urls")), # Pipeline diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index cf21bd3e1e..f91385bbca 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -371,9 +371,6 @@ def filter(self, record): # Connector OAuth # "connector_auth", "social_django", - # Doc generator - "drf_yasg", - "docs", # Plugins "plugins.apps.PluginsConfig", "feature_flag", @@ -677,8 +674,8 @@ def filter(self, record): } } }, - # Group descriptions clients show in their help; without this the spec has - # no root `tags` array and the text has nowhere to live. + # Group descriptions generated clients show in their help; without this + # the spec has no root `tags` array for the text to live in. "TAGS": [ { "name": "deployment", diff --git a/backend/docs/__init__.py b/backend/docs/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/backend/docs/urls.py b/backend/docs/urls.py deleted file mode 100644 index 83260b01db..0000000000 --- a/backend/docs/urls.py +++ /dev/null @@ -1,20 +0,0 @@ -from django.urls import path -from drf_yasg import openapi -from drf_yasg.views import get_schema_view - -schema_view = get_schema_view( - openapi.Info( - title="Unstract APIs", - default_version="v1", - description="", - ), - public=False, -) - -urlpatterns = [ - path( - "doc/", - schema_view.with_ui("redoc", cache_timeout=0), - name="schema-redoc", - ), -] diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 1cd74ecdcb..c1ca5b9ed4 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -30,7 +30,6 @@ dependencies = [ # Pinned: its rendering is the committed spec, so an upgrade rewrites the # contract published clients are generated from. "drf-spectacular==0.30.0", - "drf-yasg>=1.21.8", # For API docs "psycopg2-binary==2.9.9", "python-dotenv==1.2.2", "python-magic==0.4.27", # For file upload/download diff --git a/backend/uv.lock b/backend/uv.lock index 52bd298262..eb1952f9d2 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -930,24 +930,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/62/94/4e7721ff51cb10aa826cb27f3bf015e8d94d7f898c19c2b650d822019e5b/drf_standardized_errors-0.15.0-py3-none-any.whl", hash = "sha256:75dcfec11433a16c81f8c5948a5cd2932cd5b02f426f64ca82020a78c155b263", size = 25673, upload-time = "2025-06-09T07:47:55.042Z" }, ] -[[package]] -name = "drf-yasg" -version = "1.21.15" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "django" }, - { name = "djangorestframework" }, - { name = "inflection" }, - { name = "packaging" }, - { name = "pytz" }, - { name = "pyyaml" }, - { name = "uritemplate" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/4a/88/345135459b9cbaff0e8ee3270819e89ca92064a35a0a94a1cfce41c084db/drf_yasg-1.21.15.tar.gz", hash = "sha256:ef86838c4ef10dcd3ac1ebf2be601cbe02978b999671caa43667f7c9db961468", size = 5153419, upload-time = "2026-02-24T18:09:21.072Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/a4/400b0565cf25395f1d5e1a24e5d18dab8f4199e4212174341fea6f05747c/drf_yasg-1.21.15-py3-none-any.whl", hash = "sha256:7c7a7ab9feb0e13cdd6e25147d99adb0500a68bd96509ffd8f8cf7efd4bdc77e", size = 4856209, upload-time = "2026-02-24T18:09:18.982Z" }, -] - [[package]] name = "dropbox" version = "12.0.2" @@ -3707,7 +3689,6 @@ dependencies = [ { name = "djangorestframework" }, { name = "drf-spectacular" }, { name = "drf-standardized-errors" }, - { name = "drf-yasg" }, { name = "google-cloud-recaptcha-enterprise" }, { name = "gunicorn" }, { name = "httpx" }, @@ -3775,7 +3756,6 @@ requires-dist = [ { name = "djangorestframework", specifier = "==3.17.1" }, { name = "drf-spectacular", specifier = "==0.30.0" }, { name = "drf-standardized-errors", specifier = ">=0.12.6" }, - { name = "drf-yasg", specifier = ">=1.21.8" }, { name = "google-cloud-recaptcha-enterprise", specifier = ">=1.28.2" }, { name = "gunicorn", specifier = ">=23.0.0" }, { name = "httpx", specifier = ">=0.27.0" }, From 3198b529f3f614a1a19ef7c9878b6834685d0f0d Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 18 Aug 2026 15:46:56 +0530 Subject: [PATCH 6/8] refactor: move the deployment schema annotations out of the view The annotation serializers exist only to shape the published spec, so they sit in `api_v2/openapi_schema.py` rather than in `serializers.py`, where a request-time import of one would look ordinary. `api_deployment_views.py` keeps a single decorator. `deployment_spec_urls.py` now selects mounts from a tuple, so widening the spec to another endpoint is one entry plus its `@extend_schema`. The generated spec is unchanged: component names are all that reach it. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BFSunNN6RKRA1xo6kWkztx --- backend/api_v2/api_deployment_views.py | 153 +---------------------- backend/api_v2/deployment_spec_urls.py | 26 ++-- backend/api_v2/openapi_schema.py | 161 +++++++++++++++++++++++++ backend/backend/settings/base.py | 18 +-- 4 files changed, 181 insertions(+), 177 deletions(-) create mode 100644 backend/api_v2/openapi_schema.py diff --git a/backend/api_v2/api_deployment_views.py b/backend/api_v2/api_deployment_views.py index 4dce5967b2..8f5d0763a9 100644 --- a/backend/api_v2/api_deployment_views.py +++ b/backend/api_v2/api_deployment_views.py @@ -5,13 +5,6 @@ from django.db.models import F, OuterRef, QuerySet, Subquery from django.http import HttpResponse -from drf_spectacular.utils import ( - OpenApiParameter, - OpenApiResponse, - extend_schema, - extend_schema_serializer, - extend_schema_view, -) from permissions.membership_views import OwnerManagementMixin from permissions.permission import IsOwner, IsOwnerOrSharedUserOrSharedToOrg from permissions.resource_share_views import ResourceShareManagementMixin @@ -40,11 +33,11 @@ contains_tool_not_found_error, ) from api_v2.models import APIDeployment +from api_v2.openapi_schema import DEPLOYMENT_EXECUTION_SCHEMA from api_v2.rate_limiter import APIDeploymentRateLimiter from api_v2.serializers import ( APIDeploymentListSerializer, APIDeploymentSerializer, - APIExecutionResponseSerializer, DeploymentResponseSerializer, ExecutionQuerySerializer, ExecutionRequestSerializer, @@ -58,149 +51,7 @@ logger = logging.getLogger(__name__) -# Declares no field of its own, so every backend parameter arrives free and a -# change to the real serializer moves the spec. It exists only to carry a -# caller-facing description in place of the implementation docstring, and to -# keep the published model name stable. -@extend_schema_serializer(component_name="ExecuteRequest") -class ExecuteRequest(ExecutionRequestSerializer): - """The documents to run, and the options that shape the result. - - Supply `files`, `presigned_urls`, or both. - """ - - -class FileResult(serializers.Serializer): - file = serializers.CharField() - file_execution_id = serializers.CharField(required=False) - status = serializers.CharField(required=False) - result = serializers.JSONField(required=False) - metadata = serializers.JSONField(required=False) - metrics = serializers.JSONField(required=False) - error = serializers.CharField(required=False, allow_null=True) - - -# Subclasses the serializer that builds the response, so a field added or -# removed there moves the spec. Docstrings on these annotation serializers are -# published as the client-facing model description, so they are written for -# the caller rather than the maintainer. -class ExecutionMessage(APIExecutionResponseSerializer): - """The execution's identity and, once it has finished, its per-file - results. - """ - - # The one field that has to be restated: the real declaration is an - # untyped JSONField, which gives generated clients nothing to work with. - # The backend also sends `result: null` while pending, and without - # allow_null the generated deserialiser iterates None and crashes. - result = FileResult(many=True, required=False, allow_null=True) - - -class ExecuteResponse(serializers.Serializer): - message = ExecutionMessage() - - -class StatusResponse(serializers.Serializer): - status = serializers.CharField() - message = FileResult(many=True, required=False, allow_null=True) - - -class ErrorResponse(serializers.Serializer): - status = serializers.CharField(required=False) - message = serializers.JSONField(required=False, allow_null=True) - - -# The pattern the route itself enforces, restated so a generated client can -# reject a mistyped identifier without a round trip. -PATH_SEGMENT = {"type": "string", "pattern": r"^[\w-]+$"} - -DEPLOYMENT_PATH_PARAMETERS = [ - OpenApiParameter( - "org_name", - PATH_SEGMENT, - OpenApiParameter.PATH, - description="Organization identifier.", - ), - OpenApiParameter( - "api_name", - PATH_SEGMENT, - OpenApiParameter.PATH, - description="API deployment name.", - ), -] - - -DEPLOYMENT_AUTH = [{"deploymentKey": []}] - -# Every failure a caller has to handle. Declared explicitly because a client -# generated without them treats an authentication or rate-limit response as an -# unknown status and has nothing to branch on. -DEPLOYMENT_ERRORS = { - 400: OpenApiResponse(ErrorResponse, description="The request failed validation."), - 401: OpenApiResponse(ErrorResponse, description="The API key is not valid."), - 403: OpenApiResponse(ErrorResponse, description="No API key was supplied."), - 404: OpenApiResponse(ErrorResponse, description="No such active deployment."), - 429: OpenApiResponse( - ErrorResponse, description="Too many concurrent executions; retry later." - ), - 500: ErrorResponse, -} - -EXECUTE_DESCRIPTION = ( - "Execute an API deployment against one or more documents.\n\n" - "Supply the documents either as `files` (multipart upload) or as " - "`presigned_urls` (HTTPS S3 URLs), or both — a request carrying neither is " - f"rejected, and the two together may not exceed " - f"{ExecutionRequestSerializer.MAX_FILES_ALLOWED} documents.\n\n" - "With the default `timeout` of -1 the call returns as soon as the " - "execution is queued; read the outcome from the status endpoint." -) - -STATUS_DESCRIPTION = ( - "Read the result of a previously started execution.\n\n" - "This read is one-shot: the first call that observes a completed execution " - "acknowledges it and the stored result is discarded, so every later call " - "for that execution answers 406. Poll while the execution is pending, and " - "keep the payload of the call that returns it — it cannot be fetched again." -) - - -# The generated clients take their command names, module paths and request -# shapes from here, so this block is part of the public API surface. -@extend_schema_view( - post=extend_schema( - operation_id="execute", - tags=["deployment"], - auth=DEPLOYMENT_AUTH, - parameters=DEPLOYMENT_PATH_PARAMETERS, - request={"multipart/form-data": ExecuteRequest}, - responses={ - 200: ExecuteResponse, - 409: OpenApiResponse( - ErrorResponse, description="The deployment has no active API key." - ), - 422: ExecuteResponse, - **DEPLOYMENT_ERRORS, - }, - description=EXECUTE_DESCRIPTION, - ), - get=extend_schema( - operation_id="status", - tags=["deployment"], - auth=DEPLOYMENT_AUTH, - parameters=DEPLOYMENT_PATH_PARAMETERS + [ExecutionQuerySerializer], - responses={ - 200: StatusResponse, - 406: OpenApiResponse( - ErrorResponse, - description="The result was already consumed by an earlier call.", - ), - 422: StatusResponse, - **DEPLOYMENT_ERRORS, - }, - description=STATUS_DESCRIPTION, - ), -) +@DEPLOYMENT_EXECUTION_SCHEMA class DeploymentExecution(views.APIView): def initialize_request(self, request: Request, *args: Any, **kwargs: Any) -> Request: """To remove csrf request for public API. diff --git a/backend/api_v2/deployment_spec_urls.py b/backend/api_v2/deployment_spec_urls.py index 0aa2d998bd..9dff155d9d 100644 --- a/backend/api_v2/deployment_spec_urls.py +++ b/backend/api_v2/deployment_spec_urls.py @@ -1,27 +1,29 @@ -"""URLconf used only to generate the API deployment OpenAPI spec. +"""URLconf the published OpenAPI spec is generated against. -``api_v2.execution_urls`` is an included sub-urlconf, so generating against it -directly yields paths without the prefix it is mounted at — a spec describing -URLs the server does not serve. The mount is selected out of the served -urlconf rather than restated here, so a change to where the deployment API is -mounted moves the generated paths with it. +Each entry is an included sub-urlconf: generating against one directly yields +paths without the prefix it is mounted at, i.e. a spec describing URLs the +server does not serve. The mounts are selected out of the served urlconf +rather than restated, so moving one moves the generated paths with it. + +Widening the spec to another endpoint means annotating its view with +``@extend_schema`` and adding its urlconf here. """ from django.core.exceptions import ImproperlyConfigured from backend import base_urls -DEPLOYMENT_URLCONF = "api_v2.execution_urls" +SPEC_URLCONFS = ("api_v2.execution_urls",) urlpatterns = [ entry for entry in base_urls.urlpatterns - if getattr(getattr(entry, "urlconf_name", None), "__name__", None) - == DEPLOYMENT_URLCONF + if getattr(getattr(entry, "urlconf_name", None), "__name__", None) in SPEC_URLCONFS ] -if not urlpatterns: +missing = set(SPEC_URLCONFS) - {entry.urlconf_name.__name__ for entry in urlpatterns} +if missing: raise ImproperlyConfigured( - f"{DEPLOYMENT_URLCONF} is not mounted in backend.base_urls; the API " - "deployment spec would be generated for no routes at all." + f"{', '.join(sorted(missing))} is not mounted in backend.base_urls; the " + "spec would be generated for routes the server does not serve." ) diff --git a/backend/api_v2/openapi_schema.py b/backend/api_v2/openapi_schema.py new file mode 100644 index 0000000000..3af658481b --- /dev/null +++ b/backend/api_v2/openapi_schema.py @@ -0,0 +1,161 @@ +"""OpenAPI annotations for the API deployment endpoints. + +The serializers here shape the published spec only; none of them is used to +parse a request or build a response. They live outside ``serializers.py`` so +that nothing at request time imports one by accident. + +Their docstrings are published as the client-facing model descriptions, so +they are written for the caller rather than the maintainer. +""" + +from drf_spectacular.utils import ( + OpenApiParameter, + OpenApiResponse, + extend_schema, + extend_schema_serializer, + extend_schema_view, +) +from rest_framework import serializers + +from api_v2.serializers import ( + APIExecutionResponseSerializer, + ExecutionQuerySerializer, + ExecutionRequestSerializer, +) + + +# Declares no field of its own, so a change to the real serializer moves the +# spec. It exists to carry a caller-facing description and a stable name. +@extend_schema_serializer(component_name="ExecuteRequest") +class ExecuteRequest(ExecutionRequestSerializer): + """The documents to run, and the options that shape the result. + + Supply `files`, `presigned_urls`, or both. + """ + + +class FileResult(serializers.Serializer): + file = serializers.CharField() + file_execution_id = serializers.CharField(required=False) + status = serializers.CharField(required=False) + result = serializers.JSONField(required=False) + metadata = serializers.JSONField(required=False) + metrics = serializers.JSONField(required=False) + error = serializers.CharField(required=False, allow_null=True) + + +class ExecutionMessage(APIExecutionResponseSerializer): + """The execution's identity and, once it has finished, its per-file + results. + """ + + # Restated because the real declaration is an untyped JSONField, and + # because a pending execution sends `result: null`, which a generated + # deserialiser iterates and crashes on without allow_null. + result = FileResult(many=True, required=False, allow_null=True) + + +class ExecuteResponse(serializers.Serializer): + message = ExecutionMessage() + + +class StatusResponse(serializers.Serializer): + status = serializers.CharField() + message = FileResult(many=True, required=False, allow_null=True) + + +class ErrorResponse(serializers.Serializer): + status = serializers.CharField(required=False) + message = serializers.JSONField(required=False, allow_null=True) + + +# Restates the route's own pattern so a client rejects a mistyped identifier +# without a round trip. +PATH_SEGMENT = {"type": "string", "pattern": r"^[\w-]+$"} + +DEPLOYMENT_PATH_PARAMETERS = [ + OpenApiParameter( + "org_name", + PATH_SEGMENT, + OpenApiParameter.PATH, + description="Organization identifier.", + ), + OpenApiParameter( + "api_name", + PATH_SEGMENT, + OpenApiParameter.PATH, + description="API deployment name.", + ), +] + + +DEPLOYMENT_AUTH = [{"deploymentKey": []}] + +# A client generated without these treats an authentication or rate-limit +# response as an unknown status and has nothing to branch on. +DEPLOYMENT_ERRORS = { + 400: OpenApiResponse(ErrorResponse, description="The request failed validation."), + 401: OpenApiResponse(ErrorResponse, description="The API key is not valid."), + 403: OpenApiResponse(ErrorResponse, description="No API key was supplied."), + 404: OpenApiResponse(ErrorResponse, description="No such active deployment."), + 429: OpenApiResponse( + ErrorResponse, description="Too many concurrent executions; retry later." + ), + 500: ErrorResponse, +} + +EXECUTE_DESCRIPTION = ( + "Execute an API deployment against one or more documents.\n\n" + "Supply the documents either as `files` (multipart upload) or as " + "`presigned_urls` (HTTPS S3 URLs), or both — a request carrying neither is " + f"rejected, and the two together may not exceed " + f"{ExecutionRequestSerializer.MAX_FILES_ALLOWED} documents.\n\n" + "With the default `timeout` of -1 the call returns as soon as the " + "execution is queued; read the outcome from the status endpoint." +) + +STATUS_DESCRIPTION = ( + "Read the result of a previously started execution.\n\n" + "This read is one-shot: the first call that observes a completed execution " + "acknowledges it and the stored result is discarded, so every later call " + "for that execution answers 406. Poll while the execution is pending, and " + "keep the payload of the call that returns it — it cannot be fetched again." +) + + +# Generated clients take their command names, module paths and request shapes +# from here, so this is part of the public API surface. +DEPLOYMENT_EXECUTION_SCHEMA = extend_schema_view( + post=extend_schema( + operation_id="execute", + tags=["deployment"], + auth=DEPLOYMENT_AUTH, + parameters=DEPLOYMENT_PATH_PARAMETERS, + request={"multipart/form-data": ExecuteRequest}, + responses={ + 200: ExecuteResponse, + 409: OpenApiResponse( + ErrorResponse, description="The deployment has no active API key." + ), + 422: ExecuteResponse, + **DEPLOYMENT_ERRORS, + }, + description=EXECUTE_DESCRIPTION, + ), + get=extend_schema( + operation_id="status", + tags=["deployment"], + auth=DEPLOYMENT_AUTH, + parameters=DEPLOYMENT_PATH_PARAMETERS + [ExecutionQuerySerializer], + responses={ + 200: StatusResponse, + 406: OpenApiResponse( + ErrorResponse, + description="The result was already consumed by an earlier call.", + ), + 422: StatusResponse, + **DEPLOYMENT_ERRORS, + }, + description=STATUS_DESCRIPTION, + ), +) diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index f91385bbca..76cecc14b7 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -653,18 +653,15 @@ def filter(self, record): "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", } -# Read while generating the API deployment OpenAPI spec -# (``manage.py generate_docstudio_spec``). ``DEFAULT_SCHEMA_CLASS`` above is a -# project-wide DRF default, but DRF dereferences it only when a schema is -# generated, so neither has an effect at request time. +# Read only while generating the OpenAPI spec +# (``manage.py generate_docstudio_spec``); no effect at request time. SPECTACULAR_SETTINGS = { "TITLE": "Unstract API", "VERSION": "v1", "PREPROCESSING_HOOKS": ["drf_spectacular.hooks.preprocess_exclude_path_format"], "SERVE_INCLUDE_SCHEMA": False, - # DRF's unset ``DEFAULT_AUTHENTICATION_CLASSES`` would otherwise be - # introspected as a decision and publish session and basic auth, which - # these endpoints do not accept. + # Declared, because DRF's unset authentication default is otherwise + # introspected as a decision and publishes auth these endpoints reject. "APPEND_COMPONENTS": { "securitySchemes": { "deploymentKey": { @@ -709,13 +706,6 @@ def filter(self, record): # These path will work without organization in request ORGANIZATION_MIDDLEWARE_WHITELISTED_PATHS = [] -# API Doc Generator Settings -# https://drf-yasg.readthedocs.io/en/stable/settings.html -REDOC_SETTINGS = { - "PATH_IN_MIDDLE": True, - "REQUIRED_PROPS_FIRST": True, -} - # Social Auth Settings SOCIAL_AUTH_LOGIN_REDIRECT_URL = f"{WEB_APP_ORIGIN_URL}/oauth-status/?status=success" SOCIAL_AUTH_LOGIN_ERROR_URL = f"{WEB_APP_ORIGIN_URL}/oauth-status/?status=error" From 51c7cd45e61fc1589eabc1863f1724ef8738f99a Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Tue, 18 Aug 2026 16:12:32 +0530 Subject: [PATCH 7/8] test: name the downstream repos in the drift failure, and pin the response fields The drift test is the only gate, so its message has to reach the person or agent who then has to regenerate the client and the CLI; it now names both repos, as does the management command's `--check`. Adds one binding the annotation could not express by inheritance: the view returns the execution DTO as a dict rather than through `APIExecutionResponseSerializer`, so a renamed DTO field would reach clients as a field the server never sends. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01BFSunNN6RKRA1xo6kWkztx --- .../commands/generate_docstudio_spec.py | 9 +++++- backend/api_v2/tests/test_docstudio_spec.py | 30 +++++++++++++++---- 2 files changed, 32 insertions(+), 7 deletions(-) diff --git a/backend/api_v2/management/commands/generate_docstudio_spec.py b/backend/api_v2/management/commands/generate_docstudio_spec.py index a091073a39..8a6e6f1205 100644 --- a/backend/api_v2/management/commands/generate_docstudio_spec.py +++ b/backend/api_v2/management/commands/generate_docstudio_spec.py @@ -23,6 +23,13 @@ DEFAULT_OUT = Path(__file__).resolve().parents[4] / "specs" / "docstudio-oss.json" URLCONF = "api_v2.deployment_spec_urls" REGENERATE = "uv run python manage.py generate_docstudio_spec" +# Named in every failure message: the repos that regenerate from this file are +# the ones a spec change actually breaks, and nothing there watches this repo. +DOWNSTREAM = ( + "The published client (Zipstack/unstract-python-client) and the CLI " + "(Zipstack/unstract-cli) are generated from this file — raise the matching " + "PRs there for anything that changes an operation id, a tag or a schema." +) class SpecGenerationFailed(CommandError): @@ -77,7 +84,7 @@ def handle(self, *args: Any, **options: Any) -> None: if current != rendered: raise CommandError( f"{out} is out of date. Run `{REGENERATE}` from `backend/` " - "and commit the result." + f"and commit the result.\n\n{DOWNSTREAM}" ) self.stdout.write(f"{out} is up to date") return diff --git a/backend/api_v2/tests/test_docstudio_spec.py b/backend/api_v2/tests/test_docstudio_spec.py index 50b0e99cad..23c2e4c5c3 100644 --- a/backend/api_v2/tests/test_docstudio_spec.py +++ b/backend/api_v2/tests/test_docstudio_spec.py @@ -5,17 +5,20 @@ than in a client repo. """ +import dataclasses import json from django.urls import resolve, reverse from drf_spectacular.drainage import GENERATOR_STATS +from workflow_manager.workflow_v2.dto import ExecutionResponse from api_v2.management.commands.generate_docstudio_spec import ( DEFAULT_OUT, + DOWNSTREAM, REGENERATE, render_spec, ) - +from api_v2.serializers import APIExecutionResponseSerializer #: Keys under a path item that are operations. The rest -- `parameters`, #: `summary`, vendor extensions -- describe the path, not a call. @@ -44,13 +47,14 @@ def test_committed_spec_matches_the_code() -> None: assert DEFAULT_OUT.exists(), f"{DEFAULT_OUT} is missing" assert DEFAULT_OUT.read_text() == render_spec(), ( f"{DEFAULT_OUT} is out of date. Run `{REGENERATE}` from `backend/` and " - "commit the result." + f"commit the result.\n\n{DOWNSTREAM}" ) def test_generation_reports_no_diagnostics() -> None: """A warned-about operation is published with guessed request and response - shapes, and the drift comparison certifies the guess.""" + shapes, and the drift comparison certifies the guess. + """ render_spec() assert not GENERATOR_STATS._error_cache assert not GENERATOR_STATS._warn_cache @@ -58,7 +62,8 @@ def test_generation_reports_no_diagnostics() -> None: def test_spec_paths_are_the_urls_the_server_serves() -> None: """Resolves the real mount rather than restating it: a spec generated for - URLs the server does not serve is the failure this file exists to catch.""" + URLs the server does not serve is the failure this file exists to catch. + """ served = reverse( "api_deployment_execution", kwargs={"org_name": "ORG", "api_name": "API"} ) @@ -83,7 +88,8 @@ def test_spec_documents_the_deployment_operations() -> None: def test_operations_require_the_deployment_key() -> None: """Without this the unset DRF authentication default is published as - though it were a decision, and no generated client can authenticate.""" + though it were a decision, and no generated client can authenticate. + """ spec = _committed() scheme = spec["components"]["securitySchemes"]["deploymentKey"] @@ -101,7 +107,8 @@ def test_clients_can_branch_on_every_failure_they_will_see() -> None: def test_the_one_shot_read_is_documented_where_a_client_will_see_it() -> None: """The semantics that a status read destroys the result must reach the - generated client, not live in a source comment.""" + generated client, not live in a source comment. + """ reads = [ operation for _, _, operation in _operations(_committed()) @@ -112,3 +119,14 @@ def test_the_one_shot_read_is_documented_where_a_client_will_see_it() -> None: for status_op in reads: assert "one-shot" in status_op["description"] assert status_op["responses"]["406"]["description"].strip() + + +def test_the_documented_response_fields_are_ones_the_code_produces() -> None: + """The view returns the execution DTO as a dict rather than through this + serializer, so a renamed DTO field would otherwise reach clients as a field + the server never sends. + """ + documented = set(APIExecutionResponseSerializer().get_fields()) + produced = {field.name for field in dataclasses.fields(ExecutionResponse)} + + assert documented <= produced, documented - produced From eddd4b746765c77a3d6f64b428fd35d2261e60e7 Mon Sep 17 00:00:00 2001 From: Chandrasekharan M Date: Fri, 28 Aug 2026 11:01:45 +0530 Subject: [PATCH 8/8] fix(api): publish the error bodies and file fields the server really sends Address review on the API deployment OpenAPI spec. - Adopt drf-standardized-errors' AutoSchema and restate ErrorResponse as the {type, errors[]} body the project's exception handler actually emits; keep the hand-built {status, message} bodies for the 406/422/500 branches the views build themselves. - Mark `error`, `status_api` and `file_execution_id` nullable, drop the never-emitted FileResult.metrics and document `extracted_text`. - Declare 413/502/504 on execute, drop the unreachable 409, and scope 429 to execute; reword the 403/404 descriptions so they do not assert one cause. - Say in the status description that a pending poll answers 422. - Validate the rendered spec as OpenAPI and refuse a non-default deployment path prefix, so a typo in a hand-written fragment fails generation. - Anchor the spec's file result, status body, upload encoding and error shape to the code that produces them, and exercise the command's own branches. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01KjRpEocCvnxGUSkFivgnhk --- backend/README.md | 12 +- .../commands/generate_docstudio_spec.py | 40 ++- backend/api_v2/openapi_schema.py | 123 +++++-- backend/api_v2/serializers.py | 23 +- backend/api_v2/tests/test_docstudio_spec.py | 148 ++++++++- backend/backend/settings/base.py | 8 +- backend/mcp_server/views.py | 4 +- specs/docstudio-oss.json | 309 ++++++++++++++++-- 8 files changed, 587 insertions(+), 80 deletions(-) diff --git a/backend/README.md b/backend/README.md index 66c4c93bb3..bd3ea6f0ad 100644 --- a/backend/README.md +++ b/backend/README.md @@ -175,11 +175,15 @@ psql -d unstract_db -U unstract_dev ## API Docs -While running the backend server locally, access the API documentation that's auto generated at -the backend endpoint `/api/v1/doc/`. +The OpenAPI spec for the API deployment endpoints is committed at +[`specs/docstudio-oss.json`](../specs/docstudio-oss.json) and is the contract the published +clients and their generated SDKs are built from. It is not served at runtime — regenerate it in +the same PR as any route, serializer or schema-annotation change: -**NOTE:** There exists issues accessing this when the django server is run with gunicorn (in case of running with -a container) +```bash +uv run python manage.py generate_docstudio_spec # rewrite the committed spec +uv run python manage.py generate_docstudio_spec --check # no write, drift is an error +``` - [Account](account/api_doc.md) - [FileManagement](file_management/api_doc.md) diff --git a/backend/api_v2/management/commands/generate_docstudio_spec.py b/backend/api_v2/management/commands/generate_docstudio_spec.py index 8a6e6f1205..dc5ddabe88 100644 --- a/backend/api_v2/management/commands/generate_docstudio_spec.py +++ b/backend/api_v2/management/commands/generate_docstudio_spec.py @@ -7,9 +7,10 @@ uv run python manage.py generate_docstudio_spec # from backend/ uv run python manage.py generate_docstudio_spec --check # no write, drift is an error -The generated paths carry ``API_DEPLOYMENT_PATH_PREFIX``, so regenerate in an -environment that does not override it — the committed artifact describes the -deployment as it is served publicly, not as one installation mounts it. +The generated paths carry ``API_DEPLOYMENT_PATH_PREFIX``, so generation refuses +to produce a spec mounted anywhere but the public default: the committed +artifact describes the deployment as it is served publicly, not as one +installation chooses to mount it. """ import json @@ -19,10 +20,15 @@ from django.core.management.base import BaseCommand, CommandError from drf_spectacular.drainage import GENERATOR_STATS from drf_spectacular.generators import SchemaGenerator +from drf_spectacular.validation import validate_schema DEFAULT_OUT = Path(__file__).resolve().parents[4] / "specs" / "docstudio-oss.json" URLCONF = "api_v2.deployment_spec_urls" REGENERATE = "uv run python manage.py generate_docstudio_spec" +# The mount the deployment is served at publicly. `API_DEPLOYMENT_PATH_PREFIX` +# can move it per installation, and a spec carrying a private prefix would send +# every generated client to a URL only that installation answers. +PUBLISHED_PATH_PREFIX = "deployment" # Named in every failure message: the repos that regenerate from this file are # the ones a spec change actually breaks, and nothing there watches this repo. DOWNSTREAM = ( @@ -45,9 +51,11 @@ def render_spec() -> str: GENERATOR_STATS.reset() schema = SchemaGenerator(urlconf=URLCONF).get_schema(request=None, public=True) if GENERATOR_STATS: - # spectacular downgrades "unable to guess serializer" to a warning and - # writes a plausible, wrong operation. Nothing downstream can tell that - # apart from an annotation that is simply thin. + # An operation spectacular could not resolve is published with no + # request body and an empty response rather than dropped, which reads + # downstream as an annotation that is simply thin. Both caches are + # drained because the severity a given diagnostic carries is + # spectacular's choice, not something to rely on. diagnostics = "\n".join( f" {severity}: {message}" for severity, cache in ( @@ -60,6 +68,26 @@ def render_spec() -> str: f"The generator reported problems, so the spec would describe an " f"API nobody implements:\n{diagnostics}" ) + + off_prefix = [ + path + for path in schema["paths"] + if not path.startswith(f"/{PUBLISHED_PATH_PREFIX}/") + ] + if off_prefix: + raise SpecGenerationFailed( + f"Generated paths are not under /{PUBLISHED_PATH_PREFIX}/: " + f"{', '.join(sorted(off_prefix))}. Unset API_DEPLOYMENT_PATH_PREFIX " + f"and regenerate." + ) + + # Hand-written fragments (path parameter schemas, security schemes) reach + # the output verbatim, so nothing above would notice a typo in one. + try: + validate_schema(schema) + except Exception as error: + raise SpecGenerationFailed(f"The generated spec is not valid OpenAPI: {error}") + # Sorted keys are what make the committed artifact a usable drift signal. return json.dumps(schema, indent=2, sort_keys=True) + "\n" diff --git a/backend/api_v2/openapi_schema.py b/backend/api_v2/openapi_schema.py index 3af658481b..d3376276a0 100644 --- a/backend/api_v2/openapi_schema.py +++ b/backend/api_v2/openapi_schema.py @@ -35,12 +35,23 @@ class ExecuteRequest(ExecutionRequestSerializer): class FileResult(serializers.Serializer): + """One input document's outcome. + + Every key is present on every item; the ones that depend on the request + options or on the outcome are sent as `null` when they do not apply. + """ + file = serializers.CharField() - file_execution_id = serializers.CharField(required=False) + file_execution_id = serializers.CharField(required=False, allow_null=True) status = serializers.CharField(required=False) - result = serializers.JSONField(required=False) - metadata = serializers.JSONField(required=False) - metrics = serializers.JSONField(required=False) + result = serializers.JSONField(required=False, allow_null=True) + metadata = serializers.JSONField(required=False, allow_null=True) + extracted_text = serializers.CharField( + required=False, + allow_null=True, + help_text="The document's full extracted text. Sent only when the " + "request set `include_extracted_text`.", + ) error = serializers.CharField(required=False, allow_null=True) @@ -49,9 +60,11 @@ class ExecutionMessage(APIExecutionResponseSerializer): results. """ - # Restated because the real declaration is an untyped JSONField, and - # because a pending execution sends `result: null`, which a generated - # deserialiser iterates and crashes on without allow_null. + # The three fields below are restated because the real serializer declares + # them as required and non-nullable while the dataclass behind it coerces + # every falsy value to None, so the happy path would contradict the spec. + status_api = serializers.CharField(required=False, allow_null=True) + error = serializers.CharField(required=False, allow_null=True) result = FileResult(many=True, required=False, allow_null=True) @@ -64,9 +77,40 @@ class StatusResponse(serializers.Serializer): message = FileResult(many=True, required=False, allow_null=True) +class AcknowledgedResponse(serializers.Serializer): + """The execution's result was handed to an earlier call and discarded.""" + + status = serializers.CharField() + message = serializers.CharField() + + +class ErrorDetail(serializers.Serializer): + """One problem found with the request.""" + + code = serializers.CharField(help_text="Machine-readable problem identifier.") + detail = serializers.CharField(help_text="Human-readable description.") + attr = serializers.CharField( + allow_null=True, + help_text="The request field the problem belongs to, when it belongs to one.", + ) + + +#: Named so the generated enum is not called after the field that holds it. +ERROR_TYPES = ("validation_error", "client_error", "server_error") + + class ErrorResponse(serializers.Serializer): - status = serializers.CharField(required=False) - message = serializers.JSONField(required=False, allow_null=True) + """The body of a rejected request. + + Produced by the project-wide exception handler, so its shape is the same + for every failure listed against an operation. + """ + + # `code` is left free-form rather than enumerated: the deployment + # exceptions carry DRF's default code, not the per-status codes the + # standardized-errors package assumes. + type = serializers.ChoiceField(choices=ERROR_TYPES) + errors = ErrorDetail(many=True) # Restates the route's own pattern so a client rejects a mistyped identifier @@ -91,17 +135,39 @@ class ErrorResponse(serializers.Serializer): DEPLOYMENT_AUTH = [{"deploymentKey": []}] -# A client generated without these treats an authentication or rate-limit -# response as an unknown status and has nothing to branch on. +# A client generated without these treats an authentication or authorization +# response as an unknown status and has nothing to branch on. The descriptions +# name what the caller can act on rather than a single cause, because a +# document store consulted during the call can surface its own status here. DEPLOYMENT_ERRORS = { 400: OpenApiResponse(ErrorResponse, description="The request failed validation."), - 401: OpenApiResponse(ErrorResponse, description="The API key is not valid."), - 403: OpenApiResponse(ErrorResponse, description="No API key was supplied."), - 404: OpenApiResponse(ErrorResponse, description="No such active deployment."), + 401: OpenApiResponse( + ErrorResponse, description="No usable API key was supplied for the deployment." + ), + 403: OpenApiResponse( + ErrorResponse, description="The request was refused as unauthorized." + ), + 404: OpenApiResponse( + ErrorResponse, + description="No active deployment, or a referenced document, was found.", + ), +} + +# Only the execution endpoint fetches documents and takes a rate-limit slot, so +# these cannot arise on the status read. +EXECUTE_ERRORS = { + 413: OpenApiResponse( + ErrorResponse, description="A referenced document is larger than the limit." + ), 429: OpenApiResponse( ErrorResponse, description="Too many concurrent executions; retry later." ), - 500: ErrorResponse, + 502: OpenApiResponse( + ErrorResponse, description="A referenced document could not be fetched." + ), + 504: OpenApiResponse( + ErrorResponse, description="Fetching a referenced document timed out." + ), } EXECUTE_DESCRIPTION = ( @@ -120,6 +186,9 @@ class ErrorResponse(serializers.Serializer): "acknowledges it and the stored result is discarded, so every later call " "for that execution answers 406. Poll while the execution is pending, and " "keep the payload of the call that returns it — it cannot be fetched again." + "\n\nA still-running execution answers 422 carrying its current `status`, " + "so a polling loop should treat 422 as the normal reply and stop on 200. " + "Clients that raise on any non-2xx need to allow for that." ) @@ -134,11 +203,16 @@ class ErrorResponse(serializers.Serializer): request={"multipart/form-data": ExecuteRequest}, responses={ 200: ExecuteResponse, - 409: OpenApiResponse( - ErrorResponse, description="The deployment has no active API key." + 422: OpenApiResponse( + ExecuteResponse, description="The execution finished with an error." + ), + 500: OpenApiResponse( + ExecuteResponse, + description="The deployment could not be run; the body carries the " + "execution that failed.", ), - 422: ExecuteResponse, **DEPLOYMENT_ERRORS, + **EXECUTE_ERRORS, }, description=EXECUTE_DESCRIPTION, ), @@ -150,10 +224,19 @@ class ErrorResponse(serializers.Serializer): responses={ 200: StatusResponse, 406: OpenApiResponse( - ErrorResponse, + AcknowledgedResponse, description="The result was already consumed by an earlier call.", ), - 422: StatusResponse, + 422: OpenApiResponse( + StatusResponse, + description="The execution is still running, or it finished with an " + "error; read `status` to tell them apart.", + ), + 500: OpenApiResponse( + StatusResponse, + description="The execution could not be completed; the body carries " + "its last known state.", + ), **DEPLOYMENT_ERRORS, }, description=STATUS_DESCRIPTION, diff --git a/backend/api_v2/serializers.py b/backend/api_v2/serializers.py index e376cd401b..a8703f01d1 100644 --- a/backend/api_v2/serializers.py +++ b/backend/api_v2/serializers.py @@ -265,8 +265,27 @@ class ExecutionRequestSerializer(TagParamsSerializer): presigned_urls = ListField(child=URLField(), required=False) llm_profile_id = CharField(required=False, allow_null=True, allow_blank=True) - hitl_queue_name = CharField(required=False, allow_null=True, allow_blank=True) - hitl_packet_id = CharField(required=False, allow_null=True, allow_blank=True) + # Help text is published as the client-facing description of these fields. + hitl_queue_name = CharField( + required=False, + allow_null=True, + allow_blank=True, + help_text=( + "Document class name for the manual review queue. Requires the " + "enterprise manual-review capability; an installation without it " + "rejects the request with 400." + ), + ) + hitl_packet_id = CharField( + required=False, + allow_null=True, + allow_blank=True, + help_text=( + "Groups documents reviewed together into one packet. Requires the " + "enterprise manual-review capability; an installation without it " + "rejects the request with 400." + ), + ) custom_data = JSONField(required=False, allow_null=True) def validate_hitl_queue_name(self, value: str | None) -> str | None: diff --git a/backend/api_v2/tests/test_docstudio_spec.py b/backend/api_v2/tests/test_docstudio_spec.py index 23c2e4c5c3..218975c969 100644 --- a/backend/api_v2/tests/test_docstudio_spec.py +++ b/backend/api_v2/tests/test_docstudio_spec.py @@ -3,19 +3,32 @@ A route, serializer or schema-annotation change that is not regenerated ships a spec describing an API the server no longer serves, so drift fails here rather than in a client repo. + +Drift alone would pass on a spec that is uniformly wrong, so the tests below +also anchor the parts a client breaks on -- the upload encoding, the nullable +result, the error body -- to the code that produces them. """ import dataclasses import json +import pytest +from django.core.management import call_command +from django.core.management.base import CommandError from django.urls import resolve, reverse -from drf_spectacular.drainage import GENERATOR_STATS +from drf_spectacular.drainage import warn +from drf_spectacular.generators import SchemaGenerator +from middleware.exception import drf_logging_exc_handler +from rest_framework.exceptions import APIException, ValidationError +from rest_framework.test import APIRequestFactory +from workflow_manager.endpoint_v2.dto import FileExecutionResult from workflow_manager.workflow_v2.dto import ExecutionResponse from api_v2.management.commands.generate_docstudio_spec import ( DEFAULT_OUT, DOWNSTREAM, REGENERATE, + SpecGenerationFailed, render_spec, ) from api_v2.serializers import APIExecutionResponseSerializer @@ -24,23 +37,33 @@ #: `summary`, vendor extensions -- describe the path, not a call. _METHODS = ("get", "put", "post", "delete", "options", "head", "patch", "trace") +#: Documented on a file result but absent from the DTO: the workflow copies it +#: up from the extraction metadata when the request asks for it. +_PROMOTED_FILE_RESULT_FIELDS = {"extracted_text"} + def _committed() -> dict: return json.loads(DEFAULT_OUT.read_text()) +def _schema(name: str) -> dict: + return _committed()["components"]["schemas"][name] + + def _operations(spec: dict) -> list[tuple[str, str, dict]]: """Every (path, method, operation) the spec documents. The spec grows an endpoint at a time, and a check written against exactly one of them fails on the next addition without anything being wrong. """ - return [ + operations = [ (path, method, operation) for path, path_item in spec["paths"].items() for method, operation in path_item.items() if method in _METHODS ] + assert operations, "the spec documents no operation at all" + return operations def test_committed_spec_matches_the_code() -> None: @@ -51,13 +74,19 @@ def test_committed_spec_matches_the_code() -> None: ) -def test_generation_reports_no_diagnostics() -> None: - """A warned-about operation is published with guessed request and response - shapes, and the drift comparison certifies the guess. +def test_a_generator_diagnostic_fails_generation(monkeypatch) -> None: + """An operation spectacular could not resolve is published empty rather + than dropped, so the drift comparison would certify the gap. """ - render_spec() - assert not GENERATOR_STATS._error_cache - assert not GENERATOR_STATS._warn_cache + + def guessing_generator(self, request=None, public=False) -> dict: + warn("unable to guess serializer") + return {"openapi": "3.0.3", "paths": {}} + + monkeypatch.setattr(SchemaGenerator, "get_schema", guessing_generator) + + with pytest.raises(SpecGenerationFailed, match="unable to guess serializer"): + render_spec() def test_spec_paths_are_the_urls_the_server_serves() -> None: @@ -100,11 +129,25 @@ def test_operations_require_the_deployment_key() -> None: def test_clients_can_branch_on_every_failure_they_will_see() -> None: for path, method, operation in _operations(_committed()): - assert {"400", "401", "403", "404", "429", "500"} <= set( + assert {"400", "401", "403", "404", "500"} <= set( operation["responses"] ), f"{method} {path}" +def test_only_the_execution_endpoint_documents_the_statuses_only_it_returns() -> None: + """Fetching a document and taking a rate-limit slot happen on the execute + call alone, so declaring them on the status read hands clients branches + that can never be taken. + """ + fetch_and_rate_limit = {"413", "429", "502", "504"} + for _, _, operation in _operations(_committed()): + declared = fetch_and_rate_limit & set(operation["responses"]) + if operation["operationId"] == "execute": + assert declared == fetch_and_rate_limit + else: + assert not declared + + def test_the_one_shot_read_is_documented_where_a_client_will_see_it() -> None: """The semantics that a status read destroys the result must reach the generated client, not live in a source comment. @@ -118,15 +161,96 @@ def test_the_one_shot_read_is_documented_where_a_client_will_see_it() -> None: assert reads for status_op in reads: assert "one-shot" in status_op["description"] + # A pending poll answers 422, so a client that raises on non-2xx needs + # to be told before it wraps this endpoint in a loop. + assert "422" in status_op["description"] assert status_op["responses"]["406"]["description"].strip() +def test_documents_are_uploaded_as_binary_not_as_urls() -> None: + """A bare DRF FileField documents as `format: uri`, which generators turn + into a string parameter and no multipart upload. + """ + files = _schema("ExecuteRequest")["properties"]["files"] + + assert files["items"] == {"type": "string", "format": "binary"} + + +def test_the_result_a_pending_execution_omits_is_documented_nullable() -> None: + """Both endpoints send `result: null` until the execution finishes, and a + generated deserialiser iterates that field. + """ + assert _schema("ExecutionMessage")["properties"]["result"]["nullable"] is True + assert _schema("StatusResponse")["properties"]["message"]["nullable"] is True + + def test_the_documented_response_fields_are_ones_the_code_produces() -> None: - """The view returns the execution DTO as a dict rather than through this - serializer, so a renamed DTO field would otherwise reach clients as a field - the server never sends. + """`APIExecutionResponseSerializer` builds the live execute response, so a + field documented here that the DTO no longer carries reaches clients as a + field the server never sends. """ documented = set(APIExecutionResponseSerializer().get_fields()) produced = {field.name for field in dataclasses.fields(ExecutionResponse)} assert documented <= produced, documented - produced + + +def test_the_documented_file_result_fields_are_ones_the_code_produces() -> None: + documented = set(_schema("FileResult")["properties"]) + produced = { + field.name for field in dataclasses.fields(FileExecutionResult) + } | _PROMOTED_FILE_RESULT_FIELDS + + assert documented <= produced, documented - produced + + +def test_the_status_read_documents_the_two_keys_it_returns() -> None: + """The status view builds its body literally, so the spec is the only + place the pair is written down. + """ + status_response = _schema("StatusResponse") + + assert set(status_response["properties"]) == {"status", "message"} + assert status_response["properties"]["message"]["items"]["$ref"].endswith( + "/FileResult" + ) + + +@pytest.mark.parametrize( + "exc", + [APIException("Unauthorized"), ValidationError("at least one file is required")], +) +def test_the_documented_error_body_is_the_one_the_handler_sends(exc) -> None: + """The error shape comes from the project-wide exception handler, not from + any view, so nothing else in the spec moves when that handler changes. + """ + request = APIRequestFactory().post("/deployment/api/org/api/") + response = drf_logging_exc_handler(exc=exc, context={"request": request}) + + error_response = _schema("ErrorResponse") + error_detail = _schema("ErrorDetail") + + assert set(response.data) == set(error_response["required"]) + assert response.data["type"] in _schema("ErrorType")["enum"] + for error in response.data["errors"]: + assert set(error) == set(error_detail["required"]) + + +def test_the_check_flag_passes_on_the_committed_spec() -> None: + call_command("generate_docstudio_spec", "--check") + + +def test_the_check_flag_fails_on_a_drifted_spec(tmp_path) -> None: + drifted = tmp_path / "drifted.json" + drifted.write_text("{}\n") + + with pytest.raises(CommandError, match="out of date"): + call_command("generate_docstudio_spec", "--check", "--out", str(drifted)) + + +def test_writing_the_spec_reproduces_the_committed_file(tmp_path) -> None: + written = tmp_path / "nested" / "spec.json" + + call_command("generate_docstudio_spec", "--out", str(written)) + + assert written.read_text() == DEFAULT_OUT.read_text() diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.py index 76cecc14b7..6edee88d73 100644 --- a/backend/backend/settings/base.py +++ b/backend/backend/settings/base.py @@ -650,7 +650,10 @@ def filter(self, record): "DEFAULT_VERSION": "v1", "ALLOWED_VERSIONS": ["v1"], "VERSION_PARAM": "version", - "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema", + # The standardized-errors variant, because EXCEPTION_HANDLER above + # delegates to that package: it is the schema class that knows the error + # bodies these views actually return. + "DEFAULT_SCHEMA_CLASS": "drf_standardized_errors.openapi.AutoSchema", } # Read only while generating the OpenAPI spec @@ -671,6 +674,9 @@ def filter(self, record): } } }, + # Without this the enum component is named after the field that holds it, + # and generated clients get a class called `TypeEnum`. + "ENUM_NAME_OVERRIDES": {"ErrorType": "api_v2.openapi_schema.ERROR_TYPES"}, # Group descriptions generated clients show in their help; without this # the spec has no root `tags` array for the text to live in. "TAGS": [ diff --git a/backend/mcp_server/views.py b/backend/mcp_server/views.py index 7c85f5d237..1ff6015646 100644 --- a/backend/mcp_server/views.py +++ b/backend/mcp_server/views.py @@ -25,8 +25,8 @@ # MCP speaks JSON-RPC over one POST, so it has no REST surface worth -# describing; leaving it in would publish guessed request and response shapes -# to every client generated from the spec. +# describing; leaving it in would fail spec generation, which treats any +# generator diagnostic as fatal and cannot resolve a serializer for this view. @extend_schema(exclude=True) class MCPServerView(BaseMCPView): """MCP JSON-RPC endpoint for a single API deployment. diff --git a/specs/docstudio-oss.json b/specs/docstudio-oss.json index edf3196660..c7f8ebb4b0 100644 --- a/specs/docstudio-oss.json +++ b/specs/docstudio-oss.json @@ -1,17 +1,74 @@ { "components": { "schemas": { - "ErrorResponse": { + "AcknowledgedResponse": { + "description": "The execution's result was handed to an earlier call and discarded.", "properties": { "message": { - "nullable": true + "type": "string" }, "status": { "type": "string" } }, + "required": [ + "message", + "status" + ], + "type": "object" + }, + "ErrorDetail": { + "description": "One problem found with the request.", + "properties": { + "attr": { + "description": "The request field the problem belongs to, when it belongs to one.", + "nullable": true, + "type": "string" + }, + "code": { + "description": "Machine-readable problem identifier.", + "type": "string" + }, + "detail": { + "description": "Human-readable description.", + "type": "string" + } + }, + "required": [ + "attr", + "code", + "detail" + ], "type": "object" }, + "ErrorResponse": { + "description": "The body of a rejected request.\n\nProduced by the project-wide exception handler, so its shape is the same\nfor every failure listed against an operation.", + "properties": { + "errors": { + "items": { + "$ref": "#/components/schemas/ErrorDetail" + }, + "type": "array" + }, + "type": { + "$ref": "#/components/schemas/ErrorType" + } + }, + "required": [ + "errors", + "type" + ], + "type": "object" + }, + "ErrorType": { + "description": "* `validation_error` - validation_error\n* `client_error` - client_error\n* `server_error` - server_error", + "enum": [ + "validation_error", + "client_error", + "server_error" + ], + "type": "string" + }, "ExecuteRequest": { "description": "The documents to run, and the options that shape the result.\n\nSupply `files`, `presigned_urls`, or both.", "properties": { @@ -26,10 +83,12 @@ "type": "array" }, "hitl_packet_id": { + "description": "Groups documents reviewed together into one packet. Requires the enterprise manual-review capability; an installation without it rejects the request with 400.", "nullable": true, "type": "string" }, "hitl_queue_name": { + "description": "Document class name for the manual review queue. Requires the enterprise manual-review capability; an installation without it rejects the request with 400.", "nullable": true, "type": "string" }, @@ -89,6 +148,7 @@ "description": "The execution's identity and, once it has finished, its per-file\nresults.", "properties": { "error": { + "nullable": true, "type": "string" }, "execution_id": { @@ -105,32 +165,41 @@ "type": "array" }, "status_api": { + "nullable": true, "type": "string" } }, "required": [ - "error", "execution_id", - "execution_status", - "status_api" + "execution_status" ], "type": "object" }, "FileResult": { + "description": "One input document's outcome.\n\nEvery key is present on every item; the ones that depend on the request\noptions or on the outcome are sent as `null` when they do not apply.", "properties": { "error": { "nullable": true, "type": "string" }, + "extracted_text": { + "description": "The document's full extracted text. Sent only when the request set `include_extracted_text`.", + "nullable": true, + "type": "string" + }, "file": { "type": "string" }, "file_execution_id": { + "nullable": true, "type": "string" }, - "metadata": {}, - "metrics": {}, - "result": {}, + "metadata": { + "nullable": true + }, + "result": { + "nullable": true + }, "status": { "type": "string" } @@ -175,7 +244,7 @@ "paths": { "/deployment/api/{org_name}/{api_name}/": { "get": { - "description": "Read the result of a previously started execution.\n\nThis read is one-shot: the first call that observes a completed execution acknowledges it and the stored result is discarded, so every later call for that execution answers 406. Poll while the execution is pending, and keep the payload of the call that returns it \u2014 it cannot be fetched again.", + "description": "Read the result of a previously started execution.\n\nThis read is one-shot: the first call that observes a completed execution acknowledges it and the stored result is discarded, so every later call for that execution answers 406. Poll while the execution is pending, and keep the payload of the call that returns it \u2014 it cannot be fetched again.\n\nA still-running execution answers 422 carrying its current `status`, so a polling loop should treat 422 as the normal reply and stop on 200. Clients that raise on any non-2xx need to allow for that.", "operationId": "status", "parameters": [ { @@ -256,38 +325,106 @@ "401": { "content": { "application/json": { + "examples": { + "AuthenticationFailed": { + "value": { + "errors": [ + { + "attr": null, + "code": "authentication_failed", + "detail": "Incorrect authentication credentials." + } + ], + "type": "client_error" + } + }, + "NotAuthenticated": { + "value": { + "errors": [ + { + "attr": null, + "code": "not_authenticated", + "detail": "Authentication credentials were not provided." + } + ], + "type": "client_error" + } + } + }, "schema": { "$ref": "#/components/schemas/ErrorResponse" } } }, - "description": "The API key is not valid." + "description": "No usable API key was supplied for the deployment." }, "403": { "content": { "application/json": { + "examples": { + "PermissionDenied": { + "value": { + "errors": [ + { + "attr": null, + "code": "permission_denied", + "detail": "You do not have permission to perform this action." + } + ], + "type": "client_error" + } + } + }, "schema": { "$ref": "#/components/schemas/ErrorResponse" } } }, - "description": "No API key was supplied." + "description": "The request was refused as unauthorized." }, "404": { "content": { "application/json": { + "examples": { + "NotFound": { + "value": { + "errors": [ + { + "attr": null, + "code": "not_found", + "detail": "Not found." + } + ], + "type": "client_error" + } + } + }, "schema": { "$ref": "#/components/schemas/ErrorResponse" } } }, - "description": "No such active deployment." + "description": "No active deployment, or a referenced document, was found." }, "406": { "content": { "application/json": { + "examples": { + "NotAcceptable": { + "value": { + "errors": [ + { + "attr": null, + "code": "not_acceptable", + "detail": "Could not satisfy the request Accept header." + } + ], + "type": "client_error" + } + } + }, "schema": { - "$ref": "#/components/schemas/ErrorResponse" + "$ref": "#/components/schemas/AcknowledgedResponse" } } }, @@ -301,27 +438,31 @@ } } }, - "description": "" - }, - "429": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ErrorResponse" - } - } - }, - "description": "Too many concurrent executions; retry later." + "description": "The execution is still running, or it finished with an error; read `status` to tell them apart." }, "500": { "content": { "application/json": { + "examples": { + "APIException": { + "value": { + "errors": [ + { + "attr": null, + "code": "error", + "detail": "A server error occurred." + } + ], + "type": "server_error" + } + } + }, "schema": { - "$ref": "#/components/schemas/ErrorResponse" + "$ref": "#/components/schemas/StatusResponse" } } }, - "description": "" + "description": "The execution could not be completed; the body carries its last known state." } }, "security": [ @@ -391,34 +532,88 @@ "401": { "content": { "application/json": { + "examples": { + "AuthenticationFailed": { + "value": { + "errors": [ + { + "attr": null, + "code": "authentication_failed", + "detail": "Incorrect authentication credentials." + } + ], + "type": "client_error" + } + }, + "NotAuthenticated": { + "value": { + "errors": [ + { + "attr": null, + "code": "not_authenticated", + "detail": "Authentication credentials were not provided." + } + ], + "type": "client_error" + } + } + }, "schema": { "$ref": "#/components/schemas/ErrorResponse" } } }, - "description": "The API key is not valid." + "description": "No usable API key was supplied for the deployment." }, "403": { "content": { "application/json": { + "examples": { + "PermissionDenied": { + "value": { + "errors": [ + { + "attr": null, + "code": "permission_denied", + "detail": "You do not have permission to perform this action." + } + ], + "type": "client_error" + } + } + }, "schema": { "$ref": "#/components/schemas/ErrorResponse" } } }, - "description": "No API key was supplied." + "description": "The request was refused as unauthorized." }, "404": { "content": { "application/json": { + "examples": { + "NotFound": { + "value": { + "errors": [ + { + "attr": null, + "code": "not_found", + "detail": "Not found." + } + ], + "type": "client_error" + } + } + }, "schema": { "$ref": "#/components/schemas/ErrorResponse" } } }, - "description": "No such active deployment." + "description": "No active deployment, or a referenced document, was found." }, - "409": { + "413": { "content": { "application/json": { "schema": { @@ -426,7 +621,7 @@ } } }, - "description": "The deployment has no active API key." + "description": "A referenced document is larger than the limit." }, "422": { "content": { @@ -436,11 +631,25 @@ } } }, - "description": "" + "description": "The execution finished with an error." }, "429": { "content": { "application/json": { + "examples": { + "Throttled": { + "value": { + "errors": [ + { + "attr": null, + "code": "throttled", + "detail": "Request was throttled." + } + ], + "type": "client_error" + } + } + }, "schema": { "$ref": "#/components/schemas/ErrorResponse" } @@ -449,6 +658,30 @@ "description": "Too many concurrent executions; retry later." }, "500": { + "content": { + "application/json": { + "examples": { + "APIException": { + "value": { + "errors": [ + { + "attr": null, + "code": "error", + "detail": "A server error occurred." + } + ], + "type": "server_error" + } + } + }, + "schema": { + "$ref": "#/components/schemas/ExecuteResponse" + } + } + }, + "description": "The deployment could not be run; the body carries the execution that failed." + }, + "502": { "content": { "application/json": { "schema": { @@ -456,7 +689,17 @@ } } }, - "description": "" + "description": "A referenced document could not be fetched." + }, + "504": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorResponse" + } + } + }, + "description": "Fetching a referenced document timed out." } }, "security": [