UN-4009 [MISC] Generate and commit the API deployment OpenAPI spec in-repo

backend/README.md

@@ -175,11 +175,15 @@ psql -d unstract_db -U unstract_dev
 
 ## API Docs
 
-While running the backend server locally, access the API documentation that's auto generated at
-the backend endpoint `/api/v1/doc/`.
+The OpenAPI spec for the API deployment endpoints is committed at
+[`specs/docstudio-oss.json`](../specs/docstudio-oss.json) and is the contract the published
+clients and their generated SDKs are built from. It is not served at runtime — regenerate it in
+the same PR as any route, serializer or schema-annotation change:
 
-**NOTE:** There exists issues accessing this when the django server is run with gunicorn (in case of running with
-a container)
+```bash
+uv run python manage.py generate_docstudio_spec           # rewrite the committed spec
+uv run python manage.py generate_docstudio_spec --check   # no write, drift is an error
+```
 
 - [Account](account/api_doc.md)
 - [FileManagement](file_management/api_doc.md)

backend/api_v2/api_deployment_views.py

@@ -33,6 +33,7 @@
     contains_tool_not_found_error,
 )
 from api_v2.models import APIDeployment
+from api_v2.openapi_schema import DEPLOYMENT_EXECUTION_SCHEMA
 from api_v2.rate_limiter import APIDeploymentRateLimiter
 from api_v2.serializers import (
     APIDeploymentListSerializer,
@@ -50,6 +51,7 @@
 logger = logging.getLogger(__name__)
 
 
+@DEPLOYMENT_EXECUTION_SCHEMA
 class DeploymentExecution(views.APIView):
     def initialize_request(self, request: Request, *args: Any, **kwargs: Any) -> Request:
         """To remove csrf request for public API.

backend/api_v2/deployment_spec_urls.py

@@ -0,0 +1,29 @@
+"""URLconf the published OpenAPI spec is generated against.
+
+Each entry is an included sub-urlconf: generating against one directly yields
+paths without the prefix it is mounted at, i.e. a spec describing URLs the
+server does not serve. The mounts are selected out of the served urlconf
+rather than restated, so moving one moves the generated paths with it.
+
+Widening the spec to another endpoint means annotating its view with
+``@extend_schema`` and adding its urlconf here.
+"""
+
+from django.core.exceptions import ImproperlyConfigured
+
+from backend import base_urls
+
+SPEC_URLCONFS = ("api_v2.execution_urls",)
+
+urlpatterns = [
+    entry
+    for entry in base_urls.urlpatterns
+    if getattr(getattr(entry, "urlconf_name", None), "__name__", None) in SPEC_URLCONFS
+]
+
+missing = set(SPEC_URLCONFS) - {entry.urlconf_name.__name__ for entry in urlpatterns}
+if missing:
+    raise ImproperlyConfigured(
+        f"{', '.join(sorted(missing))} is not mounted in backend.base_urls; the "
+        "spec would be generated for routes the server does not serve."
+    )

backend/api_v2/management/commands/generate_docstudio_spec.py

@@ -0,0 +1,132 @@
+"""Regenerate the committed API deployment OpenAPI spec.
+
+The spec is the contract the published clients and their generated SDKs are
+built from, so it is committed and CI fails on drift: change a route, a
+serializer or the schema annotation, and regenerate in the same PR.
+
+    uv run python manage.py generate_docstudio_spec           # from backend/
+    uv run python manage.py generate_docstudio_spec --check   # no write, drift is an error
+
+The generated paths carry ``API_DEPLOYMENT_PATH_PREFIX``, so generation refuses
+to produce a spec mounted anywhere but the public default: the committed
+artifact describes the deployment as it is served publicly, not as one
+installation chooses to mount it.
+"""
+
+import json
+from pathlib import Path
+from typing import Any
+
+from django.core.management.base import BaseCommand, CommandError
+from drf_spectacular.drainage import GENERATOR_STATS
+from drf_spectacular.generators import SchemaGenerator
+from drf_spectacular.validation import validate_schema
+
+DEFAULT_OUT = Path(__file__).resolve().parents[4] / "specs" / "docstudio-oss.json"
+URLCONF = "api_v2.deployment_spec_urls"
+REGENERATE = "uv run python manage.py generate_docstudio_spec"
+# The mount the deployment is served at publicly. `API_DEPLOYMENT_PATH_PREFIX`
+# can move it per installation, and a spec carrying a private prefix would send
+# every generated client to a URL only that installation answers.
+PUBLISHED_PATH_PREFIX = "deployment"
+# Named in every failure message: the repos that regenerate from this file are
+# the ones a spec change actually breaks, and nothing there watches this repo.
+DOWNSTREAM = (
+    "The published client (Zipstack/unstract-python-client) and the CLI "
+    "(Zipstack/unstract-cli) are generated from this file — raise the matching "
+    "PRs there for anything that changes an operation id, a tag or a schema."
+)
+
+
+class SpecGenerationFailed(CommandError):
+    """Raised when the generator had to guess."""
+
+
+def render_spec() -> str:
+    """The committed artifact, byte for byte.
+
+    Shared with the drift test: two copies of this could disagree, and then
+    the gate rejects exactly the file the command it names produces.
+    """
+    GENERATOR_STATS.reset()
+    schema = SchemaGenerator(urlconf=URLCONF).get_schema(request=None, public=True)
+    if GENERATOR_STATS:
+        # An operation spectacular could not resolve is published with no
+        # request body and an empty response rather than dropped, which reads
+        # downstream as an annotation that is simply thin. Both caches are
+        # drained because the severity a given diagnostic carries is
+        # spectacular's choice, not something to rely on.
+        diagnostics = "\n".join(
+            f"  {severity}: {message}"
+            for severity, cache in (
+                ("error", GENERATOR_STATS._error_cache),
+                ("warning", GENERATOR_STATS._warn_cache),
+            )
+            for message in cache
+        )
+        raise SpecGenerationFailed(
+            f"The generator reported problems, so the spec would describe an "
+            f"API nobody implements:\n{diagnostics}"
+        )
+
+    off_prefix = [
+        path
+        for path in schema["paths"]
+        if not path.startswith(f"/{PUBLISHED_PATH_PREFIX}/")
+    ]
+    if off_prefix:
+        raise SpecGenerationFailed(
+            f"Generated paths are not under /{PUBLISHED_PATH_PREFIX}/: "
+            f"{', '.join(sorted(off_prefix))}. Unset API_DEPLOYMENT_PATH_PREFIX "
+            f"and regenerate."
+        )
+
+    # Hand-written fragments (path parameter schemas, security schemes) reach
+    # the output verbatim, so nothing above would notice a typo in one.
+    try:
+        validate_schema(schema)
+    except Exception as error:
+        raise SpecGenerationFailed(f"The generated spec is not valid OpenAPI: {error}")
+
+    # Sorted keys are what make the committed artifact a usable drift signal.
+    return json.dumps(schema, indent=2, sort_keys=True) + "\n"
+
+
+class Command(BaseCommand):
+    help = "Generate the API deployment OpenAPI spec."
+
+    def add_arguments(self, parser: Any) -> None:
+        parser.add_argument("--out", type=Path, default=DEFAULT_OUT)
+        parser.add_argument(
+            "--check",
+            action="store_true",
+            help="Fail if the file on disk differs, instead of writing it.",
+        )
+
+    def handle(self, *args: Any, **options: Any) -> None:
+        rendered = render_spec()
+
+        out: Path = options["out"]
+        if options["check"]:
+            current = out.read_text() if out.exists() else ""
+            if current != rendered:
+                raise CommandError(
+                    f"{out} is out of date. Run `{REGENERATE}` from `backend/` "
+                    f"and commit the result.\n\n{DOWNSTREAM}"
+                )
+            self.stdout.write(f"{out} is up to date")
+            return
+
+        out.parent.mkdir(parents=True, exist_ok=True)
+        out.write_text(rendered)
+        schema = json.loads(rendered)
+        operations = sum(
+            1
+            for methods in schema["paths"].values()
+            for method in methods
+            if method in {"get", "post", "put", "patch", "delete"}
+        )
+        self.stdout.write(
+            f"{out}: {len(schema['paths'])} paths, {operations} operations, "
+            f"{len(schema.get('components', {}).get('schemas', {}))} schemas"
+        )