UN-4011 [FEAT] Support every extraction parameter via a generated transport

.gitattributes

@@ -0,0 +1 @@
+src/unstract/llmwhisperer/sdk_llmwhisperer/** linguist-generated=true

.github/workflows/ci_test.yaml

@@ -9,6 +9,66 @@ on:
     branches: [main]
 
 jobs:
+  sdk-drift:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.6.14"
+          python-version: 3.12.9
+
+      # The generated tree is committed, so an edit to it reviews like any
+      # other change and then disappears on the next regeneration. Same for a
+      # spec change that never had the generator run over it.
+      - name: Regenerate from the committed spec
+        run: ./tools/gen_sdk.sh
+
+      # `git add -N` first: a diff alone cannot see a file the generator has
+      # newly created, which is exactly what a spec growing an endpoint does.
+      - name: Fail if the committed SDK is not what the spec generates
+        run: |
+          git add -N -- src/unstract/llmwhisperer/sdk_llmwhisperer
+          git diff --exit-code -- src/unstract/llmwhisperer/sdk_llmwhisperer
+
+  api-surface:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          # griffe compares against the latest release tag, which the default
+          # shallow clone does not fetch.
+          fetch-depth: 0
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          version: "0.6.14"
+          python-version: 3.12.9
+
+      # Signatures only, against the latest release tag rather than a vendored
+      # file -- the one compat check whose reference point moves without anyone
+      # remembering to move it. It cannot see what goes out on the wire;
+      # tests/unit/compat_test.py owns that half and is not redundant with this.
+      #
+      # Named module by module rather than by package: the generated subpackage
+      # is public to griffe, so a spec change that drops a field would report as
+      # a breakage of this client's own surface -- and the drift gate above
+      # simultaneously requires the generated tree to follow the spec.
+      - name: Check the public API against the last release
+        run: |
+          for module in unstract.llmwhisperer.client_v2 unstract.llmwhisperer.utils; do
+            uvx griffe==2.2.0 check "$module" -s src -f github -X
+          done
+
   test:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest

.gitignore

@@ -164,3 +164,4 @@ cython_debug/
 
 .pdm-python
 .python-version
+.gen-venv

.pre-commit-config.yaml

@@ -6,16 +6,22 @@ repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v5.0.0
     hooks:
+      # Excluded per hook rather than globally: a formatter's fix to generated
+      # or vendored code is lost on the next refresh, but the security hooks
+      # below have to read every file in the tree.
       - id: trailing-whitespace
+        exclude: &generated "^(src/unstract/llmwhisperer/sdk_llmwhisperer/|tests/baseline/)"
         exclude_types:
           - "markdown"
       - id: end-of-file-fixer
+        exclude: *generated
       - id: check-yaml
         args: [--unsafe]
       - id: check-added-large-files
         args: ["--maxkb=10240"]
       - id: check-case-conflict
       - id: check-docstring-first
+        exclude: *generated
       - id: check-ast
       - id: check-json
         exclude: ".vscode/launch.json"
@@ -29,32 +35,40 @@ repos:
       - id: destroyed-symlinks
       - id: forbid-new-submodules
       - id: mixed-line-ending
+        exclude: *generated
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.4
+    # Kept in step with the pinned ruff in the dev group: two versions disagree
+    # on import order and docstrings, and each undoes the other's fixes.
+    rev: v0.11.9
     hooks:
       - id: ruff
         args: [--fix]
+        exclude: *generated
       - id: ruff-format
+        exclude: *generated
 
   - repo: https://github.com/hadialqattan/pycln
     rev: v2.4.0
     hooks:
       - id: pycln
         entry: uv run pycln
         args: [--config=pyproject.toml]
+        exclude: *generated
 
   - repo: https://github.com/pycqa/docformatter
     rev: v1.7.5
     hooks:
       - id: docformatter
         language: python
+        exclude: *generated
 
   - repo: https://github.com/asottile/pyupgrade
     rev: v3.17.0
     hooks:
       - id: pyupgrade
         entry: pyupgrade --py39-plus --keep-runtime-typing
+        exclude: *generated
         types:
           - python
 

README.md

@@ -15,6 +15,27 @@ This package provides **LLMWhispererClientV2**, the client for LLMWhisperer API
 
 Documentation is available [here](https://docs.unstract.com/llmwhisperer/).
 
+### Covered surface
+
+`specs/llmwhisperer.json` describes the whole service. It is a copy of the
+service's own published spec, taken at the revision `tools/gen_sdk.sh` records —
+never edited here, because an edit is reverted by the next refresh. A fix goes
+upstream first and comes back through a re-copy.
+
+This client wraps a subset of that surface, unchanged from what it has always
+wrapped. The operations it deliberately does not expose are listed as
+`UNWRAPPED_OPERATIONS` in `tests/unit/compat_test.py`, which fails if the two
+disagree — so that list, not this paragraph, is what to read.
+
+### Page separator
+
+The query parameter carrying the page separator was renamed in LLMWhisperer
+v2.64.2; before that release the service read only the earlier, misspelled
+spelling. The client sends the value under **both** spellings, so a custom
+`page_separator` applies whatever version the service is on, and no version
+check is needed against a self-hosted deployment. The behaviour is pinned by
+`tests/unit/client_v2_test.py`; do not drop either spelling as redundant.
+
 ## Running Tests
 
 Install test dependencies and run all tests:

pyproject.toml

@@ -18,7 +18,12 @@ classifiers = [
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 requires-python = ">=3.12"
-dependencies = ["requests>=2", "tenacity>=8.0"]
+# `requests` remains a dependency for its exception classes: callers catch
+# ConnectionError and Timeout by name, and the httpx equivalents are not
+# subclasses of them.
+# httpx is pinned to the single minor series the generated transport is written
+# against: a bump has to be regenerated and re-tested, not resolved into.
+dependencies = ["httpx>=0.28,<0.29", "attrs>=23.2", "requests>=2", "tenacity>=8.0"]
 
 [dependency-groups]
 test = [
@@ -33,10 +38,14 @@ dev = [
     "mypy~=1.2.0",
     "pre-commit~=3.3.1",
     "yamllint>=1.35.1",
-    "ruff<1.0.0,>=0.2.2",
+    "ruff==0.11.9",
     "pytest>=8.0.1",
     "pycln>=2.5.0",
     "poethepoet>=0.34.0",
+    # Stub-only packages. Without them mypy reports the import itself as an
+    # error and cannot check any use of what it names.
+    "types-requests>=2.32",
+    "types-setuptools>=75",
 ]
 
 [tool.poe.tasks]
@@ -86,6 +95,10 @@ exclude = [
     "dist",
     "node_modules",
     "venv",
+    # Generated and vendored code is overwritten wholesale by its refresh
+    # script, so a lint finding there can never be fixed in place.
+    "src/unstract/llmwhisperer/sdk_llmwhisperer",
+    "tests/baseline",
 ]
 
 [tool.ruff.lint]
@@ -106,6 +119,11 @@ select = [
 fixable = ["ALL"]
 ignore = [
     "D205",
+    # docformatter pulls a closing quote back onto the last line and ruff moves
+    # it off again, so with both enabled every multi-line docstring flips on
+    # every run and pre-commit never converges. docformatter wins here because
+    # it is the one that also rewraps.
+    "D209",
     "D100",   # Missing docstring in public module
     "D101",   # Missing docstring in public class
     "D102",   # Missing docstring in public method
@@ -118,6 +136,13 @@ ignore = [
     "N818",
 ]
 
+[tool.ruff.lint.per-file-ignores]
+# The client mirrors a service that takes and returns arbitrary JSON, and its
+# published signatures say so. Narrowing these annotations would describe an API
+# that is not the one callers have.
+"src/unstract/llmwhisperer/client_v2.py" = ["ANN401"]
+"tests/**" = ["ANN401"]
+
 [tool.ruff.format]
 quote-style = "double"
 indent-style = "space"
@@ -172,4 +197,17 @@ ignore_missing_imports = true
 pretty = true
 show_column_numbers = true
 show_error_codes = true
-exclude = ["venv", ".venv"]
+exclude = ["venv", ".venv", "tests/baseline/"]
+# `unstract` is a namespace package: without a root to compute module names
+# from, the generated tree is reachable under two names and mypy refuses to
+# check either.
+mypy_path = "src"
+explicit_package_bases = true
+
+# Generated code is overwritten wholesale by its refresh script, so a finding
+# there can never be fixed in place. Silenced rather than excluded: excluding it
+# also stops mypy resolving its types for the facade, which is where the calls
+# into it -- by keyword names the spec owns -- most need checking.
+[[tool.mypy.overrides]]
+module = "unstract.llmwhisperer.sdk_llmwhisperer.*"
+ignore_errors = true