From 78512b609cd22a30fdbe9b2ba2889dbc565082f4 Mon Sep 17 00:00:00 2001 From: Anthony Ronning <101225832+AnthonyRonning@users.noreply.github.com> Date: Wed, 18 Nov 2026 17:59:41 -0800 Subject: [PATCH 1/2] Production release inputs: metrics guard image, new served name Build the serving image here: Tinfoil's Kimi image plus one file that requires the metrics key on every path under vLLM's /metrics. vLLM's own bearer check does not cover that path, and the shim's only gate for it is Tinfoil's online admin-key validator. - maple_metrics_auth.py: pure ASGI middleware, standard library only, loaded with --middleware. Refuses when no key is configured. - scripts/install_metrics_auth.py: build-time installer that fails if the base image's vLLM is not the build the guard was written against. - Release workflow builds the image, pins its digest on a release-only commit and tags it; the renderer checks the pinned configuration. - Serve the model as kimi-k3-1-pro-uncensored. - Close /v1/messages and /v1/messages/count_tokens again. - EXPERIMENTS.md becomes CHANGES.md and lists every difference from stock. Co-Authored-By: Claude Fable 5.2 --- .dockerignore | 5 + .github/workflows/ci.yml | 45 +++++ .github/workflows/tinfoil-release-publish.yml | 10 +- .github/workflows/tinfoil-release.yml | 119 ++++++++++--- CHANGES.md | 37 ++++ Dockerfile | 9 + EXPERIMENTS.md | 21 --- README.md | 34 +++- maple_metrics_auth.py | 88 ++++++++++ render_config.py | 37 ++-- scripts/install_metrics_auth.py | 69 ++++++++ tests/test_metrics_auth.py | 161 ++++++++++++++++++ tests/test_release_inputs.py | 67 ++++++++ tinfoil-config.template.yml | 8 +- tinfoil-config.yml | 8 +- 15 files changed, 642 insertions(+), 76 deletions(-) create mode 100644 .dockerignore create mode 100644 .github/workflows/ci.yml create mode 100644 CHANGES.md create mode 100644 Dockerfile delete mode 100644 EXPERIMENTS.md create mode 100644 maple_metrics_auth.py create mode 100644 scripts/install_metrics_auth.py create mode 100644 tests/test_metrics_auth.py create mode 100644 tests/test_release_inputs.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..3950f9d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +** +!Dockerfile +!maple_metrics_auth.py +!scripts/ +!scripts/install_metrics_auth.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..7bd5d7c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,45 @@ +name: Validate + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +concurrency: + group: validate-${{ github.ref }} + cancel-in-progress: true + +jobs: + validate: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Check out the repository + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + persist-credentials: false + + - name: Test the metrics guard and the release inputs + run: PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -v + + - name: Check the committed configuration + run: | + python3 render_config.py --check + git diff --exit-code -- tinfoil-config.yml + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + + # The build itself is the test that the guard installs into the real + # image: the installer fails if vLLM is not the build it expects. + - name: Build the image without publishing + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 + env: + SOURCE_DATE_EPOCH: 0 + with: + context: . + platforms: linux/amd64 + push: false diff --git a/.github/workflows/tinfoil-release-publish.yml b/.github/workflows/tinfoil-release-publish.yml index 5f32c51..1432f7f 100644 --- a/.github/workflows/tinfoil-release-publish.yml +++ b/.github/workflows/tinfoil-release-publish.yml @@ -30,18 +30,18 @@ jobs: with: persist-credentials: false - - name: Validate the tagged staging configuration + - name: Validate the tagged configuration shell: bash run: | git ls-files --error-unmatch tinfoil-config.yml >/dev/null - python3 render_config.py --check + python3 render_config.py --check --require-pinned-image git diff --exit-code -- tinfoil-config.yml - - name: Measure, attest, and publish the staging release + - name: Measure, attest, and publish the release uses: tinfoilsh/measure-image-action@59f7ab5dbe472b3097c888ebe64e6a4131fc0975 # v0.14.0 with: config-file: ${{ github.workspace }}/tinfoil-config.yml github-token: ${{ secrets.GITHUB_TOKEN }} - # Publish the isolated staging release as latest so Tinfoil can - # issue a freshness witness for its deployment digest. + # Publish the release as latest so Tinfoil can issue a freshness + # witness for its deployment digest. mark-as-latest: 'true' diff --git a/.github/workflows/tinfoil-release.yml b/.github/workflows/tinfoil-release.yml index c68c0aa..20abf53 100644 --- a/.github/workflows/tinfoil-release.yml +++ b/.github/workflows/tinfoil-release.yml @@ -1,3 +1,6 @@ +# Two-phase Tinfoil release. Builds the image, pins its digest in the +# configuration on a release-only commit, tags that commit, then dispatches +# measurement on the tag. name: Tinfoil Release run-name: Prepare Tinfoil release ${{ inputs.version }} @@ -5,7 +8,7 @@ on: workflow_dispatch: inputs: version: - description: Stable staging release version (for example, v0.1.0) + description: Stable release version (for example, v0.2.0) required: true type: string @@ -13,11 +16,14 @@ concurrency: group: tinfoil-release-prepare-${{ inputs.version }} jobs: - prepare-release: + build: runs-on: ubuntu-latest + timeout-minutes: 60 permissions: - contents: write - actions: write + contents: read + packages: write + outputs: + digest: ${{ steps.build.outputs.digest }} steps: - name: Check out the requested commit @@ -25,7 +31,8 @@ jobs: with: persist-credentials: false - - name: Validate the committed staging configuration + - name: Validate the source and the requested version + id: release shell: bash env: VERSION: ${{ inputs.version }} @@ -36,31 +43,101 @@ jobs: exit 1 fi if [[ ! "$VERSION" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$ ]]; then - echo "::error::Choose a stable version such as v0.1.0." + echo "::error::Choose a stable version such as v0.2.0." + exit 1 + fi + if git ls-remote --exit-code --tags origin "refs/tags/$VERSION" >/dev/null; then + echo "::error::Tag $VERSION already exists. Choose a new version." exit 1 + else + status=$? + if [[ "$status" != 2 ]]; then exit "$status"; fi fi + PYTHONDONTWRITEBYTECODE=1 python3 -m unittest discover -s tests -v git ls-files --error-unmatch tinfoil-config.yml >/dev/null python3 render_config.py --check git diff --exit-code -- tinfoil-config.yml + echo "image-tag=${VERSION#v}" >> "$GITHUB_OUTPUT" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata for Docker + id: meta + uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 + with: + images: ghcr.io/${{ github.repository }} + flavor: latest=false + tags: | + type=raw,value=${{ steps.release.outputs.image-tag }} + labels: | + org.opencontainers.image.version=${{ inputs.version }} - - name: Tag the validated commit and dispatch measurement + - name: Build and push the image + id: build + uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 + env: + SOURCE_DATE_EPOCH: 0 + with: + context: . + platforms: linux/amd64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + prepare-release: + needs: build + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + actions: write + + steps: + - name: Check out the source commit + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + + - name: Check out the pinned configuration updater + uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + repository: tinfoilsh/update-container-action + ref: 5e0a3789316941664cee055706799114e0180565 # v0.6.0 + path: .release-tools + persist-credentials: false + + - name: Pin the built image and create the release tag shell: bash env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} VERSION: ${{ inputs.version }} + IMAGE_REFS: | + ghcr.io/mapleprivacylabs/inference-penclaw@${{ needs.build.outputs.digest }} run: | - tag_ref="refs/tags/$VERSION" - existing_commit=$(git ls-remote --refs origin "$tag_ref" | cut -f 1) - if [[ -z "$existing_commit" ]]; then - gh api --method POST "repos/$GH_REPO/git/refs" \ - -f ref="$tag_ref" -f sha="$GITHUB_SHA" --silent - elif [[ "$existing_commit" != "$GITHUB_SHA" ]]; then - echo "::error::The release tag already points to another commit. Choose a new version." - exit 1 - elif gh release view "$VERSION" >/dev/null 2>&1; then - echo "Release $VERSION is already published." - exit 0 - fi + python3 .release-tools/update_config.py + python3 render_config.py --check --require-pinned-image + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + source_date=$(git show -s --format=%cI "$GITHUB_SHA") + export GIT_AUTHOR_DATE="$source_date" GIT_COMMITTER_DATE="$source_date" + git add tinfoil-config.yml + git commit -m "chore: prepare $VERSION release" + git tag "$VERSION" + gh auth setup-git + git push origin "refs/tags/$VERSION" - gh workflow run tinfoil-release-publish.yml --ref "$VERSION" + - name: Dispatch measurement on the new tag + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + VERSION: ${{ inputs.version }} + run: gh workflow run tinfoil-release-publish.yml --ref "$VERSION" diff --git a/CHANGES.md b/CHANGES.md new file mode 100644 index 0000000..67461f3 --- /dev/null +++ b/CHANGES.md @@ -0,0 +1,37 @@ +# Differences from stock + +v0.1.4 equals the stock Maple Kimi v0.0.3 configuration apart from the primary +checkpoint fields. Every later difference is listed here. + +`render_config.py` lists every difference from stock in `EDITS` and refuses to +render a template that differs in any other way. + +## Current configuration (v0.2.0 onwards) + +| Difference from stock | Why | +|---|---| +| `cpus` 120 instead of 32 | The host is already paid for. Measured throughput is unchanged up to 32 concurrent requests. | +| Three DSpark draft tokens instead of five | About 15% more output at 32 concurrent requests in the same clean test (see v0.1.7 below). | +| Image built by this repository from Tinfoil's image | The [Dockerfile](Dockerfile) adds one file, [`maple_metrics_auth.py`](maple_metrics_auth.py), and changes nothing else. The build fails if the base image's vLLM is not the build the guard was written against. | +| `--middleware maple_metrics_auth.MetricsAuthMiddleware` | Loads the guard. It requires the `METRICS_API_KEY` bearer on every path under `/metrics` and passes all other requests through untouched. | +| Served model name `kimi-k3-1-pro-uncensored` instead of `kimi-k3-1-pro` | The node serves Audn's abliterated checkpoint, so it answers under a name that says so. Maple offers it as a separate model beside the stock Kimi K3.1 Pro. | +| `/metrics` in the shim paths | vLLM's Prometheus metrics (queue, cache, latency, token and speculative-decoding counters) become readable with the metrics key. Without the guard this path would be open: vLLM's own bearer check does not cover it, and the shim's only gate for it is Tinfoil's online admin-key validator. | + +The same key already protects the shim's own `/.well-known/tinfoil-metrics` and +`/.well-known/metrics`. Metrics are counters and histograms; they contain no +request content. + +## Release history + +| Release | Change from the previous release | Why | +|---|---|---| +| v0.1.5 | `cpus` 32 to 112; `/metrics` added to the shim paths. **Never deployed:** the platform accepts only 2, 4, 8, 16, 32, 56, 64, 120 or 128 vCPUs and rejected 112 at update planning | The identical host class runs two 56-vCPU guests, so 32 vCPUs for eight GPUs looked low. vLLM's Prometheus metrics give running and waiting requests, token counters, latency histograms and speculative-decoding acceptance, which have never been readable on this node. | +| v0.1.6 | `cpus` 120 in place of 112; otherwise v0.1.5 | Nearest accepted size above the 112 vCPUs that two DeepSeek guests use on the identical host class. | +| v0.1.7 | DSpark draft tokens 5 to 3 | On v0.1.6 the draft tokens were accepted 65%, 40%, 23%, 13% and 7% of the time by position: the fourth and fifth add about 0.2 tokens per step while every step carries a third more rows. Result on v0.1.6 for comparison: 128-134 tokens/s for one request, 549-603 at 8, 1,358-1,378 at 32. | +| v0.1.8 | `/v1/messages` and `/v1/messages/count_tokens` added to the shim paths | The published Terminal-Bench 4.1 result for Kimi K3.1 Pro was run with Claude Code, which speaks the Anthropic Messages format. vLLM already serves that format at this build; only the shim blocked the path. No engine setting changes. | +| v0.2.0 | Image built here with the metrics guard, loaded by `--middleware`; served model name `kimi-k3-1-pro-uncensored`; `/v1/messages` and `/v1/messages/count_tokens` removed from the shim paths again | First release meant to serve customers. `/metrics` needs the metrics key from this release on. The Anthropic Messages paths were only needed for the benchmark run. | + +**Result of v0.1.7, and the configuration kept:** three draft tokens were faster at every level in the same clean test: 140-146 tokens/s for one request, 600-642 at 8, 1,568-1,612 at 32 (about 15% more than v0.1.6), and 1,493 at 32 with real prompts against 1,281. Tokens per step fell only from about 2.5 to about 2.25. + +In v0.1.6 to v0.1.8 `/metrics` was reachable without a key. Those releases were +staging experiments and carried no customer traffic. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7ebb72f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,9 @@ +# syntax=docker/dockerfile:1.6 +# Tinfoil's qualified Kimi K3.1 Pro runtime, unchanged: same vLLM build, kernels, +# confidential-computing settings and entrypoint. The only addition is one file, +# a guard that requires the Maple metrics key on vLLM's /metrics. +FROM ghcr.io/tinfoilsh/confidential-kimi-k31-pro@sha256:0a81868bb275945f23e0e71d53524dd9a416bfcde3acb14903af0c2139495bc0 + +COPY maple_metrics_auth.py scripts/install_metrics_auth.py /tmp/ +RUN python3 -B /tmp/install_metrics_auth.py && \ + rm /tmp/install_metrics_auth.py /tmp/maple_metrics_auth.py diff --git a/EXPERIMENTS.md b/EXPERIMENTS.md deleted file mode 100644 index ad43809..0000000 --- a/EXPERIMENTS.md +++ /dev/null @@ -1,21 +0,0 @@ -# Performance experiments - -Releases after v0.1.4 are throughput experiments on the staging node. The node -carries no customer traffic. v0.1.4 remains the release that equals the stock -Maple Kimi v0.0.3 configuration apart from the primary checkpoint fields. - -`render_config.py` lists every difference from stock in `EXPERIMENT_EDITS` and -refuses to render a template that differs in any other way. - -| Release | Change from the previous release | Why | -|---|---|---| -| v0.1.5 | `cpus` 32 to 112; `/metrics` added to the shim paths. **Never deployed:** the platform accepts only 2, 4, 8, 16, 32, 56, 64, 120 or 128 vCPUs and rejected 112 at update planning | The identical host class runs two 56-vCPU guests, so 32 vCPUs for eight GPUs looked low. vLLM's Prometheus metrics give running and waiting requests, token counters, latency histograms and speculative-decoding acceptance, which have never been readable on this node. | -| v0.1.6 | `cpus` 120 in place of 112; otherwise v0.1.5 | Nearest accepted size above the 112 vCPUs that two DeepSeek guests use on the identical host class. | -| v0.1.7 | DSpark draft tokens 5 to 3 | On v0.1.6 the draft tokens were accepted 65%, 40%, 23%, 13% and 7% of the time by position: the fourth and fifth add about 0.2 tokens per step while every step carries a third more rows. Result on v0.1.6 for comparison: 128-134 tokens/s for one request, 549-603 at 8, 1,358-1,378 at 32. | -| v0.1.8 | `/v1/messages` and `/v1/messages/count_tokens` added to the shim paths | The published Terminal-Bench 4.1 result for Kimi K3.1 Pro was run with Claude Code, which speaks the Anthropic Messages format. vLLM already serves that format at this build; only the shim blocked the path. No engine setting changes. | - -**Result of v0.1.7, and the configuration kept:** three draft tokens were faster at every level in the same clean test: 140-146 tokens/s for one request, 600-642 at 8, 1,568-1,612 at 32 (about 15% more than v0.1.6), and 1,493 at 32 with real prompts against 1,281. Tokens per step fell only from about 2.5 to about 2.25. - -`/metrics` is reachable without a key while `shim.authenticated` is false. It -exposes counters and histograms, not request content. Remove it again before -this configuration serves anyone. diff --git a/README.md b/README.md index 28ec650..6dbe3b2 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,18 @@ -# Penclaw Kimi K3.1 Pro staging +# Penclaw Kimi K3.1 Pro -Releases after v0.1.4 are performance experiments and differ from the stock -configuration in the ways listed in [EXPERIMENTS.md](EXPERIMENTS.md). +Releases after v0.1.4 differ from the stock configuration in the ways listed +in [CHANGES.md](CHANGES.md). From v0.2.0 the serving image is built by this +repository: Tinfoil's image plus one file that requires the metrics key on +vLLM's `/metrics`. The v0.1.4 configuration serves Audn's Penclaw checkpoint using the [Maple Kimi v0.0.3 configuration](https://github.com/MaplePrivacyLabs/inference-kimi-k3-1-pro/blob/06a62d7829bdb30237846544874830a86305e9f8/tinfoil-config.yml). It changes the primary checkpoint repository and revision, Modelwrap tuple, pack schema, and vLLM model path. It uses the same eight B300 GPUs, serving -image, Inferact DSpark draft, and native vLLM bearer authentication. The -`kimi-k3-1-pro` API model name is retained for compatibility. The signed -Penclaw release identifies the actual checkpoint served under that name. +image, Inferact DSpark draft, and native vLLM bearer authentication. Through +v0.1.8 it answered to the stock API model name `kimi-k3-1-pro`. From v0.2.0 +the API model name is `kimi-k3-1-pro-uncensored`. The signed release +identifies the actual checkpoint served under either name. The [snapshot manifest](snapshot-manifest.json) records the exact Audn checkpoint revision and Hugging Face's checksums. The measured configuration pins each @@ -41,6 +44,19 @@ that commitment. The checkpoint is distributed under the Kimi K3.1-Pro license. Pass the full `root_offset_uuid` tuple from the completed schema-2 wrap to `python3 render_config.py --mpk TUPLE`. Run `python3 render_config.py --check` before publishing. The renderer verifies that the template matches the pinned -stock configuration outside the four primary pack fields. The release workflow -checks the rendered config on the default branch and at the release tag before -measuring it. +stock configuration outside the four primary pack fields and the differences +listed in `EDITS`. The source names the image without a digest. The release +workflow builds the image, writes its digest into the configuration on a +release-only commit, tags that commit and measures it. The renderer checks the +configuration on the default branch and again, with the pinned digest, at the +release tag. + +## The serving image + +The [Dockerfile](Dockerfile) starts from the exact image the stock +configuration pins and adds [`maple_metrics_auth.py`](maple_metrics_auth.py) +through [`scripts/install_metrics_auth.py`](scripts/install_metrics_auth.py). +The installer checks the vLLM version and the three places in vLLM's source the +guard depends on, and fails the build if any has changed. Tests in `tests/` +cover the guard and check that the Dockerfile, the configuration and the guard +agree. diff --git a/maple_metrics_auth.py b/maple_metrics_auth.py new file mode 100644 index 0000000..d3cc2ea --- /dev/null +++ b/maple_metrics_auth.py @@ -0,0 +1,88 @@ +"""Require the Maple metrics key on vLLM's Prometheus endpoint. + +vLLM's own bearer check covers /v1 and a few other prefixes. It does not cover +/metrics, and vLLM serves every path that starts with /metrics from the same +Prometheus application. This middleware asks for the metrics key on all of +those paths and hands every other request to vLLM untouched. + +Loaded with `--middleware maple_metrics_auth.MetricsAuthMiddleware`. It uses +only the standard library, so it adds nothing to the image but this file. +""" + +import hashlib +import hmac +import os + +GUARDED_PREFIX = "/metrics" +KEY_VARIABLE = "METRICS_API_KEY" + +_REFUSAL_BODY = b'{"error":"Unauthorized"}' +_REFUSAL_HEADERS = [ + (b"content-type", b"application/json"), + (b"content-length", str(len(_REFUSAL_BODY)).encode("ascii")), + (b"www-authenticate", b"Bearer"), +] + + +def is_guarded(path): + """True for any spelling of a path under /metrics. + + Repeated slashes and letter case are folded first. Guarding a path that + vLLM would not have served costs nothing; missing one would expose it. + """ + folded = "/" + "/".join(part for part in path.split("/") if part) + return folded.lower().startswith(GUARDED_PREFIX) + + +class MetricsAuthMiddleware: + """Pure ASGI middleware: no response wrapping, no buffering.""" + + def __init__(self, app): + self.app = app + key = os.environ.get(KEY_VARIABLE, "") + # Without a key nothing can match, so every guarded request is refused. + self._key_hash = hashlib.sha256(key.encode("utf-8")).digest() if key else None + + def _authorized(self, scope): + if self._key_hash is None: + return False + values = [ + value + for name, value in scope.get("headers") or () + if name.lower() == b"authorization" + ] + if len(values) != 1: + return False + scheme, _, token = values[0].partition(b" ") + if scheme.lower() != b"bearer" or not token: + return False + return hmac.compare_digest(hashlib.sha256(token).digest(), self._key_hash) + + async def __call__(self, scope, receive, send): + kind = scope.get("type") + if kind not in ("http", "websocket"): + await self.app(scope, receive, send) + return + + path = scope.get("path") or "" + root_path = scope.get("root_path") or "" + guarded = is_guarded(path) or ( + bool(root_path) + and path.startswith(root_path) + and is_guarded(path[len(root_path):]) + ) + if not guarded or self._authorized(scope): + await self.app(scope, receive, send) + return + + if kind == "websocket": + await send({"type": "websocket.close", "code": 1008}) + return + await send( + { + "type": "http.response.start", + "status": 401, + "headers": list(_REFUSAL_HEADERS), + } + ) + await send({"type": "http.response.body", "body": _REFUSAL_BODY}) diff --git a/render_config.py b/render_config.py index d135be5..5855b41 100644 --- a/render_config.py +++ b/render_config.py @@ -14,15 +14,23 @@ STOCK_REPO = "moonshotai/Kimi-K3.1-Pro@6d0a0202956796543364d666ab19c6ad1a494ee1" STOCK_MPK = "8d9c355a94e60f43598713759028328bbe795336f2700799d0ff4f3898e785e0_1561355993088_97fb42e4-45a9-5745-813b-61cb6ddb043d" STOCK_CONFIG_SHA256 = "2400260beabc0fe45b1c959a29c1401fbd703d6b233efa7de883eccba29de68b" -# Performance experiments on the staging node (see EXPERIMENTS.md). Each entry is -# (text in stock v0.0.3, text in this template). Undoing every entry must give -# back the stock configuration exactly, so the template can only differ from -# stock in the ways listed here and in the primary model fields. -EXPERIMENT_EDITS = [ +IMAGE = "ghcr.io/mapleprivacylabs/inference-penclaw" +STOCK_IMAGE = "ghcr.io/tinfoilsh/confidential-kimi-k31-pro@sha256:0a81868bb275945f23e0e71d53524dd9a416bfcde3acb14903af0c2139495bc0" +# The source template names the image without a digest. The release workflow +# builds the image and writes the pinned line into the tagged configuration. +IMAGE_TEMPLATE_LINE = f' image: "{IMAGE}:build-required" # source template; the release workflow pins the built digest\n' +IMAGE_PINNED_LINE = rf' image: "{re.escape(IMAGE)}@sha256:[0-9a-f]{{64}}" # v(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\n' +# Every difference from stock v0.0.3 besides the primary model fields (see +# CHANGES.md). Each entry is (text in stock, text in this template). Undoing +# every entry must give back the stock configuration exactly, so the template +# cannot differ from stock in any way that is not listed here. +EDITS = [ ("cpus: 32\n", "cpus: 120\n"), - (" - /health\n", " - /health\n - /metrics\n"), ('"num_speculative_tokens":5', '"num_speculative_tokens":3'), - (" - /v1/responses\n", " - /v1/responses\n - /v1/messages\n - /v1/messages/count_tokens\n"), + (f' image: "{STOCK_IMAGE}" # v0.0.2\n', IMAGE_TEMPLATE_LINE), + (" - --port\n - '8001'\n", " - --middleware\n - maple_metrics_auth.MetricsAuthMiddleware\n - --port\n - '8001'\n"), + (" - /health\n", " - /health\n - /metrics\n"), + (" - --served-model-name\n - kimi-k3-1-pro\n", " - --served-model-name\n - kimi-k3-1-pro-uncensored\n"), ] @@ -52,10 +60,10 @@ def render(mpk): template = (ROOT / "tinfoil-config.template.yml").read_bytes().decode("utf-8") stock = template - for stock_text, experiment_text in EXPERIMENT_EDITS: - if stock.count(experiment_text) != 1: - raise ValueError(f"Expected one experiment edit matching {experiment_text!r}") - stock = stock.replace(experiment_text, stock_text) + for stock_text, our_text in EDITS: + if stock.count(our_text) != 1: + raise ValueError(f"Expected one listed difference matching {our_text!r}") + stock = stock.replace(our_text, stock_text) stock_values = { f"repo: {REPO}@{REVISION}": f"repo: {STOCK_REPO}", "mpk: __PENCLAW_MPK__\n schema: 2": f"mpk: {STOCK_MPK}\n schema: 1", @@ -66,7 +74,7 @@ def render(mpk): raise ValueError(f"Expected one primary model field matching {marker!r}") stock = stock.replace(marker, value) if hashlib.sha256(stock.encode("utf-8")).hexdigest() != STOCK_CONFIG_SHA256: - raise ValueError("Template differs from stock v0.0.3 outside the primary model fields and the listed experiment edits") + raise ValueError("Template differs from stock v0.0.3 outside the primary model fields and the listed differences") return template.replace("__PENCLAW_MPK__", mpk).replace("__PENCLAW_ROOT__", pack[1]) @@ -77,10 +85,15 @@ def main(): mode.add_argument("--mpk", metavar="ROOT_OFFSET_UUID", help="Tuple from the completed schema-2 wrap") mode.add_argument("--check", action="store_true", help="Check the generated config is renderer-exact") parser.add_argument("--output", type=Path, default=ROOT / "tinfoil-config.yml") + parser.add_argument("--require-pinned-image", action="store_true", help="With --check: the config must pin a built image digest") args = parser.parse_args() try: if args.check: config = args.output.read_bytes().decode("utf-8") + # A tagged release carries the pinned image; the source carries the template line. + config, pinned = re.subn(IMAGE_PINNED_LINE, lambda _: IMAGE_TEMPLATE_LINE, config) + if pinned > 1 or (args.require_pinned_image and pinned != 1): + raise ValueError("Expected exactly one pinned image line for this repository's image") packs = re.findall(r"^ mpk: (\S+)$", config, re.MULTILINE) if len(packs) != 2 or config != render(packs[0]): raise ValueError("Measured config must match the rendered primary and stock draft packs") diff --git a/scripts/install_metrics_auth.py b/scripts/install_metrics_auth.py new file mode 100644 index 0000000..00dbd30 --- /dev/null +++ b/scripts/install_metrics_auth.py @@ -0,0 +1,69 @@ +"""Install the metrics guard into the image, after checking what it relies on. + +Runs once while the image is built. It refuses to continue if the vLLM build is +not the one the guard was written against, so a changed base image fails the +build instead of shipping a guard that no longer covers /metrics. +""" + +import importlib +import importlib.util +import inspect +import shutil +from importlib.metadata import version +from pathlib import Path + +EXPECTED_VLLM = "0.1.dev24216+g266e88c05" +SOURCE = Path("/tmp/maple_metrics_auth.py") + +# Exact text the guard depends on, by file under the installed vllm package. +REQUIRED_SOURCE = { + # How `--middleware` loads a class: added with add_middleware, no wrapping. + "entrypoints/serve/middleware/register.py": ( + "for middleware in args.middleware:", + "if inspect.isclass(imported):", + "app.add_middleware(imported)", + ), + # Where Prometheus is served: every path that starts with /metrics. + "entrypoints/serve/instrumentator/metrics.py": ( + 'Mount("/metrics", make_asgi_app(registry=registry))', + 're.compile("^/metrics(?P.*)$")', + ), + # vLLM's own bearer check does not cover /metrics, which is why this exists. + "entrypoints/serve/middleware/authenticate.py": ( + 'GUARDED_PREFIX = ("/v1", "/v2", "/inference", "/cohere")', + ), +} + + +def main(): + installed = version("vllm") + if installed != EXPECTED_VLLM: + raise SystemExit(f"Expected vllm {EXPECTED_VLLM}, found {installed}") + + spec = importlib.util.find_spec("vllm") + if spec is None or not spec.submodule_search_locations: + raise SystemExit("The vllm package is not importable by this interpreter") + package = Path(next(iter(spec.submodule_search_locations))) + + for relative, needles in REQUIRED_SOURCE.items(): + text = (package / relative).read_text() + for needle in needles: + if text.count(needle) != 1: + raise SystemExit(f"{relative} no longer contains exactly one {needle!r}") + + target = package.parent / SOURCE.name + if target.exists(): + raise SystemExit(f"{target} already exists") + shutil.copyfile(SOURCE, target) + target.chmod(0o644) + + module = importlib.import_module("maple_metrics_auth") + if Path(module.__file__).resolve() != target.resolve(): + raise SystemExit(f"Imported {module.__file__}, expected {target}") + if not inspect.isclass(module.MetricsAuthMiddleware): + raise SystemExit("MetricsAuthMiddleware must be a class") + print(f"Installed {target} for vllm {installed}") + + +if __name__ == "__main__": + main() diff --git a/tests/test_metrics_auth.py b/tests/test_metrics_auth.py new file mode 100644 index 0000000..2082316 --- /dev/null +++ b/tests/test_metrics_auth.py @@ -0,0 +1,161 @@ +"""The metrics guard: what it refuses, what it lets through, and that it stays out of the way.""" + +import asyncio +import importlib +import os +import sys +import unittest +from pathlib import Path +from unittest import mock + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +KEY = "test-metrics-key" + + +def load(key=KEY): + """Build the middleware around a recording app, with the given key in the environment.""" + environment = {} if key is None else {"METRICS_API_KEY": key} + with mock.patch.dict(os.environ, environment, clear=True): + module = importlib.import_module("maple_metrics_auth") + calls = [] + + async def app(scope, receive, send): + calls.append((scope, receive, send)) + + return module.MetricsAuthMiddleware(app), calls + + +def request(path, authorization=None, kind="http", root_path="", extra_headers=()): + headers = list(extra_headers) + if authorization is not None: + headers.append((b"authorization", authorization.encode())) + return {"type": kind, "path": path, "root_path": root_path, "headers": headers, "method": "GET"} + + +def run(middleware, scope): + sent = [] + + async def receive(): + return {"type": "http.request"} + + async def send(message): + sent.append(message) + + asyncio.run(middleware(scope, receive, send)) + return sent, receive, send + + +class Refusals(unittest.TestCase): + def assert_refused(self, scope, middleware=None): + middleware, calls = (middleware, None) if middleware else load() + sent, _, _ = run(middleware, scope) + self.assertEqual([m["type"] for m in sent], ["http.response.start", "http.response.body"]) + self.assertEqual(sent[0]["status"], 401) + self.assertIn((b"www-authenticate", b"Bearer"), sent[0]["headers"]) + self.assertEqual(sent[1]["body"], b'{"error":"Unauthorized"}') + if calls is not None: + self.assertEqual(calls, []) + + def test_no_key_sent(self): + self.assert_refused(request("/metrics")) + + def test_wrong_key(self): + self.assert_refused(request("/metrics", "Bearer not-the-key")) + + def test_key_with_extra_characters(self): + for value in (f"Bearer {KEY}x", f"Bearer x{KEY}", f"Bearer {KEY} ", f"Bearer {KEY}"): + with self.subTest(value=value): + self.assert_refused(request("/metrics", value)) + + def test_other_schemes_and_empty_tokens(self): + for value in (f"Basic {KEY}", KEY, "Bearer", "Bearer ", ""): + with self.subTest(value=value): + self.assert_refused(request("/metrics", value)) + + def test_two_authorization_headers(self): + scope = request( + "/metrics", + f"Bearer {KEY}", + extra_headers=[(b"authorization", b"Bearer something-else")], + ) + self.assert_refused(scope) + + def test_every_spelling_under_metrics(self): + # vLLM routes with the pattern ^/metrics(.*)$, so all of these reach Prometheus. + for path in ("/metrics/", "/metrics/anything", "/metricsfoo", "//metrics", "/METRICS", "/Metrics/x", "metrics"): + with self.subTest(path=path): + self.assert_refused(request(path)) + + def test_root_path_prefix(self): + self.assert_refused(request("/proxy/metrics", root_path="/proxy")) + + def test_every_method(self): + for method in ("GET", "HEAD", "POST", "OPTIONS"): + with self.subTest(method=method): + scope = request("/metrics") + scope["method"] = method + self.assert_refused(scope) + + def test_no_key_configured_refuses_everything(self): + for key in (None, ""): + for authorization in (None, "Bearer ", "Bearer anything"): + with self.subTest(key=key, authorization=authorization): + middleware, calls = load(key) + self.assert_refused(request("/metrics", authorization), middleware) + self.assertEqual(calls, []) + + def test_websocket_is_closed(self): + middleware, calls = load() + sent, _, _ = run(middleware, request("/metrics", kind="websocket")) + self.assertEqual(sent, [{"type": "websocket.close", "code": 1008}]) + self.assertEqual(calls, []) + + +class PassThrough(unittest.TestCase): + def assert_passed_untouched(self, scope, key=KEY): + middleware, calls = load(key) + sent, receive, send = run(middleware, scope) + self.assertEqual(sent, []) + self.assertEqual(len(calls), 1) + # The very same objects: nothing is wrapped, so streaming and + # disconnect handling behave exactly as they do without the guard. + self.assertIs(calls[0][0], scope) + self.assertIs(calls[0][1], receive) + self.assertIs(calls[0][2], send) + + def test_metrics_with_the_key(self): + for value in (f"Bearer {KEY}", f"bearer {KEY}", f"BEARER {KEY}"): + with self.subTest(value=value): + self.assert_passed_untouched(request("/metrics", value)) + + def test_header_name_case(self): + scope = request("/metrics", extra_headers=[(b"Authorization", f"Bearer {KEY}".encode())]) + self.assert_passed_untouched(scope) + + def test_inference_paths_need_nothing_from_this_guard(self): + for path in ("/v1/chat/completions", "/v1/models", "/v1/responses", "/health", "/", "/v1/metrics"): + with self.subTest(path=path): + self.assert_passed_untouched(request(path)) + + def test_inference_paths_pass_without_a_configured_key(self): + self.assert_passed_untouched(request("/v1/chat/completions"), key=None) + + def test_lifespan_and_other_scopes(self): + self.assert_passed_untouched({"type": "lifespan"}) + + +class Module(unittest.TestCase): + def test_standard_library_only(self): + source = (Path(__file__).resolve().parent.parent / "maple_metrics_auth.py").read_text() + imports = sorted(line.split()[1] for line in source.splitlines() if line.startswith("import ")) + self.assertEqual(imports, ["hashlib", "hmac", "os"]) + self.assertNotIn("\nfrom ", source) + + def test_key_is_never_kept_in_the_clear(self): + middleware, _ = load() + self.assertNotIn(KEY, repr(vars(middleware))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_inputs.py b/tests/test_release_inputs.py new file mode 100644 index 0000000..961599c --- /dev/null +++ b/tests/test_release_inputs.py @@ -0,0 +1,67 @@ +"""The image, the measured configuration and the metrics guard must agree with each other.""" + +import re +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT)) + +import render_config # noqa: E402 + +CONFIG = (ROOT / "tinfoil-config.yml").read_text() +TEMPLATE = (ROOT / "tinfoil-config.template.yml").read_text() + + +def block(text, key): + """The list items under a top-level-or-nested `key:` line, in order.""" + match = re.search(rf"^( *){re.escape(key)}:\n((?:\1 *- .*\n)+)", text, re.MULTILINE) + if match is None: + raise AssertionError(f"No list under {key!r}") + return [line.strip()[2:] for line in match.group(2).splitlines()] + + +class ReleaseInputs(unittest.TestCase): + def test_image_is_built_from_the_stock_image(self): + dockerfile = (ROOT / "Dockerfile").read_text() + self.assertEqual(re.findall(r"^FROM (\S+)$", dockerfile, re.MULTILINE), [render_config.STOCK_IMAGE]) + + def test_image_adds_only_the_guard(self): + dockerfile = (ROOT / "Dockerfile").read_text() + instructions = [line.split()[0] for line in dockerfile.splitlines() if line and line[0].isalpha()] + self.assertEqual(instructions, ["FROM", "COPY", "RUN"]) + ignore = (ROOT / ".dockerignore").read_text().split() + self.assertEqual(ignore, ["**", "!Dockerfile", "!maple_metrics_auth.py", "!scripts/", "!scripts/install_metrics_auth.py"]) + + def test_source_names_exactly_one_unpinned_image_of_ours(self): + for text in (CONFIG, TEMPLATE): + self.assertEqual(re.findall(r'^ *image: "([^"]+)"', text, re.MULTILINE), [f"{render_config.IMAGE}:build-required"]) + + def test_public_paths(self): + self.assertEqual( + block(CONFIG, " paths"), + ["/v1/chat/completions", "/v1/models", "/v1/responses", "/health", "/metrics"], + ) + + def test_metrics_is_public_only_with_the_guard_and_its_key(self): + command = block(CONFIG, " command") + index = command.index("--middleware") + self.assertEqual(command[index + 1], "maple_metrics_auth.MetricsAuthMiddleware") + self.assertEqual(command.count("--middleware"), 1) + self.assertEqual(block(CONFIG, " secrets"), ["VLLM_API_KEY", "METRICS_API_KEY"]) + # The guard is Python middleware, so the Python API server must stay selected. + self.assertIn(" - VLLM_USE_RUST_FRONTEND: '0'\n", CONFIG) + + def test_served_model_name(self): + command = block(CONFIG, " command") + self.assertEqual(command[command.index("--served-model-name") + 1], "kimi-k3-1-pro-uncensored") + self.assertEqual(command.count("--served-model-name"), 1) + + def test_installer_expects_the_stock_vllm_build(self): + installer = (ROOT / "scripts" / "install_metrics_auth.py").read_text() + self.assertIn('EXPECTED_VLLM = "0.1.dev24216+g266e88c05"', installer) + + +if __name__ == "__main__": + unittest.main() diff --git a/tinfoil-config.template.yml b/tinfoil-config.template.yml index 8f70abd..548c08c 100644 --- a/tinfoil-config.template.yml +++ b/tinfoil-config.template.yml @@ -13,7 +13,7 @@ models: schema: 1 containers: - name: kimi-k3-1-pro - image: "ghcr.io/tinfoilsh/confidential-kimi-k31-pro@sha256:0a81868bb275945f23e0e71d53524dd9a416bfcde3acb14903af0c2139495bc0" # v0.0.2 + image: "ghcr.io/mapleprivacylabs/inference-penclaw:build-required" # source template; the release workflow pins the built digest runtime: nvidia read_only: true gpus: all @@ -50,7 +50,7 @@ containers: - --model - /tinfoil/mpk/mpk-__PENCLAW_ROOT__ - --served-model-name - - kimi-k3-1-pro + - kimi-k3-1-pro-uncensored - --tensor-parallel-size - '8' - --decode-context-parallel-size @@ -77,6 +77,8 @@ containers: - --trust-request-mm-kwargs - --scheduling-policy - priority + - --middleware + - maple_metrics_auth.MetricsAuthMiddleware - --port - '8001' - --max-model-len @@ -118,7 +120,5 @@ shim: - /v1/chat/completions - /v1/models - /v1/responses - - /v1/messages - - /v1/messages/count_tokens - /health - /metrics diff --git a/tinfoil-config.yml b/tinfoil-config.yml index 48622a1..0354194 100644 --- a/tinfoil-config.yml +++ b/tinfoil-config.yml @@ -13,7 +13,7 @@ models: schema: 1 containers: - name: kimi-k3-1-pro - image: "ghcr.io/tinfoilsh/confidential-kimi-k31-pro@sha256:0a81868bb275945f23e0e71d53524dd9a416bfcde3acb14903af0c2139495bc0" # v0.0.2 + image: "ghcr.io/mapleprivacylabs/inference-penclaw:build-required" # source template; the release workflow pins the built digest runtime: nvidia read_only: true gpus: all @@ -50,7 +50,7 @@ containers: - --model - /tinfoil/mpk/mpk-0e6d901b33b7a141f23c8aae390f636d714b605cf07a66dc6632ed723780438f - --served-model-name - - kimi-k3-1-pro + - kimi-k3-1-pro-uncensored - --tensor-parallel-size - '8' - --decode-context-parallel-size @@ -77,6 +77,8 @@ containers: - --trust-request-mm-kwargs - --scheduling-policy - priority + - --middleware + - maple_metrics_auth.MetricsAuthMiddleware - --port - '8001' - --max-model-len @@ -118,7 +120,5 @@ shim: - /v1/chat/completions - /v1/models - /v1/responses - - /v1/messages - - /v1/messages/count_tokens - /health - /metrics From d8b2e9144524439eb7d810b2a3fb338c39703a9f Mon Sep 17 00:00:00 2001 From: Anthony Ronning <101225832+AnthonyRonning@users.noreply.github.com> Date: Wed, 18 Nov 2026 18:07:11 -0800 Subject: [PATCH 2/2] Verify the installed guard from a fresh interpreter The first build copied the guard correctly but the check imported the source copy beside the installer. Import it in a separate isolated interpreter instead, the way the server will find it. Co-Authored-By: Claude --- scripts/install_metrics_auth.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/scripts/install_metrics_auth.py b/scripts/install_metrics_auth.py index 00dbd30..de628fc 100644 --- a/scripts/install_metrics_auth.py +++ b/scripts/install_metrics_auth.py @@ -5,10 +5,10 @@ build instead of shipping a guard that no longer covers /metrics. """ -import importlib import importlib.util -import inspect import shutil +import subprocess +import sys from importlib.metadata import version from pathlib import Path @@ -57,11 +57,28 @@ def main(): shutil.copyfile(SOURCE, target) target.chmod(0o644) - module = importlib.import_module("maple_metrics_auth") - if Path(module.__file__).resolve() != target.resolve(): - raise SystemExit(f"Imported {module.__file__}, expected {target}") - if not inspect.isclass(module.MetricsAuthMiddleware): - raise SystemExit("MetricsAuthMiddleware must be a class") + # Import it the way the server will: a fresh interpreter that cannot see + # this script's directory, where the source copy still sits. + probe = subprocess.run( + [ + sys.executable, + "-B", + "-I", + "-c", + "import inspect, maple_metrics_auth as m; " + "assert inspect.isclass(m.MetricsAuthMiddleware); print(m.__file__)", + ], + cwd="/", + capture_output=True, + text=True, + check=False, + ) + imported = probe.stdout.strip() + if probe.returncode != 0 or Path(imported).resolve() != target.resolve(): + raise SystemExit( + f"The installed guard does not import from {target}: " + f"exit {probe.returncode}, imported {imported!r}, {probe.stderr.strip()!r}" + ) print(f"Installed {target} for vllm {installed}")