Auth: recognise known OpenFn clients and use the keys we hold for them

.env.example

@@ -1,7 +1,30 @@
-OPENAI_API_KEY=sk-YOUR-API-KEY-HERE
+# Instance auth: /services/* is always gated on the api_key callers send, looked
+# up in lightning_clients via APOLLO_CLIENTS_DB_URL (falls back to POSTGRES_URL).
+# A known client swaps in its stored Anthropic key; an unknown sk-ant- key is
+# forwarded. See platform/src/auth/README.md.
+
+# Database holding the lightning_clients credentials table. Leave unset locally to
+# share POSTGRES_URL (one DB for everything). In production point this at a separate
+# credentials DB so client secrets don't co-locate with the docs data. The TS auth
+# code, `bun run migrate`, and the `client` CLI all resolve this var; the Python docs
+# services always use POSTGRES_URL.
+# APOLLO_CLIENTS_DB_URL=postgresql://localhost:5432/apollo_clients
+
+# Optional at-rest encryption for stored client Anthropic keys. Base64 of 32 bytes
+# (openssl rand -base64 32). See platform/src/auth/README.md.
+# APOLLO_ENC_KEY=
+
+# Shared secret for internal Apollo-to-Apollo apollo() calls. In production set
+# this to the SAME value across all processes — it is what lets self-calls through
+# the gate, and the global ANTHROPIC_API_KEY is a dev-only fallback. If unset,
+# each process mints its own token, which only works single-process-per-host.
+# APOLLO_INTERNAL_TOKEN=
+
 ANTHROPIC_API_KEY=sk-YOUR-API-KEY-HERE
+
+OPENAI_API_KEY=sk-YOUR-API-KEY-HERE
 PINECONE_KEY=YOUR-API-KEY-HERE
-POSTGRES_URL=POSTGRES_URL=postgresql://localhost:5432/apollo_dev
+POSTGRES_URL=postgresql://localhost:5432/apollo_dev
 SENTRY_DSN=YOUR-API-KEY-HERE
 GITHUB_TOKEN=KEY
 

.github/workflows/dockerize.yaml

@@ -24,6 +24,15 @@ jobs:
           echo Docker Tag: $DOCKER_TAG
 
           echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV
+
+          # Only move :latest for final releases. A version with a hyphen
+          # (e.g. 1.4.0-pre.0) is a pre-release and must not repoint latest.
+          {
+            echo "DOCKER_TAGS<<EOF"
+            echo "openfn/apollo:v${DOCKER_TAG}"
+            [[ "$DOCKER_TAG" != *-* ]] && echo "openfn/apollo:latest"
+            echo "EOF"
+          } >> $GITHUB_ENV
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v4
       - name: Set up Docker Buildx
@@ -38,6 +47,4 @@ jobs:
         with:
           context: .
           push: ${{ github.event_name != 'pull_request' }}
-          tags: |
-            openfn/apollo:latest
-            openfn/apollo:v${{ env.DOCKER_TAG }}
+          tags: ${{ env.DOCKER_TAGS }}

.github/workflows/unit-tests.yaml

@@ -33,11 +33,31 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 5
 
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: apollo
+          POSTGRES_PASSWORD: apollo
+          POSTGRES_DB: apollo_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    env:
+      POSTGRES_URL: postgres://apollo:apollo@localhost:5432/apollo_test
+
     steps:
       - uses: actions/checkout@v6
 
       - name: Set up Bun
         uses: oven-sh/setup-bun@v2
+        with:
+          bun-version-file: .tool-versions
 
       - name: Install dependencies
         run: bun install --frozen-lockfile

.tool-versions

@@ -1,2 +1,2 @@
 python 3.11.9
-bun 1.1.13
+bun 1.3.14

CHANGELOG.md

@@ -1,5 +1,11 @@
 # apollo
 
+## 1.4.0
+
+### Minor Changes
+
+- b3f8f38: Add authorisation to all service routes.
+
 ## 1.3.3
 
 ### Patch Changes

CLAUDE.md

@@ -45,6 +45,47 @@ TypeScript) service modules.
 - **Service discovery**: `platform/src/util/describe-modules.ts` - Auto-mounts
   any `services/<name>/` directory not starting with `_`. Detects service type
   by checking for `<name>.py` (Python) or `<name>.ts` (TypeScript) index file.
+- **Instance auth** (`platform/src/auth/`): `/services/*` uses a
+  map-if-known-else-forward auth hook that is always active (no flag). The auth surface
+  is split into three named concerns: the client-credential authenticate hook and Anthropic-key
+  resolver on the injectable `InstanceAuth` class (`instance-auth.ts`), the
+  internal-call exemption (`internal-token.ts`), and the shared `hashToken`
+  (`hash.ts`). The credential
+  is the `api_key` the caller (Lightning) already sends in the request body —
+  there is no bearer token and no Lightning-side change. Its SHA-256 is looked up
+  in the `lightning_clients` table via `APOLLO_CLIENTS_DB_URL` (falling back to
+  `POSTGRES_URL` when unset, so local dev needs only the one var; staging/prod point
+  `APOLLO_CLIENTS_DB_URL` at a separate credentials DB). The inbound `api_key` is
+  treated purely as a credential and is **never** forwarded to the LLM on a known
+  match: it is replaced with the matched client's stored `anthropic_api_key`, or
+  stripped (falling back to the global `ANTHROPIC_API_KEY`) when that column is
+  `NULL`. An *unknown* key is forwarded unchanged only if it is `sk-ant-`-shaped
+  (bring-your-own key); an unknown non-`sk-ant-` key is rejected with `401`
+  (likely a Lightning credential that must not leak to the LLM). No `api_key`
+  falls back to the global key. The resolver (`InstanceAuth.resolveKey`) returns a
+  tagged `KeyResolution` (`useKey`/`useGlobal`/`forward`/`passthrough`) dispatched
+  by a named switch in `services.ts`. The stored
+  `anthropic_api_key` may be plaintext or AES-256-GCM-encrypted (`enc:v1:`
+  values, decrypted with `APOLLO_ENC_KEY`; see
+  `platform/src/util/instance-key-crypto.ts` and the `client` CLI at
+  `platform/src/auth/client/`). Lookups are
+  cached in memory (~60s TTL), so the DB is hit at most once per minute per
+  process, not per request. The refresh is single-flight with
+  stale-while-revalidate, so a burst of requests at the TTL boundary shares one
+  DB read (cold start awaits it; a warm-but-stale cache is served while one
+  background refresh runs) rather than stampeding the DB. If the table can't be
+  reached, known-client swaps don't resolve and callers degrade to the
+  shape-checked forward path (the same `sk-ant-` rule applies; it does not
+  blanket-reject). The auth hook is scoped to
+  `/services/*`, so health/root endpoints outside that group are unaffected.
+  Internal Apollo-to-Apollo `apollo()` calls are exempt via a per-process
+  internal token (`APOLLO_INTERNAL_TOKEN`, minted at startup; `bridge.ts` injects
+  it into each spawned Python child's env via `getInternalToken()`, and
+  `services/util.py` echoes it back). This replaces the old loopback exemption so a
+  co-located Lightning is still required to authenticate. The authenticate hook and resolver
+  live on a single `InstanceAuth` instance constructed in `server.ts`; tests build
+  their own configured instance rather than poking module globals. Provisioning
+  lives in `platform/src/auth/`.
 
 ### Services Architecture
 

Dockerfile

@@ -3,7 +3,8 @@ FROM python:3.11-bullseye
 WORKDIR /app
 
 COPY ./pyproject.toml ./poetry.lock ./
-COPY ./package.json bun.lockb ./
+COPY ./package.json bun.lock ./
+COPY ./.tool-versions ./
 COPY ./tsconfig.json ./
 COPY ./path.config ./
 
@@ -20,7 +21,8 @@ RUN python -m pipx install poetry
 ENV PATH="${PATH}:/root/.local/bin/"
 RUN poetry install --only main --no-root
 
-RUN curl -fsSL https://bun.sh/install | bash
+RUN BUN_VERSION="$(awk '/^bun / {print $2}' .tool-versions)" \
+    && curl -fsSL https://bun.sh/install | bash -s "bun-v${BUN_VERSION}"
 ENV PATH="${PATH}:/root/.bun/bin/"
 
 RUN bun install