Build Stage 8 API v2 platform foundation

.dockerignore

@@ -0,0 +1,20 @@
+# App Engine's rendered root Dockerfile needs only the runtime package and the
+# packaging/startup inputs below. Keep tests, planning artifacts, local caches,
+# and bytecode out of the image build context.
+**
+!.dockerignore
+!Dockerfile
+!start.sh
+!Makefile
+!pyproject.toml
+!README.md
+!policyengine_api/
+!policyengine_api/**
+
+**/__pycache__/**
+**/*.py[co]
+**/*.db
+**/*.sqlite*
+**/.env*
+**/*.key
+**/*.pem

.gcloudignore

@@ -0,0 +1,23 @@
+# Keep the App Engine source upload intentionally small. The deploy script
+# renders the four root-level bundle files immediately before `gcloud app
+# deploy`; everything else needed by `make install` is listed explicitly.
+**
+!.gcloudignore
+!app.yaml
+!Dockerfile
+!start.sh
+!Makefile
+!pyproject.toml
+!README.md
+!policyengine_api/
+!policyengine_api/**
+
+# These exclusions intentionally follow the package inclusion so local bytecode
+# can never override the source tree or resurrect a deleted runtime module.
+**/__pycache__/**
+**/*.py[co]
+**/*.db
+**/*.sqlite*
+**/.env*
+**/*.key
+**/*.pem

.github/scripts/build_app_engine_image.sh

@@ -3,13 +3,14 @@
 set -euo pipefail
 
 APP_ENGINE_IMAGE_TAG="${APP_ENGINE_IMAGE_TAG:-policyengine-api-app-engine:test}"
+APP_ENGINE_PLATFORM="${APP_ENGINE_PLATFORM:-linux/amd64}"
 
 cleanup() {
-  rm -f app.yaml Dockerfile start.sh .dbpw
+  rm -f app.yaml Dockerfile start.sh
 }
 
 trap cleanup EXIT
 
 bash .github/scripts/prepare_app_engine_bundle.sh
 
-docker build -t "${APP_ENGINE_IMAGE_TAG}" .
+docker build --platform "${APP_ENGINE_PLATFORM}" -t "${APP_ENGINE_IMAGE_TAG}" .

.github/scripts/build_cloud_run_image.sh

@@ -8,7 +8,7 @@ cloud_run_set_defaults
 if [[ "${CLOUD_RUN_DRY_RUN:-0}" == "1" ]]; then
   cloud_run_run gcloud artifacts repositories describe "${CLOUD_RUN_ARTIFACT_REPOSITORY}" --project "${CLOUD_RUN_PROJECT}" --location "${CLOUD_RUN_REGION}"
   cloud_run_run gcloud auth configure-docker "${CLOUD_RUN_REGION}-docker.pkg.dev" --quiet
-  cloud_run_run docker build -f gcp/cloud_run/Dockerfile -t "${CLOUD_RUN_IMAGE_URI}" .
+  cloud_run_run docker build --platform "${CLOUD_RUN_IMAGE_PLATFORM}" -f gcp/cloud_run/Dockerfile -t "${CLOUD_RUN_IMAGE_URI}" .
   cloud_run_run docker push "${CLOUD_RUN_IMAGE_URI}"
   exit 0
 fi
@@ -24,5 +24,5 @@ EOF
 fi
 
 gcloud auth configure-docker "${CLOUD_RUN_REGION}-docker.pkg.dev" --quiet
-docker build -f gcp/cloud_run/Dockerfile -t "${CLOUD_RUN_IMAGE_URI}" .
+docker build --platform "${CLOUD_RUN_IMAGE_PLATFORM}" -f gcp/cloud_run/Dockerfile -t "${CLOUD_RUN_IMAGE_URI}" .
 docker push "${CLOUD_RUN_IMAGE_URI}"

.github/scripts/cloud_run_env.sh

@@ -8,6 +8,7 @@ cloud_run_set_defaults() {
   # Image name stays fixed across services: the production deploy reuses the
   # image built by the staging track, so it must not embed the service name.
   CLOUD_RUN_IMAGE_NAME="${CLOUD_RUN_IMAGE_NAME:-policyengine-api}"
+  CLOUD_RUN_IMAGE_PLATFORM="${CLOUD_RUN_IMAGE_PLATFORM:-linux/amd64}"
   CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT="${CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT:-policyengine-api-cr-runtime@policyengine-api.iam.gserviceaccount.com}"
   CLOUD_RUN_CPU="${CLOUD_RUN_CPU:-4}"
   CLOUD_RUN_MEMORY="${CLOUD_RUN_MEMORY:-16Gi}"
@@ -40,7 +41,12 @@ cloud_run_set_defaults() {
   CLOUD_RUN_ANTHROPIC_API_KEY_SECRET="${CLOUD_RUN_ANTHROPIC_API_KEY_SECRET:-policyengine-api-prod-anthropic-api-key:latest}"
   CLOUD_RUN_OPENAI_API_KEY_SECRET="${CLOUD_RUN_OPENAI_API_KEY_SECRET:-policyengine-api-prod-openai-api-key:latest}"
   CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET="${CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET:-policyengine-api-prod-hugging-face-token:latest}"
-
+  CLOUD_RUN_RUNTIME_CACHE_URL_SECRET="${CLOUD_RUN_RUNTIME_CACHE_URL_SECRET:-policyengine-api-prod-runtime-cache-url:latest}"
+  CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET="${CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET:-policyengine-api-prod-runtime-cache-ca:latest}"
+  CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT="${CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT:-production}"
+  CLOUD_RUN_VPC_NETWORK="${CLOUD_RUN_VPC_NETWORK:-default}"
+  CLOUD_RUN_VPC_SUBNET="${CLOUD_RUN_VPC_SUBNET:-default}"
+  CLOUD_RUN_VPC_EGRESS="${CLOUD_RUN_VPC_EGRESS:-private-ranges-only}"
   local sha
   sha="${GITHUB_SHA:-local}"
   CLOUD_RUN_IMAGE_TAG="${CLOUD_RUN_IMAGE_TAG:-${sha}}"
@@ -55,6 +61,7 @@ cloud_run_set_defaults() {
   export CLOUD_RUN_SERVICE
   export CLOUD_RUN_ARTIFACT_REPOSITORY
   export CLOUD_RUN_IMAGE_NAME
+  export CLOUD_RUN_IMAGE_PLATFORM
   export CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT
   export CLOUD_RUN_CPU
   export CLOUD_RUN_MEMORY
@@ -71,6 +78,14 @@ cloud_run_set_defaults() {
   export CLOUD_RUN_ANTHROPIC_API_KEY_SECRET
   export CLOUD_RUN_OPENAI_API_KEY_SECRET
   export CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET
+  export CLOUD_RUN_RUNTIME_CACHE_URL_SECRET
+  export CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET
+  export CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT
+  export CLOUD_RUN_VPC_NETWORK
+  export CLOUD_RUN_VPC_SUBNET
+  export CLOUD_RUN_VPC_EGRESS
+  export V2_SUPABASE_PROJECT_REF
+  export V2_SUPABASE_ENVIRONMENT
   export CLOUD_RUN_IMAGE_TAG
   export CLOUD_RUN_IMAGE_URI
   export CLOUD_RUN_TAG

.github/scripts/deploy_app_engine_version.sh

@@ -3,12 +3,12 @@
 set -euo pipefail
 
 : "${APP_ENGINE_VERSION:?APP_ENGINE_VERSION is required}"
+: "${APP_ENGINE_SERVICE_ACCOUNT:?APP_ENGINE_SERVICE_ACCOUNT is required}"
 
 APP_ENGINE_PROMOTE="${APP_ENGINE_PROMOTE:-0}"
-APP_ENGINE_SERVICE_ACCOUNT="${APP_ENGINE_SERVICE_ACCOUNT:-github-deployment@policyengine-api.iam.gserviceaccount.com}"
 
 cleanup() {
-  rm -f app.yaml Dockerfile start.sh .dbpw
+  rm -f app.yaml Dockerfile start.sh
 }
 
 trap cleanup EXIT
@@ -27,6 +27,10 @@ if [[ -n "${APP_ENGINE_PROJECT:-}" ]]; then
   deploy_args+=("--project=${APP_ENGINE_PROJECT}")
 fi
 
+if [[ -n "${APP_ENGINE_IMAGE_URL:-}" ]]; then
+  deploy_args+=("--image-url=${APP_ENGINE_IMAGE_URL}")
+fi
+
 if [[ "${APP_ENGINE_PROMOTE}" != "1" ]]; then
   deploy_args+=("--no-promote")
 fi

.github/scripts/deploy_cloud_run_candidate.sh

@@ -24,6 +24,11 @@ env_vars=(
   "SIM_COMPUTE_ECONOMY=old_gateway"
   "CLOUD_RUN_REVISION_TAG=${CLOUD_RUN_TAG}"
   "WEB_CONCURRENCY=${CLOUD_RUN_WEB_CONCURRENCY}"
+  "RUNTIME_CACHE_MODE=deployed"
+  "RUNTIME_CACHE_ENVIRONMENT=${CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT}"
+  "RUNTIME_CACHE_SERVICE=api"
+  "V2_SUPABASE_PROJECT_REF=${V2_SUPABASE_PROJECT_REF}"
+  "V2_SUPABASE_ENVIRONMENT=${V2_SUPABASE_ENVIRONMENT}"
 )
 
 if [[ -n "${OLD_SIMULATION_GATEWAY_URL:-}" ]]; then
@@ -39,6 +44,8 @@ secret_vars=(
   "ANTHROPIC_API_KEY=${CLOUD_RUN_ANTHROPIC_API_KEY_SECRET}"
   "OPENAI_API_KEY=${CLOUD_RUN_OPENAI_API_KEY_SECRET}"
   "HUGGING_FACE_TOKEN=${CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET}"
+  "RUNTIME_CACHE_URL=${CLOUD_RUN_RUNTIME_CACHE_URL_SECRET}"
+  "RUNTIME_CACHE_CA_CERT=${CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET}"
 )
 
 set_env_vars="$(IFS='|'; echo "^|^${env_vars[*]}")"
@@ -53,6 +60,9 @@ cloud_run_run gcloud run deploy "${CLOUD_RUN_SERVICE}" \
   --no-traffic \
   --allow-unauthenticated \
   --execution-environment gen2 \
+  --network "${CLOUD_RUN_VPC_NETWORK}" \
+  --subnet "${CLOUD_RUN_VPC_SUBNET}" \
+  --vpc-egress "${CLOUD_RUN_VPC_EGRESS}" \
   --service-account "${CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT}" \
   --add-cloudsql-instances "${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME}" \
   --port "${CLOUD_RUN_PORT}" \

.github/scripts/prepare_app_engine_bundle.sh

@@ -2,7 +2,5 @@
 
 set -euo pipefail
 
-python gcp/export.py
-cp gcp/policyengine_api/app.yaml .
-cp gcp/policyengine_api/Dockerfile .
+python3 gcp/export.py
 cp gcp/policyengine_api/start.sh .

.github/scripts/test_alembic_v2_lifecycle.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+uv run pytest -q \
+  tests/unit/v2/test_alembic_v2.py \
+  tests/integration/test_alembic_v2_lifecycle.py

.github/scripts/validate_app_engine_deploy_env.sh

@@ -9,13 +9,24 @@ selected_url_env="$(
 )"
 
 required=(
+  APP_ENGINE_SERVICE_ACCOUNT
   POLICYENGINE_DB_INSTANCE_CONNECTION_NAME
+  POLICYENGINE_DB_PASSWORD_SECRET_RESOURCE
+  POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN_SECRET_RESOURCE
+  ANTHROPIC_API_KEY_SECRET_RESOURCE
+  OPENAI_API_KEY_SECRET_RESOURCE
+  HUGGING_FACE_TOKEN_SECRET_RESOURCE
   SIM_ENTRYPOINT
   "${selected_url_env}"
   GATEWAY_AUTH_ISSUER
   GATEWAY_AUTH_AUDIENCE
   GATEWAY_AUTH_CLIENT_ID
   GATEWAY_AUTH_CLIENT_SECRET_RESOURCE
+  RUNTIME_CACHE_ENVIRONMENT
+  RUNTIME_CACHE_URL_SECRET_RESOURCE
+  RUNTIME_CACHE_CA_CERT_SECRET_RESOURCE
+  V2_SUPABASE_PROJECT_REF
+  V2_SUPABASE_ENVIRONMENT
 )
 
 missing=()

.github/scripts/validate_cloud_run_deploy_env.sh

@@ -29,6 +29,14 @@ cloud_run_require_env \
   CLOUD_RUN_ANTHROPIC_API_KEY_SECRET \
   CLOUD_RUN_OPENAI_API_KEY_SECRET \
   CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET \
+  CLOUD_RUN_RUNTIME_CACHE_URL_SECRET \
+  CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET \
+  CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT \
+  CLOUD_RUN_VPC_NETWORK \
+  CLOUD_RUN_VPC_SUBNET \
+  CLOUD_RUN_VPC_EGRESS \
+  V2_SUPABASE_PROJECT_REF \
+  V2_SUPABASE_ENVIRONMENT \
   SIM_ENTRYPOINT \
   ROUTE_IMPL_HEALTH \
   ROUTE_IMPL_SPECIFICATION \

.github/workflows/alembic-v2-check.yml

@@ -0,0 +1,56 @@
+name: Alembic v2 and runtime-cache checks
+
+on:
+  workflow_call:
+  workflow_dispatch:
+
+jobs:
+  postgres-redis-lifecycle:
+    name: V2 Postgres and Redis lifecycle
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:17
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: policyengine_v2_test
+          POSTGRES_DB: policyengine_v2_alembic_test
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd="pg_isready -U postgres"
+          --health-interval=5s
+          --health-timeout=5s
+          --health-retries=20
+      redis:
+        image: redis:7.2-alpine
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd="redis-cli ping"
+          --health-interval=5s
+          --health-timeout=5s
+          --health-retries=20
+    env:
+      V2_MIGRATION_DATABASE_URL: postgresql+psycopg://postgres:policyengine_v2_test@127.0.0.1:5432/policyengine_v2_alembic_test
+      V2_ALEMBIC_DISPOSABLE_TEST: "1"
+      RUNTIME_CACHE_TEST_URL: redis://127.0.0.1:6379/0
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Setup uv
+        uses: astral-sh/setup-uv@v6
+      - name: Install locked dependencies
+        run: uv sync --frozen
+      - name: Test generated-only v2 migration configuration and lifecycle
+        run: bash .github/scripts/test_alembic_v2_lifecycle.sh
+      - name: Require database at the v2 head
+        run: uv run alembic -c alembic-v2.ini current --check-heads
+      - name: Require no ungenerated v2 schema or data operations
+        run: uv run alembic -c alembic-v2.ini check
+      - name: Test real Redis cross-instance semantics
+        run: uv run pytest -q tests/integration/test_runtime_cache_redis.py

.github/workflows/pr.yml

@@ -54,6 +54,9 @@ jobs:
   alembic-v1-check:
     name: Alembic v1 qualification
     uses: ./.github/workflows/alembic-v1-check.yml
+  alembic-v2-check:
+    name: Alembic v2 and Redis qualification
+    uses: ./.github/workflows/alembic-v2-check.yml
   check-changelog:
     name: Check changelog fragment
     runs-on: ubuntu-latest