From 60021933d6fca82d4275182eab1a6b2aabc24c1c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:08:53 +0300 Subject: [PATCH 01/18] Build Stage 8 v2 platform foundation --- .dockerignore | 20 + .gcloudignore | 23 + .github/scripts/build_app_engine_image.sh | 5 +- .github/scripts/build_cloud_run_image.sh | 4 +- .github/scripts/cloud_run_env.sh | 18 + .github/scripts/deploy_app_engine_version.sh | 8 +- .github/scripts/deploy_cloud_run_candidate.sh | 10 + .github/scripts/prepare_app_engine_bundle.sh | 4 +- .../scripts/validate_app_engine_deploy_env.sh | 9 + .../scripts/validate_cloud_run_deploy_env.sh | 8 + .github/workflows/alembic-v2-check.yml | 59 + .github/workflows/pr.yml | 33 + .github/workflows/push.yml | 94 +- .gitignore | 1 + Makefile | 3 +- README.md | 29 +- alembic-v2.ini | 37 + docs/engineering/skills/alembic-migrations.md | 111 +- docs/engineering/skills/testing.md | 63 +- docs/migration/stage-8-managed-redis.md | 126 ++ docs/migration/stage-8-platform-runbook.md | 89 + docs/migration/stage-8-supabase-bootstrap.md | 68 + docs/migration/stage-8-supabase-target.md | 156 ++ gcp/Dockerfile | 1 - gcp/README.md | 24 +- gcp/cloud_run/Dockerfile | 5 +- gcp/cloud_run/start.sh | 80 +- gcp/export.py | 145 +- gcp/policyengine_api/Dockerfile | 17 +- gcp/policyengine_api/app.yaml | 22 + gcp/policyengine_api/start.sh | 27 +- migrations/v2/env.py | 89 + migrations/v2/script.py.mako | 27 + migrations/v2/versions/.gitkeep | 1 + ...1336f_establish_v2_core_schema_baseline.py | 1722 +++++++++++++++++ ...1_constrain_v2_user_country_and_report_.py | 50 + ...63_add_stage_8_platform_validation_data.py | 83 + ...7_enforce_v2_user_association_ownership.py | 79 + policyengine_api/api.py | 47 +- policyengine_api/app_engine_runtime.py | 94 + policyengine_api/asgi.py | 9 +- policyengine_api/data/local_database.py | 45 - policyengine_api/data/local_models.py | 26 - policyengine_api/data/orm.py | 123 +- policyengine_api/data/v2/__init__.py | 1 + policyengine_api/data/v2/database.py | 135 ++ policyengine_api/data/v2/migration_target.py | 208 ++ policyengine_api/data/v2/models/__init__.py | 152 ++ policyengine_api/data/v2/models/base.py | 88 + policyengine_api/data/v2/models/domain.py | 396 ++++ policyengine_api/data/v2/models/metadata.py | 340 ++++ policyengine_api/data/v2/models/reports.py | 601 ++++++ policyengine_api/data/v2/reference_data.py | 144 ++ .../data/v2/reference_data_autogenerate.py | 344 ++++ policyengine_api/data/v2/report_runs.py | 212 ++ policyengine_api/data/v2/settings.py | 217 +++ policyengine_api/data/v2/storage_bootstrap.py | 213 ++ policyengine_api/data/v2/table_inventory.py | 160 ++ policyengine_api/gcp_logging.py | 15 +- policyengine_api/runtime_cache/__init__.py | 6 + policyengine_api/runtime_cache/claims.py | 78 + policyengine_api/runtime_cache/client.py | 58 + policyengine_api/runtime_cache/core.py | 217 +++ .../runtime_cache/dependencies.py | 38 + policyengine_api/runtime_cache/fake.py | 199 ++ .../runtime_cache/repositories.py | 539 ++++++ policyengine_api/runtime_cache/settings.py | 238 +++ .../services/ai_analysis_service.py | 74 +- .../services/budget_window_cache.py | 291 ++- policyengine_api/services/economy_service.py | 59 +- .../services/household_calculation_service.py | 134 +- .../services/reform_impacts_service.py | 419 ++-- .../services/tracer_analysis_service.py | 74 +- pyproject.toml | 5 + scripts/bootstrap_v2_supabase_storage.py | 27 + scripts/check_stage8_scaffolding_hygiene.py | 72 + tests/fixtures/local_v1_database.py | 32 + tests/fixtures/services/economy_service.py | 2 + .../services/tracer_analysis_service.py | 4 +- .../services/tracer_fixture_service.py | 61 +- .../integration/test_alembic_v2_lifecycle.py | 135 ++ .../test_budget_window_in_flight_dedupe.py | 18 +- tests/integration/test_runtime_cache_redis.py | 169 ++ tests/unit/conftest.py | 10 +- tests/unit/data/test_orm_sessions.py | 9 +- tests/unit/data/test_sqlalchemy_v2.py | 76 +- tests/unit/data/test_v1_models.py | 4 +- ...st_household_and_user_policy_orm_routes.py | 73 +- tests/unit/runtime_cache/__init__.py | 1 + tests/unit/runtime_cache/test_client.py | 35 + tests/unit/runtime_cache/test_core.py | 202 ++ tests/unit/runtime_cache/test_repositories.py | 204 ++ tests/unit/runtime_cache/test_settings.py | 161 ++ .../unit/services/test_ai_analysis_service.py | 84 +- .../unit/services/test_budget_window_cache.py | 100 +- .../test_direct_orm_local_analysis.py | 77 +- tests/unit/services/test_economy_service.py | 81 +- tests/unit/services/test_execute_analysis.py | 14 +- .../test_household_calculation_service.py | 107 +- .../services/test_reform_impacts_service.py | 53 +- tests/unit/services/test_tracer_service.py | 11 +- tests/unit/test_alembic_workflows.py | 28 +- tests/unit/test_app_engine_runtime.py | 160 ++ tests/unit/test_cloud_run_deploy_scripts.py | 313 ++- tests/unit/test_gcp_logging.py | 39 + tests/unit/v2/__init__.py | 1 + tests/unit/v2/test_alembic_v2.py | 444 +++++ tests/unit/v2/test_database.py | 99 + tests/unit/v2/test_import_side_effects.py | 146 ++ tests/unit/v2/test_model_persistence.py | 91 + tests/unit/v2/test_models.py | 233 +++ .../v2/test_reference_data_autogenerate.py | 221 +++ tests/unit/v2/test_report_runs.py | 415 ++++ tests/unit/v2/test_scaffolding_hygiene.py | 40 + tests/unit/v2/test_settings.py | 137 ++ tests/unit/v2/test_stage8_activation.py | 46 + tests/unit/v2/test_storage_bootstrap.py | 186 ++ tests/unit/v2/test_table_inventory.py | 43 + uv.lock | 89 +- 119 files changed, 12568 insertions(+), 1334 deletions(-) create mode 100644 .dockerignore create mode 100644 .gcloudignore create mode 100644 .github/workflows/alembic-v2-check.yml create mode 100644 alembic-v2.ini create mode 100644 docs/migration/stage-8-managed-redis.md create mode 100644 docs/migration/stage-8-platform-runbook.md create mode 100644 docs/migration/stage-8-supabase-bootstrap.md create mode 100644 docs/migration/stage-8-supabase-target.md create mode 100644 migrations/v2/env.py create mode 100644 migrations/v2/script.py.mako create mode 100644 migrations/v2/versions/.gitkeep create mode 100644 migrations/v2/versions/47592781336f_establish_v2_core_schema_baseline.py create mode 100644 migrations/v2/versions/5f048586d8f1_constrain_v2_user_country_and_report_.py create mode 100644 migrations/v2/versions/6ee725e0c563_add_stage_8_platform_validation_data.py create mode 100644 migrations/v2/versions/b4c69674dd47_enforce_v2_user_association_ownership.py create mode 100644 policyengine_api/app_engine_runtime.py delete mode 100644 policyengine_api/data/local_database.py delete mode 100644 policyengine_api/data/local_models.py create mode 100644 policyengine_api/data/v2/__init__.py create mode 100644 policyengine_api/data/v2/database.py create mode 100644 policyengine_api/data/v2/migration_target.py create mode 100644 policyengine_api/data/v2/models/__init__.py create mode 100644 policyengine_api/data/v2/models/base.py create mode 100644 policyengine_api/data/v2/models/domain.py create mode 100644 policyengine_api/data/v2/models/metadata.py create mode 100644 policyengine_api/data/v2/models/reports.py create mode 100644 policyengine_api/data/v2/reference_data.py create mode 100644 policyengine_api/data/v2/reference_data_autogenerate.py create mode 100644 policyengine_api/data/v2/report_runs.py create mode 100644 policyengine_api/data/v2/settings.py create mode 100644 policyengine_api/data/v2/storage_bootstrap.py create mode 100644 policyengine_api/data/v2/table_inventory.py create mode 100644 policyengine_api/runtime_cache/__init__.py create mode 100644 policyengine_api/runtime_cache/claims.py create mode 100644 policyengine_api/runtime_cache/client.py create mode 100644 policyengine_api/runtime_cache/core.py create mode 100644 policyengine_api/runtime_cache/dependencies.py create mode 100644 policyengine_api/runtime_cache/fake.py create mode 100644 policyengine_api/runtime_cache/repositories.py create mode 100644 policyengine_api/runtime_cache/settings.py create mode 100644 scripts/bootstrap_v2_supabase_storage.py create mode 100644 scripts/check_stage8_scaffolding_hygiene.py create mode 100644 tests/fixtures/local_v1_database.py create mode 100644 tests/integration/test_alembic_v2_lifecycle.py create mode 100644 tests/integration/test_runtime_cache_redis.py create mode 100644 tests/unit/runtime_cache/__init__.py create mode 100644 tests/unit/runtime_cache/test_client.py create mode 100644 tests/unit/runtime_cache/test_core.py create mode 100644 tests/unit/runtime_cache/test_repositories.py create mode 100644 tests/unit/runtime_cache/test_settings.py create mode 100644 tests/unit/test_app_engine_runtime.py create mode 100644 tests/unit/test_gcp_logging.py create mode 100644 tests/unit/v2/__init__.py create mode 100644 tests/unit/v2/test_alembic_v2.py create mode 100644 tests/unit/v2/test_database.py create mode 100644 tests/unit/v2/test_import_side_effects.py create mode 100644 tests/unit/v2/test_model_persistence.py create mode 100644 tests/unit/v2/test_models.py create mode 100644 tests/unit/v2/test_reference_data_autogenerate.py create mode 100644 tests/unit/v2/test_report_runs.py create mode 100644 tests/unit/v2/test_scaffolding_hygiene.py create mode 100644 tests/unit/v2/test_settings.py create mode 100644 tests/unit/v2/test_stage8_activation.py create mode 100644 tests/unit/v2/test_storage_bootstrap.py create mode 100644 tests/unit/v2/test_table_inventory.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..db52bd363 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,20 @@ +# App Engine's rendered root Dockerfile needs only the runtime package and the +# packaging/startup inputs below. Keep tests, planning artifacts, local caches, +# and bytecode out of the image build context. +** +!.dockerignore +!Dockerfile +!start.sh +!Makefile +!pyproject.toml +!README.md +!policyengine_api/ +!policyengine_api/** + +**/__pycache__/** +**/*.py[co] +**/*.db +**/*.sqlite* +**/.env* +**/*.key +**/*.pem diff --git a/.gcloudignore b/.gcloudignore new file mode 100644 index 000000000..6f1c65bce --- /dev/null +++ b/.gcloudignore @@ -0,0 +1,23 @@ +# Keep the App Engine source upload intentionally small. The deploy script +# renders the four root-level bundle files immediately before `gcloud app +# deploy`; everything else needed by `make install` is listed explicitly. +** +!.gcloudignore +!app.yaml +!Dockerfile +!start.sh +!Makefile +!pyproject.toml +!README.md +!policyengine_api/ +!policyengine_api/** + +# These exclusions intentionally follow the package inclusion so local bytecode +# can never override the source tree or resurrect a deleted runtime module. +**/__pycache__/** +**/*.py[co] +**/*.db +**/*.sqlite* +**/.env* +**/*.key +**/*.pem diff --git a/.github/scripts/build_app_engine_image.sh b/.github/scripts/build_app_engine_image.sh index 0d1da4906..14d3a51bc 100644 --- a/.github/scripts/build_app_engine_image.sh +++ b/.github/scripts/build_app_engine_image.sh @@ -3,13 +3,14 @@ set -euo pipefail APP_ENGINE_IMAGE_TAG="${APP_ENGINE_IMAGE_TAG:-policyengine-api-app-engine:test}" +APP_ENGINE_PLATFORM="${APP_ENGINE_PLATFORM:-linux/amd64}" cleanup() { - rm -f app.yaml Dockerfile start.sh .dbpw + rm -f app.yaml Dockerfile start.sh } trap cleanup EXIT bash .github/scripts/prepare_app_engine_bundle.sh -docker build -t "${APP_ENGINE_IMAGE_TAG}" . +docker build --platform "${APP_ENGINE_PLATFORM}" -t "${APP_ENGINE_IMAGE_TAG}" . diff --git a/.github/scripts/build_cloud_run_image.sh b/.github/scripts/build_cloud_run_image.sh index 2911cafe2..6222c232a 100755 --- a/.github/scripts/build_cloud_run_image.sh +++ b/.github/scripts/build_cloud_run_image.sh @@ -8,7 +8,7 @@ cloud_run_set_defaults if [[ "${CLOUD_RUN_DRY_RUN:-0}" == "1" ]]; then cloud_run_run gcloud artifacts repositories describe "${CLOUD_RUN_ARTIFACT_REPOSITORY}" --project "${CLOUD_RUN_PROJECT}" --location "${CLOUD_RUN_REGION}" cloud_run_run gcloud auth configure-docker "${CLOUD_RUN_REGION}-docker.pkg.dev" --quiet - cloud_run_run docker build -f gcp/cloud_run/Dockerfile -t "${CLOUD_RUN_IMAGE_URI}" . + cloud_run_run docker build --platform "${CLOUD_RUN_IMAGE_PLATFORM}" -f gcp/cloud_run/Dockerfile -t "${CLOUD_RUN_IMAGE_URI}" . cloud_run_run docker push "${CLOUD_RUN_IMAGE_URI}" exit 0 fi @@ -24,5 +24,5 @@ EOF fi gcloud auth configure-docker "${CLOUD_RUN_REGION}-docker.pkg.dev" --quiet -docker build -f gcp/cloud_run/Dockerfile -t "${CLOUD_RUN_IMAGE_URI}" . +docker build --platform "${CLOUD_RUN_IMAGE_PLATFORM}" -f gcp/cloud_run/Dockerfile -t "${CLOUD_RUN_IMAGE_URI}" . docker push "${CLOUD_RUN_IMAGE_URI}" diff --git a/.github/scripts/cloud_run_env.sh b/.github/scripts/cloud_run_env.sh index 2263b3f23..13d131094 100755 --- a/.github/scripts/cloud_run_env.sh +++ b/.github/scripts/cloud_run_env.sh @@ -8,6 +8,7 @@ cloud_run_set_defaults() { # Image name stays fixed across services: the production deploy reuses the # image built by the staging track, so it must not embed the service name. CLOUD_RUN_IMAGE_NAME="${CLOUD_RUN_IMAGE_NAME:-policyengine-api}" + CLOUD_RUN_IMAGE_PLATFORM="${CLOUD_RUN_IMAGE_PLATFORM:-linux/amd64}" CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT="${CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT:-policyengine-api-cr-runtime@policyengine-api.iam.gserviceaccount.com}" CLOUD_RUN_CPU="${CLOUD_RUN_CPU:-4}" CLOUD_RUN_MEMORY="${CLOUD_RUN_MEMORY:-16Gi}" @@ -40,6 +41,14 @@ cloud_run_set_defaults() { CLOUD_RUN_ANTHROPIC_API_KEY_SECRET="${CLOUD_RUN_ANTHROPIC_API_KEY_SECRET:-policyengine-api-prod-anthropic-api-key:latest}" CLOUD_RUN_OPENAI_API_KEY_SECRET="${CLOUD_RUN_OPENAI_API_KEY_SECRET:-policyengine-api-prod-openai-api-key:latest}" CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET="${CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET:-policyengine-api-prod-hugging-face-token:latest}" + CLOUD_RUN_RUNTIME_CACHE_URL_SECRET="${CLOUD_RUN_RUNTIME_CACHE_URL_SECRET:-policyengine-api-prod-runtime-cache-url:latest}" + CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET="${CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET:-policyengine-api-prod-runtime-cache-ca:latest}" + CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT="${CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT:-production}" + CLOUD_RUN_VPC_NETWORK="${CLOUD_RUN_VPC_NETWORK:-default}" + CLOUD_RUN_VPC_SUBNET="${CLOUD_RUN_VPC_SUBNET:-default}" + CLOUD_RUN_VPC_EGRESS="${CLOUD_RUN_VPC_EGRESS:-private-ranges-only}" + V2_SUPABASE_PROJECT_REF="${V2_SUPABASE_PROJECT_REF:-kvrifaviwhzjztcbrfpy}" + V2_SUPABASE_ENVIRONMENT="${V2_SUPABASE_ENVIRONMENT:-production-foundation}" local sha sha="${GITHUB_SHA:-local}" @@ -55,6 +64,7 @@ cloud_run_set_defaults() { export CLOUD_RUN_SERVICE export CLOUD_RUN_ARTIFACT_REPOSITORY export CLOUD_RUN_IMAGE_NAME + export CLOUD_RUN_IMAGE_PLATFORM export CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT export CLOUD_RUN_CPU export CLOUD_RUN_MEMORY @@ -71,6 +81,14 @@ cloud_run_set_defaults() { export CLOUD_RUN_ANTHROPIC_API_KEY_SECRET export CLOUD_RUN_OPENAI_API_KEY_SECRET export CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET + export CLOUD_RUN_RUNTIME_CACHE_URL_SECRET + export CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET + export CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT + export CLOUD_RUN_VPC_NETWORK + export CLOUD_RUN_VPC_SUBNET + export CLOUD_RUN_VPC_EGRESS + export V2_SUPABASE_PROJECT_REF + export V2_SUPABASE_ENVIRONMENT export CLOUD_RUN_IMAGE_TAG export CLOUD_RUN_IMAGE_URI export CLOUD_RUN_TAG diff --git a/.github/scripts/deploy_app_engine_version.sh b/.github/scripts/deploy_app_engine_version.sh index 6ed6c08d6..917a569d4 100644 --- a/.github/scripts/deploy_app_engine_version.sh +++ b/.github/scripts/deploy_app_engine_version.sh @@ -3,12 +3,12 @@ set -euo pipefail : "${APP_ENGINE_VERSION:?APP_ENGINE_VERSION is required}" +: "${APP_ENGINE_SERVICE_ACCOUNT:?APP_ENGINE_SERVICE_ACCOUNT is required}" APP_ENGINE_PROMOTE="${APP_ENGINE_PROMOTE:-0}" -APP_ENGINE_SERVICE_ACCOUNT="${APP_ENGINE_SERVICE_ACCOUNT:-github-deployment@policyengine-api.iam.gserviceaccount.com}" cleanup() { - rm -f app.yaml Dockerfile start.sh .dbpw + rm -f app.yaml Dockerfile start.sh } trap cleanup EXIT @@ -27,6 +27,10 @@ if [[ -n "${APP_ENGINE_PROJECT:-}" ]]; then deploy_args+=("--project=${APP_ENGINE_PROJECT}") fi +if [[ -n "${APP_ENGINE_IMAGE_URL:-}" ]]; then + deploy_args+=("--image-url=${APP_ENGINE_IMAGE_URL}") +fi + if [[ "${APP_ENGINE_PROMOTE}" != "1" ]]; then deploy_args+=("--no-promote") fi diff --git a/.github/scripts/deploy_cloud_run_candidate.sh b/.github/scripts/deploy_cloud_run_candidate.sh index ffbe6dd8d..6baa5ed65 100755 --- a/.github/scripts/deploy_cloud_run_candidate.sh +++ b/.github/scripts/deploy_cloud_run_candidate.sh @@ -24,6 +24,11 @@ env_vars=( "SIM_COMPUTE_ECONOMY=old_gateway" "CLOUD_RUN_REVISION_TAG=${CLOUD_RUN_TAG}" "WEB_CONCURRENCY=${CLOUD_RUN_WEB_CONCURRENCY}" + "RUNTIME_CACHE_MODE=deployed" + "RUNTIME_CACHE_ENVIRONMENT=${CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT}" + "RUNTIME_CACHE_SERVICE=api" + "V2_SUPABASE_PROJECT_REF=${V2_SUPABASE_PROJECT_REF}" + "V2_SUPABASE_ENVIRONMENT=${V2_SUPABASE_ENVIRONMENT}" ) if [[ -n "${OLD_SIMULATION_GATEWAY_URL:-}" ]]; then @@ -39,6 +44,8 @@ secret_vars=( "ANTHROPIC_API_KEY=${CLOUD_RUN_ANTHROPIC_API_KEY_SECRET}" "OPENAI_API_KEY=${CLOUD_RUN_OPENAI_API_KEY_SECRET}" "HUGGING_FACE_TOKEN=${CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET}" + "RUNTIME_CACHE_URL=${CLOUD_RUN_RUNTIME_CACHE_URL_SECRET}" + "RUNTIME_CACHE_CA_CERT=${CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET}" ) set_env_vars="$(IFS='|'; echo "^|^${env_vars[*]}")" @@ -53,6 +60,9 @@ cloud_run_run gcloud run deploy "${CLOUD_RUN_SERVICE}" \ --no-traffic \ --allow-unauthenticated \ --execution-environment gen2 \ + --network "${CLOUD_RUN_VPC_NETWORK}" \ + --subnet "${CLOUD_RUN_VPC_SUBNET}" \ + --vpc-egress "${CLOUD_RUN_VPC_EGRESS}" \ --service-account "${CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT}" \ --add-cloudsql-instances "${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME}" \ --port "${CLOUD_RUN_PORT}" \ diff --git a/.github/scripts/prepare_app_engine_bundle.sh b/.github/scripts/prepare_app_engine_bundle.sh index e7b8f96f6..a5b931d2f 100644 --- a/.github/scripts/prepare_app_engine_bundle.sh +++ b/.github/scripts/prepare_app_engine_bundle.sh @@ -2,7 +2,5 @@ set -euo pipefail -python gcp/export.py -cp gcp/policyengine_api/app.yaml . -cp gcp/policyengine_api/Dockerfile . +python3 gcp/export.py cp gcp/policyengine_api/start.sh . diff --git a/.github/scripts/validate_app_engine_deploy_env.sh b/.github/scripts/validate_app_engine_deploy_env.sh index 95fdd9a2b..396496e43 100644 --- a/.github/scripts/validate_app_engine_deploy_env.sh +++ b/.github/scripts/validate_app_engine_deploy_env.sh @@ -9,13 +9,22 @@ selected_url_env="$( )" required=( + APP_ENGINE_SERVICE_ACCOUNT POLICYENGINE_DB_INSTANCE_CONNECTION_NAME + POLICYENGINE_DB_PASSWORD_SECRET_RESOURCE + POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN_SECRET_RESOURCE + ANTHROPIC_API_KEY_SECRET_RESOURCE + OPENAI_API_KEY_SECRET_RESOURCE + HUGGING_FACE_TOKEN_SECRET_RESOURCE SIM_ENTRYPOINT "${selected_url_env}" GATEWAY_AUTH_ISSUER GATEWAY_AUTH_AUDIENCE GATEWAY_AUTH_CLIENT_ID GATEWAY_AUTH_CLIENT_SECRET_RESOURCE + RUNTIME_CACHE_ENVIRONMENT + RUNTIME_CACHE_URL_SECRET_RESOURCE + RUNTIME_CACHE_CA_CERT_SECRET_RESOURCE ) missing=() diff --git a/.github/scripts/validate_cloud_run_deploy_env.sh b/.github/scripts/validate_cloud_run_deploy_env.sh index aed2784ed..772c733cb 100755 --- a/.github/scripts/validate_cloud_run_deploy_env.sh +++ b/.github/scripts/validate_cloud_run_deploy_env.sh @@ -29,6 +29,14 @@ cloud_run_require_env \ CLOUD_RUN_ANTHROPIC_API_KEY_SECRET \ CLOUD_RUN_OPENAI_API_KEY_SECRET \ CLOUD_RUN_HUGGING_FACE_TOKEN_SECRET \ + CLOUD_RUN_RUNTIME_CACHE_URL_SECRET \ + CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET \ + CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT \ + CLOUD_RUN_VPC_NETWORK \ + CLOUD_RUN_VPC_SUBNET \ + CLOUD_RUN_VPC_EGRESS \ + V2_SUPABASE_PROJECT_REF \ + V2_SUPABASE_ENVIRONMENT \ SIM_ENTRYPOINT \ ROUTE_IMPL_HEALTH \ ROUTE_IMPL_SPECIFICATION \ diff --git a/.github/workflows/alembic-v2-check.yml b/.github/workflows/alembic-v2-check.yml new file mode 100644 index 000000000..ef80ba775 --- /dev/null +++ b/.github/workflows/alembic-v2-check.yml @@ -0,0 +1,59 @@ +name: Alembic v2 and runtime-cache checks + +on: + workflow_call: + workflow_dispatch: + +jobs: + postgres-redis-lifecycle: + name: V2 Postgres and Redis lifecycle + runs-on: ubuntu-latest + services: + postgres: + image: postgres:17 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: policyengine_v2_test + POSTGRES_DB: policyengine_v2_alembic_test + ports: + - 5432:5432 + options: >- + --health-cmd="pg_isready -U postgres" + --health-interval=5s + --health-timeout=5s + --health-retries=20 + redis: + image: redis:7.2-alpine + ports: + - 6379:6379 + options: >- + --health-cmd="redis-cli ping" + --health-interval=5s + --health-timeout=5s + --health-retries=20 + env: + V2_MIGRATION_DATABASE_URL: postgresql+psycopg://postgres:policyengine_v2_test@127.0.0.1:5432/policyengine_v2_alembic_test + V2_ALEMBIC_DISPOSABLE_TEST: "1" + RUNTIME_CACHE_TEST_URL: redis://127.0.0.1:6379/0 + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Setup uv + uses: astral-sh/setup-uv@v6 + - name: Install locked dependencies + run: uv sync --frozen + - name: Test generated-only v2 migration configuration and lifecycle + run: >- + uv run pytest -q tests/unit/v2/test_alembic_v2.py + tests/unit/v2/test_reference_data_autogenerate.py + tests/integration/test_alembic_v2_lifecycle.py + - name: Require database at the v2 head + run: uv run alembic -c alembic-v2.ini current --check-heads + - name: Require no ungenerated v2 schema or data operations + run: uv run alembic -c alembic-v2.ini check + - name: Test real Redis cross-instance semantics + run: uv run pytest -q tests/integration/test_runtime_cache_redis.py diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 83d23f875..418f48395 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -54,6 +54,39 @@ jobs: alembic-v1-check: name: Alembic v1 qualification uses: ./.github/workflows/alembic-v1-check.yml + detect-v2-platform-changes: + name: Detect v2 platform changes + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + v2: ${{ steps.paths.outputs.v2 }} + steps: + - name: Checkout repo + uses: actions/checkout@v4 + - name: Detect v2 migration and cache paths + id: paths + uses: dorny/paths-filter@v3 + with: + filters: | + v2: + - 'alembic-v2.ini' + - 'migrations/v2/**' + - 'policyengine_api/data/v2/**' + - 'policyengine_api/runtime_cache/**' + - 'tests/unit/v2/**' + - 'tests/unit/runtime_cache/**' + - 'tests/integration/test_alembic_v2_lifecycle.py' + - 'tests/integration/test_runtime_cache_redis.py' + - '.github/workflows/alembic-v2-check.yml' + - 'pyproject.toml' + - 'uv.lock' + alembic-v2-check: + name: Alembic v2 and Redis qualification + needs: detect-v2-platform-changes + if: needs.detect-v2-platform-changes.outputs.v2 == 'true' + uses: ./.github/workflows/alembic-v2-check.yml check-changelog: name: Check changelog fragment runs-on: ubuntu-latest diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index cab5ff0ff..790fb05c8 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -33,6 +33,11 @@ jobs: if: github.repository == 'PolicyEngine/policyengine-api' uses: ./.github/workflows/alembic-v1-check.yml + alembic-v2-check: + name: Alembic v2 and Redis qualification + if: github.repository == 'PolicyEngine/policyengine-api' + uses: ./.github/workflows/alembic-v2-check.yml + ensure-staging-model-version-aligns-with-sim-api: name: Ensure staging model version aligns with simulation API runs-on: ubuntu-latest @@ -53,7 +58,7 @@ jobs: versioning: name: Update versioning - needs: [lint, alembic-v1-check] + needs: [lint, alembic-v1-check, alembic-v2-check] if: | (github.repository == 'PolicyEngine/policyengine-api') && !(github.event.head_commit.message == 'Update PolicyEngine API') @@ -95,6 +100,7 @@ jobs: - ensure-staging-model-version-aligns-with-sim-api - lint - alembic-v1-check + - alembic-v2-check if: | (github.repository == 'PolicyEngine/policyengine-api') && (github.event.head_commit.message == 'Update PolicyEngine API') @@ -162,6 +168,19 @@ jobs: SIM_ENTRYPOINT: ${{ vars.SIM_ENTRYPOINT }} SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }} POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} + APP_ENGINE_SERVICE_ACCOUNT: policyengine-api-ae-staging@policyengine-api.iam.gserviceaccount.com + POLICYENGINE_DB_PASSWORD_SECRET_RESOURCE: projects/policyengine-api/secrets/policyengine-api-prod-db-password/versions/latest + POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN_SECRET_RESOURCE: projects/policyengine-api/secrets/policyengine-api-prod-github-microdata-token/versions/latest + ANTHROPIC_API_KEY_SECRET_RESOURCE: projects/policyengine-api/secrets/policyengine-api-prod-anthropic-api-key/versions/latest + OPENAI_API_KEY_SECRET_RESOURCE: projects/policyengine-api/secrets/policyengine-api-prod-openai-api-key/versions/latest + HUGGING_FACE_TOKEN_SECRET_RESOURCE: projects/policyengine-api/secrets/policyengine-api-prod-hugging-face-token/versions/latest + GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }} + GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }} + GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }} + GATEWAY_AUTH_CLIENT_SECRET_RESOURCE: ${{ secrets.GATEWAY_AUTH_CLIENT_SECRET_RESOURCE }} + RUNTIME_CACHE_ENVIRONMENT: staging + RUNTIME_CACHE_URL_SECRET_RESOURCE: projects/policyengine-api/secrets/policyengine-api-staging-runtime-cache-url/versions/latest + RUNTIME_CACHE_CA_CERT_SECRET_RESOURCE: projects/policyengine-api/secrets/policyengine-api-staging-runtime-cache-ca/versions/latest permissions: contents: read id-token: write @@ -202,47 +221,15 @@ jobs: HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} - name: Validate App Engine deployment configuration run: bash .github/scripts/validate_app_engine_deploy_env.sh - env: - # Transitional: these values are still passed into the deploy bundle - # by gcp/export.py. Long-term target is a generic image plus runtime - # config / Secret Manager lookups instead of image bake-in. - GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }} - GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }} - GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }} - GATEWAY_AUTH_CLIENT_SECRET_RESOURCE: ${{ secrets.GATEWAY_AUTH_CLIENT_SECRET_RESOURCE }} - name: Build staging deploy image run: bash .github/scripts/build_app_engine_image.sh env: - # Transitional: these values are still rendered into the App Engine - # image today. Long-term target is to stop passing them into the - # image build and supply them as runtime config / Secret Manager data. APP_ENGINE_IMAGE_TAG: policyengine-api:staging-${{ steps.version.outputs.version }} - POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} - POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} - GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }} - GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }} - GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }} - GATEWAY_AUTH_CLIENT_SECRET_RESOURCE: ${{ secrets.GATEWAY_AUTH_CLIENT_SECRET_RESOURCE }} - name: Deploy staging version run: bash .github/scripts/deploy_app_engine_version.sh env: - # Transitional: deploy_app_engine_version.sh still prepares a bundle - # from these values before App Engine deploy. Long-term target is one - # generic image plus runtime config / Secret Manager. APP_ENGINE_VERSION: ${{ steps.version.outputs.version }} APP_ENGINE_PROMOTE: "0" - POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} - POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} - GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }} - GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }} - GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }} - GATEWAY_AUTH_CLIENT_SECRET_RESOURCE: ${{ secrets.GATEWAY_AUTH_CLIENT_SECRET_RESOURCE }} - name: Resolve staging version URL id: version_url run: | @@ -274,6 +261,9 @@ jobs: ROUTE_IMPL_SPECIFICATION: ${{ vars.ROUTE_IMPL_SPECIFICATION }} ROUTE_IMPL_METADATA: ${{ vars.ROUTE_IMPL_METADATA }} CLOUD_RUN_SERVICE: policyengine-api-staging + CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT: staging + CLOUD_RUN_RUNTIME_CACHE_URL_SECRET: policyengine-api-staging-runtime-cache-url:latest + CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET: policyengine-api-staging-runtime-cache-ca:latest # Staging stays scale-to-zero, single instance: it exists for per-push # validation, not capacity. Both the revision-level (--min-instances) and # service-level (--min) floors are 0. @@ -321,7 +311,7 @@ jobs: env: CLOUD_RUN_IMAGE_TAG: ${{ steps.cloud_run.outputs.image_tag }} CLOUD_RUN_TAG: ${{ steps.cloud_run.outputs.revision_tag }} - CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT: ${{ secrets.GCP_CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT }} + CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT: policyengine-api-cr-staging@policyengine-api.iam.gserviceaccount.com POLICYENGINE_DB_USER: ${{ vars.POLICYENGINE_DB_USER }} POLICYENGINE_DB_NAME: ${{ vars.POLICYENGINE_DB_NAME }} GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }} @@ -505,6 +495,19 @@ jobs: SIM_ENTRYPOINT: ${{ vars.SIM_ENTRYPOINT }} SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }} POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ${{ vars.POLICYENGINE_DB_INSTANCE_CONNECTION_NAME }} + APP_ENGINE_SERVICE_ACCOUNT: policyengine-api-ae-prod@policyengine-api.iam.gserviceaccount.com + POLICYENGINE_DB_PASSWORD_SECRET_RESOURCE: projects/policyengine-api/secrets/policyengine-api-prod-db-password/versions/latest + POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN_SECRET_RESOURCE: projects/policyengine-api/secrets/policyengine-api-prod-github-microdata-token/versions/latest + ANTHROPIC_API_KEY_SECRET_RESOURCE: projects/policyengine-api/secrets/policyengine-api-prod-anthropic-api-key/versions/latest + OPENAI_API_KEY_SECRET_RESOURCE: projects/policyengine-api/secrets/policyengine-api-prod-openai-api-key/versions/latest + HUGGING_FACE_TOKEN_SECRET_RESOURCE: projects/policyengine-api/secrets/policyengine-api-prod-hugging-face-token/versions/latest + GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }} + GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }} + GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }} + GATEWAY_AUTH_CLIENT_SECRET_RESOURCE: ${{ secrets.GATEWAY_AUTH_CLIENT_SECRET_RESOURCE }} + RUNTIME_CACHE_ENVIRONMENT: production + RUNTIME_CACHE_URL_SECRET_RESOURCE: projects/policyengine-api/secrets/policyengine-api-prod-runtime-cache-url/versions/latest + RUNTIME_CACHE_CA_CERT_SECRET_RESOURCE: projects/policyengine-api/secrets/policyengine-api-prod-runtime-cache-ca/versions/latest permissions: contents: read id-token: write @@ -531,31 +534,11 @@ jobs: uses: "google-github-actions/setup-gcloud@v2" - name: Validate App Engine deployment configuration run: bash .github/scripts/validate_app_engine_deploy_env.sh - env: - # Transitional: these values are still passed into the deploy bundle - # by gcp/export.py. Long-term target is a generic image plus runtime - # config / Secret Manager lookups instead of image bake-in. - GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }} - GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }} - GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }} - GATEWAY_AUTH_CLIENT_SECRET_RESOURCE: ${{ secrets.GATEWAY_AUTH_CLIENT_SECRET_RESOURCE }} - name: Deploy production version run: bash .github/scripts/deploy_app_engine_version.sh env: - # Transitional: deploy_app_engine_version.sh still prepares a bundle - # from these values before App Engine deploy. Long-term target is one - # generic image plus runtime config / Secret Manager. APP_ENGINE_VERSION: ${{ steps.version.outputs.version }} APP_ENGINE_PROMOTE: "0" - POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }} - POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: ${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }} - GATEWAY_AUTH_ISSUER: ${{ secrets.GATEWAY_AUTH_ISSUER }} - GATEWAY_AUTH_AUDIENCE: ${{ secrets.GATEWAY_AUTH_AUDIENCE }} - GATEWAY_AUTH_CLIENT_ID: ${{ secrets.GATEWAY_AUTH_CLIENT_ID }} - GATEWAY_AUTH_CLIENT_SECRET_RESOURCE: ${{ secrets.GATEWAY_AUTH_CLIENT_SECRET_RESOURCE }} - name: Resolve production version URL id: version_url run: | @@ -641,6 +624,9 @@ jobs: ROUTE_IMPL_SPECIFICATION: ${{ vars.ROUTE_IMPL_SPECIFICATION }} ROUTE_IMPL_METADATA: ${{ vars.ROUTE_IMPL_METADATA }} CLOUD_RUN_SERVICE: policyengine-api + CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT: production + CLOUD_RUN_RUNTIME_CACHE_URL_SECRET: policyengine-api-prod-runtime-cache-url:latest + CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET: policyengine-api-prod-runtime-cache-ca:latest # Sized by the Stage 2 qualification and the PR 4 host cutover — rationale # and numbers in docs/migration/cloud-run-operations.md ("Runtime shape and # scaling"). Warm capacity is expressed service-level (--min); the diff --git a/.gitignore b/.gitignore index 4679cd5e3..d69167e9d 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ dist/* **/*.csv.gz .env .DS_Store +/supabase/.temp/ # Ignore generated credentials from google-github-actions/auth gha-creds-*.json diff --git a/Makefile b/Makefile index d9bbc1dda..5a0b90dca 100644 --- a/Makefile +++ b/Makefile @@ -27,13 +27,12 @@ format: ruff format . deploy: - python gcp/export.py + python3 gcp/export.py gcloud config set app/cloud_build_timeout 2400 cp gcp/policyengine_api/* . y | gcloud app deploy --service-account=github-deployment@policyengine-api.iam.gserviceaccount.com rm -f app.yaml rm -f Dockerfile - rm -f .dbpw changelog: python .github/bump_version.py diff --git a/README.md b/README.md index 925405cd9..04386d112 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,11 @@ make setup-env - `OPENAI_API_KEY` - `HUGGING_FACE_TOKEN` +The database settings must resolve to an explicit durable development MySQL +database (or an authorized Cloud SQL development target). `FLASK_DEBUG` does +not select or create SQLite, and the application never bootstraps +`policyengine.db`. + If you need a local Google credential file for ADC, uncomment and set: - `GOOGLE_APPLICATION_CREDENTIALS` @@ -64,7 +69,10 @@ If you are running against an auth-protected simulation gateway outside the mana - `GATEWAY_AUTH_CLIENT_ID` - one of `GATEWAY_AUTH_CLIENT_SECRET` or `GATEWAY_AUTH_CLIENT_SECRET_RESOURCE` -Managed App Engine deploys currently still render some runtime config into the image bundle. Long-term, we intend to stop doing that and supply environment-specific config at runtime instead. +Managed App Engine deploys render non-secret runtime configuration and Secret +Manager resource names into `app.yaml`. Application secret values are resolved +in memory by the attached runtime service account before Gunicorn starts; they +are never written into the build context or image layers. ### 4. Start a server on localhost to see your changes @@ -124,7 +132,7 @@ NOTE: Any output that needs to be calculated will not work. Therefore, only hous ### 6. Testing calculations -Redis is required for API cache paths, including budget-window economy requests. The budget-window endpoint uses Redis for completed-result caching and in-flight batch deduplication; if Redis is unavailable, those requests fail instead of falling back to the database or an in-process cache. +Redis is required for API cache paths, including budget-window economy requests. The budget-window endpoint uses Redis for completed-result caching and in-flight batch deduplication; if Redis is unavailable, completed results are recoverable misses while ownership and deduplication operations fail closed instead of launching duplicate work. To test anything that utilizes Redis or the API's service workers (e.g. anything that requires society-wide calculations with the policy calculator), you'll also need to complete the following steps: @@ -142,7 +150,18 @@ brew install redis redis-server ``` -By default the API connects to Redis at `127.0.0.1:6379`, database `0`. Override this with `CACHE_REDIS_HOST`, `CACHE_REDIS_PORT`, and `CACHE_REDIS_DB` if your local Redis uses different connection settings. +Configure that separate local process explicitly; there is no localhost +fallback: + +```sh +export RUNTIME_CACHE_MODE=local +export RUNTIME_CACHE_URL=redis://127.0.0.1:6379/0 +export RUNTIME_CACHE_ENVIRONMENT=local-dev +export RUNTIME_CACHE_SERVICE=api +``` + +Deployed mode instead requires an authenticated `rediss://` URL and the +Memorystore instance CA. Do not use production cache credentials locally. 2. Start the API @@ -152,7 +171,9 @@ Run the below FLASK_DEBUG=1 python -m flask --app policyengine_api.api run ``` -App Engine staging and production deployments install and start Redis in the API container before Gunicorn starts. +App Engine and Cloud Run images start only Gunicorn. Deployed revisions connect +to their environment's managed Memorystore instance and never launch Redis in +the application container. NOTE: Calculations are not possible in the uk app without access to a specific dataset. Expect an error: "ValueError: Invalid response code 404 for url https://api.github.com/repos/policyengine/non-public-microdata/releases/tags/uk-2024-march-efo." diff --git a/alembic-v2.ini b/alembic-v2.ini new file mode 100644 index 000000000..8717e5a3d --- /dev/null +++ b/alembic-v2.ini @@ -0,0 +1,37 @@ +[alembic] +script_location = migrations/v2 +prepend_sys_path = . +path_separator = os + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index ea6891559..2f8ecac8b 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -9,11 +9,18 @@ AI **MUST NOT manually author Alembic revision scripts**. Generate every schema revision from reviewed SQLAlchemy metadata: ```bash +# API v1 / Cloud SQL MySQL uv run alembic -c alembic-v1.ini revision --autogenerate -m "" + +# API v2-alpha / Supabase Postgres +uv run alembic -c alembic-v2.ini revision --autogenerate -m "" ``` -The mandatory generation operation is `alembic revision --autogenerate`; the -explicit v1 configuration keeps these revisions in the Cloud SQL/MySQL chain. +The mandatory generation operation is `alembic revision --autogenerate`. +Always select the configuration for the intended migration domain explicitly; +never use Alembic's default configuration discovery. The v1 configuration +keeps MySQL revisions in `migrations/v1`, while the v2 configuration keeps +Postgres revisions in `migrations/v2`. If generated operations are wrong, first correct the model metadata and regenerate. AI may make minimal post-generation corrections only for dialect compatibility, @@ -26,12 +33,13 @@ those narrow review corrections, stop and request a human migration decision. ## Required checks -Before committing a migration: +Before committing a migration, run these checks against the matching domain: -1. Run `uv run alembic -c alembic-v1.ini check` and review the generated - operations. +1. Run `uv run alembic -c check` and review the generated + schema and declared-data operations. 2. Upgrade a fresh database to `head`. -3. Compare the deployed database to the ORM metadata before release. +3. Compare the database to the complete ORM metadata and, for v2, the declared + application-data source before release. 4. Downgrade one revision and upgrade to `head` again in an isolated database. 5. Confirm application startup performs no implicit DDL. @@ -58,11 +66,82 @@ database. ## Separate v1 and v2 migration domains -The current chain manages API v1 Cloud SQL/MySQL only. Stage 8 must introduce a -different configuration and a separate revision chain for the v2 -Supabase/Postgres schema. Never point `alembic-v1.ini` at Supabase, append v2 -Postgres revisions under `migrations/v1`, or use both Alembic and Supabase CLI -schema migrations as authorities for the same tables. +The two migration domains are mechanically independent: + +| Domain | Configuration | Revisions | Database | Target metadata | +| --- | --- | --- | --- | --- | +| API v1 | `alembic-v1.ini` | `migrations/v1` | Cloud SQL MySQL | existing v1 declarative metadata | +| API v2-alpha | `alembic-v2.ini` | `migrations/v2` | Supabase Postgres | reviewed `SQLModel.metadata` only | + +Never point `alembic-v1.ini` at Supabase, point `alembic-v2.ini` at Cloud SQL, +append v2 revisions under `migrations/v1`, import v1 metadata into the v2 +environment, or use both Alembic and Supabase CLI schema migrations as +authorities for the same application tables. + +## API v2-alpha database targets + +This is the isolated Supabase/Postgres migration domain with a separate revision chain; +it is never a v1 Cloud SQL target. + +Every v2 command requires an explicit, secret-supplied migration URL using a +Postgres dialect and the Psycopg 3 driver, for example +`postgresql+psycopg://...`. The v2 environment must not read +`ALEMBIC_DATABASE_URL`, infer a URL from v1 or runtime settings, select SQLite +under `FLASK_DEBUG`, or fall back to local Postgres. Missing, MySQL, SQLite, or +otherwise non-Postgres targets fail before any migration operation and without +printing the URL. + +A command against a persistent Supabase environment must also verify its +declared environment and project reference against the durable target record. +Before baseline generation or the first upgrade, it must require a successful +freshness qualification proving that the target contains no application +tables, Alembic history, or predecessor data. Missing, mismatched, ambiguous, +or non-fresh qualification stops the command; never drop, reset, reconcile, +adopt, or stamp the target automatically. + +Only a separate, explicit disposable-test mode may omit Supabase identity. It +is limited to an isolated Postgres database created for local or CI migration +lifecycle tests and must be rejected for staging, production, or any other +persistent target. + +The v2 environment imports the controlled v2 table-model package before +exposing `SQLModel.metadata`. It must compare the resulting table names with +the reviewed inventory and fail before generation or execution if expected +tables are absent or v1, predecessor, `runtime_bundles`, population, or other +unreviewed tables are registered. + +## Generated v2 application-data migrations + +Alembic is also the sole authority for versioned v2 application-data changes. +Small reference data belongs in a versioned declarative source with stable +natural identifiers and deterministic before-and-after values. The bounded v2 +autogeneration comparator and renderer compare that declaration with the +current target and emit ordered, reversible operations through the same +command used for schema revisions: + +```bash +uv run alembic -c alembic-v2.ini revision --autogenerate -m "" +``` + +Do not create a blank revision and add `bulk_insert`, SQL strings, ORM calls, +or other data operations by hand. Correct the declaration or generator and +regenerate when output is wrong. Generated data additions run only after their +required schema exists, and generated removals run before destructive schema +operations. An unsafe identifier, unknown prior value, non-deterministic +ordering, or non-reversible change must fail generation and invoke the human +decision rule above. + +`alembic check` for v2 must detect both schema drift and declared-data drift. +Application startup, model import, project provisioning, and Supabase Storage +bootstrap must never call `create_all`, create or stamp application tables, or +mutate versioned application data. The Supabase CLI is not an application +schema or seed migration authority. + +Migration credentials remain separate from future runtime credentials. The +migration identity may create and alter the v2 application schema; the runtime +identity must not. Neither credential nor a secret-bearing URL belongs in an +Alembic INI file, log, committed environment file, generated artifact, or +revision. ### Fresh database qualification @@ -114,3 +193,13 @@ post-generation correction is `if_exists=True` on the generated tracer drop: fresh databases built from the original baseline contain the table, while deployed MySQL databases do not. Integration tests cover both fresh and production-shaped schemas and verify that `execution_id` becomes non-nullable. + +### API v2 baseline native-enum cleanup + +Revision `47592781336f` was autogenerated from the reviewed SQLModel metadata. +PostgreSQL native enum types are created as part of the generated table DDL, +but Alembic does not autogenerate removal of those schema-level types after the +last dependent table is dropped. Its only post-generation correction drops the +nine generated `v2_*` enum types at the end of the baseline downgrade. The v2 +Postgres lifecycle test covers empty upgrade, downgrade to base, and re-upgrade +so stale enum types cannot make the generated baseline non-reversible. diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index f75a797ab..a6d6421ca 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -52,6 +52,63 @@ FLASK_DEBUG=1 python -m pytest tests/unit/test_migration_flags.py tests/unit/tes python -m pytest tests/unit/test_cloud_run_deploy_scripts.py tests/unit/test_capture_migration_baseline.py tests/unit/test_compare_migration_baseline.py -q ``` +For Stage 8 v2 platform foundation work, run the smallest applicable group +while iterating, then all groups before qualification. + +SQLModel schema, report/run behavior, lazy configuration, and import effects: + +```bash +uv run pytest tests/unit/v2/test_models.py tests/unit/v2/test_model_persistence.py tests/unit/v2/test_report_runs.py tests/unit/v2/test_settings.py tests/unit/v2/test_import_side_effects.py -q +``` + +The generated-only v2 Alembic extension and disposable Postgres lifecycle: + +```bash +uv run pytest tests/unit/v2/test_alembic_v2.py tests/unit/v2/test_reference_data_autogenerate.py tests/unit/test_alembic_workflows.py -q +V2_ALEMBIC_DISPOSABLE_TEST=1 V2_MIGRATION_DATABASE_URL="postgresql+psycopg://.../policyengine_v2_alembic_test" \ + uv run pytest tests/integration/test_alembic_v2_lifecycle.py -q +``` + +The disposable lifecycle must start from empty Postgres, upgrade to `head`, +check schema and declared-data drift, compare the live schema with the exact +SQLModel inventory, downgrade the reviewed boundary, and upgrade again. It +must not use the persistent Supabase qualification bypass outside explicit +disposable-test mode. Continue running the existing isolated v1 MySQL +lifecycle whenever either Alembic domain changes. + +Supabase Storage bootstrap and repository hygiene: + +```bash +uv run pytest tests/unit/v2/test_storage_bootstrap.py tests/unit/v2/test_scaffolding_hygiene.py -q +``` + +Shared-cache unit behavior and real Redis-compatible integration semantics: + +```bash +uv run pytest tests/unit/runtime_cache tests/unit/services/test_household_calculation_service.py tests/unit/services/test_tracer_service.py tests/unit/services/test_ai_analysis_service.py tests/unit/services/test_reform_impacts_service.py tests/unit/services/test_budget_window_cache.py -q +RUNTIME_CACHE_TEST_URL="redis://127.0.0.1:6379/0" uv run pytest tests/integration/test_runtime_cache_redis.py -q +``` + +The real Redis-compatible suite must be an explicit integration run and cover +two independent connections, TTL expiry, atomic household/tracer generations, +bounded lookup indexes, completed-result miss semantics, and token-safe +coordination claims. Unit tests use the deterministic in-memory fake and do +not require network credentials. + +Startup, deployment, SQLite-removal, and unchanged API migration contracts: + +```bash +FLASK_DEBUG=1 uv run pytest tests/unit/v2/test_import_side_effects.py tests/unit/v2/test_stage8_activation.py tests/unit/data/test_orm_sessions.py tests/unit/services/test_direct_orm_local_analysis.py tests/unit/test_app_engine_runtime.py tests/unit/test_cloud_run_deploy_scripts.py tests/unit/test_asgi_factory.py tests/contract/test_v1_route_contracts.py -q +python3 scripts/export_migration_contracts.py +python3 scripts/run_quality_guards.py +``` + +These checks must prove that imports, debug startup, ordinary startup, and +requests create no SQLite database or lock file; deployed startup launches no +Redis server; missing managed-cache configuration fails closed; Cloud SQL and +existing routes/compute remain primary; and generated migration-contract +artifacts remain current. + Cloud Run must receive `ROUTE_IMPL_HEALTH`, `ROUTE_IMPL_SPECIFICATION`, and `ROUTE_IMPL_METADATA` from the selected GitHub environment. Candidate resolution must verify those values on the exact revision. Staging promotion @@ -68,9 +125,9 @@ docker build -f gcp/cloud_run/Dockerfile -t policyengine-api-cloud-run:test . If the Cloud Run container startup script changes, keep the script syntax and child-process supervision assertions in `tests/unit/test_cloud_run_deploy_scripts.py` -updated. The tier 1 Redis path keeps Redis local to the container, so tests -should verify the bash entrypoint, explicit Redis/Uvicorn PID tracking, and -fail-fast behavior rather than any managed Redis integration. +updated. Stage 8 removes the tier 1 container-local Redis process, so the +tests must assert that only the API server is supervised and that deployed +configuration selects managed Redis without a localhost fallback. Staging deployment checks should run the same live integration suite against both the App Engine staging URL and the tagged Cloud Run staging URL before diff --git a/docs/migration/stage-8-managed-redis.md b/docs/migration/stage-8-managed-redis.md new file mode 100644 index 000000000..9b68a248f --- /dev/null +++ b/docs/migration/stage-8-managed-redis.md @@ -0,0 +1,126 @@ +# Stage 8 managed Redis topology + +## Decision + +Stage 8 will use **Google Cloud Memorystore for Redis** in `us-central1`, the +same region as the production and staging Cloud Run services. Each deployed +environment receives its own instance and credentials: + +| Environment | Instance ID | Tier | Capacity | Redis version | +| --- | --- | --- | --- | --- | +| Staging | `policyengine-api-cache-staging` | Basic | 1 GiB | 7.2 | +| Production | `policyengine-api-cache-prod` | Standard HA | 5 GiB | 7.2 | + +Production uses Standard Tier's cross-zone replica and automatic failover +because Redis coordinates expensive work in addition to caching completed +results. Staging uses Basic Tier to keep the validation environment +independent at lower cost. Neither environment enables read replicas. +Capacity is an initial allocation, not a durable-data commitment; metrics and +eviction pressure may justify resizing it later. + +Memorystore for Redis is preferred over Memorystore for Redis Cluster or +Memorystore for Valkey for this stage. The existing consumers use ordinary +Redis commands and require multi-key atomic operations. A single-instance +Redis-compatible endpoint preserves those semantics without introducing hash +slot constraints or a cluster-aware client during the SQLite-to-cache +migration. + +Google documents [Memorystore for Redis tiers and pricing][pricing], the +[supported Redis versions and creation flags][create], and direct Cloud Run +[connectivity to Memorystore][cloud-run-redis]. + +## Network and transport + +Both instances use all of the following settings: + +- the existing `default` VPC in the `policyengine-api` Google Cloud project; +- the `policyengine-api-memorystore-psa` automatically allocated `/24` Private + Service Access range and + `PRIVATE_SERVICE_ACCESS` connection mode; +- Redis AUTH enabled; +- in-transit encryption set to `SERVER_AUTHENTICATION`; and +- no public endpoint, localhost fallback, or container-launched Redis server. + +Cloud Run revisions will use [Direct VPC egress][direct-vpc] through the +`default` `us-central1` subnet with `private-ranges-only` routing. Direct VPC +egress avoids a permanently provisioned Serverless VPC Access connector while +keeping ordinary internet-bound traffic on its current path. The deployment +must account for Cloud Run subnet address consumption and connection resets; +the runtime client therefore needs bounded pools, timeouts, and reconnect +behavior. + +Stage 8 enables the Memorystore and service-networking APIs, creates the +non-overlapping allocation and both instances, and validates +TLS-authenticated access from more than one Cloud Run instance before rollout. + +## Authentication and secret boundaries + +The authenticated URL and all currently downloadable instance CAs are stored +in separate Secret Manager secrets: + +| Environment | URL secret | CA secret | +| --- | --- | --- | +| Staging | `policyengine-api-staging-runtime-cache-url` | `policyengine-api-staging-runtime-cache-ca` | +| Production | `policyengine-api-prod-runtime-cache-url` | `policyengine-api-prod-runtime-cache-ca` | + +They are exposed only to the Cloud Run and App Engine runtime identities for +that environment. Staging and +production must not share an endpoint or credential. Endpoint, port, expected +TLS mode, and environment are explicit deployed settings; application code +must not infer them from v1 database configuration or fall back to localhost. + +The client validates the Memorystore server certificate using in-memory +`ssl_ca_data` and Google's documented instance-specific certificate-authority +material. CA rotation requires storing every currently downloadable CA in the +CA secret before Google rotates the serving certificate. Secret-bearing URLs, AUTH values, +and certificate material must never be logged, committed, or placed in image +layers. + +## Namespace and loss semantics + +Every key is namespaced by environment, service, cache family, and cache +schema version. Result keys additionally include every result-affecting input +and relevant package version. This is defense in depth around the physically +separate instances and permits schema transitions without interpreting old +payloads as current. + +Redis remains disposable: + +- missing, expired, evicted, flushed, incompatible, or unreadable completed + results are cache misses and may be recomputed from durable sources; +- completed-result writes subtract a random zero-to-ten-percent interval from + each cache family's nominal TTL to spread normal expirations without + exceeding the configured maximum age; +- coordination and claim TTLs remain exact because they establish safety + bounds for work ownership; +- a failed completed-result cache write does not turn successful computation + into failure; and +- coordination and claim failures fail closed so multiple instances do not + silently duplicate guarded expensive work. + +No report, report run, household, policy, output, or other durable domain +record may exist only in Redis. + +## Revision rollout and rollback + +Managed-cache configuration is revision-specific. A revision cannot receive +traffic unless its environment's endpoint, AUTH secret, TLS configuration, +and VPC attachment are valid. New Stage 8 revisions use managed Redis only; +they never start or select an embedded Redis process. + +The immediately preceding application revision retains its own prior runtime +configuration. Rolling back means moving Cloud Run traffic to that known-good +revision. It does not copy, restore, or downgrade cache contents. The managed +cache may be retained or flushed because cache loss is handled as a miss, and +the dormant v2 Postgres schema remains in place unless an operator separately +invokes a reviewed migration downgrade. + +If a rollout requires incompatible cache serialization, the new revision uses +a new cache-schema namespace. During a traffic split, old and new revisions +may therefore coexist without either revision reading the other's incompatible +values. + +[cloud-run-redis]: https://cloud.google.com/memorystore/docs/redis/connect-redis-instance-cloud-run +[create]: https://cloud.google.com/memorystore/docs/redis/create-manage-instances +[direct-vpc]: https://cloud.google.com/run/docs/configuring/vpc-direct-vpc +[pricing]: https://cloud.google.com/memorystore/docs/redis/pricing diff --git a/docs/migration/stage-8-platform-runbook.md b/docs/migration/stage-8-platform-runbook.md new file mode 100644 index 000000000..1a7fef7b3 --- /dev/null +++ b/docs/migration/stage-8-platform-runbook.md @@ -0,0 +1,89 @@ +# Stage 8 v2 platform runbook + +## Scope and authority + +Stage 8 keeps Cloud SQL, existing routes, Simulation Entrypoint selection, and +existing compute primary. The Supabase Postgres schema is dormant. Redis holds +only recoverable completed results and expiring coordination state; it is never +a durable domain store. + +The recorded Supabase target is project `kvrifaviwhzjztcbrfpy`, organization +`PolicyEngine`, region `us-east-2`, environment `production-foundation`. Stop +if a supplied connection cannot be proven to resolve to that target. + +## Persistent Supabase qualification and initialization + +1. Confirm the target record in `stage-8-supabase-target.md` and its successful + fresh-state audit. A target with application tables, Alembic history, a + mismatched project reference, or ambiguous identity is not reset, adopted, + dropped, or stamped. +2. Retrieve only the migration database credential from Secret Manager and + supply `V2_MIGRATION_DATABASE_URL`, `V2_SUPABASE_PROJECT_REF`, and + `V2_SUPABASE_ENVIRONMENT` to the explicit operator process. +3. Run `uv run alembic -c alembic-v2.ini upgrade head` and then + `uv run alembic -c alembic-v2.ini check`. Do not run either operation during + application startup. +4. Only after migration succeeds, retrieve the separate Storage administration + credential and run + `uv run python3 scripts/bootstrap_v2_supabase_storage.py`. + A second identical run must be a no-op; incompatible existing configuration + is an error, not permission to replace the bucket. +5. Confirm no migration URL, password, Storage key, scratch SQL, generated + payload, dump, or one-off scaffolding file entered the repository or logs. + +Application runtime receives the non-secret dormant project identity only. It +does not receive the migration password or Storage administration key. + +## Managed-cache rollout + +Staging uses `policyengine-api-cache-staging` (Basic, 1 GiB) and production uses +`policyengine-api-cache-prod` (Standard HA, 5 GiB). Both are Redis 7.2, +AUTH-enabled, TLS-only on port 6378, and private through the `default` VPC and +`policyengine-api-memorystore-psa` allocation. + +Before sending traffic to a candidate: + +1. Verify the instance is `READY`, its AUTH and TLS modes are enabled, and all + current server CAs are present in the environment's CA secret. +2. Verify the candidate has `RUNTIME_CACHE_MODE=deployed`, the correct + environment namespace, both Secret Manager bindings, and Direct VPC egress + through `default/default` with `private-ranges-only` routing. +3. Verify Cloud Run uses the dedicated runtime service account and App Engine + uses only non-secret Secret Manager resource names. Confirm the App Engine + staging and production service accounts have repository-scoped Artifact + Registry Reader access so each can pull the reviewed candidate image, and + those identities have `secretAccessor` only on the required database + password, GitHub microdata, Anthropic, OpenAI, Hugging Face, gateway-auth, + and environment-specific cache secrets. Neither runtime identity receives + v2 migration or Storage administration access. +4. Send test traffic to at least two Cloud Run instances and verify one + connection's value is visible to another. Confirm the container has no + `redis-server` child and startup creates no SQLite database or lock file. +5. Delete only test cache keys, then verify completed-result reads recompute as + misses. Do not use a coordination failure as a miss: claims must fail closed. + +Expect a cold cache during first rollout. Monitor cache-family hits, misses, +recomputations, write failures, coordination failures, connection latency, +timeouts, evictions, memory pressure, Cloud SQL behavior, and API errors. +Completed-result writes use subtract-only TTL jitter of up to ten percent to +spread normal expirations; coordination and claim TTLs remain exact. Jitter +does not spread misses after a full flush, so bounded recomputation and atomic +claims remain the controls for complete cache loss. +Runtime cache operations emit the stable `runtime_cache_operations` structured +metric with `metric_value=1`, `cache_family`, `cache_event`, optional +`cache_operation`, and `latency_ms`. Use the value as a counter grouped by the +family and event fields and the latency field as the distribution source. +Logs must include no keys, values, URLs, AUTH strings, or CA payloads. + +## Rollback + +Move traffic to the exact preceding application revision with its own +revision-specific cache configuration. Do not recover, copy, or downgrade Redis +contents; cache loss is handled as a miss. The prior revision must not be +retrofitted with the new secret or VPC configuration during rollback. + +Leave the dormant v2 Supabase schema at its current revision during an +application rollback. Run a v2 Alembic downgrade only as a separate reviewed +operator action against the re-qualified v2 target. Never point the v1 chain at +Supabase, automatically downgrade Postgres, or delete Storage as part of an +application traffic rollback. diff --git a/docs/migration/stage-8-supabase-bootstrap.md b/docs/migration/stage-8-supabase-bootstrap.md new file mode 100644 index 000000000..2cdfbca25 --- /dev/null +++ b/docs/migration/stage-8-supabase-bootstrap.md @@ -0,0 +1,68 @@ +# Stage 8 Supabase Migration and Storage Bootstrap + +This runbook operates only on the dedicated dormant API v2-alpha target +recorded in `docs/migration/stage-8-supabase-target.md`. It is not application +startup logic. Cloud SQL and all existing production routes and compute remain +primary throughout Stage 8. + +## Required identity and credential boundaries + +The non-secret identity must be exactly: + +- environment: `production-foundation` +- project reference: `kvrifaviwhzjztcbrfpy` +- Storage API origin: `https://kvrifaviwhzjztcbrfpy.supabase.co` +- private bucket: `policyengine-v2-alpha` + +Inject the database migration password and the Storage administration key from +their separate GCP Secret Manager secrets at execution time. Do not echo them, +place them in repository files, reuse the migration URL as runtime +configuration, or expose the Storage key to the application service account. + +## Ordered explicit operations + +1. Confirm the target record and its successful freshness audit. Stop on any + identity ambiguity or unexpected application state; never reset, adopt, or + stamp the database. +2. Supply `V2_MIGRATION_DATABASE_URL`, `V2_SUPABASE_ENVIRONMENT`, and + `V2_SUPABASE_PROJECT_REF` to an explicit operator or CI migration step. +3. Run `uv run alembic -c alembic-v2.ini upgrade head`, followed by + `uv run alembic -c alembic-v2.ini check`. The v2 chain requires an online + connection so it can qualify the persistent target and verify generated + application-data before/after states. +4. Remove the migration credential from the execution environment. Supply the + separate `V2_SUPABASE_STORAGE_ADMIN_KEY` together with the recorded + identity, `V2_SUPABASE_STORAGE_URL`, and + `V2_SUPABASE_STORAGE_BUCKET=policyengine-v2-alpha`. +5. Run `uv run python3 scripts/bootstrap_v2_supabase_storage.py`. A fresh run + creates the reviewed private bucket. A repeat run verifies it and reports + `created: false`. An incompatible existing bucket stops without update, + deletion, recreation, or public exposure. +6. Remove the Storage administration credential from the execution + environment and run + `uv run python3 scripts/check_stage8_scaffolding_hygiene.py` before commit. + +The Storage initializer calls only Supabase's bucket-management endpoint. It +does not run Alembic, import application startup, modify application tables or +rows, initialize canonical metadata, upload an object, or create an access +policy. The dedicated `sb_secret_...` key is sent only in the `apikey` header; +it is not a JWT and must not be placed in `Authorization: Bearer`. The +initializer recognizes both structured current Storage errors such as +`NoSuchBucket` and legacy HTTP 404/409 responses without logging response +bodies. Supabase documents that buckets are private by default and that bucket +creation needs bucket insert permission but no object permission: + and +. +The current key and Storage error contracts are documented at + +and . + +## Repository hygiene + +One-off SQL, dumps, generated payloads, temporary environment files, Supabase +CLI state, and scratch scaffolding belong only in ignored local-artifact or +system-temporary locations and must be removed after use. If an operation is +needed again, promote it to tested idempotent tooling before committing it. +Generated Alembic revisions, declarative migration sources, this supported +initializer, tests, and durable documentation are reviewed project artifacts, +not disposable scaffolding. diff --git a/docs/migration/stage-8-supabase-target.md b/docs/migration/stage-8-supabase-target.md new file mode 100644 index 000000000..f534981d0 --- /dev/null +++ b/docs/migration/stage-8-supabase-target.md @@ -0,0 +1,156 @@ +# Stage 8 Supabase Target + +This document is the durable, non-secret identity record for the Supabase +project introduced by Stage 8 of the unified API v2-alpha migration. It does +not contain credentials, connection URLs, API keys, or one-off provisioning +output. + +## Target identity + +| Field | Value | +| --- | --- | +| Supabase organization | `PolicyEngine` | +| Organization ID | `jygirqnhxzbevhozzrzi` | +| Project name | `policyengine-api-v2-alpha` | +| Project reference | `kvrifaviwhzjztcbrfpy` | +| Region | `us-east-2` | +| Environment classification | Production foundation; dormant during Stage 8 | +| Owning team | PolicyEngine engineering | +| Stage 8 authority | No production request reads, writes, routes, or compute | +| Database host identity | `db.kvrifaviwhzjztcbrfpy.supabase.co` | +| Postgres engine | PostgreSQL 17, Supabase GA channel | +| Provisioned | 2026-08-13; observed `ACTIVE_HEALTHY` | + +## Purpose + +The project is the dedicated Postgres and Storage destination for the API +v2-alpha migration. Stage 8 establishes and qualifies the platform while Cloud +SQL and the existing API and compute paths remain primary. Later migration +stages may populate and activate the target under their own reviewed cutover +contracts. + +## Selection record + +- The authenticated Supabase account exposes one organization, `PolicyEngine`, + with organization ID `jygirqnhxzbevhozzrzi`. +- No existing project is named `policyengine-api-v2-alpha`. In particular, the + unrelated project named `database` is not reused. +- The deployed API and Cloud SQL defaults are in GCP `us-central1`. Supabase + currently offers `us-east-2`; it is selected as the nearby supported region + for this AWS-hosted project. +- A Supabase project's region is fixed at the infrastructure level, so a later + region change would require a new project and a reviewed migration. + +## Provisioning boundary + +Project creation is an explicit operator action. It may establish the project, +database service, ownership, networking, and secret placement, but it must not +create application tables or rows, stamp Alembic, or initialize a Storage +bucket. Application schema and versioned application data remain exclusively +owned by the generated v2 Alembic chain. Storage initialization is a separate, +later idempotent operation. + +The owner credential created with the project is stored in GCP Secret Manager +as `policyengine-api-v2-alpha-prod-db-owner-password`. It is a provisioning +credential, not the later migration or application-runtime identity, and its +value is never stored in this repository. + +Tasks 1.3 through 1.6 qualify connectivity, credential separation, database +freshness, and repository hygiene before any v2 baseline is generated. + +## Connectivity qualification + +- External database SSL enforcement is enabled at the Supabase project level. +- The direct database identity remains + `db.kvrifaviwhzjztcbrfpy.supabase.co:5432`. Supabase direct endpoints require + IPv6 unless the IPv4 add-on is enabled, so it is not the qualified + `us-central1` Cloud Run path at this stage. +- Authenticated operator connectivity is qualified over the IPv4 Supavisor + session endpoint `aws-0-us-east-2.pooler.supabase.com:5432`, using database + `postgres` and the project-qualified owner username. A read-only connection + reported PostgreSQL 17.6 and confirmed TLS in `pg_stat_ssl`. +- Database network restrictions currently allow `0.0.0.0/0` and `::/0` because + the existing Cloud Run service has no reviewed static egress CIDR. Narrowing + the allowlist to an invented address would make connectivity unreliable. + Mandatory TLS and credential isolation are the active boundary; a later + network restriction requires a provisioned, tested egress range. +- No IPv4 add-on, custom Postgres override, database DDL, Alembic stamp, + application row, or Storage bucket was introduced during connectivity setup. + +Supabase's connection-mode guidance is documented at +. + +## Credential boundaries + +All secret values live in GCP Secret Manager in project `policyengine-api`. +The repository records names and intended use only. + +| Access path | Identity or key | Secret Manager secret | Effective boundary | +| --- | --- | --- | --- | +| Initial project ownership and emergency administration | Supabase `postgres` owner | `policyengine-api-v2-alpha-prod-db-owner-password` | Provisioning only; not application or routine migration configuration | +| Generated v2 Alembic chain | `policyengine_v2_migrator` | `policyengine-api-v2-alpha-prod-db-migration-password` | Login, database connect, and `USAGE`/`CREATE` on `public`; no superuser, role creation, database creation, replication, or RLS bypass | +| Future ordinary v2 persistence | `policyengine_v2_runtime` | `policyengine-api-v2-alpha-prod-db-runtime-password` | Login, database connect, and `USAGE` on `public`; no schema creation, superuser, role creation, database creation, replication, or RLS bypass | +| Explicit Storage bootstrap | Supabase secret key `stage_8_storage_bootstrap` | `policyengine-api-v2-alpha-prod-storage-admin-key` | Dedicated, independently rotatable server-side credential exposed only to the Storage bootstrap operation | + +The migration role owns the default privileges for objects it later creates: +ordinary table read/write and sequence use are granted to the runtime role. +Those grants do not create an application object or row. + +Supabase secret keys are elevated server-side credentials that bypass RLS; the +platform does not represent them as Storage-only keys. Least privilege is +therefore enforced by using a distinct named key, storing it separately, and +making it available only to the explicit Storage initializer. Neither the +runtime database identity nor the migration identity receives this key. + +The Cloud Run runtime service account previously held project-wide Secret +Manager accessor rights. Before completing this credential split, that broad +binding was replaced with per-secret access to its six existing production +runtime secrets: the gateway client secret plus the database, microdata, +Anthropic, OpenAI, and Hugging Face secrets. It has no project-wide Secret +Manager role and no binding on any Stage 8 administrative secret. + +## Freshness qualification + +On 2026-08-13, the recorded project was audited through a PostgreSQL +`READ ONLY` transaction over the qualified TLS connection. + +- Connected project reference: `kvrifaviwhzjztcbrfpy`. +- Database and role: `postgres` as the provisioning owner. +- Application schema: `public` contains zero tables. +- Alembic history: no `alembic_version` table exists in any schema. +- Predecessor application state: no table matching the reviewed v2 model + groups, `runtime_bundles`, or a population table exists in `public`. +- Storage initialization: `storage.buckets` contains zero rows. +- Service-managed schemas observed: `auth`, `extensions`, `graphql`, + `graphql_public`, `pgbouncer`, `realtime`, `storage`, and `vault`. Their + platform-owned tables do not count as application state. + +The audit result is fresh. No reset, drop, stamp, reconciliation, or adoption +was required or performed. This qualification permits later v2 baseline +generation only when the migration workflow independently verifies the same +recorded target identity. + +## Provisioning hygiene + +The provisioning review completed on 2026-08-14 with these results: + +- No application table, application row, Alembic stamp, or Storage bucket was + created. +- No secret value, secret-bearing URL, SQL dump, scratch SQL, generated + payload, or temporary configuration is tracked or staged. +- `supabase/.temp/` is ignored because the Supabase CLI writes ephemeral linked + project metadata there even for management operations. Its generated file + was removed after the project reference was recorded above. +- The local `.venv` used for read-only Postgres qualification is already an + ignored development artifact and is not part of the change. +- The repository secret-pattern scan found no credential value in the changed + files. +- The four Stage 8 GCP secrets have automatic replication and purpose labels; + none grants access to the Cloud Run runtime service account. +- The dedicated Supabase Storage key exists as + `stage_8_storage_bootstrap`; only its non-secret identifier and prefix are + recorded, while the complete value exists only in GCP Secret Manager. + +The dedicated Supabase foundation is therefore ready for the generated v2 +schema work, subject to the target-identity gate implemented later in this +change. diff --git a/gcp/Dockerfile b/gcp/Dockerfile index 0af5578e8..1ccc96593 100644 --- a/gcp/Dockerfile +++ b/gcp/Dockerfile @@ -3,5 +3,4 @@ ENV VIRTUAL_ENV /env ENV PATH /env/bin:$PATH RUN apt-get update && apt-get install -y build-essential checkinstall RUN python3.12 -m pip install --upgrade pip --trusted-host pypi.python.org --trusted-host pypi.org --trusted-host files.pythonhosted.orgpip -RUN apt-get update && apt-get install -y redis-server RUN pip install git+https://github.com/policyengine/policyengine-api diff --git a/gcp/README.md b/gcp/README.md index a63f65ebd..678bc7749 100644 --- a/gcp/README.md +++ b/gcp/README.md @@ -2,7 +2,29 @@ The deployment actions build Docker images and deploy them to Google App Engine. The docker images themselves are based off a starter image (to save each API docker image having to spend 5 minutes installing the same dependencies). The starter image is the `Dockerfile` in this directory. -The App Engine API image installs `redis-server` and starts it through `gcp/policyengine_api/start.sh`. Redis is required at runtime for budget-window economy request caching and in-flight batch deduplication. The API reads `CACHE_REDIS_HOST`, `CACHE_REDIS_PORT`, and `CACHE_REDIS_DB`, defaulting to `127.0.0.1`, `6379`, and `0`. +Deployed API images run only Gunicorn. They do not install or launch Redis and +have no localhost cache fallback. Stage 8 revisions receive their environment's +Memorystore endpoint, AUTH value, and instance CA through revision-specific +Secret Manager wiring. Required runtime settings are `RUNTIME_CACHE_MODE`, +`RUNTIME_CACHE_URL`, `RUNTIME_CACHE_CA_CERT`, `RUNTIME_CACHE_ENVIRONMENT`, and +`RUNTIME_CACHE_SERVICE`; deployed mode fails closed when they are incomplete. + +The App Engine image is environment-neutral. `app.yaml` receives non-secret +configuration and Secret Manager resource names only. Before Gunicorn starts, +`policyengine_api.app_engine_runtime` resolves the database password, GitHub +microdata token, Anthropic key, OpenAI key, and Hugging Face token into the +process environment using the attached App Engine service account. Raw values +and temporary secret files must never enter the build context or image layers. + +Local development uses an explicitly launched Redis-compatible process and an +explicit durable development database. For example: + +```sh +export RUNTIME_CACHE_MODE=local +export RUNTIME_CACHE_URL=redis://127.0.0.1:6379/0 +export RUNTIME_CACHE_ENVIRONMENT=local-dev +export RUNTIME_CACHE_SERVICE=api +``` To update the starter image: * `python setup.py sdist` to build the python package diff --git a/gcp/cloud_run/Dockerfile b/gcp/cloud_run/Dockerfile index f2a38279c..1f2a09ace 100644 --- a/gcp/cloud_run/Dockerfile +++ b/gcp/cloud_run/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.12-slim RUN apt-get update \ - && apt-get install -y --no-install-recommends build-essential redis-server \ + && apt-get install -y --no-install-recommends build-essential \ && rm -rf /var/lib/apt/lists/* WORKDIR /app @@ -15,8 +15,5 @@ RUN chmod +x ./start.sh \ && pip install --no-cache-dir -e . ENV GATEWAY_AUTH_REQUIRED=1 -ENV CACHE_REDIS_HOST=127.0.0.1 -ENV CACHE_REDIS_PORT=6379 -ENV CACHE_REDIS_DB=0 CMD ["/bin/bash", "/app/start.sh"] diff --git a/gcp/cloud_run/start.sh b/gcp/cloud_run/start.sh index 45a704d8a..833c13571 100755 --- a/gcp/cloud_run/start.sh +++ b/gcp/cloud_run/start.sh @@ -2,93 +2,17 @@ set -euo pipefail PORT="${PORT:-8080}" -CACHE_REDIS_HOST="${CACHE_REDIS_HOST:-127.0.0.1}" -CACHE_REDIS_PORT="${CACHE_REDIS_PORT:-6379}" -CACHE_REDIS_DB="${CACHE_REDIS_DB:-0}" WEB_CONCURRENCY="${WEB_CONCURRENCY:-1}" -REDIS_READY_MAX_ATTEMPTS="${REDIS_READY_MAX_ATTEMPTS:-30}" -export CACHE_REDIS_HOST CACHE_REDIS_PORT CACHE_REDIS_DB - -redis_pid="" -server_pid="" - -shutdown() { - trap - INT TERM - - if [ -n "$server_pid" ] && kill -0 "$server_pid" 2>/dev/null; then - kill "$server_pid" 2>/dev/null || true - fi - - if [ -n "$redis_pid" ] && kill -0 "$redis_pid" 2>/dev/null; then - kill "$redis_pid" 2>/dev/null || true - fi - - if [ -n "$server_pid" ]; then - wait "$server_pid" 2>/dev/null || true - fi - - if [ -n "$redis_pid" ]; then - wait "$redis_pid" 2>/dev/null || true - fi -} - -trap 'shutdown; exit 143' INT TERM - -redis-server --bind "$CACHE_REDIS_HOST" \ - --port "$CACHE_REDIS_PORT" \ - --protected-mode yes \ - --maxclients 10000 \ - --timeout 0 & -redis_pid="$!" - -redis_ready_attempts=0 -until redis-cli -h "$CACHE_REDIS_HOST" -p "$CACHE_REDIS_PORT" ping >/dev/null 2>&1; do - redis_ready_attempts=$((redis_ready_attempts + 1)) - if ! kill -0 "$redis_pid" 2>/dev/null; then - echo "Redis exited before becoming ready" >&2 - shutdown - exit 1 - fi - - if [ "$redis_ready_attempts" -ge "$REDIS_READY_MAX_ATTEMPTS" ]; then - echo "Redis did not become ready after $redis_ready_attempts attempts" >&2 - shutdown - exit 1 - fi - sleep 1 -done # gunicorn's master binds the listen socket before forking workers, so the # Cloud Run TCP startup probe passes immediately instead of racing the # multi-minute app import (which happens in the worker, post-fork, because # application preloading is disabled). --timeout 0 is required: a worker mid-import does # not heartbeat, and the default 30s watchdog would kill it before boot. -gunicorn policyengine_api.asgi:app \ +exec gunicorn policyengine_api.asgi:app \ --worker-class uvicorn.workers.UvicornWorker \ --workers "$WEB_CONCURRENCY" \ --bind "0.0.0.0:${PORT}" \ --timeout 0 \ --keep-alive 5 \ - --forwarded-allow-ips '*' & -server_pid="$!" - -set +e -wait -n "$redis_pid" "$server_pid" -status="$?" -set -e - -if ! kill -0 "$redis_pid" 2>/dev/null; then - echo "Redis exited; stopping Cloud Run container" >&2 -elif ! kill -0 "$server_pid" 2>/dev/null; then - echo "API server exited; stopping Cloud Run container" >&2 -else - echo "A supervised Cloud Run process exited; stopping container" >&2 -fi - -shutdown - -if [ "$status" -eq 0 ]; then - exit 1 -fi - -exit "$status" + --forwarded-allow-ips '*' diff --git a/gcp/export.py b/gcp/export.py index 5a726e7d5..e42b859e4 100644 --- a/gcp/export.py +++ b/gcp/export.py @@ -1,80 +1,91 @@ +"""Render App Engine runtime configuration without handling secret values.""" + +from __future__ import annotations + +import json import os +from pathlib import Path + -DB_PD = os.environ["POLICYENGINE_DB_PASSWORD"] -POLICYENGINE_DB_INSTANCE_CONNECTION_NAME = os.environ[ - "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME" -] -GITHUB_MICRODATA_TOKEN = os.environ["POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN"] -ANTHROPIC_API_KEY = os.environ["ANTHROPIC_API_KEY"] -OPENAI_API_KEY = os.environ["OPENAI_API_KEY"] -HUGGING_FACE_TOKEN = os.environ["HUGGING_FACE_TOKEN"] -SIM_ENTRYPOINT = os.environ["SIM_ENTRYPOINT"] SIMULATION_URL_ENV_BY_ENTRYPOINT = { "old_gateway_direct": "OLD_SIMULATION_GATEWAY_URL", "cloud_run_simulation_entrypoint": "SIMULATION_ENTRYPOINT_URL", } -try: - selected_url_env = SIMULATION_URL_ENV_BY_ENTRYPOINT[SIM_ENTRYPOINT] -except KeyError as error: - raise ValueError( - "SIM_ENTRYPOINT must be old_gateway_direct or cloud_run_simulation_entrypoint" - ) from error -selected_url = os.environ.get(selected_url_env, "") -if not selected_url: - raise ValueError( - f"{selected_url_env} is required when SIM_ENTRYPOINT={SIM_ENTRYPOINT}" - ) -SIMULATION_ENTRYPOINT_URL = os.environ.get("SIMULATION_ENTRYPOINT_URL", "") -OLD_SIMULATION_GATEWAY_URL = os.environ.get("OLD_SIMULATION_GATEWAY_URL", "") -GATEWAY_AUTH_ISSUER = os.environ["GATEWAY_AUTH_ISSUER"] -GATEWAY_AUTH_AUDIENCE = os.environ["GATEWAY_AUTH_AUDIENCE"] -GATEWAY_AUTH_CLIENT_ID = os.environ["GATEWAY_AUTH_CLIENT_ID"] -GATEWAY_AUTH_CLIENT_SECRET_RESOURCE = os.environ["GATEWAY_AUTH_CLIENT_SECRET_RESOURCE"] +def _required(name: str) -> str: + value = os.environ.get(name, "").strip() + if not value: + raise ValueError(f"{name} is required") + return value -# Export DB_PD to .dbpw in the current directory -with open(".dbpw", "w") as f: - f.write(DB_PD) +def _render_app_config() -> str: + sim_entrypoint = _required("SIM_ENTRYPOINT") + try: + selected_url_env = SIMULATION_URL_ENV_BY_ENTRYPOINT[sim_entrypoint] + except KeyError as error: + raise ValueError( + "SIM_ENTRYPOINT must be old_gateway_direct or " + "cloud_run_simulation_entrypoint" + ) from error -app_config_location = "gcp/policyengine_api/app.yaml" -with open(app_config_location) as f: - app_config = f.read().replace( - ".policyengine_db_instance_connection_name", - POLICYENGINE_DB_INSTANCE_CONNECTION_NAME, - ) -with open(app_config_location, "w") as f: - f.write(app_config) - -# in gcp/compute_api/Dockerfile, replace .github_microdata_token with the contents of the file -for dockerfile_location in [ - "gcp/policyengine_api/Dockerfile", -]: - with open(dockerfile_location, "r") as f: - dockerfile = f.read() - dockerfile = dockerfile.replace( - ".github_microdata_token", GITHUB_MICRODATA_TOKEN - ) - dockerfile = dockerfile.replace(".anthropic_api_key", ANTHROPIC_API_KEY) - dockerfile = dockerfile.replace(".openai_api_key", OPENAI_API_KEY) - dockerfile = dockerfile.replace(".hugging_face_token", HUGGING_FACE_TOKEN) - dockerfile = dockerfile.replace( - ".simulation_entrypoint_url", SIMULATION_ENTRYPOINT_URL - ) - dockerfile = dockerfile.replace( - ".old_simulation_gateway_url", OLD_SIMULATION_GATEWAY_URL - ) - dockerfile = dockerfile.replace(".sim_entrypoint", SIM_ENTRYPOINT) - dockerfile = dockerfile.replace(".gateway_auth_issuer", GATEWAY_AUTH_ISSUER) - dockerfile = dockerfile.replace(".gateway_auth_audience", GATEWAY_AUTH_AUDIENCE) - dockerfile = dockerfile.replace( - ".gateway_auth_client_id", GATEWAY_AUTH_CLIENT_ID - ) - dockerfile = dockerfile.replace( - ".gateway_auth_client_secret_resource", - GATEWAY_AUTH_CLIENT_SECRET_RESOURCE, + selected_url = os.environ.get(selected_url_env, "").strip() + if not selected_url: + raise ValueError( + f"{selected_url_env} is required when SIM_ENTRYPOINT={sim_entrypoint}" ) - with open(dockerfile_location, "w") as f: - f.write(dockerfile) + replacements = { + ".policyengine_db_instance_connection_name": _required( + "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME" + ), + ".policyengine_db_password_secret_resource": _required( + "POLICYENGINE_DB_PASSWORD_SECRET_RESOURCE" + ), + ".policyengine_github_microdata_auth_token_secret_resource": _required( + "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN_SECRET_RESOURCE" + ), + ".anthropic_api_key_secret_resource": _required( + "ANTHROPIC_API_KEY_SECRET_RESOURCE" + ), + ".openai_api_key_secret_resource": _required("OPENAI_API_KEY_SECRET_RESOURCE"), + ".hugging_face_token_secret_resource": _required( + "HUGGING_FACE_TOKEN_SECRET_RESOURCE" + ), + ".simulation_entrypoint_url": os.environ.get( + "SIMULATION_ENTRYPOINT_URL", "" + ).strip(), + ".old_simulation_gateway_url": os.environ.get( + "OLD_SIMULATION_GATEWAY_URL", "" + ).strip(), + ".sim_entrypoint": sim_entrypoint, + ".gateway_auth_issuer": _required("GATEWAY_AUTH_ISSUER"), + ".gateway_auth_audience": _required("GATEWAY_AUTH_AUDIENCE"), + ".gateway_auth_client_id": _required("GATEWAY_AUTH_CLIENT_ID"), + ".gateway_auth_client_secret_resource": _required( + "GATEWAY_AUTH_CLIENT_SECRET_RESOURCE" + ), + ".runtime_cache_environment": _required("RUNTIME_CACHE_ENVIRONMENT"), + ".runtime_cache_url_secret_resource": _required( + "RUNTIME_CACHE_URL_SECRET_RESOURCE" + ), + ".runtime_cache_ca_cert_secret_resource": _required( + "RUNTIME_CACHE_CA_CERT_SECRET_RESOURCE" + ), + } + + template = Path("gcp/policyengine_api/app.yaml").read_text(encoding="utf-8") + for placeholder, value in replacements.items(): + quoted_placeholder = json.dumps(placeholder) + if quoted_placeholder not in template: + raise ValueError(f"App Engine template is missing {placeholder}") + template = template.replace(quoted_placeholder, json.dumps(value)) + return template + + +Path("app.yaml").write_text(_render_app_config(), encoding="utf-8") +Path("Dockerfile").write_text( + Path("gcp/policyengine_api/Dockerfile").read_text(encoding="utf-8"), + encoding="utf-8", +) diff --git a/gcp/policyengine_api/Dockerfile b/gcp/policyengine_api/Dockerfile index 87697c4d0..d74ce5541 100644 --- a/gcp/policyengine_api/Dockerfile +++ b/gcp/policyengine_api/Dockerfile @@ -1,21 +1,6 @@ FROM python:3.11 -RUN apt-get update && apt-get install -y build-essential redis-server && rm -rf /var/lib/apt/lists/* - -ENV POLICYENGINE_DB_PASSWORD .dbpw -ENV POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN .github_microdata_token -ENV ANTHROPIC_API_KEY .anthropic_api_key -ENV OPENAI_API_KEY .openai_api_key -ENV HUGGING_FACE_TOKEN .hugging_face_token -ENV SIMULATION_ENTRYPOINT_URL=".simulation_entrypoint_url" -ENV OLD_SIMULATION_GATEWAY_URL=".old_simulation_gateway_url" -ENV SIM_ENTRYPOINT=".sim_entrypoint" -ENV CREDENTIALS_JSON_API_V2 .credentials_json_api_v2 -ENV GATEWAY_AUTH_REQUIRED 1 -ENV GATEWAY_AUTH_ISSUER .gateway_auth_issuer -ENV GATEWAY_AUTH_AUDIENCE .gateway_auth_audience -ENV GATEWAY_AUTH_CLIENT_ID .gateway_auth_client_id -ENV GATEWAY_AUTH_CLIENT_SECRET_RESOURCE .gateway_auth_client_secret_resource +RUN apt-get update && apt-get install -y build-essential && rm -rf /var/lib/apt/lists/* WORKDIR /app diff --git a/gcp/policyengine_api/app.yaml b/gcp/policyengine_api/app.yaml index 2db2d191a..f84e4c2d2 100644 --- a/gcp/policyengine_api/app.yaml +++ b/gcp/policyengine_api/app.yaml @@ -20,8 +20,30 @@ liveness_check: runtime_config: operating_system: "ubuntu22" runtime_version: "22" +network: + name: default env_variables: POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: ".policyengine_db_instance_connection_name" + POLICYENGINE_DB_PASSWORD_SECRET_RESOURCE: ".policyengine_db_password_secret_resource" + POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN_SECRET_RESOURCE: ".policyengine_github_microdata_auth_token_secret_resource" + ANTHROPIC_API_KEY_SECRET_RESOURCE: ".anthropic_api_key_secret_resource" + OPENAI_API_KEY_SECRET_RESOURCE: ".openai_api_key_secret_resource" + HUGGING_FACE_TOKEN_SECRET_RESOURCE: ".hugging_face_token_secret_resource" + SIMULATION_ENTRYPOINT_URL: ".simulation_entrypoint_url" + OLD_SIMULATION_GATEWAY_URL: ".old_simulation_gateway_url" + SIM_ENTRYPOINT: ".sim_entrypoint" + GATEWAY_AUTH_REQUIRED: "1" + GATEWAY_AUTH_ISSUER: ".gateway_auth_issuer" + GATEWAY_AUTH_AUDIENCE: ".gateway_auth_audience" + GATEWAY_AUTH_CLIENT_ID: ".gateway_auth_client_id" + GATEWAY_AUTH_CLIENT_SECRET_RESOURCE: ".gateway_auth_client_secret_resource" + RUNTIME_CACHE_MODE: "deployed" + RUNTIME_CACHE_ENVIRONMENT: ".runtime_cache_environment" + RUNTIME_CACHE_SERVICE: "api" + RUNTIME_CACHE_URL_SECRET_RESOURCE: ".runtime_cache_url_secret_resource" + RUNTIME_CACHE_CA_CERT_SECRET_RESOURCE: ".runtime_cache_ca_cert_secret_resource" + V2_SUPABASE_PROJECT_REF: "kvrifaviwhzjztcbrfpy" + V2_SUPABASE_ENVIRONMENT: "production-foundation" readiness_check: path: "/readiness-check" check_interval_sec: 30 diff --git a/gcp/policyengine_api/start.sh b/gcp/policyengine_api/start.sh index 3fee8e4fe..98571cf6f 100644 --- a/gcp/policyengine_api/start.sh +++ b/gcp/policyengine_api/start.sh @@ -1,27 +1,6 @@ #!/bin/sh -# Environment variables -PORT="${PORT:-8080}" -CACHE_REDIS_HOST="${CACHE_REDIS_HOST:-127.0.0.1}" -CACHE_REDIS_PORT="${CACHE_REDIS_PORT:-6379}" -CACHE_REDIS_DB="${CACHE_REDIS_DB:-0}" -export CACHE_REDIS_HOST CACHE_REDIS_PORT CACHE_REDIS_DB - -# Start Redis with configuration for multiple clients. -redis-server --bind "$CACHE_REDIS_HOST" \ - --port "$CACHE_REDIS_PORT" \ - --protected-mode yes \ - --maxclients 10000 \ - --timeout 0 & - -# Wait for Redis to be ready -until redis-cli -h "$CACHE_REDIS_HOST" -p "$CACHE_REDIS_PORT" ping >/dev/null 2>&1; do - sleep 1 -done +set -eu -# Start the API -gunicorn -b :"$PORT" policyengine_api.api --timeout 900 --workers 5 & - -# Keep the script running and handle shutdown gracefully -trap "pkill -P $$; exit 1" INT TERM +PORT="${PORT:-8080}" -wait +exec python3 -m policyengine_api.app_engine_runtime diff --git a/migrations/v2/env.py b/migrations/v2/env.py new file mode 100644 index 000000000..d376699ba --- /dev/null +++ b/migrations/v2/env.py @@ -0,0 +1,89 @@ +"""Alembic environment for the isolated API v2-alpha Postgres schema.""" + +from logging.config import fileConfig + +from alembic import context +from alembic.script import ScriptDirectory +from sqlalchemy import create_engine, pool + +from policyengine_api.data.v2.migration_target import ( + load_v2_alembic_settings, + qualify_v2_connection, + validate_v2_head_table_inventory, +) +from policyengine_api.data.v2.models import V2_METADATA +from policyengine_api.data.v2.reference_data_autogenerate import ( + order_generated_operations, +) +from policyengine_api.data.v2.table_inventory import validate_v2_table_inventory + + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +settings = load_v2_alembic_settings() +target_metadata = V2_METADATA +validate_v2_table_inventory(target_metadata.tables) +script = ScriptDirectory.from_config(config) + + +def _include_application_object(_object, name, type_, _reflected, _compare_to) -> bool: + """Keep Alembic's own version table outside application drift.""" + + return not (type_ == "table" and name == "alembic_version") + + +def _configure(connection) -> None: + qualify_v2_connection(connection, settings) + context.configure( + connection=connection, + target_metadata=target_metadata, + compare_type=True, + compare_server_default=True, + include_schemas=False, + include_object=_include_application_object, + version_table="alembic_version", + version_table_schema="public", + process_revision_directives=order_generated_operations, + ) + migration_context = context.get_context() + previous_heads = frozenset(migration_context.get_current_heads()) + with context.begin_transaction(): + context.run_migrations() + current_heads = frozenset(migration_context.get_current_heads()) + script_heads = frozenset(script.get_heads()) + if current_heads != previous_heads and current_heads == script_heads: + validate_v2_head_table_inventory(connection) + + +def run_migrations_offline() -> None: + raise RuntimeError( + "v2 migrations require an online connection so target identity and " + "generated application-data before/after states can be verified" + ) + + +def run_migrations_online() -> None: + provided_connection = config.attributes.get("connection") + if provided_connection is not None: + _configure(provided_connection) + return + + engine = create_engine(settings.url, poolclass=pool.NullPool) + try: + # Persistent target qualification performs live reads before Alembic + # enters its migration transaction. SQLAlchemy 2.x autobegins on + # those reads, so the environment must own a committing transaction; + # otherwise a standalone command closes the connection and silently + # rolls the completed migration back. + with engine.begin() as connection: + _configure(connection) + finally: + engine.dispose() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/v2/script.py.mako b/migrations/v2/script.py.mako new file mode 100644 index 000000000..a27f8940d --- /dev/null +++ b/migrations/v2/script.py.mako @@ -0,0 +1,27 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel +${imports if imports else ""} + +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/migrations/v2/versions/.gitkeep b/migrations/v2/versions/.gitkeep new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/migrations/v2/versions/.gitkeep @@ -0,0 +1 @@ + diff --git a/migrations/v2/versions/47592781336f_establish_v2_core_schema_baseline.py b/migrations/v2/versions/47592781336f_establish_v2_core_schema_baseline.py new file mode 100644 index 000000000..01deb6e81 --- /dev/null +++ b/migrations/v2/versions/47592781336f_establish_v2_core_schema_baseline.py @@ -0,0 +1,1722 @@ +"""establish v2 core schema baseline + +Revision ID: 47592781336f +Revises: +Create Date: 2026-08-14 14:22:39.384555 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = "47592781336f" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "dynamics", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("name", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False), + sa.Column("description", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.PrimaryKeyConstraint("id", name=op.f("pk_dynamics")), + ) + op.create_table( + "households", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "country", sqlmodel.sql.sqltypes.AutoString(length=16), nullable=False + ), + sa.Column("year", sa.Integer(), nullable=False), + sa.Column("label", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=True), + sa.Column("household_data", sa.JSON(), nullable=False), + sa.CheckConstraint( + "year BETWEEN 1900 AND 2200", name=op.f("ck_households_year") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_households")), + ) + op.create_index( + op.f("ix_households_country"), "households", ["country"], unique=False + ) + op.create_table( + "tax_benefit_models", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("name", sqlmodel.sql.sqltypes.AutoString(length=32), nullable=False), + sa.Column("description", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.PrimaryKeyConstraint("id", name=op.f("pk_tax_benefit_models")), + sa.UniqueConstraint("name", name="uq_tax_benefit_models_name"), + ) + op.create_table( + "users", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "first_name", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False + ), + sa.Column( + "last_name", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False + ), + sa.Column( + "email", sqlmodel.sql.sqltypes.AutoString(length=320), nullable=False + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_users")), + sa.UniqueConstraint("email", name="uq_users_email"), + ) + op.create_index(op.f("ix_users_email"), "users", ["email"], unique=False) + op.create_table( + "datasets", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("name", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False), + sa.Column("description", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column( + "storage_path", + sqlmodel.sql.sqltypes.AutoString(length=1024), + nullable=False, + ), + sa.Column("year", sa.Integer(), nullable=False), + sa.Column("is_output_dataset", sa.Boolean(), nullable=False), + sa.Column("tax_benefit_model_id", sa.Uuid(), nullable=False), + sa.CheckConstraint("year BETWEEN 1900 AND 2200", name=op.f("ck_datasets_year")), + sa.ForeignKeyConstraint( + ["tax_benefit_model_id"], + ["tax_benefit_models.id"], + name=op.f("fk_datasets_tax_benefit_model_id_tax_benefit_models"), + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_datasets")), + sa.UniqueConstraint( + "tax_benefit_model_id", + "name", + "year", + "is_output_dataset", + name="uq_datasets_model_name_year_output", + ), + ) + op.create_index( + op.f("ix_datasets_tax_benefit_model_id"), + "datasets", + ["tax_benefit_model_id"], + unique=False, + ) + op.create_table( + "policies", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("name", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False), + sa.Column("description", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("tax_benefit_model_id", sa.Uuid(), nullable=False), + sa.ForeignKeyConstraint( + ["tax_benefit_model_id"], + ["tax_benefit_models.id"], + name=op.f("fk_policies_tax_benefit_model_id_tax_benefit_models"), + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_policies")), + ) + op.create_index( + op.f("ix_policies_tax_benefit_model_id"), + "policies", + ["tax_benefit_model_id"], + unique=False, + ) + op.create_table( + "regions", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("code", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False), + sa.Column( + "label", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False + ), + sa.Column( + "region_type", + sa.Enum( + "national", + "country", + "state", + "congressional_district", + "constituency", + "local_authority", + "city", + "place", + name="v2_region_type", + ), + nullable=False, + ), + sa.Column("requires_filter", sa.Boolean(), nullable=False), + sa.Column( + "filter_field", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=True + ), + sa.Column( + "filter_value", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=True + ), + sa.Column( + "filter_strategy", + sqlmodel.sql.sqltypes.AutoString(length=64), + nullable=True, + ), + sa.Column( + "parent_code", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=True + ), + sa.Column( + "state_code", sqlmodel.sql.sqltypes.AutoString(length=16), nullable=True + ), + sa.Column( + "state_name", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=True + ), + sa.Column("tax_benefit_model_id", sa.Uuid(), nullable=False), + sa.CheckConstraint( + "NOT requires_filter OR (filter_field IS NOT NULL AND filter_value IS NOT NULL)", + name=op.f("ck_regions_required_filter_values"), + ), + sa.ForeignKeyConstraint( + ["tax_benefit_model_id"], + ["tax_benefit_models.id"], + name=op.f("fk_regions_tax_benefit_model_id_tax_benefit_models"), + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_regions")), + sa.UniqueConstraint( + "tax_benefit_model_id", "code", name="uq_regions_model_code" + ), + ) + op.create_index( + op.f("ix_regions_tax_benefit_model_id"), + "regions", + ["tax_benefit_model_id"], + unique=False, + ) + op.create_table( + "tax_benefit_model_versions", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("model_id", sa.Uuid(), nullable=False), + sa.Column( + "version", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False + ), + sa.Column("description", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.ForeignKeyConstraint( + ["model_id"], + ["tax_benefit_models.id"], + name=op.f("fk_tax_benefit_model_versions_model_id_tax_benefit_models"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_tax_benefit_model_versions")), + sa.UniqueConstraint( + "model_id", "version", name="uq_tax_benefit_model_versions_model_version" + ), + ) + op.create_index( + op.f("ix_tax_benefit_model_versions_model_id"), + "tax_benefit_model_versions", + ["model_id"], + unique=False, + ) + op.create_table( + "user_household_associations", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("user_id", sa.Uuid(), nullable=False), + sa.Column("household_id", sa.Uuid(), nullable=False), + sa.Column( + "country", sqlmodel.sql.sqltypes.AutoString(length=16), nullable=False + ), + sa.Column("label", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=True), + sa.ForeignKeyConstraint( + ["household_id"], + ["households.id"], + name=op.f("fk_user_household_associations_household_id_households"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_user_household_associations")), + sa.UniqueConstraint( + "user_id", + "household_id", + name="uq_user_household_associations_user_household", + ), + ) + op.create_index( + op.f("ix_user_household_associations_household_id"), + "user_household_associations", + ["household_id"], + unique=False, + ) + op.create_index( + op.f("ix_user_household_associations_user_id"), + "user_household_associations", + ["user_id"], + unique=False, + ) + op.create_table( + "dataset_versions", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("name", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False), + sa.Column("description", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("dataset_id", sa.Uuid(), nullable=False), + sa.Column("tax_benefit_model_id", sa.Uuid(), nullable=False), + sa.ForeignKeyConstraint( + ["dataset_id"], + ["datasets.id"], + name=op.f("fk_dataset_versions_dataset_id_datasets"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["tax_benefit_model_id"], + ["tax_benefit_models.id"], + name=op.f("fk_dataset_versions_tax_benefit_model_id_tax_benefit_models"), + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_dataset_versions")), + sa.UniqueConstraint( + "dataset_id", "name", name="uq_dataset_versions_dataset_name" + ), + ) + op.create_index( + op.f("ix_dataset_versions_dataset_id"), + "dataset_versions", + ["dataset_id"], + unique=False, + ) + op.create_index( + op.f("ix_dataset_versions_tax_benefit_model_id"), + "dataset_versions", + ["tax_benefit_model_id"], + unique=False, + ) + op.create_table( + "household_jobs", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "country", sqlmodel.sql.sqltypes.AutoString(length=16), nullable=False + ), + sa.Column("request_data", sa.JSON(), nullable=False), + sa.Column("policy_id", sa.Uuid(), nullable=True), + sa.Column("dynamic_id", sa.Uuid(), nullable=True), + sa.Column( + "status", + sa.Enum( + "pending", + "running", + "succeeded", + "failed", + name="v2_household_job_status", + ), + nullable=False, + ), + sa.Column("error_message", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("result", sa.JSON(), nullable=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["dynamic_id"], + ["dynamics.id"], + name=op.f("fk_household_jobs_dynamic_id_dynamics"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["policy_id"], + ["policies.id"], + name=op.f("fk_household_jobs_policy_id_policies"), + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_household_jobs")), + ) + op.create_index( + "ix_household_jobs_status_created_at", + "household_jobs", + ["status", "created_at"], + unique=False, + ) + op.create_table( + "parameter_nodes", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("name", sqlmodel.sql.sqltypes.AutoString(length=512), nullable=False), + sa.Column("label", sqlmodel.sql.sqltypes.AutoString(length=512), nullable=True), + sa.Column("description", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False), + sa.ForeignKeyConstraint( + ["tax_benefit_model_version_id"], + ["tax_benefit_model_versions.id"], + name=op.f( + "fk_parameter_nodes_tax_benefit_model_version_id_tax_benefit_model_versions" + ), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_parameter_nodes")), + sa.UniqueConstraint( + "tax_benefit_model_version_id", + "name", + name="uq_parameter_nodes_model_version_name", + ), + ) + op.create_index( + op.f("ix_parameter_nodes_tax_benefit_model_version_id"), + "parameter_nodes", + ["tax_benefit_model_version_id"], + unique=False, + ) + op.create_table( + "parameters", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("name", sqlmodel.sql.sqltypes.AutoString(length=512), nullable=False), + sa.Column("label", sqlmodel.sql.sqltypes.AutoString(length=512), nullable=True), + sa.Column("description", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column( + "data_type", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=True + ), + sa.Column("unit", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=True), + sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False), + sa.ForeignKeyConstraint( + ["tax_benefit_model_version_id"], + ["tax_benefit_model_versions.id"], + name=op.f( + "fk_parameters_tax_benefit_model_version_id_tax_benefit_model_versions" + ), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_parameters")), + sa.UniqueConstraint( + "tax_benefit_model_version_id", + "name", + name="uq_parameters_model_version_name", + ), + ) + op.create_index( + op.f("ix_parameters_tax_benefit_model_version_id"), + "parameters", + ["tax_benefit_model_version_id"], + unique=False, + ) + op.create_table( + "region_datasets", + sa.Column("region_id", sa.Uuid(), nullable=False), + sa.Column("dataset_id", sa.Uuid(), nullable=False), + sa.ForeignKeyConstraint( + ["dataset_id"], + ["datasets.id"], + name=op.f("fk_region_datasets_dataset_id_datasets"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["region_id"], + ["regions.id"], + name=op.f("fk_region_datasets_region_id_regions"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint( + "region_id", "dataset_id", name=op.f("pk_region_datasets") + ), + ) + op.create_table( + "simulations", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "simulation_type", + sa.Enum("household", "economy", name="v2_simulation_type"), + nullable=False, + ), + sa.Column("dataset_id", sa.Uuid(), nullable=True), + sa.Column("household_id", sa.Uuid(), nullable=True), + sa.Column("policy_id", sa.Uuid(), nullable=True), + sa.Column("dynamic_id", sa.Uuid(), nullable=True), + sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False), + sa.Column("output_dataset_id", sa.Uuid(), nullable=True), + sa.Column("region_id", sa.Uuid(), nullable=True), + sa.Column( + "status", + sa.Enum( + "pending", "running", "succeeded", "failed", name="v2_simulation_status" + ), + nullable=False, + ), + sa.Column("error_message", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column( + "filter_field", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=True + ), + sa.Column( + "filter_value", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=True + ), + sa.Column( + "filter_strategy", + sqlmodel.sql.sqltypes.AutoString(length=64), + nullable=True, + ), + sa.Column("year", sa.Integer(), nullable=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("household_result", sa.JSON(), nullable=True), + sa.CheckConstraint( + "(simulation_type = 'household' AND household_id IS NOT NULL AND dataset_id IS NULL) OR (simulation_type = 'economy' AND dataset_id IS NOT NULL AND household_id IS NULL)", + name=op.f("ck_simulations_type_input"), + ), + sa.CheckConstraint( + "(filter_field IS NULL) = (filter_value IS NULL)", + name=op.f("ck_simulations_filter_pair"), + ), + sa.CheckConstraint( + "year IS NULL OR year BETWEEN 1900 AND 2200", + name=op.f("ck_simulations_year"), + ), + sa.ForeignKeyConstraint( + ["dataset_id"], + ["datasets.id"], + name=op.f("fk_simulations_dataset_id_datasets"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["dynamic_id"], + ["dynamics.id"], + name=op.f("fk_simulations_dynamic_id_dynamics"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["household_id"], + ["households.id"], + name=op.f("fk_simulations_household_id_households"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["output_dataset_id"], + ["datasets.id"], + name=op.f("fk_simulations_output_dataset_id_datasets"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["policy_id"], + ["policies.id"], + name=op.f("fk_simulations_policy_id_policies"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["region_id"], + ["regions.id"], + name=op.f("fk_simulations_region_id_regions"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["tax_benefit_model_version_id"], + ["tax_benefit_model_versions.id"], + name=op.f( + "fk_simulations_tax_benefit_model_version_id_tax_benefit_model_versions" + ), + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_simulations")), + ) + op.create_index( + "ix_simulations_status_created_at", + "simulations", + ["status", "created_at"], + unique=False, + ) + op.create_index( + op.f("ix_simulations_tax_benefit_model_version_id"), + "simulations", + ["tax_benefit_model_version_id"], + unique=False, + ) + op.create_table( + "user_policies", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("user_id", sa.Uuid(), nullable=False), + sa.Column("policy_id", sa.Uuid(), nullable=False), + sa.Column( + "country", sqlmodel.sql.sqltypes.AutoString(length=16), nullable=False + ), + sa.Column("label", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=True), + sa.ForeignKeyConstraint( + ["policy_id"], + ["policies.id"], + name=op.f("fk_user_policies_policy_id_policies"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_user_policies")), + sa.UniqueConstraint( + "user_id", "policy_id", name="uq_user_policies_user_policy" + ), + ) + op.create_index( + op.f("ix_user_policies_policy_id"), "user_policies", ["policy_id"], unique=False + ) + op.create_index( + op.f("ix_user_policies_user_id"), "user_policies", ["user_id"], unique=False + ) + op.create_table( + "variables", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("name", sqlmodel.sql.sqltypes.AutoString(length=512), nullable=False), + sa.Column("label", sqlmodel.sql.sqltypes.AutoString(length=512), nullable=True), + sa.Column( + "entity", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False + ), + sa.Column("description", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column( + "data_type", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=True + ), + sa.Column("possible_values", sa.JSON(), nullable=True), + sa.Column("default_value", sa.JSON(), nullable=False), + sa.Column("adds", sa.JSON(), nullable=True), + sa.Column("subtracts", sa.JSON(), nullable=True), + sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False), + sa.ForeignKeyConstraint( + ["tax_benefit_model_version_id"], + ["tax_benefit_model_versions.id"], + name=op.f( + "fk_variables_tax_benefit_model_version_id_tax_benefit_model_versions" + ), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_variables")), + sa.UniqueConstraint( + "tax_benefit_model_version_id", + "name", + name="uq_variables_model_version_name", + ), + ) + op.create_index( + op.f("ix_variables_tax_benefit_model_version_id"), + "variables", + ["tax_benefit_model_version_id"], + unique=False, + ) + op.create_table( + "parameter_values", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("parameter_id", sa.Uuid(), nullable=False), + sa.Column("value_json", sa.JSON(), nullable=False), + sa.Column("start_date", sa.DateTime(timezone=True), nullable=False), + sa.Column("end_date", sa.DateTime(timezone=True), nullable=True), + sa.Column("policy_id", sa.Uuid(), nullable=True), + sa.Column("dynamic_id", sa.Uuid(), nullable=True), + sa.CheckConstraint( + "policy_id IS NULL OR dynamic_id IS NULL", + name=op.f("ck_parameter_values_single_owner"), + ), + sa.ForeignKeyConstraint( + ["dynamic_id"], + ["dynamics.id"], + name=op.f("fk_parameter_values_dynamic_id_dynamics"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["parameter_id"], + ["parameters.id"], + name=op.f("fk_parameter_values_parameter_id_parameters"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["policy_id"], + ["policies.id"], + name=op.f("fk_parameter_values_policy_id_policies"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_parameter_values")), + ) + op.create_index( + "ix_parameter_values_parameter_period", + "parameter_values", + ["parameter_id", "start_date", "end_date"], + unique=False, + ) + op.create_table( + "reports", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "label", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False + ), + sa.Column("description", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column( + "country", sqlmodel.sql.sqltypes.AutoString(length=16), nullable=False + ), + sa.Column("type", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=True), + sa.Column("user_id", sa.Uuid(), nullable=True), + sa.Column("tax_benefit_model_id", sa.Uuid(), nullable=False), + sa.Column("policy_id", sa.Uuid(), nullable=True), + sa.Column("baseline_simulation_id", sa.Uuid(), nullable=True), + sa.Column("reform_simulation_id", sa.Uuid(), nullable=True), + sa.Column("household_id", sa.Uuid(), nullable=True), + sa.Column("dataset_id", sa.Uuid(), nullable=True), + sa.Column("region_id", sa.Uuid(), nullable=True), + sa.Column("year", sa.Integer(), nullable=True), + sa.Column("inputs", sa.JSON(), nullable=False), + sa.CheckConstraint( + "year IS NULL OR year BETWEEN 1900 AND 2200", name=op.f("ck_reports_year") + ), + sa.ForeignKeyConstraint( + ["baseline_simulation_id"], + ["simulations.id"], + name=op.f("fk_reports_baseline_simulation_id_simulations"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["dataset_id"], + ["datasets.id"], + name=op.f("fk_reports_dataset_id_datasets"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["household_id"], + ["households.id"], + name=op.f("fk_reports_household_id_households"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["policy_id"], + ["policies.id"], + name=op.f("fk_reports_policy_id_policies"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["reform_simulation_id"], + ["simulations.id"], + name=op.f("fk_reports_reform_simulation_id_simulations"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["region_id"], + ["regions.id"], + name=op.f("fk_reports_region_id_regions"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["tax_benefit_model_id"], + ["tax_benefit_models.id"], + name=op.f("fk_reports_tax_benefit_model_id_tax_benefit_models"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["users.id"], + name=op.f("fk_reports_user_id_users"), + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_reports")), + ) + op.create_index( + "ix_reports_country_type_created_at", + "reports", + ["country", "type", "created_at"], + unique=False, + ) + op.create_index( + op.f("ix_reports_tax_benefit_model_id"), + "reports", + ["tax_benefit_model_id"], + unique=False, + ) + op.create_table( + "user_simulation_associations", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("user_id", sa.Uuid(), nullable=False), + sa.Column("simulation_id", sa.Uuid(), nullable=False), + sa.Column( + "country", sqlmodel.sql.sqltypes.AutoString(length=16), nullable=False + ), + sa.Column("label", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=True), + sa.ForeignKeyConstraint( + ["simulation_id"], + ["simulations.id"], + name=op.f("fk_user_simulation_associations_simulation_id_simulations"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_user_simulation_associations")), + sa.UniqueConstraint( + "user_id", + "simulation_id", + name="uq_user_simulation_associations_user_simulation", + ), + ) + op.create_index( + op.f("ix_user_simulation_associations_simulation_id"), + "user_simulation_associations", + ["simulation_id"], + unique=False, + ) + op.create_index( + op.f("ix_user_simulation_associations_user_id"), + "user_simulation_associations", + ["user_id"], + unique=False, + ) + op.create_table( + "report_runs", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("report_id", sa.Uuid(), nullable=False), + sa.Column( + "country_package_version", + sqlmodel.sql.sqltypes.AutoString(length=128), + nullable=False, + ), + sa.Column( + "policyengine_version", + sqlmodel.sql.sqltypes.AutoString(length=128), + nullable=False, + ), + sa.Column( + "status", + sa.Enum( + "pending", "running", "succeeded", "failed", name="v2_report_run_status" + ), + nullable=False, + ), + sa.Column( + "trigger", + sa.Enum("initial", "manual", "system", name="v2_report_run_trigger"), + nullable=False, + ), + sa.Column( + "idempotency_key", + sqlmodel.sql.sqltypes.AutoString(length=255), + nullable=True, + ), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("error_message", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("markdown", sa.Text(), nullable=True), + sa.CheckConstraint( + "status NOT IN ('succeeded', 'failed') OR completed_at IS NOT NULL", + name=op.f("ck_report_runs_terminal_completion"), + ), + sa.ForeignKeyConstraint( + ["report_id"], + ["reports.id"], + name=op.f("fk_report_runs_report_id_reports"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_report_runs")), + sa.UniqueConstraint( + "report_id", "idempotency_key", name="uq_report_runs_report_idempotency_key" + ), + ) + op.create_index( + "ix_report_runs_current_output", + "report_runs", + [ + "report_id", + "status", + "country_package_version", + "policyengine_version", + "completed_at", + "id", + ], + unique=False, + ) + op.create_table( + "user_report_associations", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("user_id", sa.Uuid(), nullable=False), + sa.Column("report_id", sa.Uuid(), nullable=False), + sa.Column( + "country", sqlmodel.sql.sqltypes.AutoString(length=16), nullable=False + ), + sa.Column("label", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=True), + sa.Column("last_run_at", sa.DateTime(timezone=True), nullable=True), + sa.ForeignKeyConstraint( + ["report_id"], + ["reports.id"], + name=op.f("fk_user_report_associations_report_id_reports"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_user_report_associations")), + sa.UniqueConstraint( + "user_id", "report_id", name="uq_user_report_associations_user_report" + ), + ) + op.create_index( + op.f("ix_user_report_associations_report_id"), + "user_report_associations", + ["report_id"], + unique=False, + ) + op.create_index( + op.f("ix_user_report_associations_user_id"), + "user_report_associations", + ["user_id"], + unique=False, + ) + op.create_table( + "aggregates", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("report_run_id", sa.Uuid(), nullable=False), + sa.Column("simulation_id", sa.Uuid(), nullable=False), + sa.Column( + "variable", sqlmodel.sql.sqltypes.AutoString(length=512), nullable=False + ), + sa.Column( + "aggregate_type", + sa.Enum("sum", "mean", "count", name="v2_aggregate_type"), + nullable=False, + ), + sa.Column( + "entity", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=True + ), + sa.Column("filter_config", sa.JSON(), nullable=False), + sa.Column( + "status", + sa.Enum( + "pending", "running", "succeeded", "failed", name="v2_output_status" + ), + nullable=False, + ), + sa.Column("error_message", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("result", sa.Float(), nullable=True), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_aggregates_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["simulation_id"], + ["simulations.id"], + name=op.f("fk_aggregates_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_aggregates")), + ) + op.create_index( + "ix_aggregates_report_run_status", + "aggregates", + ["report_run_id", "status"], + unique=False, + ) + op.create_table( + "budget_summary", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("report_run_id", sa.Uuid(), nullable=False), + sa.Column("baseline_simulation_id", sa.Uuid(), nullable=False), + sa.Column("reform_simulation_id", sa.Uuid(), nullable=False), + sa.Column( + "variable_name", + sqlmodel.sql.sqltypes.AutoString(length=512), + nullable=False, + ), + sa.Column( + "entity", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False + ), + sa.Column("baseline_total", sa.Float(), nullable=True), + sa.Column("reform_total", sa.Float(), nullable=True), + sa.Column("change", sa.Float(), nullable=True), + sa.ForeignKeyConstraint( + ["baseline_simulation_id"], + ["simulations.id"], + name=op.f("fk_budget_summary_baseline_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["reform_simulation_id"], + ["simulations.id"], + name=op.f("fk_budget_summary_reform_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_budget_summary_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_budget_summary")), + sa.UniqueConstraint( + "report_run_id", + "variable_name", + "entity", + name="uq_budget_summary_run_variable_entity", + ), + ) + op.create_table( + "change_aggregates", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("report_run_id", sa.Uuid(), nullable=False), + sa.Column("baseline_simulation_id", sa.Uuid(), nullable=False), + sa.Column("reform_simulation_id", sa.Uuid(), nullable=False), + sa.Column( + "variable", sqlmodel.sql.sqltypes.AutoString(length=512), nullable=False + ), + sa.Column( + "aggregate_type", + sa.Enum("sum", "mean", "count", name="v2_aggregate_type"), + nullable=False, + ), + sa.Column( + "entity", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=True + ), + sa.Column("filter_config", sa.JSON(), nullable=False), + sa.Column("change_geq", sa.Float(), nullable=True), + sa.Column("change_leq", sa.Float(), nullable=True), + sa.Column( + "status", + sa.Enum( + "pending", "running", "succeeded", "failed", name="v2_output_status" + ), + nullable=False, + ), + sa.Column("error_message", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("result", sa.Float(), nullable=True), + sa.ForeignKeyConstraint( + ["baseline_simulation_id"], + ["simulations.id"], + name=op.f("fk_change_aggregates_baseline_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["reform_simulation_id"], + ["simulations.id"], + name=op.f("fk_change_aggregates_reform_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_change_aggregates_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_change_aggregates")), + ) + op.create_index( + "ix_change_aggregates_report_run_status", + "change_aggregates", + ["report_run_id", "status"], + unique=False, + ) + op.create_table( + "congressional_district_impacts", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("report_run_id", sa.Uuid(), nullable=False), + sa.Column("baseline_simulation_id", sa.Uuid(), nullable=False), + sa.Column("reform_simulation_id", sa.Uuid(), nullable=False), + sa.Column("average_household_income_change", sa.Float(), nullable=False), + sa.Column("relative_household_income_change", sa.Float(), nullable=False), + sa.Column("population", sa.Float(), nullable=False), + sa.Column("district_geoid", sa.Integer(), nullable=False), + sa.Column("state_fips", sa.Integer(), nullable=False), + sa.Column("district_number", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ["baseline_simulation_id"], + ["simulations.id"], + name=op.f( + "fk_congressional_district_impacts_baseline_simulation_id_simulations" + ), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["reform_simulation_id"], + ["simulations.id"], + name=op.f( + "fk_congressional_district_impacts_reform_simulation_id_simulations" + ), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_congressional_district_impacts_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_congressional_district_impacts")), + sa.UniqueConstraint( + "report_run_id", + "district_geoid", + name="uq_congressional_district_impacts_run_geoid", + ), + ) + op.create_table( + "constituency_impacts", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("report_run_id", sa.Uuid(), nullable=False), + sa.Column("baseline_simulation_id", sa.Uuid(), nullable=False), + sa.Column("reform_simulation_id", sa.Uuid(), nullable=False), + sa.Column("average_household_income_change", sa.Float(), nullable=False), + sa.Column("relative_household_income_change", sa.Float(), nullable=False), + sa.Column("population", sa.Float(), nullable=False), + sa.Column( + "constituency_code", + sqlmodel.sql.sqltypes.AutoString(length=64), + nullable=False, + ), + sa.Column( + "constituency_name", + sqlmodel.sql.sqltypes.AutoString(length=255), + nullable=False, + ), + sa.Column("x", sa.Integer(), nullable=False), + sa.Column("y", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ["baseline_simulation_id"], + ["simulations.id"], + name=op.f("fk_constituency_impacts_baseline_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["reform_simulation_id"], + ["simulations.id"], + name=op.f("fk_constituency_impacts_reform_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_constituency_impacts_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_constituency_impacts")), + sa.UniqueConstraint( + "report_run_id", + "constituency_code", + name="uq_constituency_impacts_run_code", + ), + ) + op.create_table( + "decile_impacts", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("report_run_id", sa.Uuid(), nullable=False), + sa.Column("baseline_simulation_id", sa.Uuid(), nullable=False), + sa.Column("reform_simulation_id", sa.Uuid(), nullable=False), + sa.Column( + "income_variable", + sqlmodel.sql.sqltypes.AutoString(length=512), + nullable=False, + ), + sa.Column( + "entity", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False + ), + sa.Column("decile", sa.Integer(), nullable=False), + sa.Column("quantiles", sa.Integer(), nullable=False), + sa.Column("baseline_mean", sa.Float(), nullable=True), + sa.Column("reform_mean", sa.Float(), nullable=True), + sa.Column("absolute_change", sa.Float(), nullable=True), + sa.Column("relative_change", sa.Float(), nullable=True), + sa.Column("count_better_off", sa.Float(), nullable=True), + sa.Column("count_worse_off", sa.Float(), nullable=True), + sa.Column("count_no_change", sa.Float(), nullable=True), + sa.CheckConstraint( + "decile BETWEEN 1 AND quantiles", name=op.f("ck_decile_impacts_decile") + ), + sa.CheckConstraint("quantiles > 0", name=op.f("ck_decile_impacts_quantiles")), + sa.ForeignKeyConstraint( + ["baseline_simulation_id"], + ["simulations.id"], + name=op.f("fk_decile_impacts_baseline_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["reform_simulation_id"], + ["simulations.id"], + name=op.f("fk_decile_impacts_reform_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_decile_impacts_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_decile_impacts")), + sa.UniqueConstraint( + "report_run_id", + "income_variable", + "entity", + "decile", + "quantiles", + name="uq_decile_impacts_run_measure", + ), + ) + op.create_table( + "inequality", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("report_run_id", sa.Uuid(), nullable=False), + sa.Column("simulation_id", sa.Uuid(), nullable=False), + sa.Column( + "income_variable", + sqlmodel.sql.sqltypes.AutoString(length=512), + nullable=False, + ), + sa.Column( + "entity", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False + ), + sa.Column("gini", sa.Float(), nullable=True), + sa.Column("top_10_share", sa.Float(), nullable=True), + sa.Column("top_1_share", sa.Float(), nullable=True), + sa.Column("bottom_50_share", sa.Float(), nullable=True), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_inequality_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["simulation_id"], + ["simulations.id"], + name=op.f("fk_inequality_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_inequality")), + sa.UniqueConstraint( + "report_run_id", + "simulation_id", + "income_variable", + "entity", + name="uq_inequality_run_simulation_measure", + ), + ) + op.create_table( + "intra_decile_impacts", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("report_run_id", sa.Uuid(), nullable=False), + sa.Column("baseline_simulation_id", sa.Uuid(), nullable=False), + sa.Column("reform_simulation_id", sa.Uuid(), nullable=False), + sa.Column( + "decile_type", + sa.Enum("income", "wealth", name="v2_decile_type"), + nullable=False, + ), + sa.Column("decile", sa.Integer(), nullable=False), + sa.Column("lose_more_than_5pct", sa.Float(), nullable=True), + sa.Column("lose_less_than_5pct", sa.Float(), nullable=True), + sa.Column("no_change", sa.Float(), nullable=True), + sa.Column("gain_less_than_5pct", sa.Float(), nullable=True), + sa.Column("gain_more_than_5pct", sa.Float(), nullable=True), + sa.CheckConstraint( + "decile BETWEEN 0 AND 10", name=op.f("ck_intra_decile_impacts_decile") + ), + sa.ForeignKeyConstraint( + ["baseline_simulation_id"], + ["simulations.id"], + name=op.f("fk_intra_decile_impacts_baseline_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["reform_simulation_id"], + ["simulations.id"], + name=op.f("fk_intra_decile_impacts_reform_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_intra_decile_impacts_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_intra_decile_impacts")), + sa.UniqueConstraint( + "report_run_id", + "decile_type", + "decile", + name="uq_intra_decile_impacts_run_type_decile", + ), + ) + op.create_table( + "local_authority_impacts", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("report_run_id", sa.Uuid(), nullable=False), + sa.Column("baseline_simulation_id", sa.Uuid(), nullable=False), + sa.Column("reform_simulation_id", sa.Uuid(), nullable=False), + sa.Column("average_household_income_change", sa.Float(), nullable=False), + sa.Column("relative_household_income_change", sa.Float(), nullable=False), + sa.Column("population", sa.Float(), nullable=False), + sa.Column( + "local_authority_code", + sqlmodel.sql.sqltypes.AutoString(length=64), + nullable=False, + ), + sa.Column( + "local_authority_name", + sqlmodel.sql.sqltypes.AutoString(length=255), + nullable=False, + ), + sa.Column("x", sa.Integer(), nullable=False), + sa.Column("y", sa.Integer(), nullable=False), + sa.ForeignKeyConstraint( + ["baseline_simulation_id"], + ["simulations.id"], + name=op.f("fk_local_authority_impacts_baseline_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["reform_simulation_id"], + ["simulations.id"], + name=op.f("fk_local_authority_impacts_reform_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_local_authority_impacts_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_local_authority_impacts")), + sa.UniqueConstraint( + "report_run_id", + "local_authority_code", + name="uq_local_authority_impacts_run_code", + ), + ) + op.create_table( + "poverty", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("report_run_id", sa.Uuid(), nullable=False), + sa.Column("simulation_id", sa.Uuid(), nullable=False), + sa.Column( + "poverty_type", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False + ), + sa.Column( + "entity", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False + ), + sa.Column( + "filter_variable", + sqlmodel.sql.sqltypes.AutoString(length=512), + nullable=True, + ), + sa.Column("headcount", sa.Float(), nullable=True), + sa.Column("total_population", sa.Float(), nullable=True), + sa.Column("rate", sa.Float(), nullable=True), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_poverty_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["simulation_id"], + ["simulations.id"], + name=op.f("fk_poverty_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_poverty")), + ) + op.create_index( + "ix_poverty_run_simulation_type", + "poverty", + ["report_run_id", "simulation_id", "poverty_type"], + unique=False, + ) + op.create_table( + "program_statistics", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("report_run_id", sa.Uuid(), nullable=False), + sa.Column("baseline_simulation_id", sa.Uuid(), nullable=False), + sa.Column("reform_simulation_id", sa.Uuid(), nullable=False), + sa.Column( + "program_name", sqlmodel.sql.sqltypes.AutoString(length=512), nullable=False + ), + sa.Column( + "entity", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=False + ), + sa.Column("is_tax", sa.Boolean(), nullable=False), + sa.Column("baseline_total", sa.Float(), nullable=True), + sa.Column("reform_total", sa.Float(), nullable=True), + sa.Column("change", sa.Float(), nullable=True), + sa.Column("baseline_count", sa.Float(), nullable=True), + sa.Column("reform_count", sa.Float(), nullable=True), + sa.Column("winners", sa.Float(), nullable=True), + sa.Column("losers", sa.Float(), nullable=True), + sa.ForeignKeyConstraint( + ["baseline_simulation_id"], + ["simulations.id"], + name=op.f("fk_program_statistics_baseline_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["reform_simulation_id"], + ["simulations.id"], + name=op.f("fk_program_statistics_reform_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_program_statistics_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_program_statistics")), + sa.UniqueConstraint( + "report_run_id", + "program_name", + "entity", + name="uq_program_statistics_run_program_entity", + ), + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table("program_statistics") + op.drop_index("ix_poverty_run_simulation_type", table_name="poverty") + op.drop_table("poverty") + op.drop_table("local_authority_impacts") + op.drop_table("intra_decile_impacts") + op.drop_table("inequality") + op.drop_table("decile_impacts") + op.drop_table("constituency_impacts") + op.drop_table("congressional_district_impacts") + op.drop_index( + "ix_change_aggregates_report_run_status", table_name="change_aggregates" + ) + op.drop_table("change_aggregates") + op.drop_table("budget_summary") + op.drop_index("ix_aggregates_report_run_status", table_name="aggregates") + op.drop_table("aggregates") + op.drop_index( + op.f("ix_user_report_associations_user_id"), + table_name="user_report_associations", + ) + op.drop_index( + op.f("ix_user_report_associations_report_id"), + table_name="user_report_associations", + ) + op.drop_table("user_report_associations") + op.drop_index("ix_report_runs_current_output", table_name="report_runs") + op.drop_table("report_runs") + op.drop_index( + op.f("ix_user_simulation_associations_user_id"), + table_name="user_simulation_associations", + ) + op.drop_index( + op.f("ix_user_simulation_associations_simulation_id"), + table_name="user_simulation_associations", + ) + op.drop_table("user_simulation_associations") + op.drop_index(op.f("ix_reports_tax_benefit_model_id"), table_name="reports") + op.drop_index("ix_reports_country_type_created_at", table_name="reports") + op.drop_table("reports") + op.drop_index("ix_parameter_values_parameter_period", table_name="parameter_values") + op.drop_table("parameter_values") + op.drop_index( + op.f("ix_variables_tax_benefit_model_version_id"), table_name="variables" + ) + op.drop_table("variables") + op.drop_index(op.f("ix_user_policies_user_id"), table_name="user_policies") + op.drop_index(op.f("ix_user_policies_policy_id"), table_name="user_policies") + op.drop_table("user_policies") + op.drop_index( + op.f("ix_simulations_tax_benefit_model_version_id"), table_name="simulations" + ) + op.drop_index("ix_simulations_status_created_at", table_name="simulations") + op.drop_table("simulations") + op.drop_table("region_datasets") + op.drop_index( + op.f("ix_parameters_tax_benefit_model_version_id"), table_name="parameters" + ) + op.drop_table("parameters") + op.drop_index( + op.f("ix_parameter_nodes_tax_benefit_model_version_id"), + table_name="parameter_nodes", + ) + op.drop_table("parameter_nodes") + op.drop_index("ix_household_jobs_status_created_at", table_name="household_jobs") + op.drop_table("household_jobs") + op.drop_index( + op.f("ix_dataset_versions_tax_benefit_model_id"), table_name="dataset_versions" + ) + op.drop_index(op.f("ix_dataset_versions_dataset_id"), table_name="dataset_versions") + op.drop_table("dataset_versions") + op.drop_index( + op.f("ix_user_household_associations_user_id"), + table_name="user_household_associations", + ) + op.drop_index( + op.f("ix_user_household_associations_household_id"), + table_name="user_household_associations", + ) + op.drop_table("user_household_associations") + op.drop_index( + op.f("ix_tax_benefit_model_versions_model_id"), + table_name="tax_benefit_model_versions", + ) + op.drop_table("tax_benefit_model_versions") + op.drop_index(op.f("ix_regions_tax_benefit_model_id"), table_name="regions") + op.drop_table("regions") + op.drop_index(op.f("ix_policies_tax_benefit_model_id"), table_name="policies") + op.drop_table("policies") + op.drop_index(op.f("ix_datasets_tax_benefit_model_id"), table_name="datasets") + op.drop_table("datasets") + op.drop_index(op.f("ix_users_email"), table_name="users") + op.drop_table("users") + op.drop_table("tax_benefit_models") + op.drop_index(op.f("ix_households_country"), table_name="households") + op.drop_table("households") + op.drop_table("dynamics") + # PostgreSQL-native enum types outlive their final table. Alembic generated + # their creation but not their removal, so the reviewed reversible-dialect + # correction drops only those generated v2 enum types. + sa.Enum(name="v2_aggregate_type").drop(op.get_bind()) + sa.Enum(name="v2_decile_type").drop(op.get_bind()) + sa.Enum(name="v2_household_job_status").drop(op.get_bind()) + sa.Enum(name="v2_output_status").drop(op.get_bind()) + sa.Enum(name="v2_region_type").drop(op.get_bind()) + sa.Enum(name="v2_report_run_status").drop(op.get_bind()) + sa.Enum(name="v2_report_run_trigger").drop(op.get_bind()) + sa.Enum(name="v2_simulation_status").drop(op.get_bind()) + sa.Enum(name="v2_simulation_type").drop(op.get_bind()) + # ### end Alembic commands ### diff --git a/migrations/v2/versions/5f048586d8f1_constrain_v2_user_country_and_report_.py b/migrations/v2/versions/5f048586d8f1_constrain_v2_user_country_and_report_.py new file mode 100644 index 000000000..0f69adb3c --- /dev/null +++ b/migrations/v2/versions/5f048586d8f1_constrain_v2_user_country_and_report_.py @@ -0,0 +1,50 @@ +"""constrain v2 user country and report idempotency + +Revision ID: 5f048586d8f1 +Revises: b4c69674dd47 +Create Date: 2026-08-18 12:54:00.963782 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = "5f048586d8f1" +down_revision: Union[str, None] = "b4c69674dd47" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_check_constraint( + op.f("ck_report_runs_idempotency_key_nonblank"), + "report_runs", + "idempotency_key IS NULL OR length(trim(idempotency_key)) > 0", + ) + op.add_column( + "users", + sa.Column( + "primary_country", + sqlmodel.sql.sqltypes.AutoString(length=2), + nullable=False, + ), + ) + op.create_check_constraint( + op.f("ck_users_primary_country"), "users", "primary_country IN ('us', 'uk')" + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint(op.f("ck_users_primary_country"), "users", type_="check") + op.drop_column("users", "primary_country") + op.drop_constraint( + op.f("ck_report_runs_idempotency_key_nonblank"), "report_runs", type_="check" + ) + # ### end Alembic commands ### diff --git a/migrations/v2/versions/6ee725e0c563_add_stage_8_platform_validation_data.py b/migrations/v2/versions/6ee725e0c563_add_stage_8_platform_validation_data.py new file mode 100644 index 000000000..3206aa1f7 --- /dev/null +++ b/migrations/v2/versions/6ee725e0c563_add_stage_8_platform_validation_data.py @@ -0,0 +1,83 @@ +"""add stage 8 platform validation data + +Revision ID: 6ee725e0c563 +Revises: 47592781336f +Create Date: 2026-08-14 14:29:59.265364 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = "6ee725e0c563" +down_revision: Union[str, None] = "47592781336f" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.v2_reference_row_change( + "tax_benefit_models", + key={"name": "stage8-platform-validation"}, + before=None, + after={ + "created_at": "2026-08-14T00:00:00+00:00", + "description": "Stage 8 migration lifecycle validation; canonical metadata is introduced in Stage 9.", + "id": "80000000-0000-4000-8000-000000000001", + "name": "stage8-platform-validation", + "updated_at": "2026-08-14T00:00:00+00:00", + }, + ) + op.v2_reference_row_change( + "tax_benefit_model_versions", + key={ + "model_id": "80000000-0000-4000-8000-000000000001", + "version": "stage8-platform-validation", + }, + before=None, + after={ + "created_at": "2026-08-14T00:00:00+00:00", + "description": "Stage 8 generated data-migration validation row.", + "id": "80000000-0000-4000-8000-000000000002", + "model_id": "80000000-0000-4000-8000-000000000001", + "version": "stage8-platform-validation", + }, + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.v2_reference_row_change( + "tax_benefit_model_versions", + key={ + "model_id": "80000000-0000-4000-8000-000000000001", + "version": "stage8-platform-validation", + }, + before={ + "created_at": "2026-08-14T00:00:00+00:00", + "description": "Stage 8 generated data-migration validation row.", + "id": "80000000-0000-4000-8000-000000000002", + "model_id": "80000000-0000-4000-8000-000000000001", + "version": "stage8-platform-validation", + }, + after=None, + ) + op.v2_reference_row_change( + "tax_benefit_models", + key={"name": "stage8-platform-validation"}, + before={ + "created_at": "2026-08-14T00:00:00+00:00", + "description": "Stage 8 migration lifecycle validation; canonical metadata is introduced in Stage 9.", + "id": "80000000-0000-4000-8000-000000000001", + "name": "stage8-platform-validation", + "updated_at": "2026-08-14T00:00:00+00:00", + }, + after=None, + ) + # ### end Alembic commands ### diff --git a/migrations/v2/versions/b4c69674dd47_enforce_v2_user_association_ownership.py b/migrations/v2/versions/b4c69674dd47_enforce_v2_user_association_ownership.py new file mode 100644 index 000000000..a23925e7a --- /dev/null +++ b/migrations/v2/versions/b4c69674dd47_enforce_v2_user_association_ownership.py @@ -0,0 +1,79 @@ +"""enforce v2 user association ownership + +Revision ID: b4c69674dd47 +Revises: 6ee725e0c563 +Create Date: 2026-08-18 12:25:45.926354 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = "b4c69674dd47" +down_revision: Union[str, None] = "6ee725e0c563" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_foreign_key( + op.f("fk_user_household_associations_user_id_users"), + "user_household_associations", + "users", + ["user_id"], + ["id"], + ondelete="CASCADE", + ) + op.create_foreign_key( + op.f("fk_user_policies_user_id_users"), + "user_policies", + "users", + ["user_id"], + ["id"], + ondelete="CASCADE", + ) + op.create_foreign_key( + op.f("fk_user_report_associations_user_id_users"), + "user_report_associations", + "users", + ["user_id"], + ["id"], + ondelete="CASCADE", + ) + op.create_foreign_key( + op.f("fk_user_simulation_associations_user_id_users"), + "user_simulation_associations", + "users", + ["user_id"], + ["id"], + ondelete="CASCADE", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_constraint( + op.f("fk_user_simulation_associations_user_id_users"), + "user_simulation_associations", + type_="foreignkey", + ) + op.drop_constraint( + op.f("fk_user_report_associations_user_id_users"), + "user_report_associations", + type_="foreignkey", + ) + op.drop_constraint( + op.f("fk_user_policies_user_id_users"), "user_policies", type_="foreignkey" + ) + op.drop_constraint( + op.f("fk_user_household_associations_user_id_users"), + "user_household_associations", + type_="foreignkey", + ) + # ### end Alembic commands ### diff --git a/policyengine_api/api.py b/policyengine_api/api.py index b0730e0ae..bffaf7eac 100644 --- a/policyengine_api/api.py +++ b/policyengine_api/api.py @@ -5,7 +5,6 @@ import time import sys -import os start_time = time.time() @@ -24,6 +23,7 @@ def log_timing(message): from policyengine_api.extensions import cache from policyengine_api.migration_logging import register_migration_request_logging +from policyengine_api.runtime_cache.settings import load_runtime_cache_settings log_timing("Caching utilities import completed") @@ -72,16 +72,41 @@ def log_timing(message): app = application = flask.Flask(__name__) log_timing("Flask app created") -app.config.from_mapping( - { - "CACHE_TYPE": "RedisCache", - "CACHE_KEY_PREFIX": "policyengine", - "CACHE_REDIS_HOST": os.environ.get("CACHE_REDIS_HOST", "127.0.0.1"), - "CACHE_REDIS_PORT": int(os.environ.get("CACHE_REDIS_PORT", "6379")), - "CACHE_REDIS_DB": int(os.environ.get("CACHE_REDIS_DB", "0")), - "CACHE_DEFAULT_TIMEOUT": 300, - } -) +runtime_cache_settings = load_runtime_cache_settings() +if runtime_cache_settings.enabled: + app.config.from_mapping( + { + "CACHE_TYPE": "RedisCache", + "CACHE_KEY_PREFIX": ( + "policyengine:" + f"{runtime_cache_settings.environment}:" + f"{runtime_cache_settings.service}:flask:v1:" + ), + "CACHE_REDIS_URL": ( + runtime_cache_settings.url.get_secret_value() + if runtime_cache_settings.url is not None + else None + ), + "CACHE_DEFAULT_TIMEOUT": 300, + "CACHE_OPTIONS": ( + { + "ssl_cert_reqs": "required", + "ssl_ca_data": runtime_cache_settings.ca_cert.get_secret_value(), + } + if runtime_cache_settings.tls + and runtime_cache_settings.ca_cert is not None + else {} + ), + } + ) +else: + app.config.from_mapping( + { + "CACHE_TYPE": "NullCache", + "CACHE_KEY_PREFIX": "policyengine:test:api:flask:v1:", + "CACHE_DEFAULT_TIMEOUT": 300, + } + ) cache.init_app(app) log_timing("Caching initialised") diff --git a/policyengine_api/app_engine_runtime.py b/policyengine_api/app_engine_runtime.py new file mode 100644 index 000000000..27328691c --- /dev/null +++ b/policyengine_api/app_engine_runtime.py @@ -0,0 +1,94 @@ +"""Resolve App Engine application secrets before starting Gunicorn.""" + +from __future__ import annotations + +from collections.abc import Callable, MutableMapping +from functools import lru_cache +import os +import re + + +SECRET_RESOURCE_PATTERN = re.compile(r"^projects/[^/]+/secrets/[^/]+/versions/[^/]+$") +SECRET_ENV_SOURCES = ( + ("POLICYENGINE_DB_PASSWORD", "POLICYENGINE_DB_PASSWORD_SECRET_RESOURCE"), + ( + "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN", + "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN_SECRET_RESOURCE", + ), + ("ANTHROPIC_API_KEY", "ANTHROPIC_API_KEY_SECRET_RESOURCE"), + ("OPENAI_API_KEY", "OPENAI_API_KEY_SECRET_RESOURCE"), + ("HUGGING_FACE_TOKEN", "HUGGING_FACE_TOKEN_SECRET_RESOURCE"), +) + + +class AppEngineRuntimeConfigurationError(RuntimeError): + """Raised without exposing a secret value.""" + + +@lru_cache(maxsize=None) +def _load_secret_from_secret_manager(resource_name: str) -> str: + from google.cloud import secretmanager + + client = secretmanager.SecretManagerServiceClient() + response = client.access_secret_version(request={"name": resource_name}) + return response.payload.data.decode("utf-8") + + +def hydrate_app_engine_runtime_secrets( + environ: MutableMapping[str, str] | None = None, + *, + secret_loader: Callable[[str], str] | None = None, +) -> None: + """Resolve exactly one direct or Secret Manager source for every secret.""" + + values = os.environ if environ is None else environ + loader = secret_loader or _load_secret_from_secret_manager + for value_name, resource_name in SECRET_ENV_SOURCES: + direct_value = values.get(value_name, "") + resource = values.get(resource_name, "").strip() + if direct_value and resource: + raise AppEngineRuntimeConfigurationError( + f"set exactly one of {value_name} or {resource_name}" + ) + if direct_value: + if not direct_value.strip(): + raise AppEngineRuntimeConfigurationError(f"{value_name} is empty") + continue + if not resource: + raise AppEngineRuntimeConfigurationError( + f"{value_name} or {resource_name} is required" + ) + if SECRET_RESOURCE_PATTERN.fullmatch(resource) is None: + raise AppEngineRuntimeConfigurationError(f"{resource_name} is invalid") + try: + resolved_value = loader(resource) + except Exception as error: + raise AppEngineRuntimeConfigurationError( + f"{resource_name} could not be resolved" + ) from error + if not resolved_value or not resolved_value.strip(): + raise AppEngineRuntimeConfigurationError(f"{resource_name} is empty") + values[value_name] = resolved_value + values.pop(resource_name, None) + + +def main() -> None: + hydrate_app_engine_runtime_secrets() + port = os.environ.get("PORT", "8080") + os.execvp( + "gunicorn", + [ + "gunicorn", + "-b", + f":{port}", + "policyengine_api.api", + "--timeout", + "900", + "--workers", + "5", + ], + ) + + +if __name__ == "__main__": + main() diff --git a/policyengine_api/asgi.py b/policyengine_api/asgi.py index 8843ad65d..d49ec1405 100644 --- a/policyengine_api/asgi.py +++ b/policyengine_api/asgi.py @@ -8,11 +8,18 @@ from policyengine_api.asgi_factory import create_asgi_app from policyengine_api.data.orm import close_v1_engines from policyengine_api.readiness import mark_not_ready, mark_ready +from policyengine_api.runtime_cache.client import close_runtime_cache_clients from policyengine_api.warmup import run_startup_warmup + +def _close_runtime_resources() -> None: + close_v1_engines() + close_runtime_cache_clients() + + app = application = create_asgi_app( flask_app, - shutdown_callback=close_v1_engines, + shutdown_callback=_close_runtime_resources, ) # Warm the simulation machinery before serving (see policyengine_api.warmup). diff --git a/policyengine_api/data/local_database.py b/policyengine_api/data/local_database.py deleted file mode 100644 index 2d29c0210..000000000 --- a/policyengine_api/data/local_database.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Temporary SQLAlchemy schema bootstrap for the local SQLite cache. - -The production ``policy`` table has an autoincrementing column inside a -composite primary key. SQLite only autoincrements an ``INTEGER PRIMARY KEY`` -column when it is the sole primary-key column, so the local cache has always -used ``policy.id`` as its database primary key. Keep that one physical-schema -exception explicit while deriving every other local table from the production -ORM metadata. -""" - -from __future__ import annotations - -from sqlalchemy import Column, Engine, Integer, JSON, MetaData, String, Table - -from policyengine_api.data.local_models import LocalV1Base -from policyengine_api.data.v1_models import Policy, V1Base - - -_sqlite_policy_metadata = MetaData() -Table( - Policy.__tablename__, - _sqlite_policy_metadata, - Column("id", Integer, primary_key=True, autoincrement=True), - Column("country_id", String(3), nullable=False), - Column("label", String(255)), - Column("api_version", String(10), nullable=False), - Column("policy_json", JSON, nullable=False), - Column("policy_hash", String(255), nullable=False), -) - - -def create_local_v1_schema(engine: Engine) -> None: - """Create the temporary local schema through SQLAlchemy DDL constructs.""" - - if engine.dialect.name != "sqlite": - raise ValueError("The local v1 schema is only defined for SQLite") - - production_tables = [ - table - for table in V1Base.metadata.sorted_tables - if table is not Policy.__table__ - ] - V1Base.metadata.create_all(engine, tables=production_tables) - LocalV1Base.metadata.create_all(engine) - _sqlite_policy_metadata.create_all(engine) diff --git a/policyengine_api/data/local_models.py b/policyengine_api/data/local_models.py deleted file mode 100644 index 35ca26074..000000000 --- a/policyengine_api/data/local_models.py +++ /dev/null @@ -1,26 +0,0 @@ -"""Declarative mappings for the temporary local SQLite cache only. - -These tables are deliberately outside the production API v1 metadata and -Alembic lifecycle. -""" - -from __future__ import annotations - -from typing import Any - -from sqlalchemy import Integer, JSON, String -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column - - -class LocalV1Base(DeclarativeBase): - pass - - -class Tracer(LocalV1Base): - __tablename__ = "tracers" - id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True) - household_id: Mapped[int] - policy_id: Mapped[int] - country_id: Mapped[str] = mapped_column(String(3)) - api_version: Mapped[str] = mapped_column(String(10)) - tracer_output: Mapped[Any] = mapped_column(JSON) diff --git a/policyengine_api/data/orm.py b/policyengine_api/data/orm.py index 88f4c9426..6c0501e29 100644 --- a/policyengine_api/data/orm.py +++ b/policyengine_api/data/orm.py @@ -3,21 +3,13 @@ from __future__ import annotations import atexit -import fcntl import os -from pathlib import Path from dotenv import load_dotenv from google.cloud.sql.connector import Connector, IPTypes -from sqlalchemy import Engine, create_engine, select +from sqlalchemy import Engine, create_engine from sqlalchemy.orm import Session, sessionmaker -from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, REPO -from policyengine_api.data.local_database import create_local_v1_schema -from policyengine_api.data.v1_models import Policy -from policyengine_api.utils import hash_object - - load_dotenv() DEFAULT_REMOTE_DB_USER = "policyengine" @@ -27,11 +19,9 @@ DATABASE_POOL_SIZE = 5 DATABASE_POOL_MAX_OVERFLOW = 2 DATABASE_POOL_TIMEOUT_SECONDS = 30 -LOCAL_DATABASE_PATH = REPO / "policyengine_api" / "data" / "policyengine.db" - -_v1_engines: dict[bool, Engine] = {} -_v1_session_factories: dict[bool, sessionmaker[Session]] = {} -_cloud_sql_connectors: dict[bool, Connector] = {} +_v1_engine: Engine | None = None +_v1_session_factory: sessionmaker[Session] | None = None +_cloud_sql_connector: Connector | None = None def get_remote_database_config() -> dict[str, str]: @@ -50,72 +40,14 @@ def build_session_factory(engine: Engine) -> sessionmaker[Session]: return sessionmaker(bind=engine, class_=Session, expire_on_commit=False) -def _initialize_local_database(engine: Engine) -> None: - create_local_v1_schema(engine) - - current_law_policies = [ - Policy( - id=policy_id, - country_id=country_id, - label="Current law", - api_version=COUNTRY_PACKAGE_VERSIONS[country_id], - policy_json={}, - policy_hash=hash_object({}), - ) - for policy_id, country_id in enumerate(COUNTRY_PACKAGE_VERSIONS, start=1) - ] - policy_ids = [policy.id for policy in current_law_policies] - with build_session_factory(engine).begin() as session: - existing_policy_ids = set( - session.scalars(select(Policy.id).where(Policy.id.in_(policy_ids))) - ) - session.add_all( - policy - for policy in current_law_policies - if policy.id not in existing_policy_ids - ) - - -# TODO: Remove this local-database initialization pattern and replace the local -# persistence path with a traditional cache. Application imports should -# eventually neither create a database file nor bootstrap a schema. -def _ensure_local_database( - engine: Engine, - database_path: Path = LOCAL_DATABASE_PATH, -) -> None: - lock_path = Path(f"{database_path}.init.lock") - with lock_path.open("w") as lock_file: - fcntl.flock(lock_file, fcntl.LOCK_EX) - try: - _initialize_local_database(engine) - finally: - fcntl.flock(lock_file, fcntl.LOCK_UN) - - -def _build_local_engine() -> Engine: - engine = create_engine(f"sqlite+pysqlite:///{LOCAL_DATABASE_PATH}") - try: - _ensure_local_database(engine) - except Exception: - engine.dispose() - raise - return engine - - -def _database_password() -> str: - password = os.environ["POLICYENGINE_DB_PASSWORD"] - if password == ".dbpw": - return Path(".dbpw").read_text(encoding="utf-8").strip() - return password - - def _build_remote_engine() -> Engine: + global _cloud_sql_connector config = get_remote_database_config() connector = Connector( ip_type=CLOUD_SQL_IP_TYPE, refresh_strategy="LAZY", ) - password = _database_password() + password = os.environ["POLICYENGINE_DB_PASSWORD"] def get_connection(): return connector.connect( @@ -135,45 +67,46 @@ def get_connection(): max_overflow=DATABASE_POOL_MAX_OVERFLOW, pool_timeout=DATABASE_POOL_TIMEOUT_SECONDS, ) - _cloud_sql_connectors[False] = connector + _cloud_sql_connector = connector return engine -def get_v1_engine(*, local: bool = False) -> Engine: - """Return one process-owned SQLAlchemy Engine for the selected runtime.""" +def get_v1_engine() -> Engine: + """Return the process-owned Cloud SQL engine; never select SQLite.""" - use_local = local or os.environ.get("FLASK_DEBUG") == "1" - if use_local not in _v1_engines: - _v1_engines[use_local] = ( - _build_local_engine() if use_local else _build_remote_engine() - ) - return _v1_engines[use_local] + global _v1_engine + if _v1_engine is None: + _v1_engine = _build_remote_engine() + return _v1_engine -def get_v1_session_factory(*, local: bool = False) -> sessionmaker[Session]: - """Return one configured Session factory per process-owned v1 Engine.""" +def get_v1_session_factory() -> sessionmaker[Session]: + """Return one configured Session factory for the Cloud SQL engine.""" - if local not in _v1_session_factories: - _v1_session_factories[local] = build_session_factory(get_v1_engine(local=local)) - return _v1_session_factories[local] + global _v1_session_factory + if _v1_session_factory is None: + _v1_session_factory = build_session_factory(get_v1_engine()) + return _v1_session_factory def clear_v1_session_factories() -> None: """Forget cached factories, primarily after replacing an Engine in tests.""" - _v1_session_factories.clear() + global _v1_session_factory + _v1_session_factory = None def close_v1_engines() -> None: """Release process-owned SQLAlchemy pools and Cloud SQL connectors.""" + global _cloud_sql_connector, _v1_engine clear_v1_session_factories() - for engine in _v1_engines.values(): - engine.dispose() - _v1_engines.clear() - for connector in _cloud_sql_connectors.values(): - connector.close() - _cloud_sql_connectors.clear() + if _v1_engine is not None: + _v1_engine.dispose() + _v1_engine = None + if _cloud_sql_connector is not None: + _cloud_sql_connector.close() + _cloud_sql_connector = None atexit.register(close_v1_engines) diff --git a/policyengine_api/data/v2/__init__.py b/policyengine_api/data/v2/__init__.py new file mode 100644 index 000000000..e786374c9 --- /dev/null +++ b/policyengine_api/data/v2/__init__.py @@ -0,0 +1 @@ +"""Dormant API v2-alpha persistence boundary.""" diff --git a/policyengine_api/data/v2/database.py b/policyengine_api/data/v2/database.py new file mode 100644 index 000000000..d9922ac06 --- /dev/null +++ b/policyengine_api/data/v2/database.py @@ -0,0 +1,135 @@ +"""Lazy, process-owned SQLModel engine and Session configuration for API v2.""" + +from __future__ import annotations + +import atexit +import os +from threading import Lock + +from sqlalchemy import Engine +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session, create_engine + +from policyengine_api.data.v2.settings import ( + V2ConfigurationError, + V2DatabaseSettings, + load_v2_runtime_database_settings, +) + + +DATABASE_POOL_RECYCLE_SECONDS = 1800 +DATABASE_POOL_SIZE = 5 +DATABASE_POOL_MAX_OVERFLOW = 5 +DATABASE_POOL_TIMEOUT_SECONDS = 30 + +_state_lock = Lock() +_engine: Engine | None = None +_engine_pid: int | None = None +_engine_url_fingerprint: tuple[object, ...] | None = None +_session_factory: sessionmaker[Session] | None = None + + +def _url_fingerprint(settings: V2DatabaseSettings) -> tuple[object, ...]: + """Build a private comparison value that is never rendered or logged.""" + + url = settings.connection.url + return ( + url.drivername, + url.username, + url.password, + url.host, + url.port, + url.database, + tuple(sorted(url.query.items())), + settings.target.project_ref, + settings.target.environment, + ) + + +def build_v2_engine(settings: V2DatabaseSettings) -> Engine: + """Construct an unconnected SQLModel engine for an explicit v2 target.""" + + return create_engine( + settings.connection.url, + pool_pre_ping=True, + pool_recycle=DATABASE_POOL_RECYCLE_SECONDS, + pool_size=DATABASE_POOL_SIZE, + max_overflow=DATABASE_POOL_MAX_OVERFLOW, + pool_timeout=DATABASE_POOL_TIMEOUT_SECONDS, + ) + + +def build_v2_session_factory(engine: Engine) -> sessionmaker[Session]: + """Build a factory whose ordinary sessions are SQLModel Sessions.""" + + return sessionmaker(bind=engine, class_=Session, expire_on_commit=False) + + +def _discard_inherited_state(current_pid: int) -> None: + """Drop pool state inherited through a process fork without touching parent FDs.""" + + global _engine, _engine_pid, _engine_url_fingerprint, _session_factory + + if _engine is not None: + _engine.dispose(close=False) + _engine = None + _engine_pid = current_pid + _engine_url_fingerprint = None + _session_factory = None + + +def get_v2_engine(settings: V2DatabaseSettings | None = None) -> Engine: + """Return the one lazy v2 runtime engine owned by this process.""" + + global _engine, _engine_pid, _engine_url_fingerprint + + current_pid = os.getpid() + with _state_lock: + if _engine_pid is not None and _engine_pid != current_pid: + _discard_inherited_state(current_pid) + + selected_settings = settings or load_v2_runtime_database_settings() + fingerprint = _url_fingerprint(selected_settings) + if _engine is not None: + if fingerprint != _engine_url_fingerprint: + raise V2ConfigurationError( + "the process-owned v2 engine is already bound to a different " + "explicit target" + ) + return _engine + + _engine = build_v2_engine(selected_settings) + _engine_pid = current_pid + _engine_url_fingerprint = fingerprint + return _engine + + +def get_v2_session_factory( + settings: V2DatabaseSettings | None = None, +) -> sessionmaker[Session]: + """Return the process-owned SQLModel Session factory without opening a session.""" + + global _session_factory + + engine = get_v2_engine(settings) + with _state_lock: + if _session_factory is None: + _session_factory = build_v2_session_factory(engine) + return _session_factory + + +def close_v2_database() -> None: + """Dispose this process's v2 pool and forget its lazy configuration.""" + + global _engine, _engine_pid, _engine_url_fingerprint, _session_factory + + with _state_lock: + if _engine is not None: + _engine.dispose() + _engine = None + _engine_pid = None + _engine_url_fingerprint = None + _session_factory = None + + +atexit.register(close_v2_database) diff --git a/policyengine_api/data/v2/migration_target.py b/policyengine_api/data/v2/migration_target.py new file mode 100644 index 000000000..4a527e727 --- /dev/null +++ b/policyengine_api/data/v2/migration_target.py @@ -0,0 +1,208 @@ +"""Fail-closed target selection and qualification for v2 Alembic.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import date +import os + +from sqlalchemy import Connection, inspect, text +from sqlalchemy.engine import URL, make_url +from sqlalchemy.exc import ArgumentError + +from policyengine_api.data.v2.settings import ( + POSTGRES_DRIVER, + V2_MIGRATION_DATABASE_URL, + V2_SUPABASE_ENVIRONMENT, + V2_SUPABASE_PROJECT_REF, + V2ConfigurationError, + parse_persistent_postgres_url, +) +from policyengine_api.data.v2.table_inventory import EXPECTED_V2_TABLES + + +V2_ALEMBIC_DISPOSABLE_TEST = "V2_ALEMBIC_DISPOSABLE_TEST" +DISPOSABLE_DATABASE_NAME = "policyengine_v2_alembic_test" +DISPOSABLE_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "postgres"}) +MIGRATION_ROLE = "policyengine_v2_migrator" + + +class V2MigrationTargetError(V2ConfigurationError): + """Raised before Alembic can modify an unqualified v2 target.""" + + +@dataclass(frozen=True) +class RecordedSupabaseTarget: + environment: str + project_ref: str + database_name: str + migration_role: str + freshness_audited_on: date + freshness_audit_passed: bool + + +RECORDED_SUPABASE_TARGETS = { + "production-foundation": RecordedSupabaseTarget( + environment="production-foundation", + project_ref="kvrifaviwhzjztcbrfpy", + database_name="postgres", + migration_role=MIGRATION_ROLE, + freshness_audited_on=date(2026, 8, 13), + freshness_audit_passed=True, + ) +} + + +@dataclass(frozen=True) +class V2AlembicSettings: + url: URL + disposable_test: bool + target: RecordedSupabaseTarget | None + + +def _required(environ: Mapping[str, str], name: str) -> str: + value = environ.get(name) + if value is None or not value.strip(): + raise V2MigrationTargetError(f"{name} is required") + return value.strip() + + +def _parse_url(raw_url: str) -> URL: + try: + url = make_url(raw_url) + except ArgumentError as error: + raise V2MigrationTargetError( + f"{V2_MIGRATION_DATABASE_URL} is not a valid URL" + ) from error + if url.drivername != POSTGRES_DRIVER: + raise V2MigrationTargetError( + f"{V2_MIGRATION_DATABASE_URL} must use the {POSTGRES_DRIVER} driver" + ) + return url + + +def _validate_disposable_url(url: URL) -> None: + if url.host not in DISPOSABLE_HOSTS or url.database != DISPOSABLE_DATABASE_NAME: + raise V2MigrationTargetError( + "disposable v2 Alembic mode requires the isolated local " + f"{DISPOSABLE_DATABASE_NAME} database" + ) + if not url.username or url.password is None: + raise V2MigrationTargetError( + "disposable v2 Alembic mode requires explicit test credentials" + ) + + +def _validate_persistent_url_identity( + url: URL, + target: RecordedSupabaseTarget, +) -> None: + direct_host = f"db.{target.project_ref}.supabase.co" + is_direct = url.host == direct_host + is_pooler = bool( + url.host + and url.host.endswith(".pooler.supabase.com") + and url.username + and url.username.endswith(f".{target.project_ref}") + ) + if not (is_direct or is_pooler): + raise V2MigrationTargetError( + "the v2 migration URL does not identify the recorded Supabase project" + ) + if url.database != target.database_name: + raise V2MigrationTargetError( + "the v2 migration URL does not identify the recorded database" + ) + + +def load_v2_alembic_settings( + environ: Mapping[str, str] | None = None, +) -> V2AlembicSettings: + """Select either the recorded persistent target or isolated test Postgres.""" + + values = os.environ if environ is None else environ + raw_url = _required(values, V2_MIGRATION_DATABASE_URL) + url = _parse_url(raw_url) + disposable_value = values.get(V2_ALEMBIC_DISPOSABLE_TEST) + if disposable_value not in {None, "", "0", "1"}: + raise V2MigrationTargetError( + f"{V2_ALEMBIC_DISPOSABLE_TEST} must be 1 when explicitly enabled" + ) + if disposable_value == "1": + _validate_disposable_url(url) + return V2AlembicSettings(url=url, disposable_test=True, target=None) + + persistent = parse_persistent_postgres_url( + raw_url, + setting_name=V2_MIGRATION_DATABASE_URL, + ) + environment = _required(values, V2_SUPABASE_ENVIRONMENT) + project_ref = _required(values, V2_SUPABASE_PROJECT_REF) + target = RECORDED_SUPABASE_TARGETS.get(environment) + if target is None or target.project_ref != project_ref: + raise V2MigrationTargetError( + "the requested environment and project reference do not match a " + "recorded v2 migration target" + ) + _validate_persistent_url_identity(persistent.url, target) + return V2AlembicSettings( + url=persistent.url, + disposable_test=False, + target=target, + ) + + +def qualify_v2_connection( + connection: Connection, + settings: V2AlembicSettings, +) -> None: + """Verify live target identity and first-use state before Alembic runs.""" + + if connection.dialect.name != "postgresql": + raise V2MigrationTargetError("v2 Alembic requires a Postgres connection") + if settings.disposable_test: + return + + target = settings.target + if target is None: + raise V2MigrationTargetError("persistent v2 Alembic target is missing") + identity = connection.execute(text("SELECT current_database(), current_user")).one() + if identity[0] != target.database_name or identity[1] != target.migration_role: + raise V2MigrationTargetError( + "the live database or migration identity does not match the recorded " + "v2 target" + ) + can_create = connection.execute( + text("SELECT has_schema_privilege(current_user, 'public', 'CREATE')") + ).scalar_one() + if can_create is not True: + raise V2MigrationTargetError( + "the recorded v2 migration identity lacks public schema CREATE" + ) + + public_tables = set(inspect(connection).get_table_names(schema="public")) + if "alembic_version" not in public_tables: + if not target.freshness_audit_passed: + raise V2MigrationTargetError( + "the recorded first-use freshness audit has not passed" + ) + if public_tables: + raise V2MigrationTargetError( + "the unstamped v2 target is not fresh; reset and adoption are " + "prohibited" + ) + + +def validate_v2_head_table_inventory(connection: Connection) -> None: + """Require the exact reviewed table inventory after upgrading to v2 head.""" + + public_tables = set(inspect(connection).get_table_names(schema="public")) + application_tables = public_tables - {"alembic_version"} + unexpected = application_tables - EXPECTED_V2_TABLES + missing = EXPECTED_V2_TABLES - application_tables + if unexpected or missing: + raise V2MigrationTargetError( + "the v2 head table inventory differs from the reviewed " + f"schema: missing={sorted(missing)}, unexpected={sorted(unexpected)}" + ) diff --git a/policyengine_api/data/v2/models/__init__.py b/policyengine_api/data/v2/models/__init__.py new file mode 100644 index 000000000..82cc4344f --- /dev/null +++ b/policyengine_api/data/v2/models/__init__.py @@ -0,0 +1,152 @@ +"""Controlled imports and metadata export for the reviewed v2 schema.""" + +from sqlmodel import SQLModel + +from policyengine_api.data.v2.table_inventory import validate_v2_table_inventory + + +# Apply deterministic names before any v2 table is declared. This is a local +# SQLAlchemy metadata facility that SQLModel does not wrap directly. +SQLModel.metadata.naming_convention = { + "ix": "ix_%(table_name)s_%(column_0_N_name)s", + "uq": "uq_%(table_name)s_%(column_0_N_name)s", + "ck": "%(constraint_name)s", + "fk": "fk_%(table_name)s_%(column_0_N_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s", +} + +# Import in dependency order so every reviewed table registers exactly once. +from policyengine_api.data.v2.models.metadata import ( # noqa: E402 + Dataset, + DatasetVersion, + Parameter, + ParameterNode, + ParameterValue, + Region, + RegionDatasetLink, + RegionType, + TaxBenefitModel, + TaxBenefitModelVersion, + Variable, +) +from policyengine_api.data.v2.models.domain import ( # noqa: E402 + Dynamic, + Household, + HouseholdJob, + HouseholdJobStatus, + Policy, + Simulation, + SimulationStatus, + SimulationType, + User, + UserHouseholdAssociation, + UserPolicy, + UserReportAssociation, + UserSimulationAssociation, +) +from policyengine_api.data.v2.models.reports import ( # noqa: E402 + AggregateOutput, + AggregateType, + BudgetSummary, + ChangeAggregate, + CongressionalDistrictImpact, + ConstituencyImpact, + DecileImpact, + DecileType, + Inequality, + IntraDecileImpact, + LocalAuthorityImpact, + OutputStatus, + Poverty, + ProgramStatistics, + Report, + ReportRun, + ReportRunStatus, + ReportRunTrigger, +) + + +V2_METADATA = SQLModel.metadata +V2_TABLE_MODELS = ( + AggregateOutput, + BudgetSummary, + ChangeAggregate, + CongressionalDistrictImpact, + ConstituencyImpact, + Dataset, + DatasetVersion, + DecileImpact, + Dynamic, + Household, + HouseholdJob, + Inequality, + IntraDecileImpact, + LocalAuthorityImpact, + Parameter, + ParameterNode, + ParameterValue, + Policy, + Poverty, + ProgramStatistics, + Region, + RegionDatasetLink, + Report, + ReportRun, + Simulation, + TaxBenefitModel, + TaxBenefitModelVersion, + User, + UserHouseholdAssociation, + UserPolicy, + UserReportAssociation, + UserSimulationAssociation, + Variable, +) +validate_v2_table_inventory(V2_METADATA.tables) + +__all__ = [ + "AggregateOutput", + "AggregateType", + "BudgetSummary", + "ChangeAggregate", + "CongressionalDistrictImpact", + "ConstituencyImpact", + "Dataset", + "DatasetVersion", + "DecileImpact", + "DecileType", + "Dynamic", + "Household", + "HouseholdJob", + "HouseholdJobStatus", + "Inequality", + "IntraDecileImpact", + "LocalAuthorityImpact", + "OutputStatus", + "Parameter", + "ParameterNode", + "ParameterValue", + "Policy", + "Poverty", + "ProgramStatistics", + "Region", + "RegionDatasetLink", + "RegionType", + "Report", + "ReportRun", + "ReportRunStatus", + "ReportRunTrigger", + "Simulation", + "SimulationStatus", + "SimulationType", + "TaxBenefitModel", + "TaxBenefitModelVersion", + "User", + "UserHouseholdAssociation", + "UserPolicy", + "UserReportAssociation", + "UserSimulationAssociation", + "V2_METADATA", + "V2_TABLE_MODELS", + "Variable", +] diff --git a/policyengine_api/data/v2/models/base.py b/policyengine_api/data/v2/models/base.py new file mode 100644 index 000000000..30fdd6c5a --- /dev/null +++ b/policyengine_api/data/v2/models/base.py @@ -0,0 +1,88 @@ +"""Shared non-table SQLModel fields for the reviewed v2 schema.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from types import MappingProxyType +from uuid import UUID, uuid4 + +import sqlalchemy as sa +from sqlmodel import Field, SQLModel + + +# These are the complete direct-SQLAlchemy categories permitted in the v2 +# table layer. Each remains underneath canonical SQLModel table classes and +# Field/Relationship declarations; no parallel declarative model exists. +DIRECT_SQLALCHEMY_EXCEPTIONS = MappingProxyType( + { + "timezone_aware_timestamps": ( + "SQLModel Field has no first-class TIMESTAMP WITH TIME ZONE and " + "server-default/on-update parameters." + ), + "named_enums": ( + "Postgres enum type names and value serialization require an " + "explicit SQLAlchemy Enum supplied through Field(sa_type=...)." + ), + "typed_json_and_text": ( + "JSON and unbounded text storage require explicit SQLAlchemy " + "types while remaining SQLModel Fields." + ), + "named_constraints_and_indexes": ( + "Composite uniqueness, database checks, and multi-column indexes " + "are not expressible by one SQLModel Field." + ), + "ambiguous_foreign_key_relationships": ( + "Dataset input/output and baseline/reform joins need SQLAlchemy " + "foreign_keys hints exposed by SQLModel Relationship." + ), + "transaction_conflict_recovery": ( + "Concurrent report idempotency requires a savepoint and bounded " + "IntegrityError recovery around the database uniqueness constraint." + ), + } +) + + +def utc_now() -> datetime: + """Return an aware UTC timestamp for application-created rows.""" + + return datetime.now(timezone.utc) + + +class IdentifiedModel(SQLModel): + """Non-table UUID and creation timestamp fields.""" + + id: UUID = Field(default_factory=uuid4, primary_key=True) + # SQLModel has no dedicated timezone-aware timestamp option. The local + # SQLAlchemy type keeps Postgres TIMESTAMP WITH TIME ZONE while Field + # remains the canonical column declaration surface. + created_at: datetime = Field( + default_factory=utc_now, + sa_type=sa.DateTime(timezone=True), + sa_column_kwargs={"server_default": sa.func.now()}, + ) + + +class TimestampedModel(IdentifiedModel): + """Non-table UUID plus creation/update timestamp fields.""" + + updated_at: datetime = Field( + default_factory=utc_now, + sa_type=sa.DateTime(timezone=True), + sa_column_kwargs={ + "server_default": sa.func.now(), + "onupdate": sa.func.now(), + }, + ) + + +def enum_type(enum_class: type, name: str) -> sa.Enum: + """Build a stable lowercase-value Postgres enum for a string Enum.""" + + # Native named enums and value serialization are SQLAlchemy features that + # SQLModel intentionally exposes through Field(sa_type=...). + return sa.Enum( + enum_class, + name=name, + values_callable=lambda members: [member.value for member in members], + ) diff --git a/policyengine_api/data/v2/models/domain.py b/policyengine_api/data/v2/models/domain.py new file mode 100644 index 000000000..2051ceff6 --- /dev/null +++ b/policyengine_api/data/v2/models/domain.py @@ -0,0 +1,396 @@ +"""Canonical SQLModel tables for v2 policies, households, and simulations.""" + +from datetime import datetime +from enum import Enum +from typing import TYPE_CHECKING, Any +from uuid import UUID + +import sqlalchemy as sa +from sqlmodel import Field, Relationship + +from policyengine_api.data.v2.models.base import ( + IdentifiedModel, + TimestampedModel, + enum_type, +) +from policyengine_api.data.v2.models.metadata import ( + Dataset, + Region, + TaxBenefitModel, + TaxBenefitModelVersion, +) + +if TYPE_CHECKING: + from policyengine_api.data.v2.models.metadata import ParameterValue + from policyengine_api.data.v2.models.reports import Report + + +class HouseholdJobStatus(str, Enum): + PENDING = "pending" + RUNNING = "running" + SUCCEEDED = "succeeded" + FAILED = "failed" + + +class SimulationStatus(str, Enum): + PENDING = "pending" + RUNNING = "running" + SUCCEEDED = "succeeded" + FAILED = "failed" + + +class SimulationType(str, Enum): + HOUSEHOLD = "household" + ECONOMY = "economy" + + +class User(IdentifiedModel, table=True): + __tablename__ = "users" + __table_args__ = ( + sa.UniqueConstraint("email", name="uq_users_email"), + sa.CheckConstraint( + "primary_country IN ('us', 'uk')", + name="ck_users_primary_country", + ), + ) + + first_name: str = Field(max_length=255) + last_name: str = Field(max_length=255) + email: str = Field(max_length=320, index=True) + primary_country: str = Field(max_length=2) + + reports: list["Report"] = Relationship(back_populates="user") + household_associations: list["UserHouseholdAssociation"] = Relationship( + back_populates="user", + cascade_delete=True, + ) + policy_associations: list["UserPolicy"] = Relationship( + back_populates="user", + cascade_delete=True, + ) + simulation_associations: list["UserSimulationAssociation"] = Relationship( + back_populates="user", + cascade_delete=True, + ) + report_associations: list["UserReportAssociation"] = Relationship( + back_populates="user", + cascade_delete=True, + ) + + +class Policy(TimestampedModel, table=True): + __tablename__ = "policies" + + name: str = Field(max_length=255) + description: str | None = None + tax_benefit_model_id: UUID = Field( + foreign_key="tax_benefit_models.id", + ondelete="RESTRICT", + index=True, + ) + + tax_benefit_model: TaxBenefitModel = Relationship(back_populates="policies") + parameter_values: list["ParameterValue"] = Relationship( + back_populates="policy", + cascade_delete=True, + ) + simulations: list["Simulation"] = Relationship(back_populates="policy") + household_jobs: list["HouseholdJob"] = Relationship(back_populates="policy") + reports: list["Report"] = Relationship(back_populates="policy") + user_associations: list["UserPolicy"] = Relationship( + back_populates="policy", + cascade_delete=True, + ) + + +class Dynamic(TimestampedModel, table=True): + __tablename__ = "dynamics" + + name: str = Field(max_length=255) + description: str | None = None + + parameter_values: list["ParameterValue"] = Relationship( + back_populates="dynamic", + cascade_delete=True, + ) + simulations: list["Simulation"] = Relationship(back_populates="dynamic") + household_jobs: list["HouseholdJob"] = Relationship(back_populates="dynamic") + + +class Household(TimestampedModel, table=True): + __tablename__ = "households" + __table_args__ = ( + sa.CheckConstraint( + "year BETWEEN 1900 AND 2200", + name="ck_households_year", + ), + ) + + country: str = Field(max_length=16, index=True) + year: int + label: str | None = Field(default=None, max_length=255) + household_data: dict[str, Any] = Field(sa_type=sa.JSON) + + simulations: list["Simulation"] = Relationship(back_populates="household") + reports: list["Report"] = Relationship(back_populates="household") + user_associations: list["UserHouseholdAssociation"] = Relationship( + back_populates="household", + cascade_delete=True, + ) + + +class HouseholdJob(IdentifiedModel, table=True): + __tablename__ = "household_jobs" + __table_args__ = ( + sa.Index("ix_household_jobs_status_created_at", "status", "created_at"), + ) + + country: str = Field(max_length=16) + request_data: dict[str, Any] = Field(sa_type=sa.JSON) + policy_id: UUID | None = Field( + default=None, + foreign_key="policies.id", + ondelete="SET NULL", + ) + dynamic_id: UUID | None = Field( + default=None, + foreign_key="dynamics.id", + ondelete="SET NULL", + ) + status: HouseholdJobStatus = Field( + default=HouseholdJobStatus.PENDING, + sa_type=enum_type(HouseholdJobStatus, "v2_household_job_status"), + ) + error_message: str | None = None + result: dict[str, Any] | None = Field(default=None, sa_type=sa.JSON) + started_at: datetime | None = Field( + default=None, + sa_type=sa.DateTime(timezone=True), + ) + completed_at: datetime | None = Field( + default=None, + sa_type=sa.DateTime(timezone=True), + ) + + policy: Policy | None = Relationship(back_populates="household_jobs") + dynamic: Dynamic | None = Relationship(back_populates="household_jobs") + + +class Simulation(TimestampedModel, table=True): + __tablename__ = "simulations" + __table_args__ = ( + sa.CheckConstraint( + "(simulation_type = 'household' AND household_id IS NOT NULL " + "AND dataset_id IS NULL) OR " + "(simulation_type = 'economy' AND dataset_id IS NOT NULL " + "AND household_id IS NULL)", + name="ck_simulations_type_input", + ), + sa.CheckConstraint( + "(filter_field IS NULL) = (filter_value IS NULL)", + name="ck_simulations_filter_pair", + ), + sa.CheckConstraint( + "year IS NULL OR year BETWEEN 1900 AND 2200", + name="ck_simulations_year", + ), + sa.Index("ix_simulations_status_created_at", "status", "created_at"), + ) + + simulation_type: SimulationType = Field( + default=SimulationType.ECONOMY, + sa_type=enum_type(SimulationType, "v2_simulation_type"), + ) + dataset_id: UUID | None = Field( + default=None, + foreign_key="datasets.id", + ondelete="RESTRICT", + ) + household_id: UUID | None = Field( + default=None, + foreign_key="households.id", + ondelete="RESTRICT", + ) + policy_id: UUID | None = Field( + default=None, + foreign_key="policies.id", + ondelete="SET NULL", + ) + dynamic_id: UUID | None = Field( + default=None, + foreign_key="dynamics.id", + ondelete="SET NULL", + ) + tax_benefit_model_version_id: UUID = Field( + foreign_key="tax_benefit_model_versions.id", + ondelete="RESTRICT", + index=True, + ) + output_dataset_id: UUID | None = Field( + default=None, + foreign_key="datasets.id", + ondelete="SET NULL", + ) + region_id: UUID | None = Field( + default=None, + foreign_key="regions.id", + ondelete="SET NULL", + ) + status: SimulationStatus = Field( + default=SimulationStatus.PENDING, + sa_type=enum_type(SimulationStatus, "v2_simulation_status"), + ) + error_message: str | None = None + filter_field: str | None = Field(default=None, max_length=128) + filter_value: str | None = Field(default=None, max_length=255) + filter_strategy: str | None = Field(default=None, max_length=64) + year: int | None = None + started_at: datetime | None = Field( + default=None, + sa_type=sa.DateTime(timezone=True), + ) + completed_at: datetime | None = Field( + default=None, + sa_type=sa.DateTime(timezone=True), + ) + household_result: dict[str, Any] | None = Field(default=None, sa_type=sa.JSON) + + dataset: Dataset | None = Relationship( + back_populates="input_simulations", + sa_relationship_kwargs={"foreign_keys": "Simulation.dataset_id"}, + ) + output_dataset: Dataset | None = Relationship( + back_populates="output_simulations", + sa_relationship_kwargs={"foreign_keys": "Simulation.output_dataset_id"}, + ) + household: Household | None = Relationship(back_populates="simulations") + policy: Policy | None = Relationship(back_populates="simulations") + dynamic: Dynamic | None = Relationship(back_populates="simulations") + tax_benefit_model_version: TaxBenefitModelVersion = Relationship( + back_populates="simulations" + ) + region: Region | None = Relationship(back_populates="simulations") + baseline_reports: list["Report"] = Relationship( + back_populates="baseline_simulation", + sa_relationship_kwargs={"foreign_keys": "Report.baseline_simulation_id"}, + ) + reform_reports: list["Report"] = Relationship( + back_populates="reform_simulation", + sa_relationship_kwargs={"foreign_keys": "Report.reform_simulation_id"}, + ) + user_associations: list["UserSimulationAssociation"] = Relationship( + back_populates="simulation", + cascade_delete=True, + ) + + +class UserHouseholdAssociation(TimestampedModel, table=True): + __tablename__ = "user_household_associations" + __table_args__ = ( + sa.UniqueConstraint( + "user_id", + "household_id", + name="uq_user_household_associations_user_household", + ), + ) + + user_id: UUID = Field( + foreign_key="users.id", + ondelete="CASCADE", + index=True, + ) + household_id: UUID = Field( + foreign_key="households.id", + ondelete="CASCADE", + index=True, + ) + country: str = Field(max_length=16) + label: str | None = Field(default=None, max_length=255) + + user: User = Relationship(back_populates="household_associations") + household: Household = Relationship(back_populates="user_associations") + + +class UserPolicy(TimestampedModel, table=True): + __tablename__ = "user_policies" + __table_args__ = ( + sa.UniqueConstraint( + "user_id", + "policy_id", + name="uq_user_policies_user_policy", + ), + ) + + user_id: UUID = Field( + foreign_key="users.id", + ondelete="CASCADE", + index=True, + ) + policy_id: UUID = Field( + foreign_key="policies.id", + ondelete="CASCADE", + index=True, + ) + country: str = Field(max_length=16) + label: str | None = Field(default=None, max_length=255) + + user: User = Relationship(back_populates="policy_associations") + policy: Policy = Relationship(back_populates="user_associations") + + +class UserSimulationAssociation(TimestampedModel, table=True): + __tablename__ = "user_simulation_associations" + __table_args__ = ( + sa.UniqueConstraint( + "user_id", + "simulation_id", + name="uq_user_simulation_associations_user_simulation", + ), + ) + + user_id: UUID = Field( + foreign_key="users.id", + ondelete="CASCADE", + index=True, + ) + simulation_id: UUID = Field( + foreign_key="simulations.id", + ondelete="CASCADE", + index=True, + ) + country: str = Field(max_length=16) + label: str | None = Field(default=None, max_length=255) + + user: User = Relationship(back_populates="simulation_associations") + simulation: Simulation = Relationship(back_populates="user_associations") + + +class UserReportAssociation(TimestampedModel, table=True): + __tablename__ = "user_report_associations" + __table_args__ = ( + sa.UniqueConstraint( + "user_id", + "report_id", + name="uq_user_report_associations_user_report", + ), + ) + + user_id: UUID = Field( + foreign_key="users.id", + ondelete="CASCADE", + index=True, + ) + report_id: UUID = Field( + foreign_key="reports.id", + ondelete="CASCADE", + index=True, + ) + country: str = Field(max_length=16) + label: str | None = Field(default=None, max_length=255) + last_run_at: datetime | None = Field( + default=None, + sa_type=sa.DateTime(timezone=True), + ) + + user: User = Relationship(back_populates="report_associations") + report: "Report" = Relationship(back_populates="user_associations") diff --git a/policyengine_api/data/v2/models/metadata.py b/policyengine_api/data/v2/models/metadata.py new file mode 100644 index 000000000..b57a33edf --- /dev/null +++ b/policyengine_api/data/v2/models/metadata.py @@ -0,0 +1,340 @@ +"""Canonical SQLModel tables for v2 model metadata, regions, and datasets.""" + +from datetime import datetime +from enum import Enum +from typing import TYPE_CHECKING, Any, Optional +from uuid import UUID + +import sqlalchemy as sa +from sqlmodel import Field, Relationship, SQLModel + +from policyengine_api.data.v2.models.base import ( + IdentifiedModel, + TimestampedModel, + enum_type, +) + +if TYPE_CHECKING: + from policyengine_api.data.v2.models.domain import Dynamic, Policy, Simulation + from policyengine_api.data.v2.models.reports import Report + + +class RegionType(str, Enum): + NATIONAL = "national" + COUNTRY = "country" + STATE = "state" + CONGRESSIONAL_DISTRICT = "congressional_district" + CONSTITUENCY = "constituency" + LOCAL_AUTHORITY = "local_authority" + CITY = "city" + PLACE = "place" + + +class RegionDatasetLink(SQLModel, table=True): + __tablename__ = "region_datasets" + + region_id: UUID = Field( + foreign_key="regions.id", + ondelete="CASCADE", + primary_key=True, + ) + dataset_id: UUID = Field( + foreign_key="datasets.id", + ondelete="CASCADE", + primary_key=True, + ) + + +class TaxBenefitModel(TimestampedModel, table=True): + __tablename__ = "tax_benefit_models" + __table_args__ = (sa.UniqueConstraint("name", name="uq_tax_benefit_models_name"),) + + name: str = Field(max_length=32) + description: str | None = None + + versions: list["TaxBenefitModelVersion"] = Relationship( + back_populates="model", + cascade_delete=True, + ) + datasets: list["Dataset"] = Relationship(back_populates="tax_benefit_model") + dataset_versions: list["DatasetVersion"] = Relationship( + back_populates="tax_benefit_model" + ) + regions: list["Region"] = Relationship(back_populates="tax_benefit_model") + policies: list["Policy"] = Relationship(back_populates="tax_benefit_model") + reports: list["Report"] = Relationship(back_populates="tax_benefit_model") + + +class TaxBenefitModelVersion(IdentifiedModel, table=True): + __tablename__ = "tax_benefit_model_versions" + __table_args__ = ( + sa.UniqueConstraint( + "model_id", + "version", + name="uq_tax_benefit_model_versions_model_version", + ), + ) + + model_id: UUID = Field( + foreign_key="tax_benefit_models.id", + ondelete="CASCADE", + index=True, + ) + version: str = Field(max_length=128) + description: str | None = None + + model: TaxBenefitModel = Relationship(back_populates="versions") + variables: list["Variable"] = Relationship( + back_populates="tax_benefit_model_version", + cascade_delete=True, + ) + parameters: list["Parameter"] = Relationship( + back_populates="tax_benefit_model_version", + cascade_delete=True, + ) + parameter_nodes: list["ParameterNode"] = Relationship( + back_populates="tax_benefit_model_version", + cascade_delete=True, + ) + simulations: list["Simulation"] = Relationship( + back_populates="tax_benefit_model_version" + ) + + +class Region(TimestampedModel, table=True): + __tablename__ = "regions" + __table_args__ = ( + sa.UniqueConstraint( + "tax_benefit_model_id", + "code", + name="uq_regions_model_code", + ), + sa.CheckConstraint( + "NOT requires_filter OR " + "(filter_field IS NOT NULL AND filter_value IS NOT NULL)", + name="ck_regions_required_filter_values", + ), + ) + + code: str = Field(max_length=255) + label: str = Field(max_length=255) + region_type: RegionType = Field(sa_type=enum_type(RegionType, "v2_region_type")) + requires_filter: bool = False + filter_field: str | None = Field(default=None, max_length=128) + filter_value: str | None = Field(default=None, max_length=255) + filter_strategy: str | None = Field(default=None, max_length=64) + parent_code: str | None = Field(default=None, max_length=255) + state_code: str | None = Field(default=None, max_length=16) + state_name: str | None = Field(default=None, max_length=128) + tax_benefit_model_id: UUID = Field( + foreign_key="tax_benefit_models.id", + ondelete="RESTRICT", + index=True, + ) + + tax_benefit_model: TaxBenefitModel = Relationship(back_populates="regions") + datasets: list["Dataset"] = Relationship( + back_populates="regions", + link_model=RegionDatasetLink, + ) + simulations: list["Simulation"] = Relationship(back_populates="region") + reports: list["Report"] = Relationship(back_populates="region") + + +class Dataset(TimestampedModel, table=True): + __tablename__ = "datasets" + __table_args__ = ( + sa.UniqueConstraint( + "tax_benefit_model_id", + "name", + "year", + "is_output_dataset", + name="uq_datasets_model_name_year_output", + ), + sa.CheckConstraint( + "year BETWEEN 1900 AND 2200", + name="ck_datasets_year", + ), + ) + + name: str = Field(max_length=255) + description: str | None = None + storage_path: str = Field(max_length=1024) + year: int + is_output_dataset: bool = False + tax_benefit_model_id: UUID = Field( + foreign_key="tax_benefit_models.id", + ondelete="RESTRICT", + index=True, + ) + + tax_benefit_model: TaxBenefitModel = Relationship(back_populates="datasets") + versions: list["DatasetVersion"] = Relationship( + back_populates="dataset", + cascade_delete=True, + ) + regions: list[Region] = Relationship( + back_populates="datasets", + link_model=RegionDatasetLink, + ) + input_simulations: list["Simulation"] = Relationship( + back_populates="dataset", + sa_relationship_kwargs={"foreign_keys": "Simulation.dataset_id"}, + ) + output_simulations: list["Simulation"] = Relationship( + back_populates="output_dataset", + sa_relationship_kwargs={"foreign_keys": "Simulation.output_dataset_id"}, + ) + reports: list["Report"] = Relationship(back_populates="dataset") + + +class DatasetVersion(IdentifiedModel, table=True): + __tablename__ = "dataset_versions" + __table_args__ = ( + sa.UniqueConstraint( + "dataset_id", + "name", + name="uq_dataset_versions_dataset_name", + ), + ) + + name: str = Field(max_length=128) + description: str | None = None + dataset_id: UUID = Field( + foreign_key="datasets.id", + ondelete="CASCADE", + index=True, + ) + tax_benefit_model_id: UUID = Field( + foreign_key="tax_benefit_models.id", + ondelete="RESTRICT", + index=True, + ) + + dataset: Dataset = Relationship(back_populates="versions") + tax_benefit_model: TaxBenefitModel = Relationship(back_populates="dataset_versions") + + +class Variable(IdentifiedModel, table=True): + __tablename__ = "variables" + __table_args__ = ( + sa.UniqueConstraint( + "tax_benefit_model_version_id", + "name", + name="uq_variables_model_version_name", + ), + ) + + name: str = Field(max_length=512) + label: str | None = Field(default=None, max_length=512) + entity: str = Field(max_length=128) + description: str | None = None + data_type: str | None = Field(default=None, max_length=128) + possible_values: list[str] | None = Field(default=None, sa_type=sa.JSON) + default_value: Any = Field(default=None, sa_type=sa.JSON) + adds: list[str] | None = Field(default=None, sa_type=sa.JSON) + subtracts: list[str] | None = Field(default=None, sa_type=sa.JSON) + tax_benefit_model_version_id: UUID = Field( + foreign_key="tax_benefit_model_versions.id", + ondelete="CASCADE", + index=True, + ) + + tax_benefit_model_version: TaxBenefitModelVersion = Relationship( + back_populates="variables" + ) + + +class ParameterNode(IdentifiedModel, table=True): + __tablename__ = "parameter_nodes" + __table_args__ = ( + sa.UniqueConstraint( + "tax_benefit_model_version_id", + "name", + name="uq_parameter_nodes_model_version_name", + ), + ) + + name: str = Field(max_length=512) + label: str | None = Field(default=None, max_length=512) + description: str | None = None + tax_benefit_model_version_id: UUID = Field( + foreign_key="tax_benefit_model_versions.id", + ondelete="CASCADE", + index=True, + ) + + tax_benefit_model_version: TaxBenefitModelVersion = Relationship( + back_populates="parameter_nodes" + ) + + +class Parameter(IdentifiedModel, table=True): + __tablename__ = "parameters" + __table_args__ = ( + sa.UniqueConstraint( + "tax_benefit_model_version_id", + "name", + name="uq_parameters_model_version_name", + ), + ) + + name: str = Field(max_length=512) + label: str | None = Field(default=None, max_length=512) + description: str | None = None + data_type: str | None = Field(default=None, max_length=128) + unit: str | None = Field(default=None, max_length=128) + tax_benefit_model_version_id: UUID = Field( + foreign_key="tax_benefit_model_versions.id", + ondelete="CASCADE", + index=True, + ) + + tax_benefit_model_version: TaxBenefitModelVersion = Relationship( + back_populates="parameters" + ) + values: list["ParameterValue"] = Relationship( + back_populates="parameter", + cascade_delete=True, + ) + + +class ParameterValue(IdentifiedModel, table=True): + __tablename__ = "parameter_values" + __table_args__ = ( + sa.CheckConstraint( + "policy_id IS NULL OR dynamic_id IS NULL", + name="ck_parameter_values_single_owner", + ), + sa.Index( + "ix_parameter_values_parameter_period", + "parameter_id", + "start_date", + "end_date", + ), + ) + + parameter_id: UUID = Field( + foreign_key="parameters.id", + ondelete="CASCADE", + ) + value_json: Any = Field(sa_type=sa.JSON) + start_date: datetime = Field(sa_type=sa.DateTime(timezone=True)) + end_date: datetime | None = Field( + default=None, + sa_type=sa.DateTime(timezone=True), + ) + policy_id: UUID | None = Field( + default=None, + foreign_key="policies.id", + ondelete="CASCADE", + ) + dynamic_id: UUID | None = Field( + default=None, + foreign_key="dynamics.id", + ondelete="CASCADE", + ) + + parameter: Parameter = Relationship(back_populates="values") + policy: Optional["Policy"] = Relationship(back_populates="parameter_values") + dynamic: Optional["Dynamic"] = Relationship(back_populates="parameter_values") diff --git a/policyengine_api/data/v2/models/reports.py b/policyengine_api/data/v2/models/reports.py new file mode 100644 index 000000000..568880e19 --- /dev/null +++ b/policyengine_api/data/v2/models/reports.py @@ -0,0 +1,601 @@ +"""Canonical SQLModel tables for stable reports, runs, and run outputs.""" + +from datetime import datetime +from enum import Enum +from typing import Any +from uuid import UUID + +import sqlalchemy as sa +from sqlmodel import Field, Relationship + +from policyengine_api.data.v2.models.base import ( + IdentifiedModel, + TimestampedModel, + enum_type, +) +from policyengine_api.data.v2.models.domain import ( + Household, + Policy, + Simulation, + User, + UserReportAssociation, +) +from policyengine_api.data.v2.models.metadata import ( + Dataset, + Region, + TaxBenefitModel, +) + + +class ReportRunStatus(str, Enum): + PENDING = "pending" + RUNNING = "running" + SUCCEEDED = "succeeded" + FAILED = "failed" + + +class ReportRunTrigger(str, Enum): + INITIAL = "initial" + MANUAL = "manual" + SYSTEM = "system" + + +class OutputStatus(str, Enum): + PENDING = "pending" + RUNNING = "running" + SUCCEEDED = "succeeded" + FAILED = "failed" + + +class AggregateType(str, Enum): + SUM = "sum" + MEAN = "mean" + COUNT = "count" + + +class DecileType(str, Enum): + INCOME = "income" + WEALTH = "wealth" + + +class Report(TimestampedModel, table=True): + """Stable report definition; execution state belongs to ReportRun.""" + + __tablename__ = "reports" + __table_args__ = ( + sa.CheckConstraint( + "year IS NULL OR year BETWEEN 1900 AND 2200", + name="ck_reports_year", + ), + sa.Index("ix_reports_country_type_created_at", "country", "type", "created_at"), + ) + + label: str = Field(max_length=255) + description: str | None = None + country: str = Field(max_length=16) + type: str | None = Field(default=None, max_length=128) + user_id: UUID | None = Field( + default=None, + foreign_key="users.id", + ondelete="SET NULL", + ) + tax_benefit_model_id: UUID = Field( + foreign_key="tax_benefit_models.id", + ondelete="RESTRICT", + index=True, + ) + policy_id: UUID | None = Field( + default=None, + foreign_key="policies.id", + ondelete="SET NULL", + ) + baseline_simulation_id: UUID | None = Field( + default=None, + foreign_key="simulations.id", + ondelete="SET NULL", + ) + reform_simulation_id: UUID | None = Field( + default=None, + foreign_key="simulations.id", + ondelete="SET NULL", + ) + household_id: UUID | None = Field( + default=None, + foreign_key="households.id", + ondelete="SET NULL", + ) + dataset_id: UUID | None = Field( + default=None, + foreign_key="datasets.id", + ondelete="SET NULL", + ) + region_id: UUID | None = Field( + default=None, + foreign_key="regions.id", + ondelete="SET NULL", + ) + year: int | None = None + inputs: dict[str, Any] = Field(default_factory=dict, sa_type=sa.JSON) + + user: User | None = Relationship(back_populates="reports") + tax_benefit_model: TaxBenefitModel = Relationship(back_populates="reports") + policy: Policy | None = Relationship(back_populates="reports") + baseline_simulation: Simulation | None = Relationship( + back_populates="baseline_reports", + sa_relationship_kwargs={"foreign_keys": "Report.baseline_simulation_id"}, + ) + reform_simulation: Simulation | None = Relationship( + back_populates="reform_reports", + sa_relationship_kwargs={"foreign_keys": "Report.reform_simulation_id"}, + ) + household: Household | None = Relationship(back_populates="reports") + dataset: Dataset | None = Relationship(back_populates="reports") + region: Region | None = Relationship(back_populates="reports") + runs: list["ReportRun"] = Relationship( + back_populates="report", + cascade_delete=True, + ) + user_associations: list[UserReportAssociation] = Relationship( + back_populates="report", + cascade_delete=True, + ) + + +class ReportRun(TimestampedModel, table=True): + """One immutable-version execution attempt for a stable report.""" + + __tablename__ = "report_runs" + __table_args__ = ( + sa.UniqueConstraint( + "report_id", + "idempotency_key", + name="uq_report_runs_report_idempotency_key", + ), + sa.CheckConstraint( + "status NOT IN ('succeeded', 'failed') OR completed_at IS NOT NULL", + name="ck_report_runs_terminal_completion", + ), + sa.CheckConstraint( + "idempotency_key IS NULL OR length(trim(idempotency_key)) > 0", + name="ck_report_runs_idempotency_key_nonblank", + ), + sa.Index( + "ix_report_runs_current_output", + "report_id", + "status", + "country_package_version", + "policyengine_version", + "completed_at", + "id", + ), + ) + + report_id: UUID = Field( + foreign_key="reports.id", + ondelete="CASCADE", + ) + country_package_version: str = Field(max_length=128) + policyengine_version: str = Field(max_length=128) + status: ReportRunStatus = Field( + default=ReportRunStatus.PENDING, + sa_type=enum_type(ReportRunStatus, "v2_report_run_status"), + ) + trigger: ReportRunTrigger = Field( + default=ReportRunTrigger.INITIAL, + sa_type=enum_type(ReportRunTrigger, "v2_report_run_trigger"), + ) + idempotency_key: str | None = Field(default=None, max_length=255) + started_at: datetime | None = Field( + default=None, + sa_type=sa.DateTime(timezone=True), + ) + completed_at: datetime | None = Field( + default=None, + sa_type=sa.DateTime(timezone=True), + ) + error_message: str | None = None + markdown: str | None = Field(default=None, sa_type=sa.Text) + + report: Report = Relationship(back_populates="runs") + aggregates: list["AggregateOutput"] = Relationship( + back_populates="report_run", + cascade_delete=True, + ) + change_aggregates: list["ChangeAggregate"] = Relationship( + back_populates="report_run", + cascade_delete=True, + ) + budget_summaries: list["BudgetSummary"] = Relationship( + back_populates="report_run", + cascade_delete=True, + ) + congressional_district_impacts: list["CongressionalDistrictImpact"] = Relationship( + back_populates="report_run", cascade_delete=True + ) + constituency_impacts: list["ConstituencyImpact"] = Relationship( + back_populates="report_run", + cascade_delete=True, + ) + decile_impacts: list["DecileImpact"] = Relationship( + back_populates="report_run", + cascade_delete=True, + ) + inequalities: list["Inequality"] = Relationship( + back_populates="report_run", + cascade_delete=True, + ) + intra_decile_impacts: list["IntraDecileImpact"] = Relationship( + back_populates="report_run", + cascade_delete=True, + ) + local_authority_impacts: list["LocalAuthorityImpact"] = Relationship( + back_populates="report_run", + cascade_delete=True, + ) + poverty_results: list["Poverty"] = Relationship( + back_populates="report_run", + cascade_delete=True, + ) + program_statistics: list["ProgramStatistics"] = Relationship( + back_populates="report_run", + cascade_delete=True, + ) + + +class AggregateOutput(IdentifiedModel, table=True): + __tablename__ = "aggregates" + __table_args__ = ( + sa.Index("ix_aggregates_report_run_status", "report_run_id", "status"), + ) + + report_run_id: UUID = Field( + foreign_key="report_runs.id", + ondelete="CASCADE", + ) + simulation_id: UUID = Field( + foreign_key="simulations.id", + ondelete="RESTRICT", + ) + variable: str = Field(max_length=512) + aggregate_type: AggregateType = Field( + sa_type=enum_type(AggregateType, "v2_aggregate_type") + ) + entity: str | None = Field(default=None, max_length=128) + filter_config: dict[str, Any] = Field(default_factory=dict, sa_type=sa.JSON) + status: OutputStatus = Field( + default=OutputStatus.PENDING, + sa_type=enum_type(OutputStatus, "v2_output_status"), + ) + error_message: str | None = None + result: float | None = None + + report_run: ReportRun = Relationship(back_populates="aggregates") + + +class ChangeAggregate(IdentifiedModel, table=True): + __tablename__ = "change_aggregates" + __table_args__ = ( + sa.Index( + "ix_change_aggregates_report_run_status", + "report_run_id", + "status", + ), + ) + + report_run_id: UUID = Field( + foreign_key="report_runs.id", + ondelete="CASCADE", + ) + baseline_simulation_id: UUID = Field( + foreign_key="simulations.id", + ondelete="RESTRICT", + ) + reform_simulation_id: UUID = Field( + foreign_key="simulations.id", + ondelete="RESTRICT", + ) + variable: str = Field(max_length=512) + aggregate_type: AggregateType = Field( + sa_type=enum_type(AggregateType, "v2_aggregate_type"), + ) + entity: str | None = Field(default=None, max_length=128) + filter_config: dict[str, Any] = Field(default_factory=dict, sa_type=sa.JSON) + change_geq: float | None = None + change_leq: float | None = None + status: OutputStatus = Field( + default=OutputStatus.PENDING, + sa_type=enum_type(OutputStatus, "v2_output_status"), + ) + error_message: str | None = None + result: float | None = None + + report_run: ReportRun = Relationship(back_populates="change_aggregates") + + +class BudgetSummary(IdentifiedModel, table=True): + __tablename__ = "budget_summary" + __table_args__ = ( + sa.UniqueConstraint( + "report_run_id", + "variable_name", + "entity", + name="uq_budget_summary_run_variable_entity", + ), + ) + + report_run_id: UUID = Field( + foreign_key="report_runs.id", + ondelete="CASCADE", + ) + baseline_simulation_id: UUID = Field( + foreign_key="simulations.id", + ondelete="RESTRICT", + ) + reform_simulation_id: UUID = Field( + foreign_key="simulations.id", + ondelete="RESTRICT", + ) + variable_name: str = Field(max_length=512) + entity: str = Field(max_length=128) + baseline_total: float | None = None + reform_total: float | None = None + change: float | None = None + + report_run: ReportRun = Relationship(back_populates="budget_summaries") + + +class DecileImpact(IdentifiedModel, table=True): + __tablename__ = "decile_impacts" + __table_args__ = ( + sa.UniqueConstraint( + "report_run_id", + "income_variable", + "entity", + "decile", + "quantiles", + name="uq_decile_impacts_run_measure", + ), + sa.CheckConstraint("quantiles > 0", name="ck_decile_impacts_quantiles"), + sa.CheckConstraint( + "decile BETWEEN 1 AND quantiles", + name="ck_decile_impacts_decile", + ), + ) + + report_run_id: UUID = Field( + foreign_key="report_runs.id", + ondelete="CASCADE", + ) + baseline_simulation_id: UUID = Field( + foreign_key="simulations.id", + ondelete="RESTRICT", + ) + reform_simulation_id: UUID = Field( + foreign_key="simulations.id", + ondelete="RESTRICT", + ) + income_variable: str = Field(max_length=512) + entity: str = Field(max_length=128) + decile: int + quantiles: int = 10 + baseline_mean: float | None = None + reform_mean: float | None = None + absolute_change: float | None = None + relative_change: float | None = None + count_better_off: float | None = None + count_worse_off: float | None = None + count_no_change: float | None = None + + report_run: ReportRun = Relationship(back_populates="decile_impacts") + + +class IntraDecileImpact(IdentifiedModel, table=True): + __tablename__ = "intra_decile_impacts" + __table_args__ = ( + sa.UniqueConstraint( + "report_run_id", + "decile_type", + "decile", + name="uq_intra_decile_impacts_run_type_decile", + ), + sa.CheckConstraint( + "decile BETWEEN 0 AND 10", + name="ck_intra_decile_impacts_decile", + ), + ) + + report_run_id: UUID = Field( + foreign_key="report_runs.id", + ondelete="CASCADE", + ) + baseline_simulation_id: UUID = Field( + foreign_key="simulations.id", + ondelete="RESTRICT", + ) + reform_simulation_id: UUID = Field( + foreign_key="simulations.id", + ondelete="RESTRICT", + ) + decile_type: DecileType = Field( + default=DecileType.INCOME, + sa_type=enum_type(DecileType, "v2_decile_type"), + ) + decile: int + lose_more_than_5pct: float | None = None + lose_less_than_5pct: float | None = None + no_change: float | None = None + gain_less_than_5pct: float | None = None + gain_more_than_5pct: float | None = None + + report_run: ReportRun = Relationship(back_populates="intra_decile_impacts") + + +class Inequality(IdentifiedModel, table=True): + __tablename__ = "inequality" + __table_args__ = ( + sa.UniqueConstraint( + "report_run_id", + "simulation_id", + "income_variable", + "entity", + name="uq_inequality_run_simulation_measure", + ), + ) + + report_run_id: UUID = Field( + foreign_key="report_runs.id", + ondelete="CASCADE", + ) + simulation_id: UUID = Field( + foreign_key="simulations.id", + ondelete="RESTRICT", + ) + income_variable: str = Field(max_length=512) + entity: str = Field(default="household", max_length=128) + gini: float | None = None + top_10_share: float | None = None + top_1_share: float | None = None + bottom_50_share: float | None = None + + report_run: ReportRun = Relationship(back_populates="inequalities") + + +class Poverty(IdentifiedModel, table=True): + __tablename__ = "poverty" + __table_args__ = ( + sa.Index( + "ix_poverty_run_simulation_type", + "report_run_id", + "simulation_id", + "poverty_type", + ), + ) + + report_run_id: UUID = Field( + foreign_key="report_runs.id", + ondelete="CASCADE", + ) + simulation_id: UUID = Field( + foreign_key="simulations.id", + ondelete="RESTRICT", + ) + poverty_type: str = Field(max_length=128) + entity: str = Field(default="person", max_length=128) + filter_variable: str | None = Field(default=None, max_length=512) + headcount: float | None = None + total_population: float | None = None + rate: float | None = None + + report_run: ReportRun = Relationship(back_populates="poverty_results") + + +class ProgramStatistics(IdentifiedModel, table=True): + __tablename__ = "program_statistics" + __table_args__ = ( + sa.UniqueConstraint( + "report_run_id", + "program_name", + "entity", + name="uq_program_statistics_run_program_entity", + ), + ) + + report_run_id: UUID = Field( + foreign_key="report_runs.id", + ondelete="CASCADE", + ) + baseline_simulation_id: UUID = Field( + foreign_key="simulations.id", + ondelete="RESTRICT", + ) + reform_simulation_id: UUID = Field( + foreign_key="simulations.id", + ondelete="RESTRICT", + ) + program_name: str = Field(max_length=512) + entity: str = Field(max_length=128) + is_tax: bool = False + baseline_total: float | None = None + reform_total: float | None = None + change: float | None = None + baseline_count: float | None = None + reform_count: float | None = None + winners: float | None = None + losers: float | None = None + + report_run: ReportRun = Relationship(back_populates="program_statistics") + + +class GeographicImpactBase(IdentifiedModel): + """Shared non-table fields for geographic report-run impacts.""" + + report_run_id: UUID = Field( + foreign_key="report_runs.id", + ondelete="CASCADE", + ) + baseline_simulation_id: UUID = Field( + foreign_key="simulations.id", + ondelete="RESTRICT", + ) + reform_simulation_id: UUID = Field( + foreign_key="simulations.id", + ondelete="RESTRICT", + ) + average_household_income_change: float + relative_household_income_change: float + population: float + + +class CongressionalDistrictImpact(GeographicImpactBase, table=True): + __tablename__ = "congressional_district_impacts" + __table_args__ = ( + sa.UniqueConstraint( + "report_run_id", + "district_geoid", + name="uq_congressional_district_impacts_run_geoid", + ), + ) + + district_geoid: int + state_fips: int + district_number: int + + report_run: ReportRun = Relationship( + back_populates="congressional_district_impacts" + ) + + +class ConstituencyImpact(GeographicImpactBase, table=True): + __tablename__ = "constituency_impacts" + __table_args__ = ( + sa.UniqueConstraint( + "report_run_id", + "constituency_code", + name="uq_constituency_impacts_run_code", + ), + ) + + constituency_code: str = Field(max_length=64) + constituency_name: str = Field(max_length=255) + x: int + y: int + + report_run: ReportRun = Relationship(back_populates="constituency_impacts") + + +class LocalAuthorityImpact(GeographicImpactBase, table=True): + __tablename__ = "local_authority_impacts" + __table_args__ = ( + sa.UniqueConstraint( + "report_run_id", + "local_authority_code", + name="uq_local_authority_impacts_run_code", + ), + ) + + local_authority_code: str = Field(max_length=64) + local_authority_name: str = Field(max_length=255) + x: int + y: int + + report_run: ReportRun = Relationship(back_populates="local_authority_impacts") diff --git a/policyengine_api/data/v2/reference_data.py b/policyengine_api/data/v2/reference_data.py new file mode 100644 index 000000000..47b237c01 --- /dev/null +++ b/policyengine_api/data/v2/reference_data.py @@ -0,0 +1,144 @@ +"""Versioned declarative Stage 8 application data for Alembic autogeneration.""" + +from __future__ import annotations + +from dataclasses import dataclass +from types import MappingProxyType +from typing import Any + +from policyengine_api.data.v2.table_inventory import EXPECTED_V2_TABLES + + +REFERENCE_DATA_FORMAT_VERSION = 1 +VALIDATION_MODEL_ID = "80000000-0000-4000-8000-000000000001" +VALIDATION_MODEL_VERSION_ID = "80000000-0000-4000-8000-000000000002" +VALIDATION_TIMESTAMP = "2026-08-14T00:00:00+00:00" + + +class ReferenceDataDeclarationError(ValueError): + """Raised when declarative data is not safe for deterministic generation.""" + + +@dataclass(frozen=True) +class ReferenceRow: + key: MappingProxyType + values: MappingProxyType + + @classmethod + def create(cls, *, key: dict[str, Any], values: dict[str, Any]) -> "ReferenceRow": + if not key or any(value is None for value in key.values()): + raise ReferenceDataDeclarationError( + "reference rows require non-null stable key values" + ) + overlap = set(key) & set(values) + if overlap: + raise ReferenceDataDeclarationError( + f"reference row key/value columns overlap: {sorted(overlap)}" + ) + _validate_wire_values({**key, **values}) + return cls(MappingProxyType(dict(key)), MappingProxyType(dict(values))) + + @property + def complete_values(self) -> dict[str, Any]: + return {**self.key, **self.values} + + +@dataclass(frozen=True) +class ReferenceTable: + table_name: str + key_columns: tuple[str, ...] + managed_prefix_column: str + managed_prefix: str + rows: tuple[ReferenceRow, ...] + + def __post_init__(self) -> None: + if self.table_name not in EXPECTED_V2_TABLES: + raise ReferenceDataDeclarationError( + f"unreviewed reference-data table: {self.table_name}" + ) + if not self.key_columns or self.managed_prefix_column not in self.key_columns: + raise ReferenceDataDeclarationError( + "managed prefix column must be part of the stable key" + ) + seen_keys: set[tuple[Any, ...]] = set() + for row in self.rows: + if tuple(row.key) != self.key_columns: + raise ReferenceDataDeclarationError( + f"{self.table_name} row key columns must be {self.key_columns}" + ) + prefix_value = row.key[self.managed_prefix_column] + if not isinstance(prefix_value, str) or not prefix_value.startswith( + self.managed_prefix + ): + raise ReferenceDataDeclarationError( + f"{self.table_name} managed key is outside its declared scope" + ) + stable_key = tuple(row.key[column] for column in self.key_columns) + if stable_key in seen_keys: + raise ReferenceDataDeclarationError( + f"duplicate reference-data key for {self.table_name}: {stable_key}" + ) + seen_keys.add(stable_key) + + +def _validate_wire_values(values: Any) -> None: + if values is None or isinstance(values, str | int | float | bool): + return + if isinstance(values, list | tuple): + for value in values: + _validate_wire_values(value) + return + if isinstance(values, dict): + if not all(isinstance(key, str) for key in values): + raise ReferenceDataDeclarationError( + "reference-data object keys must be strings" + ) + for value in values.values(): + _validate_wire_values(value) + return + raise ReferenceDataDeclarationError( + f"unsupported reference-data value type: {type(values).__name__}" + ) + + +REFERENCE_DATA = ( + ReferenceTable( + table_name="tax_benefit_models", + key_columns=("name",), + managed_prefix_column="name", + managed_prefix="stage8-", + rows=( + ReferenceRow.create( + key={"name": "stage8-platform-validation"}, + values={ + "id": VALIDATION_MODEL_ID, + "description": ( + "Stage 8 migration lifecycle validation; canonical " + "metadata is introduced in Stage 9." + ), + "created_at": VALIDATION_TIMESTAMP, + "updated_at": VALIDATION_TIMESTAMP, + }, + ), + ), + ), + ReferenceTable( + table_name="tax_benefit_model_versions", + key_columns=("model_id", "version"), + managed_prefix_column="version", + managed_prefix="stage8-", + rows=( + ReferenceRow.create( + key={ + "model_id": VALIDATION_MODEL_ID, + "version": "stage8-platform-validation", + }, + values={ + "id": VALIDATION_MODEL_VERSION_ID, + "description": ("Stage 8 generated data-migration validation row."), + "created_at": VALIDATION_TIMESTAMP, + }, + ), + ), + ), +) diff --git a/policyengine_api/data/v2/reference_data_autogenerate.py b/policyengine_api/data/v2/reference_data_autogenerate.py new file mode 100644 index 000000000..a3efb0676 --- /dev/null +++ b/policyengine_api/data/v2/reference_data_autogenerate.py @@ -0,0 +1,344 @@ +"""Bounded Alembic operations and comparators for declared v2 reference rows.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any +from uuid import UUID + +from alembic.autogenerate import comparators, renderers +from alembic.operations import MigrateOperation, Operations, ops +import sqlalchemy as sa + +from policyengine_api.data.v2.reference_data import REFERENCE_DATA, ReferenceTable +from policyengine_api.data.v2.table_inventory import EXPECTED_V2_TABLES + + +class ReferenceDataMigrationError(RuntimeError): + """Raised when a declared row change cannot be applied or reversed safely.""" + + +def _ordered(values: dict[str, Any] | None) -> dict[str, Any] | None: + if values is None: + return None + return {key: values[key] for key in sorted(values)} + + +@Operations.register_operation("v2_reference_row_change") +class ReferenceRowChangeOp(MigrateOperation): + """One deterministic and reversible declared-row transition.""" + + def __init__( + self, + table_name: str, + *, + key: dict[str, Any], + before: dict[str, Any] | None, + after: dict[str, Any] | None, + ) -> None: + if table_name not in EXPECTED_V2_TABLES: + raise ReferenceDataMigrationError( + f"reference-data operation targets unreviewed table {table_name}" + ) + if not key or (before is None and after is None): + raise ReferenceDataMigrationError( + "reference-data operations need a stable key and one row state" + ) + self.table_name = table_name + self.key = _ordered(key) or {} + self.before = _ordered(before) + self.after = _ordered(after) + + @classmethod + def v2_reference_row_change( + cls, + operations: Operations, + table_name: str, + *, + key: dict[str, Any], + before: dict[str, Any] | None, + after: dict[str, Any] | None, + ) -> Any: + return operations.invoke(cls(table_name, key=key, before=before, after=after)) + + def reverse(self) -> "ReferenceRowChangeOp": + return ReferenceRowChangeOp( + self.table_name, + key=self.key, + before=self.after, + after=self.before, + ) + + def to_diff_tuple(self) -> tuple[Any, ...]: + """Expose deterministic drift details to ``alembic check``.""" + + return ( + "v2_reference_row_change", + self.table_name, + self.key, + self.before, + self.after, + ) + + +def _normalize(value: Any) -> Any: + if isinstance(value, UUID): + return str(value) + if isinstance(value, datetime): + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat() + if isinstance(value, dict): + return {key: _normalize(item) for key, item in sorted(value.items())} + if isinstance(value, list | tuple): + return [_normalize(item) for item in value] + return value + + +def _coerce(column: sa.Column, value: Any) -> Any: + if value is None: + return None + if isinstance(column.type, sa.Uuid) and not isinstance(value, UUID): + return UUID(str(value)) + if isinstance(column.type, sa.DateTime) and not isinstance(value, datetime): + parsed = datetime.fromisoformat(str(value)) + return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) + return value + + +def _predicate(table: sa.Table, key: dict[str, Any]) -> sa.ColumnElement: + return sa.and_( + *( + table.c[column] == _coerce(table.c[column], value) + for column, value in key.items() + ) + ) + + +def _current_row( + bind: sa.Connection, + table: sa.Table, + key: dict[str, Any], +) -> dict[str, Any] | None: + row = ( + bind.execute(sa.select(table).where(_predicate(table, key))) + .mappings() + .one_or_none() + ) + if row is None: + return None + return {column: _normalize(value) for column, value in row.items()} + + +def _assert_before( + current: dict[str, Any] | None, + expected: dict[str, Any] | None, + *, + table_name: str, +) -> None: + if expected is None: + if current is not None: + raise ReferenceDataMigrationError( + f"{table_name} insert found an existing managed row" + ) + return + if current is None or any( + current.get(column) != _normalize(value) for column, value in expected.items() + ): + raise ReferenceDataMigrationError( + f"{table_name} row differs from the generated before state" + ) + + +def _assert_after( + current: dict[str, Any] | None, + expected: dict[str, Any] | None, + *, + table_name: str, +) -> None: + if expected is None: + if current is not None: + raise ReferenceDataMigrationError( + f"{table_name} generated delete left the managed row present" + ) + return + if current is None or any( + current.get(column) != _normalize(value) for column, value in expected.items() + ): + raise ReferenceDataMigrationError( + f"{table_name} row differs from the generated after state" + ) + + +@Operations.implementation_for(ReferenceRowChangeOp) +def _apply_reference_row_change( + operations: Operations, + operation: ReferenceRowChangeOp, +) -> None: + bind = operations.get_bind() + table = sa.Table( + operation.table_name, + sa.MetaData(), + schema="public", + autoload_with=bind, + ) + current = _current_row(bind, table, operation.key) + _assert_before(current, operation.before, table_name=operation.table_name) + + if operation.after is None: + bind.execute(table.delete().where(_predicate(table, operation.key))) + elif operation.before is None: + values = { + column: _coerce(table.c[column], value) + for column, value in operation.after.items() + } + bind.execute(table.insert().values(**values)) + else: + values = { + column: _coerce(table.c[column], value) + for column, value in operation.after.items() + if column not in operation.key + } + bind.execute( + table.update().where(_predicate(table, operation.key)).values(**values) + ) + _assert_after( + _current_row(bind, table, operation.key), + operation.after, + table_name=operation.table_name, + ) + + +@renderers.dispatch_for(ReferenceRowChangeOp) +def _render_reference_row_change( + autogen_context, operation: ReferenceRowChangeOp +) -> str: + return ( + "op.v2_reference_row_change(" + f"{operation.table_name!r}, key={operation.key!r}, " + f"before={operation.before!r}, after={operation.after!r})" + ) + + +def _managed_rows( + connection: sa.Connection, + declaration: ReferenceTable, +) -> dict[tuple[Any, ...], dict[str, Any]]: + table = sa.Table( + declaration.table_name, + sa.MetaData(), + schema="public", + autoload_with=connection, + ) + prefix_column = table.c[declaration.managed_prefix_column] + rows = connection.execute( + sa.select(table).where(prefix_column.startswith(declaration.managed_prefix)) + ).mappings() + return { + tuple(_normalize(row[column]) for column in declaration.key_columns): { + column: _normalize(value) for column, value in row.items() + } + for row in rows + } + + +def _table_differences( + connection: sa.Connection, + declaration: ReferenceTable, +) -> tuple[list[ReferenceRowChangeOp], list[ReferenceRowChangeOp]]: + live = _managed_rows(connection, declaration) + desired = { + tuple(row.key[column] for column in declaration.key_columns): row + for row in declaration.rows + } + removals: list[ReferenceRowChangeOp] = [] + upserts: list[ReferenceRowChangeOp] = [] + + for stable_key in sorted(live): + if stable_key not in desired: + key = dict(zip(declaration.key_columns, stable_key)) + removals.append( + ReferenceRowChangeOp( + declaration.table_name, + key=key, + before=live[stable_key], + after=None, + ) + ) + + for stable_key in sorted(desired): + row = desired[stable_key] + desired_values = row.complete_values + current = live.get(stable_key) + if current is None: + upserts.append( + ReferenceRowChangeOp( + declaration.table_name, + key=dict(row.key), + before=None, + after=desired_values, + ) + ) + continue + tracked_current = {column: current.get(column) for column in desired_values} + if tracked_current != desired_values: + upserts.append( + ReferenceRowChangeOp( + declaration.table_name, + key=dict(row.key), + before=tracked_current, + after=desired_values, + ) + ) + return removals, upserts + + +@comparators.dispatch_for("schema") +def compare_reference_data(autogen_context, upgrade_ops, _schemas) -> None: + """Append declared row drift after the complete schema already exists.""" + + connection = autogen_context.connection + if connection is None: + return + live_tables = set(sa.inspect(connection).get_table_names(schema="public")) + # Keep the clean schema baseline separate. The next autogeneration, after + # baseline upgrade, observes all tables and emits the data-only revision. + if not EXPECTED_V2_TABLES.issubset(live_tables): + return + + differences = [ + _table_differences(connection, declaration) for declaration in REFERENCE_DATA + ] + # Delete children before parents; insert/update parents before children. + removals = [ + operation + for table_removals, _ in reversed(differences) + for operation in table_removals + ] + upserts = [ + operation for _, table_upserts in differences for operation in table_upserts + ] + upgrade_ops.ops.extend([*removals, *upserts]) + + +def _is_destructive(operation: MigrateOperation) -> bool: + if isinstance(operation, (ops.DropTableOp, ops.DropColumnOp)): + return True + if isinstance(operation, ops.ModifyTableOps): + return any(_is_destructive(child) for child in operation.ops) + return False + + +def order_generated_operations(_context, _revision, directives) -> None: + """Place data changes after schema additions and before destructive DDL.""" + + for script in directives: + operations = script.upgrade_ops.ops + data = [op for op in operations if isinstance(op, ReferenceRowChangeOp)] + non_data = [op for op in operations if not isinstance(op, ReferenceRowChangeOp)] + constructive = [op for op in non_data if not _is_destructive(op)] + destructive = [op for op in non_data if _is_destructive(op)] + script.upgrade_ops.ops = [*constructive, *data, *destructive] + script.downgrade_ops.ops = [ + operation.reverse() for operation in reversed(script.upgrade_ops.ops) + ] diff --git a/policyengine_api/data/v2/report_runs.py b/policyengine_api/data/v2/report_runs.py new file mode 100644 index 000000000..7390c3b9a --- /dev/null +++ b/policyengine_api/data/v2/report_runs.py @@ -0,0 +1,212 @@ +"""Durable report-definition and report-run operations for API v2-alpha.""" + +from __future__ import annotations + +from collections.abc import Mapping +from datetime import datetime +from uuid import UUID + +from sqlalchemy.exc import IntegrityError +from sqlmodel import Session, select + +from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_VERSION +from policyengine_api.data.v2.models import ( + Report, + ReportRun, + ReportRunStatus, + ReportRunTrigger, +) +from policyengine_api.data.v2.models.base import utc_now + + +class ReportNotFoundError(LookupError): + """Raised when a requested durable report does not exist.""" + + +class ReportRunNotFoundError(LookupError): + """Raised when a requested durable report run does not exist.""" + + +class ReportTypeImmutableError(ValueError): + """Raised when a report type change would reinterpret existing runs.""" + + +class ReportRunStateError(ValueError): + """Raised when a worker transition is invalid for the durable run.""" + + +def _locked_report(session: Session, report_id: UUID) -> Report: + # SQLModel's select remains the query surface. Row locking is the bounded + # SQLAlchemy capability needed to serialize first-run/type updates. + statement = select(Report).where(Report.id == report_id).with_for_update() + report = session.exec(statement).one_or_none() + if report is None: + raise ReportNotFoundError(f"report {report_id} does not exist") + return report + + +def set_report_type( + session: Session, + *, + report_id: UUID, + report_type: str | None, +) -> Report: + """Set a report's optional type only while it has no execution history.""" + + report = _locked_report(session, report_id) + if report.type == report_type: + return report + + existing_run_id = session.exec( + select(ReportRun.id).where(ReportRun.report_id == report_id).limit(1) + ).first() + if existing_run_id is not None: + raise ReportTypeImmutableError( + "report type is immutable after the first report run" + ) + + report.type = report_type + session.add(report) + session.flush() + return report + + +def create_report_run( + session: Session, + *, + report_id: UUID, + country_package_version: str, + policyengine_version: str, + trigger: ReportRunTrigger, + idempotency_key: str | None = None, +) -> ReportRun: + """Create one run, or return the run for a retried idempotent request.""" + + _locked_report(session, report_id) + normalized_key = idempotency_key.strip() if idempotency_key else None + if trigger is ReportRunTrigger.MANUAL and not normalized_key: + raise ValueError("manual report reruns require an idempotency key") + if not country_package_version or not policyengine_version: + raise ValueError("report run package versions must be non-empty") + + run = ReportRun( + report_id=report_id, + country_package_version=country_package_version, + policyengine_version=policyengine_version, + trigger=trigger, + idempotency_key=normalized_key, + ) + if normalized_key is None: + session.add(run) + session.flush() + return run + + try: + # The report-scoped unique constraint is authoritative under races. + # A savepoint contains the expected conflict without rolling back + # unrelated caller work in the surrounding transaction. + with session.begin_nested(): + session.add(run) + session.flush() + except IntegrityError: + existing = session.exec( + select(ReportRun).where( + ReportRun.report_id == report_id, + ReportRun.idempotency_key == normalized_key, + ) + ).one_or_none() + if existing is None: + raise + return existing + return run + + +def begin_report_run( + session: Session, + *, + report_run_id: UUID, + started_at: datetime | None = None, +) -> ReportRun: + """Start or resume the same pending/running durable run after worker retry.""" + + run = session.exec( + select(ReportRun).where(ReportRun.id == report_run_id).with_for_update() + ).one_or_none() + if run is None: + raise ReportRunNotFoundError(f"report run {report_run_id} does not exist") + if run.status in {ReportRunStatus.SUCCEEDED, ReportRunStatus.FAILED}: + raise ReportRunStateError("a terminal report run cannot be resumed") + if run.status is ReportRunStatus.PENDING: + run.status = ReportRunStatus.RUNNING + run.started_at = started_at or utc_now() + session.add(run) + session.flush() + return run + + +def complete_report_run( + session: Session, + *, + report_run_id: UUID, + completed_at: datetime | None = None, + markdown: str | None = None, +) -> ReportRun: + """Mark the selected durable run successful without replacing history.""" + + run = begin_report_run(session, report_run_id=report_run_id) + run.status = ReportRunStatus.SUCCEEDED + run.completed_at = completed_at or utc_now() + run.markdown = markdown + run.error_message = None + session.add(run) + session.flush() + return run + + +def fail_report_run( + session: Session, + *, + report_run_id: UUID, + error_message: str, + completed_at: datetime | None = None, +) -> ReportRun: + """Mark the selected durable run failed without deleting older success.""" + + run = begin_report_run(session, report_run_id=report_run_id) + run.status = ReportRunStatus.FAILED + run.completed_at = completed_at or utc_now() + run.error_message = error_message + session.add(run) + session.flush() + return run + + +def select_current_report_run( + session: Session, + *, + report_id: UUID, + country_package_versions: Mapping[str, str] = COUNTRY_PACKAGE_VERSIONS, + policyengine_version: str = POLICYENGINE_VERSION, +) -> ReportRun | None: + """Select the deterministic current successful run for deployed versions.""" + + report = session.get(Report, report_id) + if report is None: + raise ReportNotFoundError(f"report {report_id} does not exist") + country_package_version = country_package_versions.get(report.country) + if country_package_version is None: + return None + + statement = ( + select(ReportRun) + .where( + ReportRun.report_id == report_id, + ReportRun.status == ReportRunStatus.SUCCEEDED, + ReportRun.country_package_version == country_package_version, + ReportRun.policyengine_version == policyengine_version, + ReportRun.completed_at.is_not(None), + ) + .order_by(ReportRun.completed_at.desc(), ReportRun.id.desc()) + .limit(1) + ) + return session.exec(statement).first() diff --git a/policyengine_api/data/v2/settings.py b/policyengine_api/data/v2/settings.py new file mode 100644 index 000000000..6c565282d --- /dev/null +++ b/policyengine_api/data/v2/settings.py @@ -0,0 +1,217 @@ +"""Explicit, lazy configuration for dormant API v2-alpha persistence. + +Loading these settings is an operator or selected-runtime action. Importing +this module reads no environment variables, opens no connection, and creates +no local fallback. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +import os +import re +from urllib.parse import urlsplit + +from pydantic import SecretStr +from sqlalchemy.engine import URL, make_url +from sqlalchemy.exc import ArgumentError + + +V2_RUNTIME_DATABASE_URL = "V2_RUNTIME_DATABASE_URL" +V2_MIGRATION_DATABASE_URL = "V2_MIGRATION_DATABASE_URL" +V2_SUPABASE_PROJECT_REF = "V2_SUPABASE_PROJECT_REF" +V2_SUPABASE_ENVIRONMENT = "V2_SUPABASE_ENVIRONMENT" +V2_SUPABASE_STORAGE_URL = "V2_SUPABASE_STORAGE_URL" +V2_SUPABASE_STORAGE_ADMIN_KEY = "V2_SUPABASE_STORAGE_ADMIN_KEY" +V2_SUPABASE_STORAGE_BUCKET = "V2_SUPABASE_STORAGE_BUCKET" + +POSTGRES_DRIVER = "postgresql+psycopg" +PERSISTENT_SSL_MODES = frozenset({"require", "verify-ca", "verify-full"}) +LOCAL_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"}) +PROJECT_REF_PATTERN = re.compile(r"^[a-z0-9]{20}$") +ENVIRONMENT_PATTERN = re.compile(r"^[a-z][a-z0-9-]{1,31}$") +BUCKET_PATTERN = re.compile(r"^[a-z0-9][a-z0-9._-]{1,61}[a-z0-9]$") + + +class V2ConfigurationError(RuntimeError): + """Raised when explicitly selected v2 configuration is absent or unsafe.""" + + +@dataclass(frozen=True) +class PostgresConnectionSettings: + """A validated Psycopg URL whose representation never contains secrets.""" + + _url: URL = field(repr=False) + + @property + def url(self) -> URL: + """Return SQLAlchemy's immutable URL for engine construction.""" + + return self._url + + @property + def redacted_url(self) -> str: + """Return a log-safe URL with the password hidden.""" + + return self._url.render_as_string(hide_password=True) + + def __str__(self) -> str: + return self.redacted_url + + +@dataclass(frozen=True) +class SupabaseTargetSettings: + """Non-secret identity for one persistent Supabase target.""" + + project_ref: str + environment: str + + +@dataclass(frozen=True) +class V2DatabaseSettings: + """Selected runtime or migration database and its persistent identity.""" + + connection: PostgresConnectionSettings + target: SupabaseTargetSettings + + +@dataclass(frozen=True) +class SupabaseStorageSettings: + """Storage-only administration settings, separate from database access.""" + + project_ref: str + environment: str + api_url: str + bucket: str + admin_key: SecretStr = field(repr=False) + + +def _environment(environ: Mapping[str, str] | None) -> Mapping[str, str]: + return os.environ if environ is None else environ + + +def _required(environ: Mapping[str, str], name: str) -> str: + value = environ.get(name) + if value is None or not value.strip(): + raise V2ConfigurationError(f"{name} is required") + return value.strip() + + +def _load_target(environ: Mapping[str, str]) -> SupabaseTargetSettings: + project_ref = _required(environ, V2_SUPABASE_PROJECT_REF) + environment = _required(environ, V2_SUPABASE_ENVIRONMENT) + + if PROJECT_REF_PATTERN.fullmatch(project_ref) is None: + raise V2ConfigurationError( + f"{V2_SUPABASE_PROJECT_REF} is not a valid project reference" + ) + if ENVIRONMENT_PATTERN.fullmatch(environment) is None: + raise V2ConfigurationError( + f"{V2_SUPABASE_ENVIRONMENT} is not a valid environment name" + ) + return SupabaseTargetSettings( + project_ref=project_ref, + environment=environment, + ) + + +def parse_persistent_postgres_url( + raw_url: str, + *, + setting_name: str, +) -> PostgresConnectionSettings: + """Validate an explicit persistent Postgres URL without echoing it.""" + + try: + url = make_url(raw_url) + except ArgumentError as error: + raise V2ConfigurationError(f"{setting_name} is not a valid URL") from error + + if url.drivername != POSTGRES_DRIVER: + raise V2ConfigurationError( + f"{setting_name} must use the {POSTGRES_DRIVER} driver" + ) + if not url.host or url.host.lower() in LOCAL_HOSTS: + raise V2ConfigurationError( + f"{setting_name} must name a non-local Postgres host" + ) + if not url.database or not url.username or url.password is None: + raise V2ConfigurationError( + f"{setting_name} must include an explicit database and credentials" + ) + + sslmode = url.query.get("sslmode") + if not isinstance(sslmode, str) or sslmode not in PERSISTENT_SSL_MODES: + raise V2ConfigurationError( + f"{setting_name} must require TLS with sslmode=" + "require, verify-ca, or verify-full" + ) + return PostgresConnectionSettings(url) + + +def load_v2_runtime_database_settings( + environ: Mapping[str, str] | None = None, +) -> V2DatabaseSettings: + """Load the future ordinary-runtime Postgres identity explicitly.""" + + values = _environment(environ) + connection = parse_persistent_postgres_url( + _required(values, V2_RUNTIME_DATABASE_URL), + setting_name=V2_RUNTIME_DATABASE_URL, + ) + return V2DatabaseSettings(connection=connection, target=_load_target(values)) + + +def load_v2_migration_database_settings( + environ: Mapping[str, str] | None = None, +) -> V2DatabaseSettings: + """Load the schema-migration Postgres identity explicitly.""" + + values = _environment(environ) + connection = parse_persistent_postgres_url( + _required(values, V2_MIGRATION_DATABASE_URL), + setting_name=V2_MIGRATION_DATABASE_URL, + ) + return V2DatabaseSettings(connection=connection, target=_load_target(values)) + + +def load_supabase_storage_settings( + environ: Mapping[str, str] | None = None, +) -> SupabaseStorageSettings: + """Load the separately authorized Supabase Storage administration surface.""" + + values = _environment(environ) + target = _load_target(values) + api_url = _required(values, V2_SUPABASE_STORAGE_URL) + bucket = _required(values, V2_SUPABASE_STORAGE_BUCKET) + admin_key = _required(values, V2_SUPABASE_STORAGE_ADMIN_KEY) + + parsed_url = urlsplit(api_url) + expected_host = f"{target.project_ref}.supabase.co" + if ( + parsed_url.scheme != "https" + or parsed_url.hostname != expected_host + or parsed_url.username is not None + or parsed_url.password is not None + or parsed_url.port is not None + or parsed_url.path.rstrip("/") + or parsed_url.query + or parsed_url.fragment + ): + raise V2ConfigurationError( + f"{V2_SUPABASE_STORAGE_URL} must be the HTTPS API origin for the " + "recorded project reference" + ) + if BUCKET_PATTERN.fullmatch(bucket) is None: + raise V2ConfigurationError( + f"{V2_SUPABASE_STORAGE_BUCKET} is not a valid bucket name" + ) + + return SupabaseStorageSettings( + project_ref=target.project_ref, + environment=target.environment, + api_url=api_url.rstrip("/"), + bucket=bucket, + admin_key=SecretStr(admin_key), + ) diff --git a/policyengine_api/data/v2/storage_bootstrap.py b/policyengine_api/data/v2/storage_bootstrap.py new file mode 100644 index 000000000..bcd3e1929 --- /dev/null +++ b/policyengine_api/data/v2/storage_bootstrap.py @@ -0,0 +1,213 @@ +"""Explicit, idempotent bootstrap for the Stage 8 private Storage bucket.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Protocol +from urllib.parse import quote + +import httpx + +from policyengine_api.data.v2.settings import SupabaseStorageSettings + + +RECORDED_STORAGE_TARGETS = { + "production-foundation": "kvrifaviwhzjztcbrfpy", +} +STORAGE_REQUEST_TIMEOUT_SECONDS = 10.0 + + +class StorageBootstrapError(RuntimeError): + """Raised without response bodies or credentials when bootstrap is unsafe.""" + + +class StorageHTTPClient(Protocol): + """Narrow HTTP surface needed by the Storage initializer.""" + + def get(self, url: str, **kwargs: Any) -> httpx.Response: ... + + def post(self, url: str, **kwargs: Any) -> httpx.Response: ... + + +@dataclass(frozen=True) +class StorageBucketConfiguration: + """Reviewed Stage 8 Storage bucket properties.""" + + id: str + name: str + public: bool = False + file_size_limit: int | None = None + allowed_mime_types: tuple[str, ...] | None = None + + def create_payload(self) -> dict[str, Any]: + return { + "id": self.id, + "name": self.name, + "public": self.public, + "file_size_limit": self.file_size_limit, + "allowed_mime_types": ( + list(self.allowed_mime_types) + if self.allowed_mime_types is not None + else None + ), + } + + +@dataclass(frozen=True) +class StorageBootstrapResult: + """Secret-free result suitable for operator output.""" + + bucket: str + created: bool + public: bool + environment: str + project_ref: str + + +def _qualify_target(settings: SupabaseStorageSettings) -> None: + recorded_ref = RECORDED_STORAGE_TARGETS.get(settings.environment) + if recorded_ref != settings.project_ref: + raise StorageBootstrapError( + "Storage environment and project reference do not match the " + "recorded Stage 8 target" + ) + + +def _headers(settings: SupabaseStorageSettings) -> dict[str, str]: + key = settings.admin_key.get_secret_value() + return { + "apikey": key, + "Content-Type": "application/json", + } + + +def _decode_bucket(response: httpx.Response) -> dict[str, Any]: + try: + value = response.json() + except ValueError as error: + raise StorageBootstrapError( + "Supabase Storage returned an invalid bucket response" + ) from error + if not isinstance(value, dict): + raise StorageBootstrapError( + "Supabase Storage returned an invalid bucket response" + ) + return value + + +def _storage_error(response: httpx.Response) -> tuple[str | None, str | None]: + """Return only stable, non-secret Storage error identifiers.""" + + try: + value = response.json() + except ValueError: + return None, None + if not isinstance(value, dict): + return None, None + code = value.get("code") + status_code = value.get("statusCode", value.get("httpStatusCode")) + return ( + code if isinstance(code, str) else None, + str(status_code) if status_code is not None else None, + ) + + +def _is_missing_bucket(response: httpx.Response) -> bool: + code, status_code = _storage_error(response) + return response.status_code == 404 or code == "NoSuchBucket" or status_code == "404" + + +def _is_creation_conflict(response: httpx.Response) -> bool: + code, status_code = _storage_error(response) + return ( + response.status_code == 409 + or code in {"BucketAlreadyExists", "ResourceAlreadyExists"} + or status_code == "409" + ) + + +def _verify_bucket( + observed: dict[str, Any], + expected: StorageBucketConfiguration, +) -> None: + expected_values = { + "id": expected.id, + "name": expected.name, + "public": expected.public, + "file_size_limit": expected.file_size_limit, + "allowed_mime_types": ( + list(expected.allowed_mime_types) + if expected.allowed_mime_types is not None + else None + ), + } + incompatible = { + field: {"expected": expected_value, "observed": observed.get(field)} + for field, expected_value in expected_values.items() + if observed.get(field) != expected_value + } + if incompatible: + fields = ", ".join(sorted(incompatible)) + raise StorageBootstrapError( + f"existing Storage bucket has incompatible fields: {fields}" + ) + + +def initialize_supabase_storage( + settings: SupabaseStorageSettings, + *, + client: StorageHTTPClient | None = None, +) -> StorageBootstrapResult: + """Create or verify the recorded private bucket without replacing it.""" + + _qualify_target(settings) + expected = StorageBucketConfiguration( + id=settings.bucket, + name=settings.bucket, + ) + headers = _headers(settings) + bucket_url = ( + f"{settings.api_url}/storage/v1/bucket/{quote(settings.bucket, safe='')}" + ) + collection_url = f"{settings.api_url}/storage/v1/bucket" + owns_client = client is None + active_client = client or httpx.Client( + timeout=STORAGE_REQUEST_TIMEOUT_SECONDS, + follow_redirects=False, + ) + created = False + try: + try: + response = active_client.get(bucket_url, headers=headers) + if _is_missing_bucket(response): + response = active_client.post( + collection_url, + headers=headers, + json=expected.create_payload(), + ) + if _is_creation_conflict(response): + response = active_client.get(bucket_url, headers=headers) + elif 200 <= response.status_code < 300: + created = True + response = active_client.get(bucket_url, headers=headers) + if not 200 <= response.status_code < 300: + raise StorageBootstrapError( + "Supabase Storage bucket verification failed with status " + f"{response.status_code}" + ) + _verify_bucket(_decode_bucket(response), expected) + except httpx.HTTPError as error: + raise StorageBootstrapError( + "Supabase Storage bucket verification request failed" + ) from error + finally: + if owns_client: + active_client.close() # type: ignore[attr-defined] + + return StorageBootstrapResult( + bucket=expected.id, + created=created, + public=expected.public, + environment=settings.environment, + project_ref=settings.project_ref, + ) diff --git a/policyengine_api/data/v2/table_inventory.py b/policyengine_api/data/v2/table_inventory.py new file mode 100644 index 000000000..f88340e3f --- /dev/null +++ b/policyengine_api/data/v2/table_inventory.py @@ -0,0 +1,160 @@ +"""Reviewed Stage 8 API v2-alpha application-table inventory. + +This module is deliberately independent of ORM imports. SQLModel metadata, +Alembic generation, lifecycle tests, and live-schema comparisons all use this +single allowlist. Supabase-managed tables and Alembic's version table are not +application tables and are outside this inventory. +""" + +from collections.abc import Iterable + + +V2_TABLE_GROUPS: tuple[tuple[str, frozenset[str]], ...] = ( + ( + "identity", + frozenset( + { + "users", + "user_household_associations", + "user_policies", + "user_report_associations", + "user_simulation_associations", + } + ), + ), + ( + "model_metadata", + frozenset( + { + "tax_benefit_models", + "tax_benefit_model_versions", + } + ), + ), + ( + "regions_and_datasets", + frozenset( + { + "regions", + "datasets", + "dataset_versions", + "region_datasets", + } + ), + ), + ( + "variables_and_parameters", + frozenset( + { + "variables", + "parameter_nodes", + "parameters", + "parameter_values", + } + ), + ), + ( + "policies", + frozenset( + { + "policies", + "dynamics", + } + ), + ), + ( + "households_and_simulations", + frozenset( + { + "households", + "household_jobs", + "simulations", + } + ), + ), + ( + "reports_and_outputs", + frozenset( + { + "reports", + "report_runs", + "aggregates", + "change_aggregates", + } + ), + ), + ( + "impact_results", + frozenset( + { + "budget_summary", + "congressional_district_impacts", + "constituency_impacts", + "decile_impacts", + "inequality", + "intra_decile_impacts", + "local_authority_impacts", + "poverty", + "program_statistics", + } + ), + ), +) + +EXPECTED_V2_TABLES = frozenset( + table_name for _, table_names in V2_TABLE_GROUPS for table_name in table_names +) + +# These v1-only names make accidental V1Base metadata registration especially +# clear. Names intentionally reviewed for both domains, such as `simulations` +# and `user_policies`, remain valid because the exact allowlist is authoritative. +V1_ONLY_TABLES = frozenset( + { + "analysis", + "computed_household", + "economy", + "household", + "legacy_report_output_aliases", + "policy", + "reform_impact", + "report_output_runs", + "report_outputs", + "simulation_runs", + "tracers", + "user_profiles", + } +) + +# Stage 8 explicitly rejects the former runtime-bundle indirection and any +# standalone population model. Population-valued columns on reviewed impact +# tables are unrelated to these prohibited table names. +PROHIBITED_V2_TABLES = frozenset( + { + "population", + "populations", + "runtime_bundle", + "runtime_bundles", + } +) + + +class V2TableInventoryError(RuntimeError): + """Raised when table metadata differs from the reviewed Stage 8 schema.""" + + +def validate_v2_table_inventory(table_names: Iterable[str]) -> None: + """Fail closed unless *table_names* exactly match the reviewed allowlist.""" + + actual = frozenset(table_names) + if actual == EXPECTED_V2_TABLES: + return + + missing = sorted(EXPECTED_V2_TABLES - actual) + unexpected = sorted(actual - EXPECTED_V2_TABLES) + prohibited = sorted(actual & PROHIBITED_V2_TABLES) + v1_only = sorted(actual & V1_ONLY_TABLES) + raise V2TableInventoryError( + "API v2-alpha table inventory mismatch: " + f"missing={missing}, unexpected={unexpected}, " + f"prohibited={prohibited}, v1_only={v1_only}" + ) diff --git a/policyengine_api/gcp_logging.py b/policyengine_api/gcp_logging.py index 630afb132..996dda4fd 100644 --- a/policyengine_api/gcp_logging.py +++ b/policyengine_api/gcp_logging.py @@ -1,4 +1,5 @@ import logging +import os from typing import Optional @@ -12,6 +13,9 @@ def __init__(self, logger_name: str): self._fallback_logger = logging.getLogger(logger_name) def _get_google_logger(self): + if not (os.environ.get("GAE_ENV") or os.environ.get("K_SERVICE")): + self._initialization_failed = True + return None if self._google_logger is not None: return self._google_logger if self._initialization_failed: @@ -34,8 +38,15 @@ def log_struct( ) -> None: google_logger = self._get_google_logger() if google_logger is not None: - google_logger.log_struct(info, severity=severity, labels=labels) - return + try: + google_logger.log_struct(info, severity=severity, labels=labels) + return + except Exception: + # Observability must never invalidate a successful request or + # cache operation. App Engine and Cloud Run collect stderr as + # a fallback when the structured logging API is unavailable. + self._google_logger = None + self._initialization_failed = True level = getattr(logging, severity.upper(), logging.INFO) self._fallback_logger.log(level, "%s", info) diff --git a/policyengine_api/runtime_cache/__init__.py b/policyengine_api/runtime_cache/__init__.py new file mode 100644 index 000000000..95255b102 --- /dev/null +++ b/policyengine_api/runtime_cache/__init__.py @@ -0,0 +1,6 @@ +"""Shared recoverable runtime-cache boundary for API services.""" + +from policyengine_api.runtime_cache.client import get_runtime_cache_client +from policyengine_api.runtime_cache.settings import load_runtime_cache_settings + +__all__ = ["get_runtime_cache_client", "load_runtime_cache_settings"] diff --git a/policyengine_api/runtime_cache/claims.py b/policyengine_api/runtime_cache/claims.py new file mode 100644 index 000000000..6f3c48b70 --- /dev/null +++ b/policyengine_api/runtime_cache/claims.py @@ -0,0 +1,78 @@ +"""Fail-closed atomic ownership claims for expensive shared work.""" + +from __future__ import annotations + +import time + +from policyengine_api.runtime_cache.core import ( + CacheBackend, + CacheCoordinationError, + record_cache_event, +) + + +COMPARE_AND_DELETE = """ +if redis.call('get', KEYS[1]) == ARGV[1] then + return redis.call('del', KEYS[1]) +end +return 0 +""".strip() + + +class ExpiringClaimStore: + def __init__( + self, + client: CacheBackend, + *, + family: str = "coordination", + ) -> None: + self.client = client + self.family = family + + def acquire(self, key: str, token: str, *, ttl_seconds: int) -> bool: + if not token or ttl_seconds <= 0: + raise ValueError("claim token and positive TTL are required") + started_at = time.perf_counter() + try: + acquired = bool(self.client.set(key, token, nx=True, ex=ttl_seconds)) + except Exception as error: + record_cache_event( + family=self.family, + event="coordination-failed", + operation="claim-acquire", + started_at=started_at, + severity="WARNING", + ) + raise CacheCoordinationError( + "shared-cache ownership could not be established" + ) from error + record_cache_event( + family=self.family, + event="claim-acquired" if acquired else "claim-contended", + operation="claim-acquire", + started_at=started_at, + ) + return acquired + + def release(self, key: str, token: str) -> bool: + started_at = time.perf_counter() + try: + released = bool(self.client.eval(COMPARE_AND_DELETE, 1, key, token)) + except Exception as error: + record_cache_event( + family=self.family, + event="coordination-failed", + operation="claim-release", + started_at=started_at, + severity="WARNING", + ) + raise CacheCoordinationError( + "shared-cache ownership could not be released safely" + ) from error + record_cache_event( + family=self.family, + event="claim-released" if released else "claim-release-rejected", + operation="claim-release", + started_at=started_at, + ) + return released diff --git a/policyengine_api/runtime_cache/client.py b/policyengine_api/runtime_cache/client.py new file mode 100644 index 000000000..24c1d39f9 --- /dev/null +++ b/policyengine_api/runtime_cache/client.py @@ -0,0 +1,58 @@ +"""Lazy, process-owned Redis client construction.""" + +from __future__ import annotations + +import os +from threading import Lock +from typing import Any + +import redis + +from policyengine_api.runtime_cache.settings import ( + RuntimeCacheSettings, + load_runtime_cache_settings, +) + + +_clients: dict[int, redis.Redis] = {} +_client_lock = Lock() + + +def build_runtime_cache_client(settings: RuntimeCacheSettings) -> redis.Redis: + if not settings.enabled or settings.url is None: + raise RuntimeError("the shared runtime cache is disabled") + kwargs: dict[str, Any] = { + "decode_responses": True, + "max_connections": settings.max_connections, + "socket_connect_timeout": settings.connect_timeout_seconds, + "socket_timeout": settings.operation_timeout_seconds, + "health_check_interval": 30, + "retry_on_timeout": False, + } + if settings.tls: + kwargs["ssl_cert_reqs"] = "required" + if settings.ca_cert is None: + raise RuntimeError("the shared runtime cache TLS CA is missing") + kwargs["ssl_ca_data"] = settings.ca_cert.get_secret_value() + return redis.Redis.from_url(settings.url.get_secret_value(), **kwargs) + + +def get_runtime_cache_client( + settings: RuntimeCacheSettings | None = None, +) -> redis.Redis: + """Return one client per process without connecting during import.""" + + process_id = os.getpid() + if process_id not in _clients: + with _client_lock: + if process_id not in _clients: + _clients[process_id] = build_runtime_cache_client( + settings or load_runtime_cache_settings() + ) + return _clients[process_id] + + +def close_runtime_cache_clients() -> None: + for client in _clients.values(): + client.close() + _clients.clear() diff --git a/policyengine_api/runtime_cache/core.py b/policyengine_api/runtime_cache/core.py new file mode 100644 index 000000000..0422a158c --- /dev/null +++ b/policyengine_api/runtime_cache/core.py @@ -0,0 +1,217 @@ +"""Versioned envelopes, deterministic namespaced keys, and cache semantics.""" + +from __future__ import annotations + +from dataclasses import dataclass +import hashlib +import json +import random +import time +from typing import Any, Callable, Protocol + +from policyengine_api.gcp_logging import logger + + +CACHE_KEY_ROOT = "policyengine" +COMPLETED_RESULT_TTL_JITTER_FRACTION = 0.10 + + +class CacheBackend(Protocol): + def get(self, key: str) -> Any: ... + def set(self, key: str, value: str, **kwargs: Any) -> Any: ... + def delete(self, *keys: str) -> Any: ... + def pipeline(self, transaction: bool = True): ... + def eval(self, script: str, numkeys: int, *keys_and_args: str) -> Any: ... + + +class CacheCoordinationError(RuntimeError): + """A fail-closed ownership or deduplication failure.""" + + +@dataclass(frozen=True) +class CacheNamespace: + environment: str + service: str + + def key( + self, + family: str, + schema_version: int, + inputs: dict[str, Any], + *, + suffix: str | None = None, + ) -> str: + encoded = json.dumps( + inputs, + sort_keys=True, + separators=(",", ":"), + default=str, + ) + digest = hashlib.sha256(encoded.encode("utf-8")).hexdigest() + key = ( + f"{CACHE_KEY_ROOT}:{self.environment}:{self.service}:" + f"{family}:v{schema_version}:{digest}" + ) + return f"{key}:{suffix}" if suffix else key + + def family_key(self, family: str, schema_version: int, name: str) -> str: + return ( + f"{CACHE_KEY_ROOT}:{self.environment}:{self.service}:" + f"{family}:v{schema_version}:{name}" + ) + + +def encode_envelope( + family: str, + schema_version: int, + payload: Any, +) -> str: + return json.dumps( + { + "family": family, + "payload": payload, + "schema_version": schema_version, + }, + sort_keys=True, + separators=(",", ":"), + default=str, + ) + + +def decode_envelope( + value: str | bytes | None, + *, + family: str, + schema_version: int, +) -> Any | None: + if value is None: + return None + try: + if isinstance(value, bytes): + value = value.decode("utf-8") + decoded = json.loads(value) + except (UnicodeDecodeError, TypeError, ValueError): + return None + if not isinstance(decoded, dict): + return None + if decoded.get("family") != family: + return None + if decoded.get("schema_version") != schema_version: + return None + return decoded.get("payload") + + +def jittered_ttl( + ttl_seconds: int, + *, + jitter_fraction: float = COMPLETED_RESULT_TTL_JITTER_FRACTION, + choose_reduction: Callable[[int, int], int] = random.randint, +) -> int: + """Return a subtract-only jittered TTL for recoverable completed results. + + Coordination and claim TTLs are safety bounds and must not use this helper. + """ + + if ttl_seconds <= 0: + raise ValueError("cache TTL must be positive") + if not 0 <= jitter_fraction < 1: + raise ValueError("TTL jitter fraction must be at least 0 and less than 1") + max_reduction = int(ttl_seconds * jitter_fraction) + reduction = choose_reduction(0, max_reduction) + if not 0 <= reduction <= max_reduction: + raise ValueError("TTL jitter reduction is outside the permitted range") + return ttl_seconds - reduction + + +def record_cache_event( + *, + family: str, + event: str, + started_at: float, + severity: str = "INFO", + operation: str | None = None, +) -> None: + """Emit one metric-ready event without accepting cache keys or values.""" + + payload: dict[str, str | int | float] = { + "message": "Runtime cache operation", + "metric_name": "runtime_cache_operations", + "metric_value": 1, + "cache_family": family, + "cache_event": event, + "latency_ms": round((time.perf_counter() - started_at) * 1000, 3), + } + if operation is not None: + payload["cache_operation"] = operation + logger.log_struct(payload, severity=severity) + + +class RecoverableJSONCache: + """Completed-result cache where failures and invalid values are misses.""" + + def __init__( + self, + client: CacheBackend, + namespace: CacheNamespace, + *, + family: str, + schema_version: int, + ttl_seconds: int, + ) -> None: + if ttl_seconds <= 0: + raise ValueError("cache TTL must be positive") + self.client = client + self.namespace = namespace + self.family = family + self.schema_version = schema_version + self.ttl_seconds = ttl_seconds + + def key(self, inputs: dict[str, Any]) -> str: + return self.namespace.key(self.family, self.schema_version, inputs) + + def get(self, inputs: dict[str, Any]) -> Any | None: + started_at = time.perf_counter() + try: + value = self.client.get(self.key(inputs)) + except Exception: + record_cache_event( + family=self.family, + event="connection-failed", + started_at=started_at, + severity="WARNING", + ) + return None + payload = decode_envelope( + value, + family=self.family, + schema_version=self.schema_version, + ) + record_cache_event( + family=self.family, + event="hit" if payload is not None else "miss", + started_at=started_at, + ) + return payload + + def set(self, inputs: dict[str, Any], payload: Any) -> bool: + started_at = time.perf_counter() + try: + result = self.client.set( + self.key(inputs), + encode_envelope(self.family, self.schema_version, payload), + ex=jittered_ttl(self.ttl_seconds), + ) + except Exception: + record_cache_event( + family=self.family, + event="write-failed", + started_at=started_at, + severity="WARNING", + ) + return False + record_cache_event( + family=self.family, + event="write", + started_at=started_at, + ) + return bool(result) diff --git a/policyengine_api/runtime_cache/dependencies.py b/policyengine_api/runtime_cache/dependencies.py new file mode 100644 index 000000000..c5cf2a97d --- /dev/null +++ b/policyengine_api/runtime_cache/dependencies.py @@ -0,0 +1,38 @@ +"""Lazy default cache backend and namespace dependencies.""" + +from __future__ import annotations + +from dataclasses import dataclass +from functools import lru_cache + +from policyengine_api.runtime_cache.client import get_runtime_cache_client +from policyengine_api.runtime_cache.core import CacheBackend, CacheNamespace +from policyengine_api.runtime_cache.fake import DisabledCacheBackend +from policyengine_api.runtime_cache.settings import load_runtime_cache_settings + + +@dataclass(frozen=True) +class RuntimeCacheContext: + client: CacheBackend + namespace: CacheNamespace + enabled: bool + + +@lru_cache(maxsize=1) +def get_runtime_cache_context() -> RuntimeCacheContext: + settings = load_runtime_cache_settings() + namespace = CacheNamespace(settings.environment, settings.service) + client: CacheBackend = ( + get_runtime_cache_client(settings) + if settings.enabled + else DisabledCacheBackend() + ) + return RuntimeCacheContext( + client=client, + namespace=namespace, + enabled=settings.enabled, + ) + + +def clear_runtime_cache_context() -> None: + get_runtime_cache_context.cache_clear() diff --git a/policyengine_api/runtime_cache/fake.py b/policyengine_api/runtime_cache/fake.py new file mode 100644 index 000000000..044a07713 --- /dev/null +++ b/policyengine_api/runtime_cache/fake.py @@ -0,0 +1,199 @@ +"""Deterministic in-memory cache backend for pure unit tests only.""" + +from __future__ import annotations + +from collections.abc import Callable +from threading import RLock +from typing import Any + + +class InMemoryCacheBackend: + """Small Redis-like fake with explicit time advancement and atomic pipelines.""" + + def __init__(self, *, now: Callable[[], float] | None = None) -> None: + self._external_now = now + self._time = 0.0 + self._values: dict[str, Any] = {} + self._expires: dict[str, float] = {} + self._sorted_sets: dict[str, dict[str, float]] = {} + self._lock = RLock() + + def _now(self) -> float: + return self._external_now() if self._external_now else self._time + + def advance(self, seconds: float) -> None: + if self._external_now is not None: + raise RuntimeError("cannot advance an externally clocked fake") + self._time += seconds + + def _purge(self, key: str) -> None: + expires_at = self._expires.get(key) + if expires_at is not None and expires_at <= self._now(): + self._values.pop(key, None) + self._sorted_sets.pop(key, None) + self._expires.pop(key, None) + + def get(self, key: str) -> Any: + with self._lock: + self._purge(key) + return self._values.get(key) + + def set( + self, + key: str, + value: Any, + *, + ex: int | None = None, + nx: bool = False, + ) -> bool | None: + with self._lock: + self._purge(key) + if nx and key in self._values: + return None + self._values[key] = value + if ex is not None: + self._expires[key] = self._now() + ex + else: + self._expires.pop(key, None) + return True + + def delete(self, *keys: str) -> int: + with self._lock: + deleted = 0 + for key in keys: + self._purge(key) + if key in self._values or key in self._sorted_sets: + deleted += 1 + self._values.pop(key, None) + self._sorted_sets.pop(key, None) + self._expires.pop(key, None) + return deleted + + def expire(self, key: str, seconds: int) -> bool: + with self._lock: + self._purge(key) + if key not in self._values and key not in self._sorted_sets: + return False + self._expires[key] = self._now() + seconds + return True + + def zadd(self, key: str, mapping: dict[str, float]) -> int: + with self._lock: + self._purge(key) + values = self._sorted_sets.setdefault(key, {}) + added = sum(member not in values for member in mapping) + values.update(mapping) + return added + + def zrevrange(self, key: str, start: int, end: int) -> list[str]: + with self._lock: + self._purge(key) + ordered = sorted( + self._sorted_sets.get(key, {}).items(), + key=lambda item: (item[1], item[0]), + reverse=True, + ) + stop = None if end == -1 else end + 1 + return [member for member, _ in ordered[start:stop]] + + def zrem(self, key: str, *members: str) -> int: + with self._lock: + values = self._sorted_sets.get(key, {}) + removed = sum(member in values for member in members) + for member in members: + values.pop(member, None) + return removed + + def zremrangebyrank(self, key: str, start: int, end: int) -> int: + with self._lock: + self._purge(key) + ordered = sorted( + self._sorted_sets.get(key, {}).items(), + key=lambda item: (item[1], item[0]), + ) + stop = None if end == -1 else end + 1 + members = [member for member, _ in ordered[start:stop]] + return self.zrem(key, *members) + + def eval( + self, + _script: str, + numkeys: int, + *keys_and_args: str, + ) -> int: + if numkeys != 1 or len(keys_and_args) != 2: + raise ValueError("fake supports one-key compare-and-delete only") + key, expected = keys_and_args + with self._lock: + if self.get(key) != expected: + return 0 + return self.delete(key) + + def pipeline(self, transaction: bool = True) -> "InMemoryPipeline": + return InMemoryPipeline(self, transaction=transaction) + + +class InMemoryPipeline: + def __init__(self, backend: InMemoryCacheBackend, *, transaction: bool) -> None: + self.backend = backend + self.transaction = transaction + self.operations: list[tuple[str, tuple[Any, ...], dict[str, Any]]] = [] + + def __getattr__(self, name: str): + def queue(*args: Any, **kwargs: Any) -> "InMemoryPipeline": + self.operations.append((name, args, kwargs)) + return self + + return queue + + def execute(self) -> list[Any]: + lock = self.backend._lock if self.transaction else RLock() + with lock: + results = [ + getattr(self.backend, name)(*args, **kwargs) + for name, args, kwargs in self.operations + ] + self.operations.clear() + return results + + def __enter__(self) -> "InMemoryPipeline": + return self + + def __exit__(self, *_args: Any) -> None: + self.operations.clear() + + +class DisabledCacheBackend(InMemoryCacheBackend): + """No-storage backend for unselected unit-test/application cache mode.""" + + def get(self, key: str) -> None: + return None + + def set(self, key: str, value: Any, **kwargs: Any) -> bool: + return False + + def delete(self, *keys: str) -> int: + return 0 + + def zadd(self, key: str, mapping: dict[str, float]) -> int: + return 0 + + def zrevrange(self, key: str, start: int, end: int) -> list[str]: + return [] + + def zrem(self, key: str, *members: str) -> int: + return 0 + + def zremrangebyrank(self, key: str, start: int, end: int) -> int: + return 0 + + def expire(self, key: str, seconds: int) -> bool: + return False + + def eval( + self, + _script: str, + numkeys: int, + *keys_and_args: str, + ) -> int: + raise RuntimeError("the shared runtime cache is disabled") diff --git a/policyengine_api/runtime_cache/repositories.py b/policyengine_api/runtime_cache/repositories.py new file mode 100644 index 000000000..0352b7c2e --- /dev/null +++ b/policyengine_api/runtime_cache/repositories.py @@ -0,0 +1,539 @@ +"""Typed repositories for recoverable API runtime state.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass, replace +from datetime import datetime, timezone +import hashlib +import re +import time +from typing import Any + +from policyengine_api.runtime_cache.claims import ExpiringClaimStore +from policyengine_api.runtime_cache.core import ( + CacheBackend, + CacheNamespace, + RecoverableJSONCache, + decode_envelope, + encode_envelope, + jittered_ttl, + record_cache_event, +) + + +HOUSEHOLD_TRACE_SCHEMA_VERSION = 1 +HOUSEHOLD_TRACE_TTL_SECONDS = 86_400 +AI_ANALYSIS_SCHEMA_VERSION = 1 +AI_ANALYSIS_TTL_SECONDS = 604_800 +REFORM_IMPACT_SCHEMA_VERSION = 1 +REFORM_IMPACT_TTL_SECONDS = 2_592_000 +REFORM_IMPACT_INDEX_LIMIT = 1_000 +REFORM_IMPACT_START_CLAIM_TTL_SECONDS = 300 + + +@dataclass(frozen=True) +class HouseholdTraceIdentity: + country_id: str + household_id: int + policy_id: int + household_hash: str + policy_hash: str + country_package_version: str + policyengine_version: str + + +@dataclass(frozen=True) +class HouseholdTraceValue: + household: dict[str, Any] + tracer_output: list[str] + + +class HouseholdTraceCache: + """One atomic value for a computed household and its matching tracer.""" + + def __init__(self, client: CacheBackend, namespace: CacheNamespace) -> None: + self._cache = RecoverableJSONCache( + client, + namespace, + family="household-trace", + schema_version=HOUSEHOLD_TRACE_SCHEMA_VERSION, + ttl_seconds=HOUSEHOLD_TRACE_TTL_SECONDS, + ) + + def cache_key(self, identity: HouseholdTraceIdentity) -> str: + return self._cache.key(asdict(identity)) + + def get(self, identity: HouseholdTraceIdentity) -> HouseholdTraceValue | None: + payload = self._cache.get(asdict(identity)) + if not isinstance(payload, dict): + return None + household = payload.get("household") + tracer_output = payload.get("tracer_output") + if not isinstance(household, dict) or not isinstance(tracer_output, list): + return None + if not all(isinstance(line, str) for line in tracer_output): + return None + return HouseholdTraceValue( + household=household, + tracer_output=tracer_output, + ) + + def set( + self, + identity: HouseholdTraceIdentity, + value: HouseholdTraceValue, + ) -> bool: + return self._cache.set(asdict(identity), asdict(value)) + + +@dataclass(frozen=True) +class CachedAnalysis: + prompt: str + analysis: str + status: str = "ok" + + +class AIAnalysisCache: + def __init__(self, client: CacheBackend, namespace: CacheNamespace) -> None: + self._cache = RecoverableJSONCache( + client, + namespace, + family="ai-analysis", + schema_version=AI_ANALYSIS_SCHEMA_VERSION, + ttl_seconds=AI_ANALYSIS_TTL_SECONDS, + ) + + @staticmethod + def _inputs(prompt: str, model: str) -> dict[str, str]: + return {"model": model, "prompt": prompt} + + def get(self, prompt: str, *, model: str) -> CachedAnalysis | None: + payload = self._cache.get(self._inputs(prompt, model)) + if not isinstance(payload, dict): + return None + if payload.get("prompt") != prompt or not isinstance( + payload.get("analysis"), str + ): + return None + return CachedAnalysis( + prompt=prompt, + analysis=payload["analysis"], + status=str(payload.get("status", "ok")), + ) + + def set(self, value: CachedAnalysis, *, model: str) -> bool: + return self._cache.set(self._inputs(value.prompt, model), asdict(value)) + + +@dataclass(frozen=True) +class CachedReformImpact: + reform_impact_id: int + baseline_policy_id: int + reform_policy_id: int + country_id: str + region: str + dataset: str + time_period: str + options_json: dict[str, Any] | None + options_hash: str | None + api_version: str + reform_impact_json: dict[str, Any] + status: str + message: str | None + start_time: datetime | None + end_time: datetime | None + execution_id: str + + +def _datetime_to_wire(value: datetime | None) -> str | None: + if value is None: + return None + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat() + + +def _datetime_from_wire(value: Any) -> datetime | None: + if value is None: + return None + if not isinstance(value, str): + raise ValueError("invalid cached datetime") + parsed = datetime.fromisoformat(value) + if parsed.tzinfo is not None: + parsed = parsed.astimezone(timezone.utc).replace(tzinfo=None) + return parsed + + +def _impact_to_wire(impact: CachedReformImpact) -> dict[str, Any]: + values = asdict(impact) + values["start_time"] = _datetime_to_wire(impact.start_time) + values["end_time"] = _datetime_to_wire(impact.end_time) + return values + + +def _impact_from_wire(payload: Any) -> CachedReformImpact | None: + if not isinstance(payload, dict): + return None + try: + return CachedReformImpact( + reform_impact_id=int(payload["reform_impact_id"]), + baseline_policy_id=int(payload["baseline_policy_id"]), + reform_policy_id=int(payload["reform_policy_id"]), + country_id=str(payload["country_id"]), + region=str(payload["region"]), + dataset=str(payload["dataset"]), + time_period=str(payload["time_period"]), + options_json=payload.get("options_json"), + options_hash=payload.get("options_hash"), + api_version=str(payload["api_version"]), + reform_impact_json=payload["reform_impact_json"], + status=str(payload["status"]), + message=payload.get("message"), + start_time=_datetime_from_wire(payload.get("start_time")), + end_time=_datetime_from_wire(payload.get("end_time")), + execution_id=str(payload["execution_id"]), + ) + except (KeyError, TypeError, ValueError): + return None + + +def _like_matches(value: str, pattern: str) -> bool: + expression: list[str] = ["^"] + escaped = False + for character in pattern: + if escaped: + expression.append(re.escape(character)) + escaped = False + elif character == "\\": + escaped = True + elif character == "%": + expression.append(".*") + elif character == "_": + expression.append(".") + else: + expression.append(re.escape(character)) + if escaped: + expression.append(re.escape("\\")) + expression.append("$") + return re.match("".join(expression), value) is not None + + +class ReformImpactCache: + """Expiring reform-impact values with bounded expiring lookup indexes.""" + + family = "reform-impact" + + def __init__(self, client: CacheBackend, namespace: CacheNamespace) -> None: + self.client = client + self.namespace = namespace + self._start_claims = ExpiringClaimStore(client, family=self.family) + + def _start_claim_key( + self, + *, + country_id: str, + reform_policy_id: int, + baseline_policy_id: int, + region: str, + dataset: str, + time_period: str, + api_version: str, + options_hash: str, + target: str, + ) -> str: + return self.namespace.key( + "reform-impact-start-claim", + REFORM_IMPACT_SCHEMA_VERSION, + { + "api_version": api_version, + "baseline_policy_id": baseline_policy_id, + "country_id": country_id, + "dataset": dataset, + "options_hash": options_hash, + "reform_policy_id": reform_policy_id, + "region": region, + "target": target, + "time_period": time_period, + }, + ) + + def claim_start( + self, + *, + country_id: str, + reform_policy_id: int, + baseline_policy_id: int, + region: str, + dataset: str, + time_period: str, + api_version: str, + options_hash: str, + target: str, + claim_token: str, + ) -> bool: + """Atomically claim ownership of one reform-impact submission.""" + + return self._start_claims.acquire( + self._start_claim_key( + country_id=country_id, + reform_policy_id=reform_policy_id, + baseline_policy_id=baseline_policy_id, + region=region, + dataset=dataset, + time_period=time_period, + api_version=api_version, + options_hash=options_hash, + target=target, + ), + claim_token, + ttl_seconds=REFORM_IMPACT_START_CLAIM_TTL_SECONDS, + ) + + def release_start( + self, + *, + country_id: str, + reform_policy_id: int, + baseline_policy_id: int, + region: str, + dataset: str, + time_period: str, + api_version: str, + options_hash: str, + target: str, + claim_token: str, + ) -> bool: + """Release a start claim only when its ownership token still matches.""" + + return self._start_claims.release( + self._start_claim_key( + country_id=country_id, + reform_policy_id=reform_policy_id, + baseline_policy_id=baseline_policy_id, + region=region, + dataset=dataset, + time_period=time_period, + api_version=api_version, + options_hash=options_hash, + target=target, + ), + claim_token, + ) + + def _record_key(self, execution_id: str) -> str: + return self.namespace.key( + self.family, + REFORM_IMPACT_SCHEMA_VERSION, + {"execution_id": execution_id}, + ) + + def _scope_index(self, impact: CachedReformImpact) -> str: + return self.namespace.key( + "reform-impact-index", + REFORM_IMPACT_SCHEMA_VERSION, + { + "api_version": impact.api_version, + "baseline_policy_id": impact.baseline_policy_id, + "country_id": impact.country_id, + "dataset": impact.dataset, + "reform_policy_id": impact.reform_policy_id, + "region": impact.region, + "time_period": impact.time_period, + }, + ) + + def _recent_index(self) -> str: + return self.namespace.family_key( + "reform-impact-index", + REFORM_IMPACT_SCHEMA_VERSION, + "recent", + ) + + @staticmethod + def _score(impact: CachedReformImpact) -> float: + if impact.start_time is None: + return time.time() + value = impact.start_time + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.timestamp() + + def set(self, impact: CachedReformImpact) -> bool: + record_key = self._record_key(impact.execution_id) + indexes = (self._scope_index(impact), self._recent_index()) + try: + ttl_seconds = jittered_ttl(REFORM_IMPACT_TTL_SECONDS) + with self.client.pipeline(transaction=True) as pipeline: + pipeline.set( + record_key, + encode_envelope( + self.family, + REFORM_IMPACT_SCHEMA_VERSION, + _impact_to_wire(impact), + ), + ex=ttl_seconds, + ) + for index in indexes: + pipeline.zadd(index, {record_key: self._score(impact)}) + pipeline.zremrangebyrank( + index, + 0, + -(REFORM_IMPACT_INDEX_LIMIT + 1), + ) + pipeline.expire(index, ttl_seconds) + pipeline.execute() + except Exception: + record_cache_event( + family=self.family, + event="write-failed", + started_at=time.perf_counter(), + severity="WARNING", + ) + return False + return True + + def get_by_execution_id(self, execution_id: str) -> CachedReformImpact | None: + try: + value = self.client.get(self._record_key(execution_id)) + except Exception: + return None + return _impact_from_wire( + decode_envelope( + value, + family=self.family, + schema_version=REFORM_IMPACT_SCHEMA_VERSION, + ) + ) + + def _from_index(self, index: str, limit: int) -> list[CachedReformImpact]: + try: + keys = self.client.zrevrange(index, 0, max(limit - 1, 0)) # type: ignore[attr-defined] + values = [self.client.get(key) for key in keys] + except Exception: + return [] + impacts = [ + _impact_from_wire( + decode_envelope( + value, + family=self.family, + schema_version=REFORM_IMPACT_SCHEMA_VERSION, + ) + ) + for value in values + ] + return [impact for impact in impacts if impact is not None] + + def recent(self, limit: int) -> list[CachedReformImpact]: + return self._from_index(self._recent_index(), limit) + + def matching( + self, + *, + country_id: str, + reform_policy_id: int, + baseline_policy_id: int, + region: str, + dataset: str, + time_period: str, + api_version: str, + options_hash: str, + options_hash_pattern: str | None = None, + ) -> list[CachedReformImpact]: + probe = CachedReformImpact( + reform_impact_id=0, + baseline_policy_id=baseline_policy_id, + reform_policy_id=reform_policy_id, + country_id=country_id, + region=region, + dataset=dataset, + time_period=time_period, + options_json=None, + options_hash=options_hash, + api_version=api_version, + reform_impact_json={}, + status="", + message=None, + start_time=None, + end_time=None, + execution_id="", + ) + impacts = self._from_index( + self._scope_index(probe), + REFORM_IMPACT_INDEX_LIMIT, + ) + selected = [ + impact + for impact in impacts + if impact.options_hash == options_hash + or ( + options_hash_pattern is not None + and impact.options_hash is not None + and _like_matches(impact.options_hash, options_hash_pattern) + ) + ] + return sorted( + selected, + key=lambda impact: ( + impact.options_hash == options_hash, + impact.start_time or datetime.min, + impact.reform_impact_id, + ), + reverse=True, + ) + + def update( + self, + execution_id: str, + **changes: Any, + ) -> CachedReformImpact | None: + impact = self.get_by_execution_id(execution_id) + if impact is None: + return None + updated = replace(impact, **changes) + return updated if self.set(updated) else None + + def delete_matching_computing( + self, + *, + country_id: str, + reform_policy_id: int, + baseline_policy_id: int, + region: str, + dataset: str, + time_period: str, + options_hash: str, + ) -> None: + # Deletion historically omits api_version, so search the bounded recent + # index and remove only matching in-flight values. + impacts = self.recent(REFORM_IMPACT_INDEX_LIMIT) + selected = [ + impact + for impact in impacts + if impact.country_id == country_id + and impact.reform_policy_id == reform_policy_id + and impact.baseline_policy_id == baseline_policy_id + and impact.region == region + and impact.dataset == dataset + and impact.time_period == time_period + and impact.options_hash == options_hash + and impact.status == "computing" + ] + if not selected: + return + try: + with self.client.pipeline(transaction=True) as pipeline: + for impact in selected: + key = self._record_key(impact.execution_id) + pipeline.delete(key) + pipeline.zrem(self._scope_index(impact), key) + pipeline.zrem(self._recent_index(), key) + pipeline.execute() + except Exception: + return + + +def reform_impact_id(execution_id: str) -> int: + """Stable positive cache-local identifier for the historical response field.""" + + digest = hashlib.sha256(execution_id.encode("utf-8")).hexdigest()[:15] + return int(digest, 16) diff --git a/policyengine_api/runtime_cache/settings.py b/policyengine_api/runtime_cache/settings.py new file mode 100644 index 000000000..0cfb68c23 --- /dev/null +++ b/policyengine_api/runtime_cache/settings.py @@ -0,0 +1,238 @@ +"""Explicit, secret-safe shared Redis configuration.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass, field +from functools import lru_cache +import os +import re +from typing import Callable +from urllib.parse import SplitResult, urlsplit + +from pydantic import SecretStr + + +RUNTIME_CACHE_MODE = "RUNTIME_CACHE_MODE" +RUNTIME_CACHE_URL = "RUNTIME_CACHE_URL" +RUNTIME_CACHE_CA_CERT = "RUNTIME_CACHE_CA_CERT" +RUNTIME_CACHE_URL_SECRET_RESOURCE = "RUNTIME_CACHE_URL_SECRET_RESOURCE" +RUNTIME_CACHE_CA_CERT_SECRET_RESOURCE = "RUNTIME_CACHE_CA_CERT_SECRET_RESOURCE" +RUNTIME_CACHE_ENVIRONMENT = "RUNTIME_CACHE_ENVIRONMENT" +RUNTIME_CACHE_SERVICE = "RUNTIME_CACHE_SERVICE" + +CACHE_MODES = frozenset({"disabled", "local", "deployed"}) +LOCAL_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"}) +NAME_PATTERN = re.compile(r"^[a-z][a-z0-9-]{1,31}$") +SECRET_RESOURCE_PATTERN = re.compile(r"^projects/[^/]+/secrets/[^/]+/versions/[^/]+$") +DEFAULT_MAX_CONNECTIONS = 20 +DEFAULT_CONNECT_TIMEOUT_SECONDS = 1.0 +DEFAULT_OPERATION_TIMEOUT_SECONDS = 2.0 + + +class RuntimeCacheConfigurationError(RuntimeError): + """Raised without echoing a secret-bearing URL.""" + + +@dataclass(frozen=True) +class RuntimeCacheSettings: + mode: str + environment: str + service: str + url: SecretStr | None = field(repr=False) + ca_cert: SecretStr | None = field(repr=False) + tls: bool + max_connections: int = DEFAULT_MAX_CONNECTIONS + connect_timeout_seconds: float = DEFAULT_CONNECT_TIMEOUT_SECONDS + operation_timeout_seconds: float = DEFAULT_OPERATION_TIMEOUT_SECONDS + + @property + def enabled(self) -> bool: + return self.mode != "disabled" + + +@lru_cache(maxsize=None) +def _load_secret_from_secret_manager(resource_name: str) -> str: + """Resolve App Engine cache secrets without placing them in image layers.""" + + from google.cloud import secretmanager + + client = secretmanager.SecretManagerServiceClient() + response = client.access_secret_version(request={"name": resource_name}) + return response.payload.data.decode("utf-8") + + +def _required(values: Mapping[str, str], name: str) -> str: + value = values.get(name) + if value is None or not value.strip(): + raise RuntimeCacheConfigurationError(f"{name} is required") + return value.strip() + + +def _resolve_secret_source( + values: Mapping[str, str], + *, + value_name: str, + resource_name: str, + secret_loader: Callable[[str], str], +) -> str: + direct_value = values.get(value_name, "") + resource = values.get(resource_name, "").strip() + if direct_value and resource: + raise RuntimeCacheConfigurationError( + f"set exactly one of {value_name} or {resource_name}" + ) + if direct_value: + return direct_value + if not resource: + raise RuntimeCacheConfigurationError( + f"{value_name} or {resource_name} is required" + ) + if SECRET_RESOURCE_PATTERN.fullmatch(resource) is None: + raise RuntimeCacheConfigurationError(f"{resource_name} is invalid") + try: + value = secret_loader(resource) + except Exception as error: + raise RuntimeCacheConfigurationError( + f"{resource_name} could not be resolved" + ) from error + if not value: + raise RuntimeCacheConfigurationError(f"{resource_name} is empty") + return value + + +def _is_deployed(values: Mapping[str, str]) -> bool: + return bool(values.get("K_SERVICE") or values.get("GAE_ENV")) + + +def _validate_name(value: str, setting: str) -> str: + if NAME_PATTERN.fullmatch(value) is None: + raise RuntimeCacheConfigurationError(f"{setting} is invalid") + return value + + +def _parse_url(raw_url: str, *, mode: str) -> SplitResult: + try: + parsed = urlsplit(raw_url) + port = parsed.port + except ValueError as error: + raise RuntimeCacheConfigurationError( + f"{RUNTIME_CACHE_URL} is invalid" + ) from error + if parsed.scheme not in {"redis", "rediss"} or not parsed.hostname: + raise RuntimeCacheConfigurationError( + f"{RUNTIME_CACHE_URL} must use redis:// or rediss://" + ) + if parsed.query or parsed.fragment or parsed.path not in {"", "/0"}: + raise RuntimeCacheConfigurationError( + f"{RUNTIME_CACHE_URL} must select database 0 without query or fragment" + ) + if port is None: + raise RuntimeCacheConfigurationError( + f"{RUNTIME_CACHE_URL} must include an explicit port" + ) + if mode == "deployed": + if parsed.scheme != "rediss": + raise RuntimeCacheConfigurationError( + f"{RUNTIME_CACHE_URL} must require TLS in deployed mode" + ) + if parsed.hostname.lower() in LOCAL_HOSTS: + raise RuntimeCacheConfigurationError( + f"{RUNTIME_CACHE_URL} cannot use localhost in deployed mode" + ) + if parsed.password is None: + raise RuntimeCacheConfigurationError( + f"{RUNTIME_CACHE_URL} requires authentication in deployed mode" + ) + elif parsed.hostname.lower() not in LOCAL_HOSTS: + raise RuntimeCacheConfigurationError( + f"{RUNTIME_CACHE_URL} local mode requires an explicit local endpoint" + ) + return parsed + + +def _parse_ca_cert(values: Mapping[str, str], *, tls: bool) -> SecretStr | None: + raw_ca_cert = values.get(RUNTIME_CACHE_CA_CERT, "") + if not tls: + if raw_ca_cert.strip(): + raise RuntimeCacheConfigurationError( + f"{RUNTIME_CACHE_CA_CERT} requires a TLS cache URL" + ) + return None + ca_cert = _required(values, RUNTIME_CACHE_CA_CERT) + if ( + not ca_cert.startswith("-----BEGIN CERTIFICATE-----") + or not ca_cert.endswith("-----END CERTIFICATE-----") + or "\x00" in ca_cert + ): + raise RuntimeCacheConfigurationError( + f"{RUNTIME_CACHE_CA_CERT} must contain a PEM certificate bundle" + ) + return SecretStr(ca_cert) + + +def load_runtime_cache_settings( + environ: Mapping[str, str] | None = None, + *, + secret_loader: Callable[[str], str] | None = None, +) -> RuntimeCacheSettings: + """Load disabled, explicit-local, or fail-closed deployed cache settings.""" + + values = os.environ if environ is None else environ + raw_mode = values.get(RUNTIME_CACHE_MODE, "").strip() + mode = raw_mode or ("deployed" if _is_deployed(values) else "disabled") + if mode not in CACHE_MODES: + raise RuntimeCacheConfigurationError( + f"{RUNTIME_CACHE_MODE} must be disabled, local, or deployed" + ) + if mode == "disabled": + if values.get(RUNTIME_CACHE_URL) or values.get( + RUNTIME_CACHE_URL_SECRET_RESOURCE + ): + raise RuntimeCacheConfigurationError( + "runtime cache secret configuration requires an explicit " + "local or deployed mode" + ) + return RuntimeCacheSettings( + mode=mode, + environment="test", + service="api", + url=None, + ca_cert=None, + tls=False, + ) + + load_secret = secret_loader or _load_secret_from_secret_manager + raw_url = _resolve_secret_source( + values, + value_name=RUNTIME_CACHE_URL, + resource_name=RUNTIME_CACHE_URL_SECRET_RESOURCE, + secret_loader=load_secret, + ) + parsed = _parse_url(raw_url, mode=mode) + tls = parsed.scheme == "rediss" + ca_values = dict(values) + if tls: + ca_values[RUNTIME_CACHE_CA_CERT] = _resolve_secret_source( + values, + value_name=RUNTIME_CACHE_CA_CERT, + resource_name=RUNTIME_CACHE_CA_CERT_SECRET_RESOURCE, + secret_loader=load_secret, + ) + ca_cert = _parse_ca_cert(ca_values, tls=tls) + environment = _validate_name( + _required(values, RUNTIME_CACHE_ENVIRONMENT), + RUNTIME_CACHE_ENVIRONMENT, + ) + service = _validate_name( + _required(values, RUNTIME_CACHE_SERVICE), + RUNTIME_CACHE_SERVICE, + ) + return RuntimeCacheSettings( + mode=mode, + environment=environment, + service=service, + url=SecretStr(raw_url), + ca_cert=ca_cert, + tls=tls, + ) diff --git a/policyengine_api/services/ai_analysis_service.py b/policyengine_api/services/ai_analysis_service.py index 6dd0bee93..d0114ef7b 100644 --- a/policyengine_api/services/ai_analysis_service.py +++ b/policyengine_api/services/ai_analysis_service.py @@ -1,15 +1,21 @@ import json import os from collections.abc import Generator +import time from typing import Callable import anthropic from pydantic import BaseModel -from sqlalchemy import select -from sqlalchemy.orm import Session, sessionmaker -from policyengine_api.data.orm import get_v1_session_factory -from policyengine_api.data.v1_models import Analysis +from policyengine_api.runtime_cache.dependencies import get_runtime_cache_context +from policyengine_api.runtime_cache.core import record_cache_event +from policyengine_api.runtime_cache.repositories import ( + AIAnalysisCache, + CachedAnalysis, +) + + +AI_ANALYSIS_MODEL = "claude-sonnet-4-20250514" class StreamEvent(BaseModel): @@ -31,36 +37,20 @@ class AIAnalysisService: def __init__( self, - session_factory: sessionmaker[Session] | None = None, + analysis_cache: AIAnalysisCache | None = None, claude_client_factory: Callable[[], anthropic.Anthropic] | None = None, ) -> None: - self._injected_session_factory = session_factory + if analysis_cache is None: + context = get_runtime_cache_context() + analysis_cache = AIAnalysisCache(context.client, context.namespace) + self._analysis_cache = analysis_cache self._claude_client_factory = claude_client_factory - @property - def _sessions(self) -> sessionmaker[Session]: - return self._injected_session_factory or get_v1_session_factory(local=True) - def get_existing_analysis( self, prompt: str, - ) -> Analysis | None: - with self._sessions() as session: - return self._get_existing_analysis(session, prompt) - - @staticmethod - def _get_existing_analysis( - session: Session, - prompt: str, - ) -> Analysis | None: - return session.scalar( - select(Analysis) - .where( - Analysis.prompt == prompt, - Analysis.status.in_(("complete", "ok")), - ) - .order_by(Analysis.prompt_id.desc()) - ) + ) -> CachedAnalysis | None: + return self._analysis_cache.get(prompt, model=AI_ANALYSIS_MODEL) def trigger_ai_analysis( self, @@ -73,9 +63,10 @@ def trigger_ai_analysis( ) def generate(): + recompute_started_at = time.perf_counter() response_text = "" with claude_client.messages.stream( - model="claude-sonnet-4-20250514", + model=AI_ANALYSIS_MODEL, max_tokens=1500, temperature=0.0, system="You are an AI assistant analyzing policy data. Explain policies clearly and factually. Do not provide commentary, opinions, or quotes. Focus only on describing what the policies do and their direct impacts.", @@ -83,6 +74,12 @@ def generate(): ) as stream: for event in stream: if event.type == "error": + record_cache_event( + family="ai-analysis", + event="recompute-failed", + started_at=recompute_started_at, + severity="WARNING", + ) yield ( json.dumps( ErrorEvent(error=event.error["type"]).model_dump() @@ -95,13 +92,18 @@ def generate(): yield ( json.dumps(TextEvent(stream=event.text).model_dump()) + "\n" ) - with self._sessions.begin() as session: - session.add( - Analysis( - prompt=prompt, - analysis=response_text, - status="ok", - ) - ) + record_cache_event( + family="ai-analysis", + event="recompute", + started_at=recompute_started_at, + ) + self._analysis_cache.set( + CachedAnalysis( + prompt=prompt, + analysis=response_text, + status="ok", + ), + model=AI_ANALYSIS_MODEL, + ) return generate() diff --git a/policyengine_api/services/budget_window_cache.py b/policyengine_api/services/budget_window_cache.py index 7c19f5921..0d2171428 100644 --- a/policyengine_api/services/budget_window_cache.py +++ b/policyengine_api/services/budget_window_cache.py @@ -1,43 +1,47 @@ -import hashlib -import json -import os +"""Shared, namespaced budget-window result cache and coordination claims.""" + +import time from typing import Any -import redis +from policyengine_api.runtime_cache.claims import ExpiringClaimStore +from policyengine_api.runtime_cache.core import ( + CacheBackend, + CacheCoordinationError, + CacheNamespace, + decode_envelope, + encode_envelope, + jittered_ttl, + record_cache_event, +) +from policyengine_api.runtime_cache.dependencies import get_runtime_cache_context -from policyengine_api.gcp_logging import logger -BUDGET_WINDOW_CACHE_PREFIX = "budget_window:v1" +BUDGET_WINDOW_CACHE_FAMILY = "budget-window" +BUDGET_WINDOW_CACHE_SCHEMA_VERSION = 1 BUDGET_WINDOW_STARTING_PREFIX = "starting:" -BUDGET_WINDOW_STARTING_TTL_SECONDS = int( - os.environ.get("BUDGET_WINDOW_STARTING_TTL_SECONDS", "300") -) -BUDGET_WINDOW_BATCH_TTL_SECONDS = int( - os.environ.get("BUDGET_WINDOW_BATCH_TTL_SECONDS", "86400") -) -BUDGET_WINDOW_RESULT_TTL_SECONDS = int( - os.environ.get("BUDGET_WINDOW_RESULT_TTL_SECONDS", "2592000") -) +BUDGET_WINDOW_STARTING_TTL_SECONDS = 300 +BUDGET_WINDOW_BATCH_TTL_SECONDS = 86_400 +BUDGET_WINDOW_RESULT_TTL_SECONDS = 2_592_000 class BudgetWindowCache: - """Redis-backed cache and in-flight mapping for budget-window requests.""" - - def __init__(self, client: redis.Redis | None = None): - self._client = client - - @property - def client(self) -> redis.Redis: - if self._client is None: - self._client = redis.Redis( - host=os.environ.get("CACHE_REDIS_HOST", "127.0.0.1"), - port=int(os.environ.get("CACHE_REDIS_PORT", "6379")), - db=int(os.environ.get("CACHE_REDIS_DB", "0")), - decode_responses=True, - socket_connect_timeout=1, - socket_timeout=1, - ) - return self._client + """Recoverable results plus fail-closed expensive-work coordination.""" + + def __init__( + self, + client: CacheBackend | None = None, + namespace: CacheNamespace | None = None, + ) -> None: + if client is None or namespace is None: + context = get_runtime_cache_context() + client = client or context.client + namespace = namespace or context.namespace + self.client = client + self.namespace = namespace + self._claims = ExpiringClaimStore( + client, + family=BUDGET_WINDOW_CACHE_FAMILY, + ) def build_key( self, @@ -51,118 +55,209 @@ def build_key( options_hash: str | None, api_version: str, ) -> str: - key_payload = { - "country_id": country_id, - "reform_policy_id": reform_policy_id, - "baseline_policy_id": baseline_policy_id, - "region": region, - "dataset": dataset, - "time_period": time_period, - "options_hash": options_hash, - "api_version": api_version, - } - encoded = json.dumps( - key_payload, - sort_keys=True, - separators=(",", ":"), - default=str, + return self.namespace.key( + BUDGET_WINDOW_CACHE_FAMILY, + BUDGET_WINDOW_CACHE_SCHEMA_VERSION, + { + "api_version": api_version, + "baseline_policy_id": baseline_policy_id, + "country_id": country_id, + "dataset": dataset, + "options_hash": options_hash, + "reform_policy_id": reform_policy_id, + "region": region, + "time_period": time_period, + }, ) - digest = hashlib.sha256(encoded.encode("utf-8")).hexdigest() - return f"{BUDGET_WINDOW_CACHE_PREFIX}:{country_id}:{digest}" - def _result_key(self, cache_key: str) -> str: + @staticmethod + def _result_key(cache_key: str) -> str: return f"{cache_key}:result" - def _batch_key(self, cache_key: str) -> str: - return f"{cache_key}:batch_job_id" + @staticmethod + def _batch_key(cache_key: str) -> str: + return f"{cache_key}:batch-job-id" - def _handle_cache_error(self, operation: str, error: Exception) -> None: - logger.log_struct( - { - "message": f"Budget-window Redis cache {operation} failed", - "error": str(error), - }, + @staticmethod + def _handle_cache_error( + operation: str, + *, + event: str, + started_at: float, + ) -> None: + record_cache_event( + family=BUDGET_WINDOW_CACHE_FAMILY, + event=event, + operation=operation, + started_at=started_at, severity="WARNING", ) def get_completed_result(self, cache_key: str) -> dict[str, Any] | None: + started_at = time.perf_counter() try: payload = self.client.get(self._result_key(cache_key)) - except Exception as error: - self._handle_cache_error("read", error) - raise - - if not payload: - return None - - try: - result = json.loads(payload) - except (TypeError, ValueError) as error: - self._handle_cache_error("decode", error) + except Exception: + self._handle_cache_error( + "read-result", + event="connection-failed", + started_at=started_at, + ) return None - + result = decode_envelope( + payload, + family=BUDGET_WINDOW_CACHE_FAMILY, + schema_version=BUDGET_WINDOW_CACHE_SCHEMA_VERSION, + ) + if payload is not None and result is None: + self._handle_cache_error( + "decode-result", + event="decode-failed", + started_at=started_at, + ) + else: + record_cache_event( + family=BUDGET_WINDOW_CACHE_FAMILY, + event="hit" if isinstance(result, dict) else "miss", + operation="read-result", + started_at=started_at, + ) return result if isinstance(result, dict) else None - def set_completed_result(self, cache_key: str, result: dict[str, Any]) -> None: + def set_completed_result( + self, + cache_key: str, + result: dict[str, Any], + ) -> bool: + started_at = time.perf_counter() try: - self.client.set( + stored = self.client.set( self._result_key(cache_key), - json.dumps(result), - ex=BUDGET_WINDOW_RESULT_TTL_SECONDS, + encode_envelope( + BUDGET_WINDOW_CACHE_FAMILY, + BUDGET_WINDOW_CACHE_SCHEMA_VERSION, + result, + ), + ex=jittered_ttl(BUDGET_WINDOW_RESULT_TTL_SECONDS), ) - except Exception as error: - self._handle_cache_error("write result", error) - raise + except Exception: + self._handle_cache_error( + "write-result", + event="write-failed", + started_at=started_at, + ) + return False + record_cache_event( + family=BUDGET_WINDOW_CACHE_FAMILY, + event="write", + operation="write-result", + started_at=started_at, + ) + return bool(stored) def get_batch_job_id(self, cache_key: str) -> str | None: + started_at = time.perf_counter() try: value = self.client.get(self._batch_key(cache_key)) except Exception as error: - self._handle_cache_error("read batch id", error) - raise - + self._handle_cache_error( + "read-batch-id", + event="coordination-failed", + started_at=started_at, + ) + raise CacheCoordinationError( + "budget-window coordination state is unavailable" + ) from error if not isinstance(value, str) or not value: + record_cache_event( + family=BUDGET_WINDOW_CACHE_FAMILY, + event="coordination-miss", + operation="read-batch-id", + started_at=started_at, + ) return None if value.startswith(BUDGET_WINDOW_STARTING_PREFIX): + record_cache_event( + family=BUDGET_WINDOW_CACHE_FAMILY, + event="claim-contended", + operation="read-batch-id", + started_at=started_at, + ) return None + record_cache_event( + family=BUDGET_WINDOW_CACHE_FAMILY, + event="coordination-hit", + operation="read-batch-id", + started_at=started_at, + ) return value def claim_batch_start(self, cache_key: str, claim_token: str) -> bool: try: - claimed = self.client.set( + return self._claims.acquire( self._batch_key(cache_key), f"{BUDGET_WINDOW_STARTING_PREFIX}{claim_token}", - nx=True, - ex=BUDGET_WINDOW_STARTING_TTL_SECONDS, + ttl_seconds=BUDGET_WINDOW_STARTING_TTL_SECONDS, ) - except Exception as error: - self._handle_cache_error("claim", error) + except CacheCoordinationError: raise - return bool(claimed) - def store_batch_job_id(self, cache_key: str, batch_job_id: str) -> None: + started_at = time.perf_counter() try: - self.client.set( + stored = self.client.set( self._batch_key(cache_key), batch_job_id, ex=BUDGET_WINDOW_BATCH_TTL_SECONDS, ) except Exception as error: - self._handle_cache_error("write batch id", error) - raise + self._handle_cache_error( + "write-batch-id", + event="coordination-failed", + started_at=started_at, + ) + raise CacheCoordinationError( + "budget-window coordination state is unavailable" + ) from error + if not stored: + self._handle_cache_error( + "write-batch-id", + event="coordination-failed", + started_at=started_at, + ) + raise CacheCoordinationError( + "budget-window batch identifier could not be stored" + ) + record_cache_event( + family=BUDGET_WINDOW_CACHE_FAMILY, + event="coordination-write", + operation="write-batch-id", + started_at=started_at, + ) def clear_starting_claim(self, cache_key: str, claim_token: str) -> None: try: - batch_key = self._batch_key(cache_key) - value = self.client.get(batch_key) - if value == f"{BUDGET_WINDOW_STARTING_PREFIX}{claim_token}": - self.client.delete(batch_key) - except Exception as error: - self._handle_cache_error("clear claim", error) + self._claims.release( + self._batch_key(cache_key), + f"{BUDGET_WINDOW_STARTING_PREFIX}{claim_token}", + ) + except CacheCoordinationError: + return def clear_batch_job_id(self, cache_key: str) -> None: + started_at = time.perf_counter() try: self.client.delete(self._batch_key(cache_key)) - except Exception as error: - self._handle_cache_error("clear batch id", error) + except Exception: + self._handle_cache_error( + "clear-batch-id", + event="coordination-failed", + started_at=started_at, + ) + return + record_cache_event( + family=BUDGET_WINDOW_CACHE_FAMILY, + event="coordination-cleared", + operation="clear-batch-id", + started_at=started_at, + ) diff --git a/policyengine_api/services/economy_service.py b/policyengine_api/services/economy_service.py index 24b88b9a9..d1a6c6bcd 100644 --- a/policyengine_api/services/economy_service.py +++ b/policyengine_api/services/economy_service.py @@ -238,14 +238,12 @@ def __init__( self, *, primary_session_factory=None, - local_session_factory=None, policy_service_: PolicyService | None = None, reform_impacts_service_: ReformImpactsService | None = None, budget_window_cache_: BudgetWindowCache | None = None, simulation_entrypoint_=None, ) -> None: self._primary_session_factory = primary_session_factory - self._local_session_factory = local_session_factory self._injected_policy_service = policy_service_ self._injected_reform_impacts_service = reform_impacts_service_ self._injected_budget_window_cache = budget_window_cache_ @@ -260,9 +258,7 @@ def _policies(self) -> PolicyService: @property def _reform_impacts(self) -> ReformImpactsService: if self._injected_reform_impacts_service is None: - self._injected_reform_impacts_service = ReformImpactsService( - self._local_session_factory - ) + self._injected_reform_impacts_service = ReformImpactsService() return self._injected_reform_impacts_service @property @@ -737,6 +733,15 @@ def _get_or_create_economic_impact( if impact_action == ImpactAction.CREATE: self._resolve_runtime_bundle_for_setup_options(setup_options) + if not self._claim_reform_impact_start(setup_options): + logger.log_struct( + { + "message": "Another request owns this reform-impact submission", + **setup_options.model_dump(), + }, + severity="INFO", + ) + return EconomicImpactResult.computing() logger.log_struct( { "message": "No previous economic impact record found in db; creating new simulation run", @@ -744,9 +749,12 @@ def _get_or_create_economic_impact( }, severity="INFO", ) - return self._handle_create_impact( - setup_options=setup_options, - ) + try: + return self._handle_create_impact( + setup_options=setup_options, + ) + finally: + self._release_reform_impact_start(setup_options) raise ValueError(f"Unexpected impact action: {impact_action}") @@ -773,6 +781,41 @@ def _resolve_runtime_bundle_for_setup_options( runtime_app_name=setup_options.runtime_app_name, ) + def _reform_impact_start_claim_arguments( + self, + setup_options: EconomicImpactSetupOptions, + ) -> dict[str, Any]: + if not setup_options.options_hash: + raise ValueError("resolved reform-impact options hash is required") + return { + "country_id": setup_options.country_id, + "policy_id": setup_options.reform_policy_id, + "baseline_policy_id": setup_options.baseline_policy_id, + "region": setup_options.region, + "dataset": setup_options.dataset, + "time_period": setup_options.time_period, + "options_hash": setup_options.options_hash, + "api_version": setup_options.api_version, + "target": setup_options.target, + "claim_token": setup_options.process_id, + } + + def _claim_reform_impact_start( + self, + setup_options: EconomicImpactSetupOptions, + ) -> bool: + return self._reform_impacts.claim_reform_impact_start( + **self._reform_impact_start_claim_arguments(setup_options) + ) + + def _release_reform_impact_start( + self, + setup_options: EconomicImpactSetupOptions, + ) -> None: + self._reform_impacts.release_reform_impact_start( + **self._reform_impact_start_claim_arguments(setup_options) + ) + def _build_budget_window_progress_message( self, *, diff --git a/policyengine_api/services/household_calculation_service.py b/policyengine_api/services/household_calculation_service.py index 0179dc2f9..ddd004862 100644 --- a/policyengine_api/services/household_calculation_service.py +++ b/policyengine_api/services/household_calculation_service.py @@ -3,19 +3,25 @@ from copy import deepcopy from dataclasses import dataclass from datetime import date +import time from typing import Any, Callable from sqlalchemy import select from sqlalchemy.orm import Session, sessionmaker -from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS -from policyengine_api.data.local_models import Tracer +from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_VERSION from policyengine_api.data.orm import get_v1_session_factory from policyengine_api.data.v1_models import ( - ComputedHousehold, Household, Policy, ) +from policyengine_api.runtime_cache.dependencies import get_runtime_cache_context +from policyengine_api.runtime_cache.core import record_cache_event +from policyengine_api.runtime_cache.repositories import ( + HouseholdTraceCache, + HouseholdTraceIdentity, + HouseholdTraceValue, +) from policyengine_api.utils.deprecated_inputs import drop_deprecated_inputs from policyengine_api.utils.input_validation import find_unrecognized_inputs @@ -93,23 +99,20 @@ class HouseholdCalculationService: def __init__( self, primary_session_factory: sessionmaker[Session] | None = None, - local_session_factory: sessionmaker[Session] | None = None, + cache: HouseholdTraceCache | None = None, country_provider: Callable[[], dict] | None = None, ) -> None: self._injected_primary_session_factory = primary_session_factory - self._injected_local_session_factory = local_session_factory + if cache is None: + context = get_runtime_cache_context() + cache = HouseholdTraceCache(context.client, context.namespace) + self._cache = cache self._country_provider = country_provider @property def _primary_sessions(self) -> sessionmaker[Session]: return self._injected_primary_session_factory or get_v1_session_factory() - @property - def _local_sessions(self) -> sessionmaker[Session]: - return self._injected_local_session_factory or get_v1_session_factory( - local=True - ) - def _countries(self) -> dict: if self._country_provider is not None: return self._country_provider() @@ -117,22 +120,22 @@ def _countries(self) -> dict: return COUNTRIES - def _get_cached_household( - self, + @staticmethod + def _cache_identity( country_id: str, - household_id: int, - policy_id: int, + household: Household, + policy: Policy, api_version: str, - ) -> ComputedHousehold | None: - with self._local_sessions() as session: - return session.scalar( - select(ComputedHousehold).where( - ComputedHousehold.household_id == household_id, - ComputedHousehold.policy_id == policy_id, - ComputedHousehold.country_id == country_id, - ComputedHousehold.api_version == api_version, - ) - ) + ) -> HouseholdTraceIdentity: + return HouseholdTraceIdentity( + country_id=country_id, + household_id=household.id, + policy_id=policy.id, + household_hash=household.household_hash, + policy_hash=policy.policy_hash, + country_package_version=api_version, + policyengine_version=POLICYENGINE_VERSION, + ) def _get_inputs( self, @@ -157,39 +160,16 @@ def _get_inputs( def _store_result( self, - country_id: str, - household_id: int, - policy_id: int, - api_version: str, + identity: HouseholdTraceIdentity, calculation: CalculationResult, ) -> None: - with self._local_sessions.begin() as session: - identity = (household_id, policy_id, country_id) - computed_household = session.get(ComputedHousehold, identity) - if computed_household is None: - computed_household = ComputedHousehold( - country_id=country_id, - household_id=household_id, - policy_id=policy_id, - computed_household_json=calculation.household, - api_version=api_version, - status="complete", - ) - session.add(computed_household) - else: - computed_household.computed_household_json = calculation.household - computed_household.api_version = api_version - computed_household.status = "complete" - if calculation.tracer_output: - session.add( - Tracer( - household_id=household_id, - policy_id=policy_id, - country_id=country_id, - api_version=api_version, - tracer_output=calculation.tracer_output, - ) - ) + self._cache.set( + identity, + HouseholdTraceValue( + household=calculation.household, + tracer_output=calculation.tracer_output, + ), + ) def calculate_stored_household( self, @@ -198,24 +178,24 @@ def calculate_stored_household( policy_id: int, ) -> HouseholdCalculationResult: api_version = COUNTRY_PACKAGE_VERSIONS[country_id] - cached = self._get_cached_household( + household, policy = self._get_inputs(country_id, household_id, policy_id) + if household is None: + raise HouseholdNotFoundError(household_id) + if policy is None: + raise PolicyNotFoundError(policy_id) + cache_identity = self._cache_identity( country_id, - household_id, - policy_id, + household, + policy, api_version, ) + cached = self._cache.get(cache_identity) if cached is not None: return HouseholdCalculationResult( - household=cached.computed_household_json, + household=cached.household, cached=True, ) - household, policy = self._get_inputs(country_id, household_id, policy_id) - if household is None: - raise HouseholdNotFoundError(household_id) - if policy is None: - raise PolicyNotFoundError(policy_id) - countries = self._countries() country = countries.get(country_id) household_json = add_yearly_variables( @@ -233,7 +213,17 @@ def calculate_stored_household( if invalid_inputs: raise InvalidHouseholdInputsError(invalid_inputs) - raw_calculation = country.calculate(household_json, policy.policy_json) + calculation_started_at = time.perf_counter() + try: + raw_calculation = country.calculate(household_json, policy.policy_json) + except Exception: + record_cache_event( + family="household-trace", + event="recompute-failed", + started_at=calculation_started_at, + severity="WARNING", + ) + raise if isinstance(raw_calculation, CalculationResult): calculation = raw_calculation elif hasattr(raw_calculation, "household"): @@ -248,11 +238,13 @@ def calculate_stored_household( household=raw_calculation, tracer_output=[], ) + record_cache_event( + family="household-trace", + event="recompute", + started_at=calculation_started_at, + ) self._store_result( - country_id, - household_id, - policy_id, - api_version, + cache_identity, calculation, ) return HouseholdCalculationResult( diff --git a/policyengine_api/services/reform_impacts_service.py b/policyengine_api/services/reform_impacts_service.py index d338e8687..790c830c5 100644 --- a/policyengine_api/services/reform_impacts_service.py +++ b/policyengine_api/services/reform_impacts_service.py @@ -1,35 +1,31 @@ +"""Recoverable reform-impact cache service backed by shared Redis.""" + import datetime from typing import Any -from sqlalchemy import delete, or_, select -from sqlalchemy.orm import Session, sessionmaker - -from policyengine_api.data.orm import get_v1_session_factory -from policyengine_api.data.v1_models import ReformImpact +from policyengine_api.runtime_cache.core import CacheCoordinationError +from policyengine_api.runtime_cache.dependencies import get_runtime_cache_context +from policyengine_api.runtime_cache.repositories import ( + CachedReformImpact, + ReformImpactCache, + reform_impact_id, +) class ReformImpactsService: - """Reform-impact operations with service-owned local transactions.""" - - def __init__( - self, - session_factory: sessionmaker[Session] | None = None, - ) -> None: - self._injected_session_factory = session_factory + """Preserve historical lookup contracts over an expiring cache repository.""" - @property - def _sessions(self) -> sessionmaker[Session]: - return self._injected_session_factory or get_v1_session_factory(local=True) + def __init__(self, cache: ReformImpactCache | None = None) -> None: + if cache is None: + context = get_runtime_cache_context() + cache = ReformImpactCache(context.client, context.namespace) + self._cache = cache - def get_recent_reform_impacts(self, max_results: int) -> list[ReformImpact]: - with self._sessions() as session: - return list( - session.scalars( - select(ReformImpact) - .order_by(ReformImpact.start_time.desc()) - .limit(max_results) - ) - ) + def get_recent_reform_impacts( + self, + max_results: int, + ) -> list[CachedReformImpact]: + return self._cache.recent(max_results) def get_all_reform_impacts( self, @@ -41,19 +37,17 @@ def get_all_reform_impacts( time_period, options_hash, api_version, - ) -> list[ReformImpact]: - with self._sessions() as session: - return self._get_all_reform_impacts( - session, - country_id, - policy_id, - baseline_policy_id, - region, - dataset, - time_period, - options_hash, - api_version, - ) + ) -> list[CachedReformImpact]: + return self._cache.matching( + country_id=country_id, + reform_policy_id=policy_id, + baseline_policy_id=baseline_policy_id, + region=region, + dataset=dataset, + time_period=time_period, + api_version=api_version, + options_hash=options_hash, + ) def get_all_reform_impacts_by_options_hash_prefix( self, @@ -66,223 +60,82 @@ def get_all_reform_impacts_by_options_hash_prefix( options_hash, options_hash_prefix, api_version, - ) -> list[ReformImpact]: - with self._sessions() as session: - return self._get_all_reform_impacts_by_options_hash_prefix( - session, - country_id, - policy_id, - baseline_policy_id, - region, - dataset, - time_period, - options_hash, - options_hash_prefix, - api_version, - ) - - def set_reform_impact( - self, - country_id, - policy_id, - baseline_policy_id, - region, - dataset, - time_period, - options: dict[str, Any], - options_hash, - status, - api_version, - reform_impact_json: dict[str, Any], - start_time, - execution_id: str, - ) -> ReformImpact: - with self._sessions.begin() as session: - return self._set_reform_impact( - session, - country_id, - policy_id, - baseline_policy_id, - region, - dataset, - time_period, - options, - options_hash, - status, - api_version, - reform_impact_json, - start_time, - execution_id, - ) - - def delete_reform_impact( - self, - country_id, - policy_id, - baseline_policy_id, - region, - dataset, - time_period, - options_hash, - ) -> None: - with self._sessions.begin() as session: - self._delete_reform_impact( - session, - country_id, - policy_id, - baseline_policy_id, - region, - dataset, - time_period, - options_hash, - ) - - def set_error_reform_impact( - self, - country_id, - policy_id, - baseline_policy_id, - region, - dataset, - time_period, - options_hash, - message, - execution_id: str, - ) -> ReformImpact | None: - with self._sessions.begin() as session: - return self._set_error_reform_impact( - session, - country_id, - policy_id, - baseline_policy_id, - region, - dataset, - time_period, - options_hash, - message, - execution_id, - ) - - def set_complete_reform_impact( - self, - country_id, - reform_policy_id, - baseline_policy_id, - region, - dataset, - time_period, - options_hash, - reform_impact_json: dict[str, Any], - execution_id, - ) -> ReformImpact | None: - with self._sessions.begin() as session: - return self._set_complete_reform_impact( - session, - country_id, - reform_policy_id, - baseline_policy_id, - region, - dataset, - time_period, - options_hash, - reform_impact_json, - execution_id, - ) - - @staticmethod - def _filters( - country_id, - policy_id, - baseline_policy_id, - region, - dataset, - time_period, - api_version=None, - ): - filters = { - "country_id": country_id, - "reform_policy_id": policy_id, - "baseline_policy_id": baseline_policy_id, - "region": region, - "dataset": dataset, - "time_period": time_period, - } - if api_version is not None: - filters["api_version"] = api_version - return filters - - @staticmethod - def _scope(statement, **filters): - return statement.where( - *(getattr(ReformImpact, key) == value for key, value in filters.items()) + ) -> list[CachedReformImpact]: + return self._cache.matching( + country_id=country_id, + reform_policy_id=policy_id, + baseline_policy_id=baseline_policy_id, + region=region, + dataset=dataset, + time_period=time_period, + api_version=api_version, + options_hash=options_hash, + options_hash_pattern=options_hash_prefix, ) - def _get_all_reform_impacts( + def claim_reform_impact_start( self, - session: Session, - country_id, - policy_id, - baseline_policy_id, - region, - dataset, - time_period, - options_hash, - api_version, - ) -> list[ReformImpact]: - filters = self._filters( - country_id, - policy_id, - baseline_policy_id, - region, - dataset, - time_period, - api_version, - ) - statement = self._scope(select(ReformImpact), **filters).where( - ReformImpact.options_hash == options_hash + *, + country_id: str, + policy_id: int, + baseline_policy_id: int, + region: str, + dataset: str, + time_period: str, + options_hash: str, + api_version: str, + target: str, + claim_token: str, + ) -> bool: + """Fail closed unless this request atomically owns job submission.""" + + return self._cache.claim_start( + country_id=country_id, + reform_policy_id=policy_id, + baseline_policy_id=baseline_policy_id, + region=region, + dataset=dataset, + time_period=time_period, + api_version=api_version, + options_hash=options_hash, + target=target, + claim_token=claim_token, ) - return list(session.scalars(statement.order_by(ReformImpact.start_time.desc()))) - def _get_all_reform_impacts_by_options_hash_prefix( + def release_reform_impact_start( self, - session: Session, - country_id, - policy_id, - baseline_policy_id, - region, - dataset, - time_period, - options_hash, - options_hash_prefix, - api_version, - ) -> list[ReformImpact]: - filters = self._filters( - country_id, - policy_id, - baseline_policy_id, - region, - dataset, - time_period, - api_version, - ) - statement = self._scope(select(ReformImpact), **filters).where( - or_( - ReformImpact.options_hash == options_hash, - ReformImpact.options_hash.like(options_hash_prefix, escape="\\"), - ) - ) - return list( - session.scalars( - statement.order_by( - (ReformImpact.options_hash == options_hash).desc(), - ReformImpact.start_time.desc(), - ) + *, + country_id: str, + policy_id: int, + baseline_policy_id: int, + region: str, + dataset: str, + time_period: str, + options_hash: str, + api_version: str, + target: str, + claim_token: str, + ) -> None: + """Best-effort release; an unavailable cache safely falls back to expiry.""" + + try: + self._cache.release_start( + country_id=country_id, + reform_policy_id=policy_id, + baseline_policy_id=baseline_policy_id, + region=region, + dataset=dataset, + time_period=time_period, + api_version=api_version, + options_hash=options_hash, + target=target, + claim_token=claim_token, ) - ) + except CacheCoordinationError: + pass - def _set_reform_impact( + def set_reform_impact( self, - session: Session, country_id, policy_id, baseline_policy_id, @@ -296,8 +149,9 @@ def _set_reform_impact( reform_impact_json: dict[str, Any], start_time, execution_id: str, - ) -> ReformImpact: - impact = ReformImpact( + ) -> CachedReformImpact: + impact = CachedReformImpact( + reform_impact_id=reform_impact_id(execution_id), country_id=country_id, reform_policy_id=policy_id, baseline_policy_id=baseline_policy_id, @@ -309,16 +163,16 @@ def _set_reform_impact( status=status, api_version=api_version, reform_impact_json=reform_impact_json, + message=None, start_time=start_time, + end_time=None, execution_id=execution_id, ) - session.add(impact) - session.flush() + self._cache.set(impact) return impact - def _delete_reform_impact( + def delete_reform_impact( self, - session: Session, country_id, policy_id, baseline_policy_id, @@ -327,24 +181,18 @@ def _delete_reform_impact( time_period, options_hash, ) -> None: - filters = self._filters( - country_id, - policy_id, - baseline_policy_id, - region, - dataset, - time_period, - ) - session.execute( - self._scope(delete(ReformImpact), **filters).where( - ReformImpact.options_hash == options_hash, - ReformImpact.status == "computing", - ) + self._cache.delete_matching_computing( + country_id=country_id, + reform_policy_id=policy_id, + baseline_policy_id=baseline_policy_id, + region=region, + dataset=dataset, + time_period=time_period, + options_hash=options_hash, ) - def _set_error_reform_impact( + def set_error_reform_impact( self, - session: Session, country_id, policy_id, baseline_policy_id, @@ -354,7 +202,7 @@ def _set_error_reform_impact( options_hash, message, execution_id: str, - ) -> ReformImpact | None: + ) -> CachedReformImpact | None: del ( country_id, policy_id, @@ -364,21 +212,15 @@ def _set_error_reform_impact( time_period, options_hash, ) - impact = session.scalar( - select(ReformImpact) - .where(ReformImpact.execution_id == execution_id) - .order_by(ReformImpact.reform_impact_id.desc()) + return self._cache.update( + execution_id, + status="error", + message=message, + end_time=self._now(), ) - if impact is None: - return None - impact.status = "error" - impact.message = message - impact.end_time = self._now() - return impact - def _set_complete_reform_impact( + def set_complete_reform_impact( self, - session: Session, country_id, reform_policy_id, baseline_policy_id, @@ -388,7 +230,7 @@ def _set_complete_reform_impact( options_hash, reform_impact_json: dict[str, Any], execution_id, - ) -> ReformImpact | None: + ) -> CachedReformImpact | None: del ( country_id, reform_policy_id, @@ -398,18 +240,13 @@ def _set_complete_reform_impact( time_period, options_hash, ) - impact = session.scalar( - select(ReformImpact) - .where(ReformImpact.execution_id == execution_id) - .order_by(ReformImpact.reform_impact_id.desc()) + return self._cache.update( + execution_id, + status="ok", + message="Completed", + reform_impact_json=reform_impact_json, + end_time=self._now(), ) - if impact is None: - return None - impact.status = "ok" - impact.message = "Completed" - impact.reform_impact_json = reform_impact_json - impact.end_time = self._now() - return impact @staticmethod def _now() -> datetime.datetime: diff --git a/policyengine_api/services/tracer_analysis_service.py b/policyengine_api/services/tracer_analysis_service.py index 575cb6a4f..7e3ff361e 100644 --- a/policyengine_api/services/tracer_analysis_service.py +++ b/policyengine_api/services/tracer_analysis_service.py @@ -1,15 +1,47 @@ -from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS +from collections.abc import Callable +from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_VERSION from typing import Generator, Literal import re import anthropic from policyengine_api.services.ai_analysis_service import AIAnalysisService from werkzeug.exceptions import NotFound from sqlalchemy import select +from sqlalchemy.orm import Session, sessionmaker -from policyengine_api.data.local_models import Tracer +from policyengine_api.data.orm import get_v1_session_factory +from policyengine_api.data.v1_models import Household, Policy +from policyengine_api.runtime_cache.dependencies import get_runtime_cache_context +from policyengine_api.runtime_cache.repositories import ( + AIAnalysisCache, + HouseholdTraceCache, + HouseholdTraceIdentity, +) class TracerAnalysisService(AIAnalysisService): + def __init__( + self, + primary_session_factory: sessionmaker[Session] | None = None, + household_trace_cache: HouseholdTraceCache | None = None, + analysis_cache: AIAnalysisCache | None = None, + claude_client_factory: Callable[[], anthropic.Anthropic] | None = None, + ) -> None: + context = get_runtime_cache_context() + super().__init__( + analysis_cache=analysis_cache + or AIAnalysisCache(context.client, context.namespace), + claude_client_factory=claude_client_factory, + ) + self._primary_session_factory = primary_session_factory + self._household_trace_cache = household_trace_cache or HouseholdTraceCache( + context.client, + context.namespace, + ) + + @property + def _primary_sessions(self) -> sessionmaker[Session]: + return self._primary_session_factory or get_v1_session_factory() + def execute_analysis( self, country_id: str, @@ -76,22 +108,36 @@ def get_tracer( api_version: str, ) -> list: try: - with self._sessions() as session: - tracer = session.scalar( - select(Tracer) - .where( - Tracer.household_id == int(household_id), - Tracer.policy_id == int(policy_id), - Tracer.country_id == country_id, - Tracer.api_version == api_version, + with self._primary_sessions() as session: + household = session.scalar( + select(Household).where( + Household.id == int(household_id), + Household.country_id == country_id, ) - .order_by(Tracer.id.desc()) ) - - if tracer is None: + policy = session.scalar( + select(Policy).where( + Policy.id == int(policy_id), + Policy.country_id == country_id, + ) + ) + if household is None or policy is None: + raise NotFound("No household simulation tracer found") + cached = self._household_trace_cache.get( + HouseholdTraceIdentity( + country_id=country_id, + household_id=household.id, + policy_id=policy.id, + household_hash=household.household_hash, + policy_hash=policy.policy_hash, + country_package_version=api_version, + policyengine_version=POLICYENGINE_VERSION, + ) + ) + if cached is None or not cached.tracer_output: raise NotFound("No household simulation tracer found") - return tracer.tracer_output + return cached.tracer_output except Exception as e: print(f"Error getting existing tracer analysis: {str(e)}") diff --git a/pyproject.toml b/pyproject.toml index fd5d1b7f1..7974ae8e6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -52,6 +52,8 @@ dependencies = [ "streamlit", "uvicorn[standard]>=0.32,<1", "werkzeug", + "sqlmodel>=0.0.39,<0.1", + "psycopg[binary]>=3.3,<4", ] [project.optional-dependencies] @@ -68,6 +70,9 @@ dev = [ [tool.hatch.build.targets.wheel] packages = ["policyengine_api"] +[tool.ruff.lint.per-file-ignores] +"migrations/*/versions/*.py" = ["F401"] + [tool.towncrier] package = "policyengine_api" directory = "changelog.d" diff --git a/scripts/bootstrap_v2_supabase_storage.py b/scripts/bootstrap_v2_supabase_storage.py new file mode 100644 index 000000000..77f133ffe --- /dev/null +++ b/scripts/bootstrap_v2_supabase_storage.py @@ -0,0 +1,27 @@ +"""Explicit operator entry point for Stage 8 Supabase Storage bootstrap.""" + +import json + +from policyengine_api.data.v2.settings import load_supabase_storage_settings +from policyengine_api.data.v2.storage_bootstrap import initialize_supabase_storage + + +def main() -> None: + settings = load_supabase_storage_settings() + result = initialize_supabase_storage(settings) + print( + json.dumps( + { + "bucket": result.bucket, + "created": result.created, + "environment": result.environment, + "project_ref": result.project_ref, + "public": result.public, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/check_stage8_scaffolding_hygiene.py b/scripts/check_stage8_scaffolding_hygiene.py new file mode 100644 index 000000000..0074e4046 --- /dev/null +++ b/scripts/check_stage8_scaffolding_hygiene.py @@ -0,0 +1,72 @@ +"""Reject staged one-off Supabase scaffolding and secret-shaped artifacts.""" + +from __future__ import annotations + +from pathlib import PurePosixPath +import subprocess + + +PROHIBITED_PREFIXES = ( + ".agent-artifacts/", + ".artifacts/", + "supabase/", +) +PROHIBITED_SUFFIXES = ( + ".dump", + ".key", + ".pem", + ".sql", + ".sql.gz", +) +PROHIBITED_NAMES = { + ".env", + "bootstrap-payload.json", + "scaffold-payload.json", +} +ONE_OFF_MARKERS = ("one-off", "one_off", "scratch") + + +def prohibited_staged_paths(paths: list[str]) -> list[str]: + """Return obvious disposable or secret-shaped paths in stable order.""" + + rejected: set[str] = set() + for raw_path in paths: + path = PurePosixPath(raw_path) + normalized = path.as_posix() + if normalized.startswith("./"): + normalized = normalized[2:] + lower = normalized.lower() + if any(normalized.startswith(prefix) for prefix in PROHIBITED_PREFIXES): + rejected.add(normalized) + if lower.endswith(PROHIBITED_SUFFIXES): + rejected.add(normalized) + if path.name.lower() in PROHIBITED_NAMES: + rejected.add(normalized) + if any(marker in path.name.lower() for marker in ONE_OFF_MARKERS): + rejected.add(normalized) + return sorted(rejected) + + +def staged_paths() -> list[str]: + completed = subprocess.run( + ["git", "diff", "--cached", "--name-only", "--diff-filter=ACMR"], + check=True, + capture_output=True, + text=True, + ) + return [line for line in completed.stdout.splitlines() if line] + + +def main() -> None: + rejected = prohibited_staged_paths(staged_paths()) + if rejected: + formatted = "\n".join(f"- {path}" for path in rejected) + raise SystemExit( + "Stage 8 staged-file hygiene rejected one-off or secret-shaped " + f"artifacts:\n{formatted}" + ) + print("Stage 8 staged-file hygiene passed.") + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/local_v1_database.py b/tests/fixtures/local_v1_database.py new file mode 100644 index 000000000..826d730cf --- /dev/null +++ b/tests/fixtures/local_v1_database.py @@ -0,0 +1,32 @@ +"""SQLite-only v1 schema helper confined to explicit test fixtures.""" + +from sqlalchemy import Column, Engine, Integer, JSON, MetaData, String, Table + +from policyengine_api.data.v1_models import Policy, V1Base + + +_sqlite_policy_metadata = MetaData() +Table( + Policy.__tablename__, + _sqlite_policy_metadata, + Column("id", Integer, primary_key=True, autoincrement=True), + Column("country_id", String(3), nullable=False), + Column("label", String(255)), + Column("api_version", String(10), nullable=False), + Column("policy_json", JSON, nullable=False), + Column("policy_hash", String(255), nullable=False), +) + + +def create_test_v1_schema(engine: Engine) -> None: + if engine.dialect.name != "sqlite": + raise ValueError("the test-only v1 schema helper requires SQLite") + V1Base.metadata.create_all( + engine, + tables=[ + table + for table in V1Base.metadata.sorted_tables + if table is not Policy.__table__ + ], + ) + _sqlite_policy_metadata.create_all(engine) diff --git a/tests/fixtures/services/economy_service.py b/tests/fixtures/services/economy_service.py index 8e1bfe0e9..93f60ca90 100644 --- a/tests/fixtures/services/economy_service.py +++ b/tests/fixtures/services/economy_service.py @@ -117,6 +117,8 @@ def mock_reform_impacts_service(): mock_service = MagicMock() mock_service.get_all_reform_impacts_by_options_hash_prefix.return_value = [] mock_service.get_all_reform_impacts.return_value = [] + mock_service.claim_reform_impact_start.return_value = True + mock_service.release_reform_impact_start.return_value = None mock_service.set_reform_impact.return_value = None mock_service.set_complete_reform_impact.return_value = None mock_service.set_error_reform_impact.return_value = None diff --git a/tests/fixtures/services/tracer_analysis_service.py b/tests/fixtures/services/tracer_analysis_service.py index 4118f233d..e93634aff 100644 --- a/tests/fixtures/services/tracer_analysis_service.py +++ b/tests/fixtures/services/tracer_analysis_service.py @@ -3,7 +3,7 @@ TracerAnalysisService, ) from unittest.mock import patch -from policyengine_api.data.v1_models import Analysis +from policyengine_api.runtime_cache.repositories import CachedAnalysis valid_tracer_output = [ " snap<2027, (default)> = [6769.799]", @@ -68,7 +68,7 @@ def mock_get_existing_analysis(): with patch.object( TracerAnalysisService, "get_existing_analysis", - return_value=Analysis( + return_value=CachedAnalysis( prompt="prompt", analysis="Existing static analysis", status="ok", diff --git a/tests/fixtures/services/tracer_fixture_service.py b/tests/fixtures/services/tracer_fixture_service.py index 2d3d015d0..67a871fb1 100644 --- a/tests/fixtures/services/tracer_fixture_service.py +++ b/tests/fixtures/services/tracer_fixture_service.py @@ -1,6 +1,16 @@ import pytest import json -from policyengine_api.data.local_models import Tracer +from types import SimpleNamespace + +from policyengine_api.constants import POLICYENGINE_VERSION +from policyengine_api.data.v1_models import Household, Policy +from policyengine_api.runtime_cache.core import CacheNamespace +from policyengine_api.runtime_cache.fake import InMemoryCacheBackend +from policyengine_api.runtime_cache.repositories import ( + HouseholdTraceCache, + HouseholdTraceIdentity, + HouseholdTraceValue, +) valid_tracer = { "tracer_output": [ @@ -23,14 +33,51 @@ @pytest.fixture -def test_tracer_data(orm_session): - tracer = Tracer( +def test_tracer_data(orm_session_factory): + with orm_session_factory.begin() as session: + session.add_all( + [ + Household( + id=int(valid_tracer_row["household_id"]), + country_id=valid_tracer_row["country_id"], + label=None, + api_version=valid_tracer_row["api_version"], + household_json={}, + household_hash="household-hash", + ), + Policy( + id=int(valid_tracer_row["policy_id"]), + country_id=valid_tracer_row["country_id"], + label=None, + api_version=valid_tracer_row["api_version"], + policy_json={}, + policy_hash="policy-hash", + ), + ] + ) + cache = HouseholdTraceCache( + InMemoryCacheBackend(), + CacheNamespace("test", "api"), + ) + cache.set( + HouseholdTraceIdentity( + household_id=int(valid_tracer_row["household_id"]), + policy_id=int(valid_tracer_row["policy_id"]), + country_id=valid_tracer_row["country_id"], + household_hash="household-hash", + policy_hash="policy-hash", + country_package_version=valid_tracer_row["api_version"], + policyengine_version=POLICYENGINE_VERSION, + ), + HouseholdTraceValue( + household={}, + tracer_output=json.loads(valid_tracer_row["tracer_output"]), + ), + ) + return SimpleNamespace( household_id=int(valid_tracer_row["household_id"]), policy_id=int(valid_tracer_row["policy_id"]), country_id=valid_tracer_row["country_id"], api_version=valid_tracer_row["api_version"], - tracer_output=json.loads(valid_tracer_row["tracer_output"]), + cache=cache, ) - orm_session.add(tracer) - orm_session.commit() - return tracer diff --git a/tests/integration/test_alembic_v2_lifecycle.py b/tests/integration/test_alembic_v2_lifecycle.py new file mode 100644 index 000000000..b424cb5cc --- /dev/null +++ b/tests/integration/test_alembic_v2_lifecycle.py @@ -0,0 +1,135 @@ +"""Exercise the isolated v2 Alembic lifecycle against disposable Postgres.""" + +import os + +from alembic import command +from alembic.autogenerate import compare_metadata +from alembic.config import Config +from alembic.migration import MigrationContext +import pytest +from sqlalchemy import create_engine, inspect, text + +from policyengine_api.constants import REPO +from policyengine_api.data.v2.migration_target import ( + V2_ALEMBIC_DISPOSABLE_TEST, + V2MigrationTargetError, + load_v2_alembic_settings, +) +from policyengine_api.data.v2.models import V2_METADATA +from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL +from policyengine_api.data.v2.table_inventory import EXPECTED_V2_TABLES + + +BASELINE_REVISION = "47592781336f" +PREVIOUS_HEAD_REVISION = "b4c69674dd47" +HEAD_REVISION = "5f048586d8f1" + + +def _disposable_url() -> str: + database_url = os.environ.get(V2_MIGRATION_DATABASE_URL, "") + if not database_url: + pytest.skip(f"{V2_MIGRATION_DATABASE_URL} is not set") + settings = load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: database_url, + V2_ALEMBIC_DISPOSABLE_TEST: os.environ.get(V2_ALEMBIC_DISPOSABLE_TEST, ""), + } + ) + if not settings.disposable_test: + pytest.fail("v2 lifecycle tests require disposable-test mode") + return settings.url.render_as_string(hide_password=False) + + +def _config() -> Config: + return Config(str(REPO / "alembic-v2.ini")) + + +def _assert_head(engine) -> None: + assert set(inspect(engine).get_table_names(schema="public")) == ( + EXPECTED_V2_TABLES | {"alembic_version"} + ) + with engine.connect() as connection: + context = MigrationContext.configure(connection) + assert context.get_current_revision() == HEAD_REVISION + assert compare_metadata(context, V2_METADATA) == [] + model_count = connection.execute( + text( + "SELECT count(*) FROM public.tax_benefit_models " + "WHERE name = 'stage8-platform-validation'" + ) + ).scalar_one() + version_count = connection.execute( + text( + "SELECT count(*) FROM public.tax_benefit_model_versions " + "WHERE version = 'stage8-platform-validation'" + ) + ).scalar_one() + assert (model_count, version_count) == (1, 1) + + +def test_empty_upgrade_check_boundary_downgrade_and_reupgrade() -> None: + database_url = _disposable_url() + config = _config() + engine = create_engine(database_url) + + try: + command.downgrade(config, "base") + assert set(inspect(engine).get_table_names(schema="public")) <= { + "alembic_version" + } + + command.upgrade(config, "head") + command.check(config) + _assert_head(engine) + + command.downgrade(config, BASELINE_REVISION) + with engine.connect() as connection: + context = MigrationContext.configure(connection) + assert context.get_current_revision() == BASELINE_REVISION + boundary_drift = compare_metadata(context, V2_METADATA) + boundary_kinds = [difference[0] for difference in boundary_drift] + assert boundary_kinds.count("v2_reference_row_change") == 2 + assert boundary_kinds.count("add_fk") == 4 + assert boundary_kinds.count("add_column") == 1 + assert boundary_kinds.count("add_constraint") == 2 + assert len(boundary_kinds) == 9 + model_count = connection.execute( + text( + "SELECT count(*) FROM public.tax_benefit_models " + "WHERE name = 'stage8-platform-validation'" + ) + ).scalar_one() + assert model_count == 0 + + command.upgrade(config, "head") + command.check(config) + _assert_head(engine) + finally: + command.upgrade(config, "head") + engine.dispose() + + +def test_upgrade_to_head_validates_the_resulting_table_inventory() -> None: + database_url = _disposable_url() + config = _config() + engine = create_engine(database_url) + + try: + command.downgrade(config, PREVIOUS_HEAD_REVISION) + with engine.begin() as connection: + connection.execute(text("CREATE TABLE unreviewed_runtime_table (id INT)")) + + with pytest.raises( + V2MigrationTargetError, + match="v2 head table inventory.*unreviewed_runtime_table", + ): + command.upgrade(config, "head") + + with engine.connect() as connection: + context = MigrationContext.configure(connection) + assert context.get_current_revision() == PREVIOUS_HEAD_REVISION + finally: + with engine.begin() as connection: + connection.execute(text("DROP TABLE IF EXISTS unreviewed_runtime_table")) + command.upgrade(config, "head") + engine.dispose() diff --git a/tests/integration/test_budget_window_in_flight_dedupe.py b/tests/integration/test_budget_window_in_flight_dedupe.py index 5d6cb82ea..c944224b6 100644 --- a/tests/integration/test_budget_window_in_flight_dedupe.py +++ b/tests/integration/test_budget_window_in_flight_dedupe.py @@ -1,23 +1,11 @@ from unittest.mock import MagicMock from flask import Flask +from policyengine_api.runtime_cache.fake import InMemoryCacheBackend -class FakeRedis: - def __init__(self): - self.values = {} - - def get(self, key): - return self.values.get(key) - - def set(self, key, value, nx=False, ex=None): - if nx and key in self.values: - return False - self.values[key] = value - return True - - def delete(self, key): - self.values.pop(key, None) +class FakeRedis(InMemoryCacheBackend): + pass def _create_client(economy_bp): diff --git a/tests/integration/test_runtime_cache_redis.py b/tests/integration/test_runtime_cache_redis.py new file mode 100644 index 000000000..04bbc3a30 --- /dev/null +++ b/tests/integration/test_runtime_cache_redis.py @@ -0,0 +1,169 @@ +"""Real Redis qualification for cross-connection, TTL, atomicity, and claims.""" + +from datetime import datetime +import os +import time +from urllib.parse import urlsplit +from uuid import uuid4 + +import pytest +import redis + +from policyengine_api.runtime_cache.claims import ExpiringClaimStore +from policyengine_api.runtime_cache.core import CacheNamespace, RecoverableJSONCache +from policyengine_api.runtime_cache.repositories import ( + CachedReformImpact, + HouseholdTraceCache, + HouseholdTraceIdentity, + HouseholdTraceValue, + ReformImpactCache, + reform_impact_id, +) + + +RUNTIME_CACHE_TEST_URL = "RUNTIME_CACHE_TEST_URL" + + +@pytest.fixture +def redis_pair(): + raw_url = os.environ.get(RUNTIME_CACHE_TEST_URL, "") + if not raw_url: + pytest.skip(f"{RUNTIME_CACHE_TEST_URL} is not set") + parsed = urlsplit(raw_url) + if parsed.scheme != "redis" or parsed.hostname not in {"127.0.0.1", "localhost"}: + pytest.fail("real cache integration tests require explicit local Redis") + if parsed.path not in {"", "/0"}: + pytest.fail("real cache integration tests require disposable database 0") + + first = redis.Redis.from_url(raw_url, decode_responses=True) + second = redis.Redis.from_url(raw_url, decode_responses=True) + first.ping() + prefix = f"policyengine:stage8-{uuid4().hex[:8]}:api:" + try: + yield first, second, CacheNamespace(prefix.split(":")[1], "api") + finally: + keys = list(first.scan_iter(match=f"{prefix}*")) + if keys: + first.delete(*keys) + first.close() + second.close() + + +def test_cross_connection_visibility_and_real_expiry(redis_pair) -> None: + first, second, namespace = redis_pair + writer = RecoverableJSONCache( + first, + namespace, + family="integration", + schema_version=1, + ttl_seconds=3, + ) + reader = RecoverableJSONCache( + second, + namespace, + family="integration", + schema_version=1, + ttl_seconds=3, + ) + assert writer.set({"input": "same"}, {"value": 42}) + assert reader.get({"input": "same"}) == {"value": 42} + time.sleep(3.1) + assert reader.get({"input": "same"}) is None + + +def test_atomic_household_tracer_value_is_shared_between_connections( + redis_pair, +) -> None: + first, second, namespace = redis_pair + identity = HouseholdTraceIdentity( + country_id="us", + household_id=1, + policy_id=2, + household_hash="household", + policy_hash="policy", + country_package_version="1.2.3", + policyengine_version="4.5.6", + ) + value = HouseholdTraceValue( + household={"people": {"you": {}}}, + tracer_output=["trace"], + ) + assert HouseholdTraceCache(first, namespace).set(identity, value) + assert HouseholdTraceCache(second, namespace).get(identity) == value + + +def test_real_claim_is_exclusive_token_safe_and_expires(redis_pair) -> None: + first, second, namespace = redis_pair + key = namespace.family_key("claims", 1, "work") + first_claims = ExpiringClaimStore(first) + second_claims = ExpiringClaimStore(second) + assert first_claims.acquire(key, "first", ttl_seconds=1) + assert not second_claims.acquire(key, "second", ttl_seconds=1) + assert not second_claims.release(key, "second") + time.sleep(1.1) + assert second_claims.acquire(key, "second", ttl_seconds=1) + assert second_claims.release(key, "second") + + +def test_reform_submission_claim_is_shared_across_connections(redis_pair) -> None: + first, second, namespace = redis_pair + writer = ReformImpactCache(first, namespace) + contender = ReformImpactCache(second, namespace) + arguments = { + "country_id": "us", + "reform_policy_id": 2, + "baseline_policy_id": 1, + "region": "us", + "dataset": "default", + "time_period": "2026", + "api_version": "v1", + "options_hash": "resolved-hash", + "target": "general", + } + + assert writer.claim_start(**arguments, claim_token="writer") + assert not contender.claim_start(**arguments, claim_token="contender") + assert not contender.release_start(**arguments, claim_token="contender") + assert writer.release_start(**arguments, claim_token="writer") + assert contender.claim_start(**arguments, claim_token="contender") + + +def test_real_reform_indexes_are_cross_connection_bounded_and_expiring( + redis_pair, + monkeypatch, +) -> None: + import policyengine_api.runtime_cache.repositories as module + + first, second, namespace = redis_pair + monkeypatch.setattr(module, "REFORM_IMPACT_INDEX_LIMIT", 2) + monkeypatch.setattr(module, "REFORM_IMPACT_TTL_SECONDS", 1) + writer = ReformImpactCache(first, namespace) + reader = ReformImpactCache(second, namespace) + for day in range(1, 4): + execution_id = f"job-{day}" + assert writer.set( + CachedReformImpact( + reform_impact_id=reform_impact_id(execution_id), + baseline_policy_id=1, + reform_policy_id=2, + country_id="us", + region="us", + dataset="default", + time_period="2026", + options_json={}, + options_hash=f"hash-{day}", + api_version="v1", + reform_impact_json={}, + status="computing", + message=None, + start_time=datetime(2026, 1, day), + end_time=None, + execution_id=execution_id, + ) + ) + assert [impact.execution_id for impact in reader.recent(10)] == [ + "job-3", + "job-2", + ] + time.sleep(1.1) + assert reader.recent(10) == [] diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index b703a0b76..b72075604 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -8,9 +8,8 @@ os.environ.setdefault("FLASK_DEBUG", "1") from policyengine_api.data import orm -from policyengine_api.data.local_database import create_local_v1_schema -from policyengine_api.data.local_models import LocalV1Base from policyengine_api.data.v1_models import V1Base +from tests.fixtures.local_v1_database import create_test_v1_schema @pytest.fixture(scope="session") @@ -20,7 +19,7 @@ def test_engine(): connect_args={"check_same_thread": False}, poolclass=StaticPool, ) - create_local_v1_schema(engine) + create_test_v1_schema(engine) yield engine engine.dispose() @@ -29,13 +28,12 @@ def test_engine(): def isolated_orm_database(test_engine, monkeypatch): """Bind runtime factories to a clean in-memory ORM database per test.""" - monkeypatch.setattr(orm, "get_v1_engine", lambda *, local=False: test_engine) + monkeypatch.setattr(orm, "get_v1_engine", lambda: test_engine) orm.clear_v1_session_factories() factory = orm.get_v1_session_factory() with factory.begin() as session: - local_tables = LocalV1Base.metadata.sorted_tables production_tables = V1Base.metadata.sorted_tables - for table in reversed([*production_tables, *local_tables]): + for table in reversed(production_tables): session.execute(table.delete()) try: yield diff --git a/tests/unit/data/test_orm_sessions.py b/tests/unit/data/test_orm_sessions.py index 681593c7b..1d814e6c7 100644 --- a/tests/unit/data/test_orm_sessions.py +++ b/tests/unit/data/test_orm_sessions.py @@ -45,25 +45,20 @@ def test_session_factory_begin_commits_and_rolls_back(): assert session.scalars(text("SELECT id FROM item ORDER BY id")).all() == [1] -def test_runtime_factories_are_cached_and_separate(monkeypatch): +def test_runtime_factory_is_cached_and_has_no_local_selector(monkeypatch): remote_engine = create_engine("sqlite+pysqlite:///:memory:") - local_engine = create_engine("sqlite+pysqlite:///:memory:") monkeypatch.setattr( orm_module, "get_v1_engine", - lambda *, local=False: local_engine if local else remote_engine, + lambda: remote_engine, ) orm_module.clear_v1_session_factories() try: remote = get_v1_session_factory() - local = get_v1_session_factory(local=True) assert remote is get_v1_session_factory() - assert local is get_v1_session_factory(local=True) - assert remote is not local assert remote.kw["bind"] is remote_engine - assert local.kw["bind"] is local_engine finally: orm_module.clear_v1_session_factories() diff --git a/tests/unit/data/test_sqlalchemy_v2.py b/tests/unit/data/test_sqlalchemy_v2.py index 89d9689e2..59dad5287 100644 --- a/tests/unit/data/test_sqlalchemy_v2.py +++ b/tests/unit/data/test_sqlalchemy_v2.py @@ -1,11 +1,9 @@ from unittest.mock import Mock import sqlalchemy -from sqlalchemy import create_engine, func, inspect, select -from sqlalchemy.orm import Session +from pathlib import Path import policyengine_api.data.orm as orm -from policyengine_api.data.v1_models import Policy def test_sqlalchemy_v2_or_newer_is_installed(): @@ -73,66 +71,32 @@ def connector_factory(**options): ] -def test_database_password_can_be_loaded_from_file(monkeypatch, tmp_path): - monkeypatch.chdir(tmp_path) - monkeypatch.setenv("POLICYENGINE_DB_PASSWORD", ".dbpw") - (tmp_path / ".dbpw").write_text("file-password\n", encoding="utf-8") - - assert orm._database_password() == "file-password" - - -def test_local_schema_preserves_the_documented_sqlite_policy_key_exception(): - from policyengine_api.data.local_database import create_local_v1_schema - - engine = create_engine("sqlite+pysqlite:///:memory:") - try: - create_local_v1_schema(engine) - - policy_key = inspect(engine).get_pk_constraint("policy") - assert policy_key["constrained_columns"] == ["id"] - assert "tracers" in inspect(engine).get_table_names() - finally: - engine.dispose() - - -def test_local_initializer_bootstraps_schema_and_current_law_rows(tmp_path): - database_path = tmp_path / "local.db" - engine = create_engine(f"sqlite+pysqlite:///{database_path}") - - try: - orm._initialize_local_database(engine) - with Session(engine) as session: - assert session.scalar(select(func.count()).select_from(Policy)) > 0 - assert session.scalars(select(Policy)).first().policy_json == {} - finally: - engine.dispose() - - -def test_local_initializer_is_idempotent(tmp_path): - engine = create_engine(f"sqlite+pysqlite:///{tmp_path / 'local.db'}") - try: - orm._initialize_local_database(engine) - orm._initialize_local_database(engine) - - with Session(engine) as session: - assert session.scalar(select(func.count()).select_from(Policy)) == len( - orm.COUNTRY_PACKAGE_VERSIONS - ) - finally: - engine.dispose() +def test_production_orm_has_no_sqlite_debug_or_local_schema_path(): + source = Path(orm.__file__).read_text(encoding="utf-8") + for prohibited in ( + "sqlite", + "FLASK_DEBUG", + "local=True", + "local_database", + "create_all", + "policyengine.db", + ".init.lock", + ".dbpw", + ): + assert prohibited not in source def test_close_v1_engines_disposes_pools_and_connectors(monkeypatch): engine = Mock() connector = Mock() - monkeypatch.setattr(orm, "_v1_engines", {False: engine}) - monkeypatch.setattr(orm, "_cloud_sql_connectors", {False: connector}) - monkeypatch.setattr(orm, "_v1_session_factories", {False: Mock()}) + monkeypatch.setattr(orm, "_v1_engine", engine) + monkeypatch.setattr(orm, "_cloud_sql_connector", connector) + monkeypatch.setattr(orm, "_v1_session_factory", Mock()) orm.close_v1_engines() engine.dispose.assert_called_once_with() connector.close.assert_called_once_with() - assert orm._v1_engines == {} - assert orm._cloud_sql_connectors == {} - assert orm._v1_session_factories == {} + assert orm._v1_engine is None + assert orm._cloud_sql_connector is None + assert orm._v1_session_factory is None diff --git a/tests/unit/data/test_v1_models.py b/tests/unit/data/test_v1_models.py index 24155d04d..d61c5cacb 100644 --- a/tests/unit/data/test_v1_models.py +++ b/tests/unit/data/test_v1_models.py @@ -1,7 +1,6 @@ from sqlalchemy.dialects import mysql from sqlalchemy.schema import CreateTable -from policyengine_api.data.local_models import LocalV1Base from policyengine_api.data.v1_models import V1Base @@ -26,9 +25,8 @@ def test_v1_metadata_contains_every_legacy_table(): assert set(V1Base.metadata.tables) == EXPECTED_TABLES -def test_tracer_metadata_is_local_only(): +def test_tracer_table_is_absent_after_sqlite_cache_removal(): assert "tracers" not in V1Base.metadata.tables - assert set(LocalV1Base.metadata.tables) == {"tracers"} def test_v1_metadata_compiles_for_the_production_mysql_dialect(): diff --git a/tests/unit/routes/test_household_and_user_policy_orm_routes.py b/tests/unit/routes/test_household_and_user_policy_orm_routes.py index 5850aac74..889944e16 100644 --- a/tests/unit/routes/test_household_and_user_policy_orm_routes.py +++ b/tests/unit/routes/test_household_and_user_policy_orm_routes.py @@ -4,15 +4,20 @@ from unittest.mock import Mock, patch from flask import Flask -from sqlalchemy import select -from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS +from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_VERSION from policyengine_api.data.v1_models import ( - ComputedHousehold, Household, Policy, UserPolicy, ) +from policyengine_api.runtime_cache.core import CacheNamespace +from policyengine_api.runtime_cache.fake import InMemoryCacheBackend +from policyengine_api.runtime_cache.repositories import ( + HouseholdTraceCache, + HouseholdTraceIdentity, + HouseholdTraceValue, +) from policyengine_api.routes.household_routes import get_household_under_policy from policyengine_api.routes.policy_routes import ( get_user_policy, @@ -27,18 +32,52 @@ def test_household_under_policy_returns_cached_json_object(orm_session_factory): stored_result = {"people": {"you": {"net_income": {"2026": 42}}}} with orm_session_factory.begin() as session: - session.add( - ComputedHousehold( - household_id=1, - policy_id=2, - country_id="us", - api_version=COUNTRY_PACKAGE_VERSIONS["us"], - computed_household_json=stored_result, - status="complete", - ) + session.add_all( + [ + Household( + id=1, + country_id="us", + label=None, + api_version=COUNTRY_PACKAGE_VERSIONS["us"], + household_json={}, + household_hash="household-hash", + ), + Policy( + id=2, + country_id="us", + label=None, + api_version=COUNTRY_PACKAGE_VERSIONS["us"], + policy_json={}, + policy_hash="policy-hash", + ), + ] ) + cache = HouseholdTraceCache( + InMemoryCacheBackend(), + CacheNamespace("test", "api"), + ) + cache.set( + HouseholdTraceIdentity( + country_id="us", + household_id=1, + policy_id=2, + household_hash="household-hash", + policy_hash="policy-hash", + country_package_version=COUNTRY_PACKAGE_VERSIONS["us"], + policyengine_version=POLICYENGINE_VERSION, + ), + HouseholdTraceValue(household=stored_result, tracer_output=[]), + ) + service = HouseholdCalculationService( + primary_session_factory=orm_session_factory, + cache=cache, + ) - response = get_household_under_policy("us", "1", "2") + with patch( + "policyengine_api.routes.household_routes.household_calculation_service", + service, + ): + response = get_household_under_policy("us", "1", "2") assert response["result"] == {"people": {"you": {"net_income": {"2026": 42}}}} @@ -78,7 +117,10 @@ def test_household_under_policy_calculates_and_caches_json_as_an_object( ) service = HouseholdCalculationService( primary_session_factory=orm_session_factory, - local_session_factory=orm_session_factory, + cache=HouseholdTraceCache( + InMemoryCacheBackend(), + CacheNamespace("test", "api"), + ), country_provider=lambda: {"us": country}, ) @@ -93,9 +135,6 @@ def test_household_under_policy_calculates_and_caches_json_as_an_object( {"people": {"you": {}}}, {"gov.example.parameter": 1}, ) - with orm_session_factory() as session: - cached = session.scalar(select(ComputedHousehold)) - assert cached.computed_household_json == calculated def test_user_policy_endpoints_round_trip_through_orm_session_factory( diff --git a/tests/unit/runtime_cache/__init__.py b/tests/unit/runtime_cache/__init__.py new file mode 100644 index 000000000..3c5affe9b --- /dev/null +++ b/tests/unit/runtime_cache/__init__.py @@ -0,0 +1 @@ +"""Unit tests for the shared runtime-cache boundary.""" diff --git a/tests/unit/runtime_cache/test_client.py b/tests/unit/runtime_cache/test_client.py new file mode 100644 index 000000000..8ea557de4 --- /dev/null +++ b/tests/unit/runtime_cache/test_client.py @@ -0,0 +1,35 @@ +"""Lazy shared Redis client construction tests.""" + +import redis + +from policyengine_api.runtime_cache.client import build_runtime_cache_client +from policyengine_api.runtime_cache.settings import load_runtime_cache_settings + +from .test_settings import TEST_CA_CERT + + +def test_tls_client_receives_memorystore_ca_in_memory(monkeypatch) -> None: + captured: dict[str, object] = {} + sentinel = object() + + def fake_from_url(url: str, **kwargs): + captured.update(url=url, **kwargs) + return sentinel + + monkeypatch.setattr(redis.Redis, "from_url", staticmethod(fake_from_url)) + settings = load_runtime_cache_settings( + { + "RUNTIME_CACHE_MODE": "deployed", + "RUNTIME_CACHE_URL": "rediss://:secret@10.0.0.2:6378/0", + "RUNTIME_CACHE_CA_CERT": TEST_CA_CERT, + "RUNTIME_CACHE_ENVIRONMENT": "production", + "RUNTIME_CACHE_SERVICE": "api", + } + ) + + assert build_runtime_cache_client(settings) is sentinel + assert captured["ssl_cert_reqs"] == "required" + assert captured["ssl_ca_data"] == TEST_CA_CERT + assert captured["max_connections"] == 20 + assert captured["socket_connect_timeout"] == 1.0 + assert captured["socket_timeout"] == 2.0 diff --git a/tests/unit/runtime_cache/test_core.py b/tests/unit/runtime_cache/test_core.py new file mode 100644 index 000000000..953a738ec --- /dev/null +++ b/tests/unit/runtime_cache/test_core.py @@ -0,0 +1,202 @@ +"""Versioned envelope, key, fake, and claim behavior.""" + +from unittest.mock import MagicMock + +import pytest + +from policyengine_api.runtime_cache.claims import ExpiringClaimStore +from policyengine_api.runtime_cache.core import ( + CacheCoordinationError, + CacheNamespace, + RecoverableJSONCache, + decode_envelope, + encode_envelope, + jittered_ttl, +) +from policyengine_api.runtime_cache.fake import InMemoryCacheBackend + + +def test_keys_include_namespace_family_schema_and_every_input() -> None: + namespace = CacheNamespace("staging", "api") + first = namespace.key("computed", 1, {"country": "us", "policy": 1}) + reordered = namespace.key("computed", 1, {"policy": 1, "country": "us"}) + changed_input = namespace.key("computed", 1, {"country": "us", "policy": 2}) + changed_schema = namespace.key("computed", 2, {"country": "us", "policy": 1}) + + assert first == reordered + assert first.startswith("policyengine:staging:api:computed:v1:") + assert len({first, changed_input, changed_schema}) == 3 + + +def test_envelopes_fail_closed_on_family_schema_or_encoding_mismatch() -> None: + encoded = encode_envelope("computed", 1, {"ok": True}) + assert decode_envelope(encoded, family="computed", schema_version=1) == {"ok": True} + assert decode_envelope(encoded, family="other", schema_version=1) is None + assert decode_envelope(encoded, family="computed", schema_version=2) is None + assert decode_envelope("{broken", family="computed", schema_version=1) is None + + +def test_completed_result_ttl_jitter_is_bounded_and_subtract_only() -> None: + def choose_minimum(lower: int, _upper: int) -> int: + return lower + + def choose_maximum(_lower: int, upper: int) -> int: + return upper + + assert jittered_ttl(100, choose_reduction=choose_minimum) == 100 + assert jittered_ttl(100, choose_reduction=choose_maximum) == 90 + assert jittered_ttl(9, choose_reduction=choose_maximum) == 9 + + with pytest.raises(ValueError, match="positive"): + jittered_ttl(0) + with pytest.raises(ValueError, match="fraction"): + jittered_ttl(100, jitter_fraction=1) + + +def test_recoverable_cache_applies_jitter_to_completed_result_writes( + monkeypatch, +) -> None: + import policyengine_api.runtime_cache.core as module + + monkeypatch.setattr(module, "jittered_ttl", lambda _ttl: 91) + backend = InMemoryCacheBackend() + cache = RecoverableJSONCache( + backend, + CacheNamespace("test", "api"), + family="result", + schema_version=1, + ttl_seconds=100, + ) + inputs = {"input": 1} + + assert cache.set(inputs, {"answer": 42}) + assert backend._expires[cache.key(inputs)] == 91 + + +def test_deterministic_fake_expires_values_and_recoverable_cache_misses() -> None: + backend = InMemoryCacheBackend() + cache = RecoverableJSONCache( + backend, + CacheNamespace("test", "api"), + family="result", + schema_version=1, + ttl_seconds=10, + ) + inputs = {"version": "1", "payload": {"x": 1}} + assert cache.get(inputs) is None + assert cache.set(inputs, {"answer": 42}) is True + assert cache.get(inputs) == {"answer": 42} + backend.advance(10) + assert cache.get(inputs) is None + + +def test_cache_events_are_metric_ready_and_cannot_include_keys_or_values( + monkeypatch, +) -> None: + mock_logger = MagicMock() + monkeypatch.setattr("policyengine_api.runtime_cache.core.logger", mock_logger) + backend = InMemoryCacheBackend() + cache = RecoverableJSONCache( + backend, + CacheNamespace("test", "api"), + family="result", + schema_version=1, + ttl_seconds=10, + ) + inputs = {"token": "sensitive-input"} + payload = {"answer": "sensitive-output"} + + assert cache.get(inputs) is None + assert cache.set(inputs, payload) + assert cache.get(inputs) == payload + + records = [call.args[0] for call in mock_logger.log_struct.call_args_list] + assert [record["cache_event"] for record in records] == [ + "miss", + "write", + "hit", + ] + for record in records: + assert record["metric_name"] == "runtime_cache_operations" + assert record["metric_value"] == 1 + assert record["cache_family"] == "result" + assert record["latency_ms"] >= 0 + assert "sensitive-input" not in repr(record) + assert "sensitive-output" not in repr(record) + + +def test_cache_connection_and_write_failures_emit_metric_events(monkeypatch) -> None: + mock_logger = MagicMock() + monkeypatch.setattr("policyengine_api.runtime_cache.core.logger", mock_logger) + + class Broken: + def get(self, _key): + raise OSError("unavailable") + + def set(self, _key, _value, **_kwargs): + raise OSError("unavailable") + + cache = RecoverableJSONCache( + Broken(), + CacheNamespace("test", "api"), + family="result", + schema_version=1, + ttl_seconds=10, + ) + + assert cache.get({"input": 1}) is None + assert not cache.set({"input": 1}, {"output": 2}) + + records = [call.args[0] for call in mock_logger.log_struct.call_args_list] + assert [record["cache_event"] for record in records] == [ + "connection-failed", + "write-failed", + ] + assert all(record["metric_value"] == 1 for record in records) + assert all( + call.kwargs["severity"] == "WARNING" + for call in mock_logger.log_struct.call_args_list + ) + + +def test_claims_are_exclusive_expiring_and_token_safe() -> None: + backend = InMemoryCacheBackend() + claims = ExpiringClaimStore(backend) + assert claims.acquire("claim", "owner-a", ttl_seconds=5) is True + assert claims.acquire("claim", "owner-b", ttl_seconds=5) is False + assert claims.release("claim", "owner-b") is False + assert backend.get("claim") == "owner-a" + backend.advance(5) + assert claims.acquire("claim", "owner-b", ttl_seconds=5) is True + assert claims.release("claim", "owner-b") is True + + +def test_claim_failures_are_coordination_errors_not_cache_misses(monkeypatch) -> None: + mock_logger = MagicMock() + monkeypatch.setattr("policyengine_api.runtime_cache.core.logger", mock_logger) + + class Broken: + def set(self, *_args, **_kwargs): + raise OSError("unavailable") + + def eval(self, *_args, **_kwargs): + raise OSError("unavailable") + + claims = ExpiringClaimStore(Broken()) + with pytest.raises(CacheCoordinationError): + claims.acquire("secret-key-123", "secret-token-456", ttl_seconds=5) + with pytest.raises(CacheCoordinationError): + claims.release("secret-key-123", "secret-token-456") + + records = [call.args[0] for call in mock_logger.log_struct.call_args_list] + assert [record["cache_event"] for record in records] == [ + "coordination-failed", + "coordination-failed", + ] + assert {record["cache_operation"] for record in records} == { + "claim-acquire", + "claim-release", + } + assert all(record["metric_value"] == 1 for record in records) + assert "secret-key-123" not in repr(records) + assert "secret-token-456" not in repr(records) diff --git a/tests/unit/runtime_cache/test_repositories.py b/tests/unit/runtime_cache/test_repositories.py new file mode 100644 index 000000000..2a2a973f1 --- /dev/null +++ b/tests/unit/runtime_cache/test_repositories.py @@ -0,0 +1,204 @@ +"""Typed runtime-cache repository tests.""" + +from datetime import datetime + +from policyengine_api.runtime_cache.core import CacheNamespace +from policyengine_api.runtime_cache.fake import InMemoryCacheBackend +from policyengine_api.runtime_cache.repositories import ( + AIAnalysisCache, + CachedAnalysis, + CachedReformImpact, + HouseholdTraceCache, + HouseholdTraceIdentity, + HouseholdTraceValue, + ReformImpactCache, + REFORM_IMPACT_START_CLAIM_TTL_SECONDS, + reform_impact_id, +) + + +def _namespace() -> CacheNamespace: + return CacheNamespace("test", "api") + + +def _identity(**changes) -> HouseholdTraceIdentity: + values = { + "country_id": "us", + "household_id": 1, + "policy_id": 2, + "household_hash": "household-a", + "policy_hash": "policy-a", + "country_package_version": "1.2.3", + "policyengine_version": "4.5.6", + } + values.update(changes) + return HouseholdTraceIdentity(**values) + + +def test_household_and_tracer_share_one_atomic_versioned_value() -> None: + backend = InMemoryCacheBackend() + cache = HouseholdTraceCache(backend, _namespace()) + value = HouseholdTraceValue( + household={"people": {"you": {"income": {"2026": 42}}}}, + tracer_output=["income <2026>"], + ) + identity = _identity() + + assert cache.set(identity, value) is True + assert cache.get(identity) == value + assert list(backend._values) == [cache.cache_key(identity)] + assert cache.cache_key(identity) != cache.cache_key( + _identity(household_hash="household-b") + ) + assert cache.cache_key(identity) != cache.cache_key( + _identity(country_package_version="9.9.9") + ) + + +def test_analysis_cache_is_model_and_prompt_specific_and_expiring() -> None: + backend = InMemoryCacheBackend() + cache = AIAnalysisCache(backend, _namespace()) + value = CachedAnalysis(prompt="explain", analysis="answer") + assert cache.set(value, model="model-a") is True + assert cache.get("explain", model="model-a") == value + assert cache.get("explain", model="model-b") is None + assert cache.get("different", model="model-a") is None + + +def _impact(execution_id: str, options_hash: str, day: int) -> CachedReformImpact: + return CachedReformImpact( + reform_impact_id=reform_impact_id(execution_id), + baseline_policy_id=1, + reform_policy_id=2, + country_id="us", + region="us", + dataset="default", + time_period="2026", + options_json={"hash": options_hash}, + options_hash=options_hash, + api_version="v1", + reform_impact_json={}, + status="computing", + message=None, + start_time=datetime(2026, 1, day), + end_time=None, + execution_id=execution_id, + ) + + +def _claim_arguments(**changes): + values = { + "country_id": "us", + "reform_policy_id": 2, + "baseline_policy_id": 1, + "region": "us", + "dataset": "default", + "time_period": "2026", + "api_version": "v1", + "options_hash": "hash", + "target": "general", + } + values.update(changes) + return values + + +def test_reform_impact_start_claim_is_atomic_exact_ttl_and_token_safe() -> None: + backend = InMemoryCacheBackend() + cache = ReformImpactCache(backend, _namespace()) + arguments = _claim_arguments() + + assert cache.claim_start(**arguments, claim_token="owner") is True + assert cache.claim_start(**arguments, claim_token="contender") is False + assert ( + cache.claim_start( + **_claim_arguments(target="cliff"), + claim_token="cliff-owner", + ) + is True + ) + assert set(backend._expires.values()) == {REFORM_IMPACT_START_CLAIM_TTL_SECONDS} + + assert cache.release_start(**arguments, claim_token="contender") is False + assert cache.release_start(**arguments, claim_token="owner") is True + assert cache.claim_start(**arguments, claim_token="next-owner") is True + + +def test_reform_impact_indexes_are_bounded_expiring_and_query_compatible( + monkeypatch, +) -> None: + import policyengine_api.runtime_cache.repositories as module + + monkeypatch.setattr(module, "REFORM_IMPACT_INDEX_LIMIT", 2) + backend = InMemoryCacheBackend() + cache = ReformImpactCache(backend, _namespace()) + exact = _impact("exact", "hash-exact", 1) + compatible = _impact("compatible", "hash-compatible", 2) + newest = _impact("newest", "hash-newest", 3) + assert cache.set(exact) + assert cache.set(compatible) + assert cache.set(newest) + + assert [value.execution_id for value in cache.recent(10)] == [ + "newest", + "compatible", + ] + results = cache.matching( + country_id="us", + reform_policy_id=2, + baseline_policy_id=1, + region="us", + dataset="default", + time_period="2026", + api_version="v1", + options_hash="hash-compatible", + options_hash_pattern="hash-%", + ) + assert [value.execution_id for value in results] == [ + "compatible", + "newest", + ] + backend.advance(module.REFORM_IMPACT_TTL_SECONDS) + assert cache.recent(10) == [] + + +def test_reform_impact_record_and_indexes_share_one_jittered_ttl( + monkeypatch, +) -> None: + import policyengine_api.runtime_cache.repositories as module + + monkeypatch.setattr(module, "jittered_ttl", lambda _ttl: 123) + backend = InMemoryCacheBackend() + cache = ReformImpactCache(backend, _namespace()) + + assert cache.set(_impact("jittered", "hash", 1)) + assert len(backend._expires) == 3 + assert set(backend._expires.values()) == {123} + + +def test_reform_impact_updates_and_deletes_only_matching_computing_values() -> None: + backend = InMemoryCacheBackend() + cache = ReformImpactCache(backend, _namespace()) + deleted = _impact("delete", "delete-hash", 1) + retained = _impact("retain", "retain-hash", 2) + cache.set(deleted) + cache.set(retained) + + completed = cache.update( + "retain", + status="ok", + message="Completed", + reform_impact_json={"result": 1}, + ) + assert completed is not None + assert completed.status == "ok" + cache.delete_matching_computing( + country_id="us", + reform_policy_id=2, + baseline_policy_id=1, + region="us", + dataset="default", + time_period="2026", + options_hash="delete-hash", + ) + assert cache.get_by_execution_id("delete") is None + assert cache.get_by_execution_id("retain") is not None diff --git a/tests/unit/runtime_cache/test_settings.py b/tests/unit/runtime_cache/test_settings.py new file mode 100644 index 000000000..dad8012ec --- /dev/null +++ b/tests/unit/runtime_cache/test_settings.py @@ -0,0 +1,161 @@ +"""Fail-closed runtime-cache configuration tests.""" + +import pytest + +from policyengine_api.runtime_cache.settings import ( + RUNTIME_CACHE_CA_CERT, + RUNTIME_CACHE_CA_CERT_SECRET_RESOURCE, + RUNTIME_CACHE_ENVIRONMENT, + RUNTIME_CACHE_MODE, + RUNTIME_CACHE_SERVICE, + RUNTIME_CACHE_URL, + RUNTIME_CACHE_URL_SECRET_RESOURCE, + RuntimeCacheConfigurationError, + load_runtime_cache_settings, +) + + +TEST_CA_CERT = """-----BEGIN CERTIFICATE----- +stage-8-test-ca +-----END CERTIFICATE-----""" + + +def test_unselected_local_or_test_environment_is_disabled_without_fallback() -> None: + settings = load_runtime_cache_settings({}) + assert settings.enabled is False + assert settings.url is None + + +def test_platform_marker_selects_deployed_mode_and_requires_configuration() -> None: + with pytest.raises(RuntimeCacheConfigurationError, match=RUNTIME_CACHE_URL): + load_runtime_cache_settings({"K_SERVICE": "policyengine-api"}) + + +@pytest.mark.parametrize( + "url", + [ + "redis://:password@10.0.0.2:6379/0", + "rediss://:password@127.0.0.1:6379/0", + "rediss://10.0.0.2:6379/0", + "rediss://:password@10.0.0.2/0", + "rediss://:password@10.0.0.2:6379/1", + ], +) +def test_deployed_mode_requires_tls_auth_nonlocal_host_port_and_database_zero( + url: str, +) -> None: + with pytest.raises(RuntimeCacheConfigurationError): + load_runtime_cache_settings( + { + RUNTIME_CACHE_MODE: "deployed", + RUNTIME_CACHE_URL: url, + RUNTIME_CACHE_ENVIRONMENT: "production", + RUNTIME_CACHE_SERVICE: "api", + } + ) + + +def test_valid_deployed_configuration_is_secret_safe() -> None: + secret = "do-not-print-cache-password" + ca_secret = "do-not-print-cache-ca" + settings = load_runtime_cache_settings( + { + RUNTIME_CACHE_MODE: "deployed", + RUNTIME_CACHE_URL: f"rediss://:{secret}@10.0.0.2:6378/0", + RUNTIME_CACHE_CA_CERT: TEST_CA_CERT.replace("stage-8-test-ca", ca_secret), + RUNTIME_CACHE_ENVIRONMENT: "production", + RUNTIME_CACHE_SERVICE: "api", + } + ) + assert settings.enabled is True + assert settings.tls is True + assert secret not in repr(settings) + assert ca_secret not in repr(settings) + + +def test_deployed_tls_configuration_requires_a_valid_ca_bundle() -> None: + values = { + RUNTIME_CACHE_MODE: "deployed", + RUNTIME_CACHE_URL: "rediss://:password@10.0.0.2:6378/0", + RUNTIME_CACHE_ENVIRONMENT: "production", + RUNTIME_CACHE_SERVICE: "api", + } + with pytest.raises(RuntimeCacheConfigurationError, match=RUNTIME_CACHE_CA_CERT): + load_runtime_cache_settings(values) + + with pytest.raises(RuntimeCacheConfigurationError, match="PEM"): + load_runtime_cache_settings( + {**values, RUNTIME_CACHE_CA_CERT: "not-a-certificate"} + ) + + +def test_app_engine_resolves_separate_secret_resources_without_echoing_them() -> None: + url_resource = "projects/project/secrets/cache-url/versions/latest" + ca_resource = "projects/project/secrets/cache-ca/versions/latest" + observed: list[str] = [] + + def load_secret(resource: str) -> str: + observed.append(resource) + if resource == url_resource: + return "rediss://:secret@10.0.0.2:6378/0" + if resource == ca_resource: + return TEST_CA_CERT + raise AssertionError(resource) + + settings = load_runtime_cache_settings( + { + RUNTIME_CACHE_MODE: "deployed", + RUNTIME_CACHE_URL_SECRET_RESOURCE: url_resource, + RUNTIME_CACHE_CA_CERT_SECRET_RESOURCE: ca_resource, + RUNTIME_CACHE_ENVIRONMENT: "staging", + RUNTIME_CACHE_SERVICE: "api", + }, + secret_loader=load_secret, + ) + + assert observed == [url_resource, ca_resource] + assert settings.url is not None + assert settings.ca_cert is not None + + +def test_runtime_cache_rejects_ambiguous_or_invalid_secret_resources() -> None: + values = { + RUNTIME_CACHE_MODE: "deployed", + RUNTIME_CACHE_URL: "rediss://:secret@10.0.0.2:6378/0", + RUNTIME_CACHE_URL_SECRET_RESOURCE: ( + "projects/project/secrets/cache-url/versions/latest" + ), + RUNTIME_CACHE_CA_CERT: TEST_CA_CERT, + RUNTIME_CACHE_ENVIRONMENT: "production", + RUNTIME_CACHE_SERVICE: "api", + } + with pytest.raises(RuntimeCacheConfigurationError, match="exactly one"): + load_runtime_cache_settings(values) + + values.pop(RUNTIME_CACHE_URL) + values[RUNTIME_CACHE_URL_SECRET_RESOURCE] = "not-a-resource" + with pytest.raises(RuntimeCacheConfigurationError, match="invalid"): + load_runtime_cache_settings(values) + + +def test_local_mode_requires_explicit_local_url_and_namespace() -> None: + settings = load_runtime_cache_settings( + { + RUNTIME_CACHE_MODE: "local", + RUNTIME_CACHE_URL: "redis://127.0.0.1:6379/0", + RUNTIME_CACHE_ENVIRONMENT: "local-dev", + RUNTIME_CACHE_SERVICE: "api", + } + ) + assert settings.enabled is True + assert settings.tls is False + + with pytest.raises(RuntimeCacheConfigurationError, match="local endpoint"): + load_runtime_cache_settings( + { + RUNTIME_CACHE_MODE: "local", + RUNTIME_CACHE_URL: "redis://cache.example.com:6379/0", + RUNTIME_CACHE_ENVIRONMENT: "local-dev", + RUNTIME_CACHE_SERVICE: "api", + } + ) diff --git a/tests/unit/services/test_ai_analysis_service.py b/tests/unit/services/test_ai_analysis_service.py index 13867f6e3..27d9949b7 100644 --- a/tests/unit/services/test_ai_analysis_service.py +++ b/tests/unit/services/test_ai_analysis_service.py @@ -1,79 +1,59 @@ import json -from contextlib import contextmanager from types import SimpleNamespace import pytest -from sqlalchemy import select -from policyengine_api.data.v1_models import Analysis -from policyengine_api.services.ai_analysis_service import AIAnalysisService +from policyengine_api.runtime_cache.core import CacheNamespace +from policyengine_api.runtime_cache.fake import InMemoryCacheBackend +from policyengine_api.runtime_cache.repositories import AIAnalysisCache +from policyengine_api.services.ai_analysis_service import ( + AI_ANALYSIS_MODEL, + AIAnalysisService, +) from tests.fixtures.services.ai_analysis_service import parse_to_chunks pytest_plugins = ["tests.fixtures.services.ai_analysis_service"] +def _cache() -> AIAnalysisCache: + return AIAnalysisCache( + InMemoryCacheBackend(), + CacheNamespace("test", "api"), + ) + + class TestTriggerAIAnalysis: - def test_claude_stream_runs_without_an_open_database_session( - self, - orm_session_factory, - ): - class TrackingSessions: - def __init__(self, delegate): - self.delegate = delegate - self.active = 0 - - @contextmanager - def __call__(self): - self.active += 1 - try: - with self.delegate() as session: - yield session - finally: - self.active -= 1 - - @contextmanager - def begin(self): - self.active += 1 - try: - with self.delegate.begin() as session: - yield session - finally: - self.active -= 1 - - sessions = TrackingSessions(orm_session_factory) + def test_claude_stream_caches_only_after_successful_completion(self): + cache = _cache() class ClaudeStream: def __enter__(self): - assert sessions.active == 0 return self def __exit__(self, *args): return None def __iter__(self): - assert sessions.active == 0 + assert cache.get("prompt", model=AI_ANALYSIS_MODEL) is None yield SimpleNamespace(type="text", text="analysis") claude_client = SimpleNamespace( messages=SimpleNamespace(stream=lambda **kwargs: ClaudeStream()) ) service = AIAnalysisService( - sessions, + cache, claude_client_factory=lambda: claude_client, ) assert list(service.trigger_ai_analysis("prompt")) == [ json.dumps({"type": "text", "stream": "analysis"}) + "\n" ] - assert sessions.active == 0 - - with orm_session_factory() as session: - stored = session.scalar(select(Analysis).where(Analysis.prompt == "prompt")) + stored = cache.get("prompt", model=AI_ANALYSIS_MODEL) assert stored is not None assert stored.analysis == "analysis" def test_trigger_ai_analysis_given_successful_streaming( - self, mock_stream_text_events, orm_session_factory + self, mock_stream_text_events ): # GIVEN a series of successful text messages from the Claude API expected_response = "This is a historical quote." @@ -82,7 +62,8 @@ def test_trigger_ai_analysis_given_successful_streaming( # WHEN we call trigger_ai_analysis prompt = "Tell me a historical quote" - generator = AIAnalysisService(orm_session_factory).trigger_ai_analysis(prompt) + cache = _cache() + generator = AIAnalysisService(cache).trigger_ai_analysis(prompt) # THEN it should yield the expected chunks results = list(generator) @@ -95,11 +76,7 @@ def test_trigger_ai_analysis_given_successful_streaming( ) assert chunk == expected_chunk - # Verify the database was updated with the complete response - with orm_session_factory() as session: - analysis_record = session.scalar( - select(Analysis).where(Analysis.prompt == prompt) - ) + analysis_record = cache.get(prompt, model=AI_ANALYSIS_MODEL) assert analysis_record is not None assert analysis_record.analysis == expected_response @@ -113,15 +90,14 @@ def test_trigger_ai_analysis_given_successful_streaming( "unknown_error", ], ) - def test_trigger_ai_analysis_given_error( - self, mock_stream_error_event, orm_session_factory, error_type - ): + def test_trigger_ai_analysis_given_error(self, mock_stream_error_event, error_type): # GIVEN an overloaded_error event from the Claude API mock_stream_error_event(error_type) # WHEN we call trigger_ai_analysis prompt = "Tell me a historical quote about erroneous systems" - generator = AIAnalysisService(orm_session_factory).trigger_ai_analysis(prompt) + cache = _cache() + generator = AIAnalysisService(cache).trigger_ai_analysis(prompt) # THEN it should yield the expected error message results = list(generator) @@ -138,10 +114,4 @@ def test_trigger_ai_analysis_given_error( ) assert results[0] == expected_error - # Verify the database was not updated - with orm_session_factory() as session: - analysis_record = session.scalar( - select(Analysis).where(Analysis.prompt == prompt) - ) - - assert analysis_record is None + assert cache.get(prompt, model=AI_ANALYSIS_MODEL) is None diff --git a/tests/unit/services/test_budget_window_cache.py b/tests/unit/services/test_budget_window_cache.py index c53f47383..d71467bc2 100644 --- a/tests/unit/services/test_budget_window_cache.py +++ b/tests/unit/services/test_budget_window_cache.py @@ -2,24 +2,19 @@ import pytest -from policyengine_api.services.budget_window_cache import BudgetWindowCache +from policyengine_api.runtime_cache.core import CacheCoordinationError +from policyengine_api.runtime_cache.fake import InMemoryCacheBackend +from policyengine_api.services.budget_window_cache import ( + BUDGET_WINDOW_BATCH_TTL_SECONDS, + BUDGET_WINDOW_STARTING_TTL_SECONDS, + BudgetWindowCache, +) -class FakeRedis: - def __init__(self): - self.values = {} - - def get(self, key): - return self.values.get(key) - - def set(self, key, value, nx=False, ex=None): - if nx and key in self.values: - return False - self.values[key] = value - return True - - def delete(self, key): - self.values.pop(key, None) +class FakeRedis(InMemoryCacheBackend): + @property + def values(self): + return self._values class RaisingRedis: @@ -40,6 +35,9 @@ def delete(self, key): if self.method == "delete": raise RuntimeError("redis unavailable") + def eval(self, *_args, **_kwargs): + raise RuntimeError("redis unavailable") + def test_build_key_is_stable_for_request_identity(): cache = BudgetWindowCache(client=FakeRedis()) @@ -66,7 +64,7 @@ def test_build_key_is_stable_for_request_identity(): ) assert first == second - assert first.startswith("budget_window:v1:us:") + assert first.startswith("policyengine:test:api:budget-window:v1:") def test_claim_batch_start_allows_one_starter(): @@ -95,6 +93,32 @@ def test_completed_result_round_trips(): assert cache.get_completed_result("budget_window:v1:us:key") == result +def test_completed_result_ttl_is_jittered_but_coordination_ttls_are_exact( + monkeypatch, +): + import policyengine_api.services.budget_window_cache as module + + monkeypatch.setattr(module, "jittered_ttl", lambda _ttl: 123) + redis_client = FakeRedis() + cache = BudgetWindowCache(client=redis_client) + cache_key = "budget_window:v1:us:key" + + assert cache.set_completed_result(cache_key, {"ok": True}) + assert redis_client._expires[f"{cache_key}:result"] == 123 + + assert cache.claim_batch_start(cache_key, "process-1") + assert ( + redis_client._expires[f"{cache_key}:batch-job-id"] + == BUDGET_WINDOW_STARTING_TTL_SECONDS + ) + + cache.store_batch_job_id(cache_key, "batch-1") + assert ( + redis_client._expires[f"{cache_key}:batch-job-id"] + == BUDGET_WINDOW_BATCH_TTL_SECONDS + ) + + def test_get_completed_result_returns_none_for_empty_payload(): redis_client = FakeRedis() redis_client.values["budget_window:v1:us:key:result"] = "" @@ -106,7 +130,7 @@ def test_get_completed_result_returns_none_for_empty_payload(): def test_get_completed_result_returns_none_for_invalid_json(monkeypatch): mock_logger = MagicMock() monkeypatch.setattr( - "policyengine_api.services.budget_window_cache.logger", + "policyengine_api.runtime_cache.core.logger", mock_logger, ) redis_client = FakeRedis() @@ -117,30 +141,28 @@ def test_get_completed_result_returns_none_for_invalid_json(monkeypatch): assert mock_logger.log_struct.call_args.kwargs["severity"] == "WARNING" -def test_get_completed_result_reraises_read_errors(monkeypatch): +def test_get_completed_result_treats_read_errors_as_misses(monkeypatch): mock_logger = MagicMock() monkeypatch.setattr( - "policyengine_api.services.budget_window_cache.logger", + "policyengine_api.runtime_cache.core.logger", mock_logger, ) cache = BudgetWindowCache(client=RaisingRedis(method="get")) - with pytest.raises(RuntimeError, match="redis unavailable"): - cache.get_completed_result("budget_window:v1:us:key") + assert cache.get_completed_result("budget_window:v1:us:key") is None assert mock_logger.log_struct.call_args.kwargs["severity"] == "WARNING" -def test_set_completed_result_reraises_write_errors(monkeypatch): +def test_set_completed_result_does_not_invalidate_compute_on_write_error(monkeypatch): mock_logger = MagicMock() monkeypatch.setattr( - "policyengine_api.services.budget_window_cache.logger", + "policyengine_api.runtime_cache.core.logger", mock_logger, ) cache = BudgetWindowCache(client=RaisingRedis(method="set")) - with pytest.raises(RuntimeError, match="redis unavailable"): - cache.set_completed_result("budget_window:v1:us:key", {"ok": True}) + assert not cache.set_completed_result("budget_window:v1:us:key", {"ok": True}) assert mock_logger.log_struct.call_args.kwargs["severity"] == "WARNING" @@ -149,25 +171,25 @@ def test_get_batch_job_id_ignores_empty_non_string_and_starting_values(): redis_client = FakeRedis() cache = BudgetWindowCache(client=redis_client) - redis_client.values["budget_window:v1:us:key:batch_job_id"] = "" + redis_client.values["budget_window:v1:us:key:batch-job-id"] = "" assert cache.get_batch_job_id("budget_window:v1:us:key") is None - redis_client.values["budget_window:v1:us:key:batch_job_id"] = 123 + redis_client.values["budget_window:v1:us:key:batch-job-id"] = 123 assert cache.get_batch_job_id("budget_window:v1:us:key") is None - redis_client.values["budget_window:v1:us:key:batch_job_id"] = "starting:process-1" + redis_client.values["budget_window:v1:us:key:batch-job-id"] = "starting:process-1" assert cache.get_batch_job_id("budget_window:v1:us:key") is None def test_get_batch_job_id_reraises_read_errors(monkeypatch): mock_logger = MagicMock() monkeypatch.setattr( - "policyengine_api.services.budget_window_cache.logger", + "policyengine_api.runtime_cache.core.logger", mock_logger, ) cache = BudgetWindowCache(client=RaisingRedis(method="get")) - with pytest.raises(RuntimeError, match="redis unavailable"): + with pytest.raises(CacheCoordinationError): cache.get_batch_job_id("budget_window:v1:us:key") assert mock_logger.log_struct.call_args.kwargs["severity"] == "WARNING" @@ -176,12 +198,12 @@ def test_get_batch_job_id_reraises_read_errors(monkeypatch): def test_claim_batch_start_reraises_claim_errors(monkeypatch): mock_logger = MagicMock() monkeypatch.setattr( - "policyengine_api.services.budget_window_cache.logger", + "policyengine_api.runtime_cache.core.logger", mock_logger, ) cache = BudgetWindowCache(client=RaisingRedis(method="set")) - with pytest.raises(RuntimeError, match="redis unavailable"): + with pytest.raises(CacheCoordinationError): cache.claim_batch_start("budget_window:v1:us:key", "process-1") assert mock_logger.log_struct.call_args.kwargs["severity"] == "WARNING" @@ -190,12 +212,12 @@ def test_claim_batch_start_reraises_claim_errors(monkeypatch): def test_store_batch_job_id_reraises_write_errors(monkeypatch): mock_logger = MagicMock() monkeypatch.setattr( - "policyengine_api.services.budget_window_cache.logger", + "policyengine_api.runtime_cache.core.logger", mock_logger, ) cache = BudgetWindowCache(client=RaisingRedis(method="set")) - with pytest.raises(RuntimeError, match="redis unavailable"): + with pytest.raises(CacheCoordinationError): cache.store_batch_job_id("budget_window:v1:us:key", "fc-parent") assert mock_logger.log_struct.call_args.kwargs["severity"] == "WARNING" @@ -209,19 +231,19 @@ def test_clear_starting_claim_deletes_only_matching_token(): cache.clear_starting_claim("budget_window:v1:us:key", "process-2") assert ( - redis_client.values["budget_window:v1:us:key:batch_job_id"] + redis_client.values["budget_window:v1:us:key:batch-job-id"] == "starting:process-1" ) cache.clear_starting_claim("budget_window:v1:us:key", "process-1") - assert "budget_window:v1:us:key:batch_job_id" not in redis_client.values + assert "budget_window:v1:us:key:batch-job-id" not in redis_client.values def test_clear_starting_claim_logs_and_swallows_errors(monkeypatch): mock_logger = MagicMock() monkeypatch.setattr( - "policyengine_api.services.budget_window_cache.logger", + "policyengine_api.runtime_cache.core.logger", mock_logger, ) cache = BudgetWindowCache(client=RaisingRedis(method="get")) @@ -234,7 +256,7 @@ def test_clear_starting_claim_logs_and_swallows_errors(monkeypatch): def test_clear_batch_job_id_logs_and_swallows_errors(monkeypatch): mock_logger = MagicMock() monkeypatch.setattr( - "policyengine_api.services.budget_window_cache.logger", + "policyengine_api.runtime_cache.core.logger", mock_logger, ) cache = BudgetWindowCache(client=RaisingRedis(method="delete")) diff --git a/tests/unit/services/test_direct_orm_local_analysis.py b/tests/unit/services/test_direct_orm_local_analysis.py index c6a4b0ef6..3f3ed165b 100644 --- a/tests/unit/services/test_direct_orm_local_analysis.py +++ b/tests/unit/services/test_direct_orm_local_analysis.py @@ -1,33 +1,45 @@ +"""Regression guards replacing the former direct local-ORM cache tests.""" + from datetime import datetime -from policyengine_api.data.local_models import Tracer -from policyengine_api.data.v1_models import Analysis, ReformImpact -from policyengine_api.services.ai_analysis_service import AIAnalysisService +from policyengine_api.runtime_cache.core import CacheNamespace +from policyengine_api.runtime_cache.fake import InMemoryCacheBackend +from policyengine_api.runtime_cache.repositories import ( + AIAnalysisCache, + CachedAnalysis, + CachedReformImpact, + ReformImpactCache, +) +from policyengine_api.services.ai_analysis_service import ( + AI_ANALYSIS_MODEL, + AIAnalysisService, +) from policyengine_api.services.reform_impacts_service import ReformImpactsService -from policyengine_api.services.tracer_analysis_service import TracerAnalysisService -def test_ai_analysis_service_returns_the_latest_mapped_analysis( - orm_session, - orm_session_factory, -): - orm_session.add_all( - [ - Analysis(prompt="prompt", analysis="old", status="ok"), - Analysis(prompt="prompt", analysis="new", status="complete"), - ] +def _context(): + return InMemoryCacheBackend(), CacheNamespace("test", "api") + + +def test_ai_analysis_service_returns_typed_cached_analysis() -> None: + backend, namespace = _context() + cache = AIAnalysisCache(backend, namespace) + cache.set( + CachedAnalysis(prompt="prompt", analysis="new"), + model=AI_ANALYSIS_MODEL, ) - orm_session.flush() - orm_session.commit() - analysis = AIAnalysisService(orm_session_factory).get_existing_analysis("prompt") + analysis = AIAnalysisService(cache).get_existing_analysis("prompt") - assert isinstance(analysis, Analysis) + assert isinstance(analysis, CachedAnalysis) assert analysis.analysis == "new" -def test_reform_impact_service_writes_mapped_entity(orm_session_factory): - impact = ReformImpactsService(orm_session_factory).set_reform_impact( +def test_reform_impact_service_writes_typed_expiring_cache_entity() -> None: + backend, namespace = _context() + impact = ReformImpactsService( + ReformImpactCache(backend, namespace) + ).set_reform_impact( country_id="us", policy_id=2, baseline_policy_id=1, @@ -43,30 +55,5 @@ def test_reform_impact_service_writes_mapped_entity(orm_session_factory): execution_id="job", ) - assert isinstance(impact, ReformImpact) + assert isinstance(impact, CachedReformImpact) assert impact.options_json == {"dataset": "default"} - - -def test_tracer_service_reads_python_json_from_mapped_entity( - orm_session, - orm_session_factory, -): - orm_session.add( - Tracer( - household_id=1, - policy_id=2, - country_id="us", - api_version="1", - tracer_output=["net_income <2026>", " dependency"], - ) - ) - orm_session.commit() - - tracer = TracerAnalysisService(orm_session_factory).get_tracer( - "us", - "1", - "2", - "1", - ) - - assert tracer == ["net_income <2026>", " dependency"] diff --git a/tests/unit/services/test_economy_service.py b/tests/unit/services/test_economy_service.py index af9f4b14b..08448a458 100644 --- a/tests/unit/services/test_economy_service.py +++ b/tests/unit/services/test_economy_service.py @@ -4,6 +4,7 @@ import httpx import pytest +from policyengine_api.runtime_cache.core import CacheCoordinationError from policyengine_api.services.economy_service import ( BUDGET_WINDOW_MAX_END_YEAR, BUDGET_WINDOW_MAX_YEARS, @@ -305,6 +306,16 @@ def test__given_no_previous_impact__creates_new_simulation( assert result.status == ImpactStatus.COMPUTING assert result.data is None mock_simulation_entrypoint.run.assert_called_once() + mock_reform_impacts_service.claim_reform_impact_start.assert_called_once() + assert ( + mock_reform_impacts_service.claim_reform_impact_start.call_args.kwargs[ + "options_hash" + ] + == MOCK_OPTIONS_HASH + ) + mock_reform_impacts_service.release_reform_impact_start.assert_called_once_with( + **mock_reform_impacts_service.claim_reform_impact_start.call_args.kwargs + ) mock_reform_impacts_service.set_reform_impact.assert_called_once() write_values = ( mock_reform_impacts_service.set_reform_impact.call_args.kwargs @@ -312,6 +323,75 @@ def test__given_no_previous_impact__creates_new_simulation( assert write_values["options"] == MOCK_OPTIONS assert write_values["reform_impact_json"] == {} + def test__given_existing_start_claim__does_not_submit_duplicate_simulation( + self, + economy_service, + base_params, + mock_country_package_versions, + mock_policyengine_version, + mock_policy_service, + mock_reform_impacts_service, + mock_simulation_entrypoint, + mock_logger, + mock_datetime, + mock_numpy_random, + ): + mock_reform_impacts_service.claim_reform_impact_start.return_value = False + + result = economy_service.get_economic_impact(**base_params) + + assert result.status is ImpactStatus.COMPUTING + mock_simulation_entrypoint.run.assert_not_called() + mock_reform_impacts_service.set_reform_impact.assert_not_called() + mock_reform_impacts_service.release_reform_impact_start.assert_not_called() + + def test__given_start_claim_cache_failure__fails_before_submission( + self, + economy_service, + base_params, + mock_country_package_versions, + mock_policyengine_version, + mock_policy_service, + mock_reform_impacts_service, + mock_simulation_entrypoint, + mock_logger, + mock_datetime, + mock_numpy_random, + ): + mock_reform_impacts_service.claim_reform_impact_start.side_effect = ( + CacheCoordinationError("cache unavailable") + ) + + with pytest.raises(CacheCoordinationError, match="cache unavailable"): + economy_service.get_economic_impact(**base_params) + + mock_simulation_entrypoint.run.assert_not_called() + mock_reform_impacts_service.set_reform_impact.assert_not_called() + + def test__given_simulation_submission_failure__releases_start_claim( + self, + economy_service, + base_params, + mock_country_package_versions, + mock_policyengine_version, + mock_policy_service, + mock_reform_impacts_service, + mock_simulation_entrypoint, + mock_logger, + mock_datetime, + mock_numpy_random, + ): + mock_simulation_entrypoint.run.side_effect = RuntimeError( + "submission failed" + ) + + with pytest.raises(RuntimeError, match="submission failed"): + economy_service.get_economic_impact(**base_params) + + mock_reform_impacts_service.release_reform_impact_start.assert_called_once_with( + **mock_reform_impacts_service.claim_reform_impact_start.call_args.kwargs + ) + def test__given_policies_created_through_orm__submits_decoded_json( self, orm_session_factory, @@ -346,7 +426,6 @@ def test__given_policies_created_through_orm__submits_decoded_json( service = EconomyService( primary_session_factory=orm_session_factory, - local_session_factory=orm_session_factory, policy_service_=policy_service, reform_impacts_service_=reform_impacts, simulation_entrypoint_=simulation_gateway, diff --git a/tests/unit/services/test_execute_analysis.py b/tests/unit/services/test_execute_analysis.py index e4e2e7714..25a2d7a59 100644 --- a/tests/unit/services/test_execute_analysis.py +++ b/tests/unit/services/test_execute_analysis.py @@ -13,7 +13,6 @@ class TestExecuteAnalysis: def test_execute_analysis_static( self, - orm_session_factory, mock_get_tracer, mock_parse_tracer_output, mock_get_existing_analysis, @@ -25,16 +24,15 @@ def test_execute_analysis_static( THEN then a static analysis with the "static" flag should be returned. """ - analysis, analysis_type = TracerAnalysisService( - orm_session_factory - ).execute_analysis(country_id, household_id, policy_id, target_variable) + analysis, analysis_type = TracerAnalysisService().execute_analysis( + country_id, household_id, policy_id, target_variable + ) assert analysis == "Existing static analysis" assert analysis_type == "static" def test_execute_analysis_streaming( self, - orm_session_factory, mock_get_tracer, mock_parse_tracer_output, mock_get_existing_analysis, @@ -50,9 +48,9 @@ def test_execute_analysis_streaming( # When existing analysis value is None mock_get_existing_analysis.return_value = None - analysis, analysis_type = TracerAnalysisService( - orm_session_factory - ).execute_analysis(country_id, household_id, policy_id, target_variable) + analysis, analysis_type = TracerAnalysisService().execute_analysis( + country_id, household_id, policy_id, target_variable + ) expected_streaming_output = ["stream chunk 1", "stream chunk 2"] streaming_output = list(analysis) diff --git a/tests/unit/services/test_household_calculation_service.py b/tests/unit/services/test_household_calculation_service.py index e849861b8..32db86a18 100644 --- a/tests/unit/services/test_household_calculation_service.py +++ b/tests/unit/services/test_household_calculation_service.py @@ -1,17 +1,20 @@ from contextlib import contextmanager from pathlib import Path from types import SimpleNamespace +from unittest.mock import MagicMock -import pytest -from sqlalchemy import select - -from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS -from policyengine_api.data.local_models import Tracer +from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS, POLICYENGINE_VERSION from policyengine_api.data.v1_models import ( - ComputedHousehold, Household, Policy, ) +from policyengine_api.runtime_cache.core import CacheNamespace +from policyengine_api.runtime_cache.fake import InMemoryCacheBackend +from policyengine_api.runtime_cache.repositories import ( + HouseholdTraceCache, + HouseholdTraceIdentity, + HouseholdTraceValue, +) from policyengine_api.services.household_calculation_service import ( HouseholdCalculationService, ) @@ -68,6 +71,25 @@ def _seed_inputs(factory): ) +def _cache() -> HouseholdTraceCache: + return HouseholdTraceCache( + InMemoryCacheBackend(), + CacheNamespace("test", "api"), + ) + + +def _identity() -> HouseholdTraceIdentity: + return HouseholdTraceIdentity( + country_id="us", + household_id=1, + policy_id=2, + household_hash="household-hash", + policy_hash="policy-hash", + country_package_version=COUNTRY_PACKAGE_VERSIONS["us"], + policyengine_version=POLICYENGINE_VERSION, + ) + + def test_household_route_and_country_do_not_manage_persistence(): route_source = (PACKAGE_ROOT / "routes" / "household_routes.py").read_text( encoding="utf-8" @@ -80,12 +102,15 @@ def test_household_route_and_country_do_not_manage_persistence(): assert "Tracer(" not in country_source -def test_calculation_closes_reads_before_compute_and_persists_local_results( +def test_calculation_closes_reads_before_compute_and_caches_atomic_results( orm_session_factory, + monkeypatch, ): + mock_logger = MagicMock() + monkeypatch.setattr("policyengine_api.runtime_cache.core.logger", mock_logger) _seed_inputs(orm_session_factory) primary = TrackingSessionFactory(orm_session_factory) - local = TrackingSessionFactory(orm_session_factory) + cache = _cache() class Country: metadata = { @@ -95,7 +120,6 @@ class Country: def calculate(self, household, policy): assert primary.active_scopes == 0 - assert local.active_scopes == 0 return SimpleNamespace( household={"people": {"you": {"net_income": {"2026": 42}}}}, tracer_output=["net_income <2026>"], @@ -103,34 +127,30 @@ def calculate(self, household, policy): service = HouseholdCalculationService( primary_session_factory=primary, - local_session_factory=local, + cache=cache, country_provider=lambda: {"us": Country()}, ) result = service.calculate_stored_household("us", 1, 2) assert result.household["people"]["you"]["net_income"]["2026"] == 42 - with orm_session_factory() as session: - cached = session.scalar(select(ComputedHousehold)) - tracer = session.scalar(select(Tracer)) - assert cached.computed_household_json == result.household - assert tracer.tracer_output == ["net_income <2026>"] + cached = cache.get(_identity()) + assert cached is not None + assert cached.household == result.household + assert cached.tracer_output == ["net_income <2026>"] + assert "recompute" in { + call.args[0]["cache_event"] for call in mock_logger.log_struct.call_args_list + } def test_calculation_uses_local_cache_without_recomputing(orm_session_factory): _seed_inputs(orm_session_factory) calculated = {"people": {"you": {"net_income": {"2026": 42}}}} - with orm_session_factory.begin() as session: - session.add( - ComputedHousehold( - household_id=1, - policy_id=2, - country_id="us", - api_version=COUNTRY_PACKAGE_VERSIONS["us"], - computed_household_json=calculated, - status="complete", - ) - ) + cache = _cache() + cache.set( + _identity(), + HouseholdTraceValue(household=calculated, tracer_output=[]), + ) country = SimpleNamespace( metadata={"variables": {}, "entities": {}}, calculate=lambda *_: (_ for _ in ()).throw( @@ -139,7 +159,7 @@ def test_calculation_uses_local_cache_without_recomputing(orm_session_factory): ) service = HouseholdCalculationService( primary_session_factory=orm_session_factory, - local_session_factory=orm_session_factory, + cache=cache, country_provider=lambda: {"us": country}, ) @@ -149,9 +169,8 @@ def test_calculation_uses_local_cache_without_recomputing(orm_session_factory): assert result.cached is True -def test_local_computed_household_and_tracer_write_roll_back_together( +def test_failed_cache_write_does_not_invalidate_successful_calculation( orm_session_factory, - monkeypatch, ): _seed_inputs(orm_session_factory) country = SimpleNamespace( @@ -164,26 +183,20 @@ def test_local_computed_household_and_tracer_write_roll_back_together( tracer_output=["trace"], ), ) + + class BrokenBackend(InMemoryCacheBackend): + def set(self, *_args, **_kwargs): + raise OSError("cache unavailable") + service = HouseholdCalculationService( primary_session_factory=orm_session_factory, - local_session_factory=orm_session_factory, + cache=HouseholdTraceCache( + BrokenBackend(), + CacheNamespace("test", "api"), + ), country_provider=lambda: {"us": country}, ) - session_type = orm_session_factory.class_ - original_flush = session_type.flush - - def fail_tracer_flush(session, *args, **kwargs): - has_tracer = any(isinstance(value, Tracer) for value in session.new) - original_flush(session, *args, **kwargs) - if has_tracer: - raise RuntimeError("tracer insert failed") - - monkeypatch.setattr(session_type, "flush", fail_tracer_flush) - with pytest.raises(RuntimeError, match="tracer insert failed"): - service.calculate_stored_household("us", 1, 2) - - monkeypatch.setattr(session_type, "flush", original_flush) - with orm_session_factory() as session: - assert session.scalar(select(ComputedHousehold)) is None - assert session.scalar(select(Tracer)) is None + result = service.calculate_stored_household("us", 1, 2) + assert result.household == {"people": {"you": {}}} + assert result.cached is False diff --git a/tests/unit/services/test_reform_impacts_service.py b/tests/unit/services/test_reform_impacts_service.py index 15b5029d5..420dff9c0 100644 --- a/tests/unit/services/test_reform_impacts_service.py +++ b/tests/unit/services/test_reform_impacts_service.py @@ -1,15 +1,21 @@ from datetime import datetime import pytest -from sqlalchemy import select -from policyengine_api.data.v1_models import ReformImpact +from policyengine_api.runtime_cache.core import CacheNamespace +from policyengine_api.runtime_cache.fake import InMemoryCacheBackend +from policyengine_api.runtime_cache.repositories import ReformImpactCache from policyengine_api.services.reform_impacts_service import ReformImpactsService @pytest.fixture -def service(orm_session_factory): - return ReformImpactsService(orm_session_factory) +def service(): + return ReformImpactsService( + ReformImpactCache( + InMemoryCacheBackend(), + CacheNamespace("test", "api"), + ) + ) def _create_impact( @@ -56,6 +62,31 @@ def test_get_recent_reform_impacts_orders_and_limits_results(service): assert older.reform_impact_id != newer.reform_impact_id +def test_reform_impact_start_claim_is_exclusive_and_releasable(service): + arguments = { + "country_id": "us", + "policy_id": 2, + "baseline_policy_id": 1, + "region": "us", + "dataset": "default", + "time_period": "2026", + "options_hash": "resolved-hash", + "api_version": "1", + "target": "general", + } + + assert service.claim_reform_impact_start(**arguments, claim_token="owner") + assert not service.claim_reform_impact_start( + **arguments, + claim_token="contender", + ) + service.release_reform_impact_start(**arguments, claim_token="owner") + assert service.claim_reform_impact_start( + **arguments, + claim_token="contender", + ) + + def test_reform_impact_service_round_trips_models_and_transitions(service): exact = _create_impact( service, @@ -130,7 +161,6 @@ def test_reform_impact_service_round_trips_models_and_transitions(service): def test_reform_impact_service_deletes_only_matching_computing_rows( service, - orm_session_factory, ): _create_impact( service, @@ -155,14 +185,11 @@ def test_reform_impact_service_deletes_only_matching_computing_rows( "delete-hash", ) - with orm_session_factory() as session: - assert ( - session.scalar( - select(ReformImpact).where(ReformImpact.execution_id == "delete-job") - ) - is None - ) - assert session.get(ReformImpact, retained.reform_impact_id) is not None + remaining = service.get_recent_reform_impacts(10) + assert all(impact.execution_id != "delete-job" for impact in remaining) + assert any( + impact.reform_impact_id == retained.reform_impact_id for impact in remaining + ) def test_reform_impact_transitions_return_none_for_missing_execution(service): diff --git a/tests/unit/services/test_tracer_service.py b/tests/unit/services/test_tracer_service.py index 8f5202fa7..8b5482275 100644 --- a/tests/unit/services/test_tracer_service.py +++ b/tests/unit/services/test_tracer_service.py @@ -15,7 +15,10 @@ def test_get_tracer_valid( ): # Test get_tracer successfully retrieves valid data from the database. - result = TracerAnalysisService(orm_session_factory).get_tracer( + result = TracerAnalysisService( + primary_session_factory=orm_session_factory, + household_trace_cache=test_tracer_data.cache, + ).get_tracer( test_tracer_data.country_id, test_tracer_data.household_id, test_tracer_data.policy_id, @@ -40,7 +43,9 @@ def test_get_tracer_not_found(orm_session_factory): invalid_api_version, ] with pytest.raises(NotFound): - TracerAnalysisService(orm_session_factory).get_tracer(*data_not_in_db) + TracerAnalysisService(primary_session_factory=orm_session_factory).get_tracer( + *data_not_in_db + ) def test_get_tracer_database_error(orm_session_factory): @@ -56,6 +61,6 @@ def test_get_tracer_database_error(orm_session_factory): valid_api_version, ] with pytest.raises(Exception): - TracerAnalysisService(orm_session_factory).get_tracer( + TracerAnalysisService(primary_session_factory=orm_session_factory).get_tracer( *missing_parameter_causing_database_exception, ) diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index deab1d7be..8665e59e0 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -61,6 +61,16 @@ def test_pr_always_runs_reusable_alembic_check(): assert "uses: ./.github/workflows/alembic-v1-check.yml" in workflow assert "detect-v1-alembic-changes:" not in workflow assert "needs.detect-v1-alembic-changes" not in workflow + assert "detect-v2-platform-changes:" in workflow + assert "alembic-v2-check:" in workflow + assert "uses: ./.github/workflows/alembic-v2-check.yml" in workflow + for path in ( + "alembic-v2.ini", + "migrations/v2/**", + "policyengine_api/data/v2/**", + "policyengine_api/runtime_cache/**", + ): + assert path in workflow def test_push_always_runs_lint_and_alembic_qualification_before_versioning(): @@ -69,7 +79,9 @@ def test_push_always_runs_lint_and_alembic_qualification_before_versioning(): assert "lint:" in workflow assert "alembic-v1-check:" in workflow assert "uses: ./.github/workflows/alembic-v1-check.yml" in workflow - assert "needs: [lint, alembic-v1-check]" in workflow + assert "alembic-v2-check:" in workflow + assert "uses: ./.github/workflows/alembic-v2-check.yml" in workflow + assert "needs: [lint, alembic-v1-check, alembic-v2-check]" in workflow assert "github.repository == 'PolicyEngine/policyengine-uk'" not in workflow @@ -106,6 +118,20 @@ def test_reusable_alembic_check_uses_the_installed_python_environment(): assert "uv run" not in workflow +def test_reusable_v2_check_uses_disposable_postgres_and_real_redis(): + workflow = _workflow("alembic-v2-check.yml") + + assert "workflow_call:" in workflow + assert "workflow_dispatch:" in workflow + assert "postgres:17" in workflow + assert "redis:7.2-alpine" in workflow + assert "V2_ALEMBIC_DISPOSABLE_TEST" in workflow + assert "alembic-v2.ini" in workflow + assert "test_alembic_v2_lifecycle.py" in workflow + assert "test_runtime_cache_redis.py" in workflow + assert "uv sync --frozen" in workflow + + def test_release_migration_fails_closed_and_gates_both_staging_deploys(): workflow = _workflow("push.yml") orchestration_script = ( diff --git a/tests/unit/test_app_engine_runtime.py b/tests/unit/test_app_engine_runtime.py new file mode 100644 index 000000000..9af420dae --- /dev/null +++ b/tests/unit/test_app_engine_runtime.py @@ -0,0 +1,160 @@ +from __future__ import annotations + +from unittest.mock import Mock + +import pytest + +from policyengine_api import app_engine_runtime + + +SECRET_RESOURCE_PREFIX = "projects/test-project/secrets" + + +def _direct_secret_env() -> dict[str, str]: + return { + value_name: f"direct-{index}" + for index, (value_name, _) in enumerate( + app_engine_runtime.SECRET_ENV_SOURCES, + start=1, + ) + } + + +def _resource_secret_env() -> dict[str, str]: + return { + resource_name: f"{SECRET_RESOURCE_PREFIX}/secret-{index}/versions/latest" + for index, (_, resource_name) in enumerate( + app_engine_runtime.SECRET_ENV_SOURCES, + start=1, + ) + } + + +def test_direct_secrets_do_not_call_secret_manager(): + environ = _direct_secret_env() + loader = Mock(side_effect=AssertionError("loader must not be called")) + + app_engine_runtime.hydrate_app_engine_runtime_secrets( + environ, + secret_loader=loader, + ) + + loader.assert_not_called() + + +def test_secret_resources_are_resolved_only_into_process_environment(): + environ = _resource_secret_env() + calls = [] + + def load_secret(resource: str) -> str: + calls.append(resource) + return f"resolved-{len(calls)}" + + app_engine_runtime.hydrate_app_engine_runtime_secrets( + environ, + secret_loader=load_secret, + ) + + assert calls == list(_resource_secret_env().values()) + for index, (value_name, _) in enumerate( + app_engine_runtime.SECRET_ENV_SOURCES, + start=1, + ): + assert environ[value_name] == f"resolved-{index}" + assert not set(_resource_secret_env()) & set(environ) + + app_engine_runtime.hydrate_app_engine_runtime_secrets( + environ, + secret_loader=Mock(side_effect=AssertionError("loader must not be called")), + ) + + +@pytest.mark.parametrize( + ("mutate", "expected_message"), + [ + ( + lambda env, value, resource: env.update({value: "direct"}), + "set exactly one of", + ), + ( + lambda env, value, resource: env.pop(resource), + "is required", + ), + ( + lambda env, value, resource: env.update({resource: "not-a-resource"}), + "is invalid", + ), + ], +) +def test_secret_source_configuration_fails_closed(mutate, expected_message): + environ = _resource_secret_env() + value_name, resource_name = app_engine_runtime.SECRET_ENV_SOURCES[0] + mutate(environ, value_name, resource_name) + + with pytest.raises( + app_engine_runtime.AppEngineRuntimeConfigurationError, + match=expected_message, + ): + app_engine_runtime.hydrate_app_engine_runtime_secrets( + environ, + secret_loader=lambda _: "resolved", + ) + + +@pytest.mark.parametrize( + ("loader", "expected_message"), + [ + (lambda _: "", "is empty"), + (Mock(side_effect=PermissionError), "could not be resolved"), + ], +) +def test_secret_resolution_fails_closed_without_exposing_values( + loader, + expected_message, +): + environ = _resource_secret_env() + + with pytest.raises( + app_engine_runtime.AppEngineRuntimeConfigurationError, + match=expected_message, + ) as error: + app_engine_runtime.hydrate_app_engine_runtime_secrets( + environ, + secret_loader=loader, + ) + + assert "resolved-secret-value" not in str(error.value) + + +def test_main_hydrates_secrets_before_replacing_process(monkeypatch): + calls = [] + monkeypatch.setattr( + app_engine_runtime, + "hydrate_app_engine_runtime_secrets", + lambda: calls.append("hydrate"), + ) + monkeypatch.setenv("PORT", "9090") + + def execvp(executable, args): + calls.append((executable, args)) + + monkeypatch.setattr(app_engine_runtime.os, "execvp", execvp) + + app_engine_runtime.main() + + assert calls == [ + "hydrate", + ( + "gunicorn", + [ + "gunicorn", + "-b", + ":9090", + "policyengine_api.api", + "--timeout", + "900", + "--workers", + "5", + ], + ), + ] diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index d891c15bd..202f837f5 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -40,6 +40,28 @@ "raw-openai-secret-value", "raw-hf-secret-value", ) +APP_ENGINE_SECRET_RESOURCES = { + "POLICYENGINE_DB_PASSWORD_SECRET_RESOURCE": ( + "projects/policyengine-api/secrets/" + "policyengine-api-prod-db-password/versions/latest" + ), + "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN_SECRET_RESOURCE": ( + "projects/policyengine-api/secrets/" + "policyengine-api-prod-github-microdata-token/versions/latest" + ), + "ANTHROPIC_API_KEY_SECRET_RESOURCE": ( + "projects/policyengine-api/secrets/" + "policyengine-api-prod-anthropic-api-key/versions/latest" + ), + "OPENAI_API_KEY_SECRET_RESOURCE": ( + "projects/policyengine-api/secrets/" + "policyengine-api-prod-openai-api-key/versions/latest" + ), + "HUGGING_FACE_TOKEN_SECRET_RESOURCE": ( + "projects/policyengine-api/secrets/" + "policyengine-api-prod-hugging-face-token/versions/latest" + ), +} def _script_env(**overrides: str) -> dict[str, str]: @@ -63,6 +85,27 @@ def _gateway_auth_env() -> dict[str, str]: } +def _runtime_cache_resource_env() -> dict[str, str]: + return { + "APP_ENGINE_SERVICE_ACCOUNT": ( + "policyengine-api-ae-prod@policyengine-api.iam.gserviceaccount.com" + ), + "RUNTIME_CACHE_ENVIRONMENT": "production", + "RUNTIME_CACHE_URL_SECRET_RESOURCE": ( + "projects/policyengine-api/secrets/" + "policyengine-api-prod-runtime-cache-url/versions/latest" + ), + "RUNTIME_CACHE_CA_CERT_SECRET_RESOURCE": ( + "projects/policyengine-api/secrets/" + "policyengine-api-prod-runtime-cache-ca/versions/latest" + ), + } + + +def _app_engine_secret_resource_env() -> dict[str, str]: + return dict(APP_ENGINE_SECRET_RESOURCES) + + def _required_runtime_env() -> dict[str, str]: return { "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": PRODUCTION_CLOUD_SQL_INSTANCE, @@ -77,6 +120,8 @@ def _required_runtime_env() -> dict[str, str]: "ROUTE_IMPL_HEALTH": "fastapi_native", "ROUTE_IMPL_SPECIFICATION": "fastapi_native", "ROUTE_IMPL_METADATA": "fastapi_native", + **_app_engine_secret_resource_env(), + **_runtime_cache_resource_env(), **_gateway_auth_env(), } @@ -465,23 +510,38 @@ def test_cloud_run_dockerfile_runs_startup_with_bash(): assert 'CMD ["/bin/sh", "/app/start.sh"]' not in dockerfile -def test_cloud_run_startup_supervises_redis_and_server_children(): - start_script = (REPO / "gcp/cloud_run/start.sh").read_text(encoding="utf-8") +def test_deployed_startup_execs_only_the_api_server(): + startup_commands = { + "gcp/cloud_run/start.sh": "exec gunicorn", + "gcp/policyengine_api/start.sh": ( + "exec python3 -m policyengine_api.app_engine_runtime" + ), + } + for relative_path, expected_command in startup_commands.items(): + start_script = (REPO / relative_path).read_text(encoding="utf-8") + + assert expected_command in start_script + assert "redis-server" not in start_script + assert "redis-cli" not in start_script + assert "CACHE_REDIS_HOST" not in start_script + assert "CACHE_REDIS_PORT" not in start_script + assert "CACHE_REDIS_DB" not in start_script + assert "wait" not in start_script + assert "pkill" not in start_script + + +def test_production_images_do_not_install_or_configure_embedded_redis(): + for relative_path in ( + "gcp/Dockerfile", + "gcp/cloud_run/Dockerfile", + "gcp/policyengine_api/Dockerfile", + ): + dockerfile = (REPO / relative_path).read_text(encoding="utf-8") - assert "#!/usr/bin/env bash" in start_script - assert 'redis_pid="$!"' in start_script - assert 'server_pid="$!"' in start_script - assert "REDIS_READY_MAX_ATTEMPTS" in start_script - assert "Redis exited before becoming ready" in start_script - assert "Redis did not become ready" in start_script - assert "Redis exited; stopping Cloud Run container" in start_script - assert "API server exited; stopping Cloud Run container" in start_script - assert 'wait -n "$redis_pid" "$server_pid"' in start_script - assert 'kill -0 "$redis_pid"' in start_script - assert 'kill -0 "$server_pid"' in start_script - assert "trap 'shutdown; exit 143' INT TERM" in start_script - assert "pkill" not in start_script - assert re.search(r"(?m)^ *wait 2>/dev/null", start_script) is None + assert "redis-server" not in dockerfile + assert "CACHE_REDIS_HOST" not in dockerfile + assert "CACHE_REDIS_PORT" not in dockerfile + assert "CACHE_REDIS_DB" not in dockerfile def test_production_gunicorn_workers_do_not_inherit_database_pools(): @@ -497,10 +557,14 @@ def test_production_gunicorn_workers_do_not_inherit_database_pools(): def test_app_engine_startup_allows_all_workers_to_finish_booting(): start_script = (REPO / "gcp/policyengine_api/start.sh").read_text(encoding="utf-8") + runtime_module = (REPO / "policyengine_api/app_engine_runtime.py").read_text( + encoding="utf-8" + ) app_config = (REPO / "gcp/policyengine_api/app.yaml").read_text(encoding="utf-8") - assert "--timeout 900" in start_script - assert "--workers 5" in start_script + assert "python3 -m policyengine_api.app_engine_runtime" in start_script + assert '"--timeout",\n "900"' in runtime_module + assert '"--workers",\n "5"' in runtime_module assert "initial_delay_sec: 1800" in app_config assert "app_start_timeout_sec: 1800" in app_config @@ -536,6 +600,8 @@ def test_validate_cloud_run_deploy_env_accepts_direct_mode_from_environment(): ROUTE_IMPL_SPECIFICATION="fastapi_native", ROUTE_IMPL_METADATA="fastapi_native", POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=PRODUCTION_CLOUD_SQL_INSTANCE, + **_app_engine_secret_resource_env(), + **_runtime_cache_resource_env(), **_gateway_auth_env(), ), ) @@ -616,6 +682,8 @@ def test_validate_cloud_run_deploy_env_requires_only_selected_url( ROUTE_IMPL_SPECIFICATION="fastapi_native", ROUTE_IMPL_METADATA="fastapi_native", POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=PRODUCTION_CLOUD_SQL_INSTANCE, + **_app_engine_secret_resource_env(), + **_runtime_cache_resource_env(), **_gateway_auth_env(), ) missing_result = _run_script( @@ -652,6 +720,8 @@ def test_validate_app_engine_deploy_env_accepts_direct_mode_from_environment(): SIM_ENTRYPOINT="old_gateway_direct", OLD_SIMULATION_GATEWAY_URL="https://old-gateway.example.test", POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=PRODUCTION_CLOUD_SQL_INSTANCE, + **_app_engine_secret_resource_env(), + **_runtime_cache_resource_env(), **_gateway_auth_env(), ), ) @@ -682,6 +752,8 @@ def test_validate_app_engine_deploy_env_requires_only_selected_url( env = _script_env( SIM_ENTRYPOINT=entrypoint, POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=PRODUCTION_CLOUD_SQL_INSTANCE, + **_app_engine_secret_resource_env(), + **_runtime_cache_resource_env(), **_gateway_auth_env(), ) missing_result = _run_script( @@ -702,18 +774,112 @@ def test_app_engine_bundle_contains_runtime_environment_placeholders(): dockerfile = (REPO / "gcp/policyengine_api/Dockerfile").read_text(encoding="utf-8") app_config = (REPO / "gcp/policyengine_api/app.yaml").read_text(encoding="utf-8") export_script = (REPO / "gcp/export.py").read_text(encoding="utf-8") + bundle_script = (REPO / ".github/scripts/prepare_app_engine_bundle.sh").read_text( + encoding="utf-8" + ) - assert 'ENV SIMULATION_ENTRYPOINT_URL=".simulation_entrypoint_url"' in dockerfile - assert 'ENV OLD_SIMULATION_GATEWAY_URL=".old_simulation_gateway_url"' in dockerfile - assert 'ENV SIM_ENTRYPOINT=".sim_entrypoint"' in dockerfile + assert "ENV " not in dockerfile assert ( "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: " '".policyengine_db_instance_connection_name"' in app_config ) - assert '".old_simulation_gateway_url", OLD_SIMULATION_GATEWAY_URL' in export_script - assert '".sim_entrypoint", SIM_ENTRYPOINT' in export_script - assert '".policyengine_db_instance_connection_name",' in export_script - assert "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME," in export_script + assert 'SIMULATION_ENTRYPOINT_URL: ".simulation_entrypoint_url"' in app_config + assert 'SIM_ENTRYPOINT: ".sim_entrypoint"' in app_config + assert '".policyengine_db_instance_connection_name": _required(' in export_script + assert "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME" in export_script + for resource_env in APP_ENGINE_SECRET_RESOURCES: + assert f'{resource_env}: ".{resource_env.lower()}"' in app_config + assert f'"{resource_env}"' in export_script + for prohibited in ( + "POLICYENGINE_DB_PASSWORD = os.environ", + "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN = os.environ", + "ANTHROPIC_API_KEY = os.environ", + "OPENAI_API_KEY = os.environ", + "HUGGING_FACE_TOKEN = os.environ", + 'open(".dbpw"', + ): + assert prohibited not in export_script + for direct_secret_env in CLOUD_RUN_SECRET_MAPPINGS: + assert ( + re.search( + rf'["\']{re.escape(direct_secret_env)}["\']', + export_script, + ) + is None + ) + assert 'RUNTIME_CACHE_MODE: "deployed"' in app_config + assert 'V2_SUPABASE_PROJECT_REF: "kvrifaviwhzjztcbrfpy"' in app_config + assert 'V2_SUPABASE_ENVIRONMENT: "production-foundation"' in app_config + assert ( + 'RUNTIME_CACHE_URL_SECRET_RESOURCE: ".runtime_cache_url_secret_resource"' + in app_config + ) + assert ( + "RUNTIME_CACHE_CA_CERT_SECRET_RESOURCE: " + '".runtime_cache_ca_cert_secret_resource"' in app_config + ) + assert '"RUNTIME_CACHE_URL_SECRET_RESOURCE"' in export_script + assert "python3 gcp/export.py" in bundle_script + + +@pytest.mark.parametrize( + ("ignore_file", "required_rules"), + [ + ( + ".gcloudignore", + {"!.gcloudignore", "!app.yaml"}, + ), + ( + ".dockerignore", + {"!.dockerignore"}, + ), + ], +) +def test_app_engine_contexts_include_only_runtime_inputs( + ignore_file, + required_rules, +): + context_rules = set((REPO / ignore_file).read_text(encoding="utf-8").splitlines()) + + assert "**" in context_rules + assert ( + required_rules + | { + "!Dockerfile", + "!start.sh", + "!Makefile", + "!pyproject.toml", + "!README.md", + "!policyengine_api/", + "!policyengine_api/**", + "**/__pycache__/**", + "**/*.py[co]", + "**/*.db", + "**/*.sqlite*", + "**/.env*", + "**/*.key", + "**/*.pem", + } + <= context_rules + ) + assert not any( + rule.startswith(("!tests", "!docs", "!.github", "!openspec")) + for rule in context_rules + ) + + +def test_app_engine_deploy_can_use_an_existing_artifact_registry_image(): + build_script = (REPO / ".github/scripts/build_app_engine_image.sh").read_text( + encoding="utf-8" + ) + deploy_script = (REPO / ".github/scripts/deploy_app_engine_version.sh").read_text( + encoding="utf-8" + ) + + assert 'APP_ENGINE_PLATFORM="${APP_ENGINE_PLATFORM:-linux/amd64}"' in build_script + assert 'docker build --platform "${APP_ENGINE_PLATFORM}"' in build_script + assert 'if [[ -n "${APP_ENGINE_IMAGE_URL:-}" ]]' in deploy_script + assert 'deploy_args+=("--image-url=${APP_ENGINE_IMAGE_URL}")' in deploy_script @pytest.mark.parametrize( @@ -776,6 +942,12 @@ def test_app_engine_export_requires_only_selected_url( selected_url_env: selected_url, } env.pop(unselected_url_env) + source_dockerfile = (tmp_path / "gcp/policyengine_api/Dockerfile").read_text( + encoding="utf-8" + ) + source_app_config = (tmp_path / "gcp/policyengine_api/app.yaml").read_text( + encoding="utf-8" + ) result = subprocess.run( [sys.executable, "gcp/export.py"], @@ -787,18 +959,29 @@ def test_app_engine_export_requires_only_selected_url( ) assert result.returncode == 0, result.stderr - rendered = (tmp_path / "gcp/policyengine_api/Dockerfile").read_text( - encoding="utf-8" - ) - rendered_app_config = (tmp_path / "gcp/policyengine_api/app.yaml").read_text( - encoding="utf-8" - ) - assert f'ENV {selected_url_env}="{selected_url}"' in rendered - assert f'ENV {unselected_url_env}=""' in rendered + rendered = (tmp_path / "Dockerfile").read_text(encoding="utf-8") + rendered_app_config = (tmp_path / "app.yaml").read_text(encoding="utf-8") + assert rendered == source_dockerfile + assert f'{selected_url_env}: "{selected_url}"' in rendered_app_config + assert f'{unselected_url_env}: ""' in rendered_app_config assert ( f"POLICYENGINE_DB_INSTANCE_CONNECTION_NAME: " f'"{PRODUCTION_CLOUD_SQL_INSTANCE}"' in rendered_app_config ) + assert 'RUNTIME_CACHE_ENVIRONMENT: "production"' in rendered_app_config + assert ( + "projects/policyengine-api/secrets/" + "policyengine-api-prod-runtime-cache-url/versions/latest" in rendered_app_config + ) + for resource in APP_ENGINE_SECRET_RESOURCES.values(): + assert resource in rendered_app_config + assert not (tmp_path / ".dbpw").exists() + assert (tmp_path / "gcp/policyengine_api/Dockerfile").read_text( + encoding="utf-8" + ) == source_dockerfile + assert (tmp_path / "gcp/policyengine_api/app.yaml").read_text( + encoding="utf-8" + ) == source_app_config def test_build_cloud_run_image_dry_run_uses_cloud_run_dockerfile(): @@ -817,6 +1000,7 @@ def test_build_cloud_run_image_dry_run_uses_cloud_run_dockerfile(): assert result.returncode == 0, result.stderr assert "gcp/cloud_run/Dockerfile" in result.stdout + assert "--platform linux/amd64" in result.stdout assert "docker push" in result.stdout assert ( "us-central1-docker.pkg.dev/policyengine-api/policyengine-api/" @@ -848,6 +1032,24 @@ def test_deploy_cloud_run_candidate_dry_run_never_shifts_traffic(): in result.stdout ) assert "--set-secrets" in result.stdout + assert "--network default" in result.stdout + assert "--subnet default" in result.stdout + assert "--vpc-egress private-ranges-only" in result.stdout + assert "RUNTIME_CACHE_MODE=deployed" in result.stdout + assert "RUNTIME_CACHE_ENVIRONMENT=production" in result.stdout + assert "RUNTIME_CACHE_SERVICE=api" in result.stdout + assert ( + "RUNTIME_CACHE_URL=policyengine-api-prod-runtime-cache-url:latest" + in result.stdout + ) + assert ( + "RUNTIME_CACHE_CA_CERT=policyengine-api-prod-runtime-cache-ca:latest" + in result.stdout + ) + assert "V2_SUPABASE_PROJECT_REF=kvrifaviwhzjztcbrfpy" in result.stdout + assert "V2_SUPABASE_ENVIRONMENT=production-foundation" in result.stdout + assert "V2_DATABASE_URL" not in result.stdout + assert "V2_STORAGE_ADMIN_KEY" not in result.stdout for env_name, secret_ref in CLOUD_RUN_SECRET_MAPPINGS.items(): assert f"{env_name}={secret_ref}" in result.stdout for raw_secret_value in RAW_CLOUD_RUN_SECRET_VALUES: @@ -867,6 +1069,19 @@ def test_deploy_cloud_run_candidate_dry_run_never_shifts_traffic(): assert result.stdout.count(f"{selector}=fastapi_native") == 1 +def test_staging_and_production_use_distinct_cloud_run_runtime_identities(): + workflow = _push_workflow() + staging = _workflow_job_block(workflow, "deploy-cloud-run-staging") + production = _workflow_job_block(workflow, "deploy-cloud-run-candidate") + + assert ( + "policyengine-api-cr-staging@policyengine-api.iam.gserviceaccount.com" + in staging + ) + assert "GCP_CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT" not in staging + assert "GCP_CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT" in production + + def test_deploy_cloud_run_candidate_uses_configured_database_instance(): configured_instance = "project:region:configured-instance" env = { @@ -1621,7 +1836,12 @@ def test_push_workflow_uses_dedicated_cloud_run_runtime_service_account(): "CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT: ${{ secrets.GCP_DEPLOY_SERVICE_ACCOUNT }}" ) - assert runtime_account_secret in cloud_run_staging + assert ( + "CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT: " + "policyengine-api-cr-staging@policyengine-api.iam.gserviceaccount.com" + in cloud_run_staging + ) + assert runtime_account_secret not in cloud_run_staging assert runtime_account_secret in cloud_run_production assert deploy_account_secret not in cloud_run_staging assert deploy_account_secret not in cloud_run_production @@ -1647,6 +1867,31 @@ def test_push_workflow_does_not_pass_raw_secrets_to_cloud_run_deploy_jobs(): assert raw_secret_env not in cloud_run_production +def test_push_workflow_app_engine_deploys_use_secret_resources_not_values(): + workflow = _push_workflow() + staging = _workflow_job_block(workflow, "deploy-staging") + production = _workflow_job_block(workflow, "deploy-production-candidate") + + for name, resource in APP_ENGINE_SECRET_RESOURCES.items(): + expected = f"{name}: {resource}" + assert expected in staging + assert expected in production + + raw_secret_envs = ( + "POLICYENGINE_DB_PASSWORD: ${{ secrets.POLICYENGINE_DB_PASSWORD }}", + ( + "POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN: " + "${{ secrets.POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN }}" + ), + "ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}", + "OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}", + "HUGGING_FACE_TOKEN: ${{ secrets.HUGGING_FACE_TOKEN }}", + ) + for raw_secret_env in raw_secret_envs: + assert staging.count(raw_secret_env) == 1 # push-time tests only + assert raw_secret_env not in production + + def test_sync_cloud_run_secrets_workflow_is_manual_and_environment_gated(): workflow = _sync_secrets_workflow() diff --git a/tests/unit/test_gcp_logging.py b/tests/unit/test_gcp_logging.py new file mode 100644 index 000000000..7c5ef98b4 --- /dev/null +++ b/tests/unit/test_gcp_logging.py @@ -0,0 +1,39 @@ +from unittest.mock import Mock + +from policyengine_api.gcp_logging import _LazyGoogleLogger + + +def test_local_logging_uses_stderr_without_initializing_google(monkeypatch): + monkeypatch.delenv("GAE_ENV", raising=False) + monkeypatch.delenv("K_SERVICE", raising=False) + logger = _LazyGoogleLogger("test-local") + logger._fallback_logger = Mock() + payload = {"message": "cache miss"} + + logger.log_struct(payload, severity="WARNING", labels={"cache": "analysis"}) + + assert logger._initialization_failed is True + assert logger._google_logger is None + logger._fallback_logger.log.assert_called_once_with(30, "%s", payload) + + +def test_remote_logging_failure_falls_back_and_disables_retries(monkeypatch): + monkeypatch.setenv("K_SERVICE", "policyengine-api") + remote_logger = Mock() + remote_logger.log_struct.side_effect = ConnectionError("logging unavailable") + logger = _LazyGoogleLogger("test-deployed") + logger._google_logger = remote_logger + logger._fallback_logger = Mock() + payload = {"message": "cache write"} + + logger.log_struct(payload, severity="INFO", labels={"cache": "household"}) + logger.log_struct(payload, severity="INFO", labels={"cache": "household"}) + + remote_logger.log_struct.assert_called_once_with( + payload, + severity="INFO", + labels={"cache": "household"}, + ) + assert logger._initialization_failed is True + assert logger._google_logger is None + assert logger._fallback_logger.log.call_count == 2 diff --git a/tests/unit/v2/__init__.py b/tests/unit/v2/__init__.py new file mode 100644 index 000000000..ec6c36827 --- /dev/null +++ b/tests/unit/v2/__init__.py @@ -0,0 +1 @@ +"""Focused API v2-alpha unit tests.""" diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py new file mode 100644 index 000000000..be0aeb3b8 --- /dev/null +++ b/tests/unit/v2/test_alembic_v2.py @@ -0,0 +1,444 @@ +"""Unit guards for the isolated and qualified v2 Alembic environment.""" + +from io import StringIO +from pathlib import Path +import re +import shutil +from types import SimpleNamespace +from unittest.mock import MagicMock + +from alembic import command +from alembic.config import Config +from alembic.script import ScriptDirectory +from alembic.script.revision import ResolutionError +from alembic.util import CommandError +import pytest + +from policyengine_api.constants import REPO +from policyengine_api.data.v1_models import V1Base +from policyengine_api.data.v2.migration_target import ( + DISPOSABLE_DATABASE_NAME, + MIGRATION_ROLE, + RECORDED_SUPABASE_TARGETS, + V2_ALEMBIC_DISPOSABLE_TEST, + V2AlembicSettings, + V2MigrationTargetError, + load_v2_alembic_settings, + qualify_v2_connection, + validate_v2_head_table_inventory, +) +from policyengine_api.data.v2.models import V2_METADATA +from policyengine_api.data.v2.settings import ( + V2_MIGRATION_DATABASE_URL, + V2_SUPABASE_ENVIRONMENT, + V2_SUPABASE_PROJECT_REF, +) +from policyengine_api.data.v2.table_inventory import EXPECTED_V2_TABLES + + +PROJECT_REF = "kvrifaviwhzjztcbrfpy" +POOLER_URL = ( + "postgresql+psycopg://policyengine_v2_migrator." + f"{PROJECT_REF}:test-password@aws-0-us-east-2.pooler.supabase.com:5432/" + "postgres?sslmode=require" +) +DISPOSABLE_URL = ( + "postgresql+psycopg://postgres:test-password@127.0.0.1:5432/" + f"{DISPOSABLE_DATABASE_NAME}" +) + + +def _clear_v2_environment(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ( + V2_MIGRATION_DATABASE_URL, + V2_ALEMBIC_DISPOSABLE_TEST, + V2_SUPABASE_ENVIRONMENT, + V2_SUPABASE_PROJECT_REF, + "ALEMBIC_DATABASE_URL", + ): + monkeypatch.delenv(name, raising=False) + + +def test_v2_configuration_contains_no_database_url() -> None: + config_text = (REPO / "alembic-v2.ini").read_text(encoding="utf-8") + + assert "sqlalchemy.url" not in config_text + assert "migrations/v2" in config_text + + +def test_v2_alembic_requires_its_explicit_url( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _clear_v2_environment(monkeypatch) + monkeypatch.setenv( + "ALEMBIC_DATABASE_URL", + "mysql+pymysql://v1:test-password@localhost/v1", + ) + config = Config(str(REPO / "alembic-v2.ini"), output_buffer=StringIO()) + + with pytest.raises(V2MigrationTargetError, match=V2_MIGRATION_DATABASE_URL): + command.upgrade(config, "head", sql=True) + + +@pytest.mark.parametrize( + "url", + [ + "sqlite+pysqlite:///:memory:", + "mysql+pymysql://user:password@localhost/database", + "postgresql+asyncpg://user:password@localhost/database", + ], +) +def test_v2_alembic_rejects_non_psycopg_postgres_urls(url: str) -> None: + with pytest.raises(V2MigrationTargetError, match=r"postgresql\+psycopg"): + load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: url, + V2_ALEMBIC_DISPOSABLE_TEST: "1", + } + ) + + +def test_disposable_mode_requires_the_exact_isolated_local_database() -> None: + with pytest.raises(V2MigrationTargetError, match=DISPOSABLE_DATABASE_NAME): + load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: ( + "postgresql+psycopg://postgres:password@127.0.0.1/postgres" + ), + V2_ALEMBIC_DISPOSABLE_TEST: "1", + } + ) + + +def test_disposable_mode_cannot_target_supabase() -> None: + with pytest.raises(V2MigrationTargetError, match="isolated local"): + load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: POOLER_URL, + V2_ALEMBIC_DISPOSABLE_TEST: "1", + } + ) + + +def test_v2_alembic_rejects_offline_execution_even_in_disposable_mode( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _clear_v2_environment(monkeypatch) + monkeypatch.setenv(V2_MIGRATION_DATABASE_URL, DISPOSABLE_URL) + monkeypatch.setenv(V2_ALEMBIC_DISPOSABLE_TEST, "1") + config = Config(str(REPO / "alembic-v2.ini"), output_buffer=StringIO()) + + with pytest.raises(RuntimeError, match="online connection"): + command.upgrade(config, "head", sql=True) + + +def test_persistent_target_requires_the_recorded_environment_and_project() -> None: + with pytest.raises(V2MigrationTargetError, match="recorded"): + load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: POOLER_URL, + V2_SUPABASE_ENVIRONMENT: "production-foundation", + V2_SUPABASE_PROJECT_REF: "aaaaaaaaaaaaaaaaaaaa", + } + ) + + +def test_pooler_identity_resolves_the_recorded_persistent_target() -> None: + settings = load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: POOLER_URL, + V2_SUPABASE_ENVIRONMENT: "production-foundation", + V2_SUPABASE_PROJECT_REF: PROJECT_REF, + } + ) + + assert settings.disposable_test is False + assert settings.target is not None + assert settings.target.project_ref == PROJECT_REF + assert settings.url.database == "postgres" + + +def test_persistent_mode_rejects_an_ambiguous_non_supabase_host() -> None: + with pytest.raises(V2MigrationTargetError, match="recorded Supabase project"): + load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: ( + "postgresql+psycopg://policyengine_v2_migrator:password@" + "db.example.com/postgres?sslmode=require" + ), + V2_SUPABASE_ENVIRONMENT: "production-foundation", + V2_SUPABASE_PROJECT_REF: PROJECT_REF, + } + ) + + +def test_target_errors_never_echo_url_passwords() -> None: + secret = "do-not-echo-this-password" + with pytest.raises(V2MigrationTargetError) as raised: + load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: ( + f"postgresql+psycopg://user:{secret}@localhost/postgres" + ), + V2_ALEMBIC_DISPOSABLE_TEST: "1", + } + ) + + assert secret not in str(raised.value) + + +def test_v2_environment_loads_only_the_exact_sqlmodel_inventory() -> None: + env_source = (REPO / "migrations" / "v2" / "env.py").read_text(encoding="utf-8") + + assert V2_METADATA is not V1Base.metadata + assert set(V2_METADATA.tables) == EXPECTED_V2_TABLES + assert "V1Base" not in env_source + assert "migrations/v1" not in env_source + assert "validate_v2_table_inventory" in env_source + + +def test_v2_files_are_mechanically_separate_from_v1() -> None: + v2_files = { + path.relative_to(REPO) + for path in (REPO / "migrations" / "v2").rglob("*") + if path.is_file() + } + + assert Path("migrations/v2/env.py") in v2_files + assert Path("migrations/v2/script.py.mako") in v2_files + assert all("migrations/v1" not in str(path) for path in v2_files) + + +def test_v2_revision_chain_is_linear_generated_and_correction_bounded() -> None: + config = Config(str(REPO / "alembic-v2.ini")) + script = ScriptDirectory.from_config(config) + assert script.get_heads() == ["5f048586d8f1"] + assert [revision.revision for revision in script.walk_revisions()] == [ + "5f048586d8f1", + "b4c69674dd47", + "6ee725e0c563", + "47592781336f", + ] + + baseline = ( + REPO + / "migrations/v2/versions/47592781336f_establish_v2_core_schema_baseline.py" + ).read_text(encoding="utf-8") + data = ( + REPO + / "migrations/v2/versions/6ee725e0c563_add_stage_8_platform_validation_data.py" + ).read_text(encoding="utf-8") + ownership = ( + REPO / "migrations/v2/versions/" + "b4c69674dd47_enforce_v2_user_association_ownership.py" + ).read_text(encoding="utf-8") + constraints = ( + REPO + / "migrations/v2/versions/5f048586d8f1_constrain_v2_user_country_and_report_.py" + ).read_text(encoding="utf-8") + revisions = baseline + data + ownership + constraints + assert all( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" in source + for source in (baseline, data, ownership, constraints) + ) + assert "op.execute(" not in revisions + assert "op.bulk_insert(" not in revisions + assert data.count("op.v2_reference_row_change(") == 4 + assert "op.create_table(" not in data + assert "op.drop_table(" not in data + assert ownership.count("op.create_foreign_key(") == 4 + assert ownership.count("op.drop_constraint(") == 4 + assert 'ondelete="CASCADE"' in ownership + assert constraints.count("op.add_column(") == 1 + assert constraints.count("op.create_check_constraint(") == 2 + assert "ck_users_primary_country" in constraints + assert "ck_report_runs_idempotency_key_nonblank" in constraints + + corrected_enum_names = set( + re.findall( + r"sa\.Enum\(name=[\"']([^\"']+)[\"']\)\.drop\(op\.get_bind\(\)\)", + revisions, + ) + ) + assert corrected_enum_names == { + "v2_aggregate_type", + "v2_decile_type", + "v2_household_job_status", + "v2_output_status", + "v2_region_type", + "v2_report_run_status", + "v2_report_run_trigger", + "v2_simulation_status", + "v2_simulation_type", + } + + +def test_alembic_rejects_unknown_missing_and_divergent_history(tmp_path: Path) -> None: + original = REPO / "migrations/v2" + missing = tmp_path / "missing" + shutil.copytree(original, missing) + (missing / "versions/47592781336f_establish_v2_core_schema_baseline.py").unlink() + missing_config = Config() + missing_config.set_main_option("script_location", str(missing)) + with pytest.raises((KeyError, ResolutionError)): + list(ScriptDirectory.from_config(missing_config).walk_revisions()) + + divergent = tmp_path / "divergent" + shutil.copytree(original, divergent) + source = divergent / "versions/6ee725e0c563_add_stage_8_platform_validation_data.py" + duplicate = source.read_text(encoding="utf-8").replace( + "6ee725e0c563", "aaaaaaaaaaaa" + ) + (divergent / "versions/aaaaaaaaaaaa_divergent.py").write_text( + duplicate, + encoding="utf-8", + ) + divergent_config = Config() + divergent_config.set_main_option("script_location", str(divergent)) + divergent_script = ScriptDirectory.from_config(divergent_config) + with pytest.raises(CommandError, match="multiple heads"): + divergent_script.get_current_head() + with pytest.raises((CommandError, ResolutionError)): + divergent_script.get_revision("bbbbbbbbbbbb") + + +def _persistent_connection( + monkeypatch: pytest.MonkeyPatch, + *, + public_tables: set[str], +): + import policyengine_api.data.v2.migration_target as module + + connection = MagicMock() + connection.dialect.name = "postgresql" + connection.execute.side_effect = [ + SimpleNamespace(one=lambda: ("postgres", MIGRATION_ROLE)), + SimpleNamespace(scalar_one=lambda: True), + ] + monkeypatch.setattr( + module, + "inspect", + lambda _: SimpleNamespace(get_table_names=lambda schema: list(public_tables)), + ) + return connection + + +def test_persistent_first_use_requires_recorded_successful_freshness_audit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + target = RECORDED_SUPABASE_TARGETS["production-foundation"] + unaudited = target.__class__( + environment=target.environment, + project_ref=target.project_ref, + database_name=target.database_name, + migration_role=target.migration_role, + freshness_audited_on=target.freshness_audited_on, + freshness_audit_passed=False, + ) + settings = V2AlembicSettings( + url=load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: POOLER_URL, + V2_SUPABASE_ENVIRONMENT: "production-foundation", + V2_SUPABASE_PROJECT_REF: PROJECT_REF, + } + ).url, + disposable_test=False, + target=unaudited, + ) + + with pytest.raises(V2MigrationTargetError, match="freshness audit"): + qualify_v2_connection( + _persistent_connection(monkeypatch, public_tables=set()), + settings, + ) + + +def test_persistent_target_rejects_unstamped_nonfresh_inventory( + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings = load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: POOLER_URL, + V2_SUPABASE_ENVIRONMENT: "production-foundation", + V2_SUPABASE_PROJECT_REF: PROJECT_REF, + } + ) + + with pytest.raises(V2MigrationTargetError): + qualify_v2_connection( + _persistent_connection( + monkeypatch, + public_tables={"unreviewed_predecessor"}, + ), + settings, + ) + + +def test_persistent_target_allows_stamped_previous_revision_inventory( + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings = load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: POOLER_URL, + V2_SUPABASE_ENVIRONMENT: "production-foundation", + V2_SUPABASE_PROJECT_REF: PROJECT_REF, + } + ) + + qualify_v2_connection( + _persistent_connection( + monkeypatch, + public_tables={"alembic_version", "reports"}, + ), + settings, + ) + + +@pytest.mark.parametrize( + "public_tables", + [ + {"alembic_version", "reports"}, + {"alembic_version", *EXPECTED_V2_TABLES, "runtime_bundles"}, + ], +) +def test_v2_head_rejects_divergent_table_inventory( + monkeypatch: pytest.MonkeyPatch, + public_tables: set[str], +) -> None: + connection = _persistent_connection( + monkeypatch, + public_tables=public_tables, + ) + + with pytest.raises(V2MigrationTargetError, match="v2 head table inventory"): + validate_v2_head_table_inventory(connection) + + +def test_v2_head_accepts_exact_table_inventory( + monkeypatch: pytest.MonkeyPatch, +) -> None: + connection = _persistent_connection( + monkeypatch, + public_tables={"alembic_version", *EXPECTED_V2_TABLES}, + ) + + validate_v2_head_table_inventory(connection) + + +def test_v2_environment_contains_no_reset_adopt_or_restamp_path() -> None: + env_source = (REPO / "migrations/v2/env.py").read_text(encoding="utf-8") + assert "create_all" not in env_source + assert "command.stamp" not in env_source + assert "drop_all" not in env_source + assert "DROP SCHEMA" not in env_source + + +def test_v2_environment_commits_after_persistent_target_qualification() -> None: + env_source = (REPO / "migrations/v2/env.py").read_text(encoding="utf-8") + + assert "with engine.begin() as connection:" in env_source + assert "with engine.connect() as connection:" not in env_source + assert "current_heads != previous_heads" in env_source + assert "current_heads == script_heads" in env_source + assert "validate_v2_head_table_inventory(connection)" in env_source diff --git a/tests/unit/v2/test_database.py b/tests/unit/v2/test_database.py new file mode 100644 index 000000000..53b0fe864 --- /dev/null +++ b/tests/unit/v2/test_database.py @@ -0,0 +1,99 @@ +"""Tests for lazy process-owned v2 engine and SQLModel Session factories.""" + +from collections.abc import Iterator + +import pytest +from sqlmodel import Session + +from policyengine_api.data.v2 import database +from policyengine_api.data.v2.settings import ( + V2_MIGRATION_DATABASE_URL, + V2_RUNTIME_DATABASE_URL, + V2_SUPABASE_ENVIRONMENT, + V2_SUPABASE_PROJECT_REF, + V2ConfigurationError, + load_v2_migration_database_settings, + load_v2_runtime_database_settings, +) + + +def _environment(*, username: str = "runtime") -> dict[str, str]: + return { + V2_SUPABASE_PROJECT_REF: "abcdefghijklmnopqrst", + V2_SUPABASE_ENVIRONMENT: "production-foundation", + V2_RUNTIME_DATABASE_URL: ( + f"postgresql+psycopg://{username}:test-password@db.example.com:5432/" + "postgres?sslmode=require" + ), + V2_MIGRATION_DATABASE_URL: ( + "postgresql+psycopg://migrator:test-password@db.example.com:5432/" + "postgres?sslmode=require" + ), + } + + +@pytest.fixture(autouse=True) +def _clear_v2_database_state() -> Iterator[None]: + database.close_v2_database() + yield + database.close_v2_database() + + +def test_engine_construction_is_lazy_and_reused_without_connecting() -> None: + settings = load_v2_runtime_database_settings(_environment()) + + first = database.get_v2_engine(settings) + second = database.get_v2_engine(settings) + + assert first is second + assert first.url.drivername == "postgresql+psycopg" + assert first.pool.checkedout() == 0 + + +def test_session_factory_builds_sqlmodel_sessions() -> None: + settings = load_v2_runtime_database_settings(_environment()) + factory = database.get_v2_session_factory(settings) + + with factory() as session: + assert isinstance(session, Session) + assert session.bind is database.get_v2_engine(settings) + + +def test_process_engine_cannot_silently_change_target() -> None: + first = load_v2_runtime_database_settings(_environment(username="runtime")) + second = load_v2_runtime_database_settings(_environment(username="other-runtime")) + database.get_v2_engine(first) + + with pytest.raises(V2ConfigurationError, match="different explicit target"): + database.get_v2_engine(second) + + +def test_migration_credentials_are_not_selected_as_runtime_fallback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + environment = _environment() + migration = load_v2_migration_database_settings(environment) + for key, value in environment.items(): + if key != V2_RUNTIME_DATABASE_URL: + monkeypatch.setenv(key, value) + monkeypatch.delenv(V2_RUNTIME_DATABASE_URL, raising=False) + + with pytest.raises(V2ConfigurationError, match=V2_RUNTIME_DATABASE_URL): + database.get_v2_engine() + + assert migration.connection.url.username == "migrator" + + +def test_forked_process_state_builds_a_new_engine( + monkeypatch: pytest.MonkeyPatch, +) -> None: + settings = load_v2_runtime_database_settings(_environment()) + parent_engine = database.get_v2_engine(settings) + parent_pid = database._engine_pid + assert parent_pid is not None + monkeypatch.setattr(database.os, "getpid", lambda: parent_pid + 1) + + child_engine = database.get_v2_engine(settings) + + assert child_engine is not parent_engine + assert database._engine_pid == parent_pid + 1 diff --git a/tests/unit/v2/test_import_side_effects.py b/tests/unit/v2/test_import_side_effects.py new file mode 100644 index 000000000..947d1e324 --- /dev/null +++ b/tests/unit/v2/test_import_side_effects.py @@ -0,0 +1,146 @@ +"""Import/startup tests for dormant and fail-closed v2 configuration.""" + +import os +from pathlib import Path +import subprocess +import sys + +import pytest + +from policyengine_api.data.v2.settings import ( + V2_RUNTIME_DATABASE_URL, + V2_SUPABASE_ENVIRONMENT, + V2_SUPABASE_PROJECT_REF, + V2ConfigurationError, + load_v2_runtime_database_settings, +) + + +V2_ENVIRONMENT_NAMES = ( + "V2_RUNTIME_DATABASE_URL", + "V2_MIGRATION_DATABASE_URL", + "V2_SUPABASE_PROJECT_REF", + "V2_SUPABASE_ENVIRONMENT", + "V2_SUPABASE_STORAGE_URL", + "V2_SUPABASE_STORAGE_ADMIN_KEY", + "V2_SUPABASE_STORAGE_BUCKET", +) + + +def _environment_without_v2() -> dict[str, str]: + environment = os.environ.copy() + for name in V2_ENVIRONMENT_NAMES: + environment.pop(name, None) + environment["POLICYENGINE_API_STARTUP_WARMUP"] = "0" + environment.pop("FLASK_DEBUG", None) + environment.pop("K_SERVICE", None) + environment.pop("GAE_ENV", None) + for name in ( + "RUNTIME_CACHE_MODE", + "RUNTIME_CACHE_URL", + "RUNTIME_CACHE_CA_CERT", + "RUNTIME_CACHE_URL_SECRET_RESOURCE", + "RUNTIME_CACHE_CA_CERT_SECRET_RESOURCE", + "RUNTIME_CACHE_ENVIRONMENT", + "RUNTIME_CACHE_SERVICE", + ): + environment.pop(name, None) + return environment + + +def test_importing_v2_modules_opens_no_network_and_creates_no_files( + tmp_path: Path, +) -> None: + script = """ +import pathlib +import socket + +def reject_connect(*args, **kwargs): + raise AssertionError("module import attempted a network connection") + +socket.socket.connect = reject_connect +before = set(pathlib.Path.cwd().iterdir()) +import policyengine_api.data.v2.settings +import policyengine_api.data.v2.database +from policyengine_api.data.v2.models import V2_METADATA +after = set(pathlib.Path.cwd().iterdir()) +assert before == after +assert len(V2_METADATA.tables) == 33 +""" + + result = subprocess.run( + [sys.executable, "-c", script], + cwd=tmp_path, + env=_environment_without_v2(), + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert list(tmp_path.iterdir()) == [] + + +def test_default_cloud_sql_startup_requires_no_supabase_configuration( + tmp_path: Path, +) -> None: + result = subprocess.run( + [sys.executable, "-c", "import policyengine_api.asgi"], + cwd=tmp_path, + env=_environment_without_v2(), + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert not (tmp_path / "policyengine.db").exists() + assert not list(tmp_path.glob("*.init.lock")) + + +@pytest.mark.parametrize("debug", ["0", "1"]) +def test_import_startup_and_request_never_create_runtime_sqlite( + tmp_path: Path, + debug: str, +) -> None: + environment = _environment_without_v2() + environment["FLASK_DEBUG"] = debug + script = """ +from pathlib import Path +from policyengine_api.api import app + +response = app.test_client().get('/liveness-check') +assert response.status_code == 200 +assert not Path('policyengine.db').exists() +assert not list(Path.cwd().glob('*.db')) +assert not list(Path.cwd().glob('*.init.lock')) +""" + result = subprocess.run( + [sys.executable, "-c", script], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert list(tmp_path.iterdir()) == [] + + +def test_runtime_sqlite_modules_are_absent_from_production_package() -> None: + data_root = Path(__file__).parents[3] / "policyengine_api/data" + assert not (data_root / "local_database.py").exists() + assert not (data_root / "local_models.py").exists() + + +def test_selected_v2_runtime_without_its_url_fails_closed() -> None: + with pytest.raises(V2ConfigurationError, match=V2_RUNTIME_DATABASE_URL): + load_v2_runtime_database_settings( + { + V2_SUPABASE_PROJECT_REF: "abcdefghijklmnopqrst", + V2_SUPABASE_ENVIRONMENT: "production-foundation", + "ALEMBIC_DATABASE_URL": "mysql+pymysql://v1:secret@db/v1", + "FLASK_DEBUG": "1", + } + ) diff --git a/tests/unit/v2/test_model_persistence.py b/tests/unit/v2/test_model_persistence.py new file mode 100644 index 000000000..b8879b1e7 --- /dev/null +++ b/tests/unit/v2/test_model_persistence.py @@ -0,0 +1,91 @@ +"""Canonical SQLModel persistence and bounded SQLAlchemy escape-hatch tests.""" + +from pathlib import Path + +import pytest +from sqlalchemy.exc import IntegrityError +from sqlmodel import Session, create_engine, select + +from policyengine_api.data.v2.models import ( + TaxBenefitModel, + TaxBenefitModelVersion, + User, + V2_METADATA, +) +from policyengine_api.data.v2.models.base import DIRECT_SQLALCHEMY_EXCEPTIONS + + +def test_ordinary_persistence_uses_sqlmodel_session_select_and_exec() -> None: + # SQLite exists only as an injected, in-memory unit-test fixture. Runtime + # v2 settings reject it and application code never selects it. + engine = create_engine("sqlite://") + V2_METADATA.create_all(engine) + model = TaxBenefitModel(name="test-country", description="Test model") + version = TaxBenefitModelVersion(model=model, version="1.2.3") + + with Session(engine) as session: + session.add(version) + session.commit() + statement = select(TaxBenefitModelVersion).where( + TaxBenefitModelVersion.version == "1.2.3" + ) + stored = session.exec(statement).one() + + assert stored.model.name == "test-country" + assert stored.id == version.id + + engine.dispose() + + +def test_direct_sqlalchemy_categories_are_complete_and_documented() -> None: + assert set(DIRECT_SQLALCHEMY_EXCEPTIONS) == { + "timezone_aware_timestamps", + "named_enums", + "typed_json_and_text", + "named_constraints_and_indexes", + "ambiguous_foreign_key_relationships", + "transaction_conflict_recovery", + } + assert all(DIRECT_SQLALCHEMY_EXCEPTIONS.values()) + + +def test_user_primary_country_can_change_between_us_and_uk_only() -> None: + engine = create_engine("sqlite://") + V2_METADATA.create_all(engine) + + with Session(engine) as session: + user = User( + first_name="Ada", + last_name="Lovelace", + email="ada@example.test", + primary_country="us", + ) + session.add(user) + session.commit() + + user.primary_country = "uk" + session.add(user) + session.commit() + assert user.primary_country == "uk" + + user.primary_country = "ca" + session.add(user) + with pytest.raises(IntegrityError): + session.commit() + + engine.dispose() + + +def test_v2_models_do_not_create_a_parallel_sqlalchemy_orm_layer() -> None: + models_directory = ( + Path(__file__).parents[3] / "policyengine_api" / "data" / "v2" / "models" + ) + model_sources = "\n".join( + path.read_text(encoding="utf-8") + for path in sorted(models_directory.glob("*.py")) + ) + + assert "declarative_base" not in model_sources + assert "mapped_column" not in model_sources + assert "sqlalchemy.orm.Session" not in model_sources + assert "class_=sqlalchemy" not in model_sources diff --git a/tests/unit/v2/test_models.py b/tests/unit/v2/test_models.py new file mode 100644 index 000000000..0f5003ecb --- /dev/null +++ b/tests/unit/v2/test_models.py @@ -0,0 +1,233 @@ +"""Structural contract tests for the complete reviewed v2 SQLModel schema.""" + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql +from sqlalchemy.orm import configure_mappers +from sqlalchemy.schema import CreateTable + +from policyengine_api.data.v1_models import V1Base +from policyengine_api.data.v2.models import ( + User, + UserHouseholdAssociation, + UserPolicy, + UserReportAssociation, + UserSimulationAssociation, + V2_METADATA, + V2_TABLE_MODELS, +) +from policyengine_api.data.v2.table_inventory import EXPECTED_V2_TABLES + + +RUN_OUTPUT_TABLES = frozenset( + { + "aggregates", + "budget_summary", + "change_aggregates", + "congressional_district_impacts", + "constituency_impacts", + "decile_impacts", + "inequality", + "intra_decile_impacts", + "local_authority_impacts", + "poverty", + "program_statistics", + } +) + + +def test_controlled_models_match_the_exact_reviewed_inventory() -> None: + model_table_names = {model.__table__.name for model in V2_TABLE_MODELS} + + assert V2_METADATA is not V1Base.metadata + assert set(V2_METADATA.tables) == EXPECTED_V2_TABLES + assert model_table_names == EXPECTED_V2_TABLES + assert len(V2_TABLE_MODELS) == len(EXPECTED_V2_TABLES) + + +def test_every_table_has_named_primary_foreign_and_relational_constraints() -> None: + for table in V2_METADATA.tables.values(): + assert table.primary_key.columns + assert table.primary_key.name + for constraint in table.constraints: + assert constraint.name, f"unnamed constraint on {table.name}" + for index in table.indexes: + assert index.name, f"unnamed index on {table.name}" + for foreign_key in table.foreign_keys: + assert foreign_key.constraint.name + assert foreign_key.ondelete in {"CASCADE", "RESTRICT", "SET NULL"} + assert foreign_key.column.table.name in EXPECTED_V2_TABLES + + +def test_every_declared_relationship_has_a_complete_back_populates_pair() -> None: + configure_mappers() + + for model in V2_TABLE_MODELS: + mapper = sa.inspect(model) + for relationship in mapper.relationships: + assert relationship.back_populates, ( + f"{model.__name__}.{relationship.key} lacks back_populates" + ) + inverse = relationship.mapper.relationships[relationship.back_populates] + assert inverse.back_populates == relationship.key + assert inverse.mapper is mapper + + +def test_every_user_association_has_relational_integrity() -> None: + configure_mappers() + user_mapper = sa.inspect(User) + associations = ( + ( + UserHouseholdAssociation, + "user_household_associations", + "household_associations", + ), + (UserPolicy, "user_policies", "policy_associations"), + ( + UserSimulationAssociation, + "user_simulation_associations", + "simulation_associations", + ), + (UserReportAssociation, "user_report_associations", "report_associations"), + ) + + for model, table_name, user_collection in associations: + user_id = V2_METADATA.tables[table_name].c.user_id + foreign_keys = list(user_id.foreign_keys) + assert len(foreign_keys) == 1 + assert foreign_keys[0].target_fullname == "users.id" + assert foreign_keys[0].ondelete == "CASCADE" + + association_user = sa.inspect(model).relationships["user"] + assert association_user.back_populates == user_collection + assert user_mapper.relationships[user_collection].back_populates == "user" + + +def test_all_datetime_columns_are_timezone_aware() -> None: + datetime_columns = [ + column + for table in V2_METADATA.tables.values() + for column in table.columns + if isinstance(column.type, sa.DateTime) + ] + + assert datetime_columns + assert all(column.type.timezone for column in datetime_columns) + + +def test_report_definition_and_run_columns_are_separated() -> None: + report = V2_METADATA.tables["reports"] + report_run = V2_METADATA.tables["report_runs"] + + assert report.c.type.nullable + assert { + "country", + "type", + "tax_benefit_model_id", + "policy_id", + "baseline_simulation_id", + "reform_simulation_id", + "household_id", + "dataset_id", + "region_id", + "year", + "inputs", + }.issubset(report.c.keys()) + assert { + "country_package_version", + "policyengine_version", + "status", + "trigger", + "idempotency_key", + "started_at", + "completed_at", + "error_message", + "markdown", + }.issubset(report_run.c.keys()) + assert { + "country_package_version", + "policyengine_version", + "status", + "error_message", + "markdown", + }.isdisjoint(report.c.keys()) + assert not report_run.c.country_package_version.nullable + assert not report_run.c.policyengine_version.nullable + + +def test_user_primary_country_is_required_and_limited_to_supported_values() -> None: + users = V2_METADATA.tables["users"] + + assert not users.c.primary_country.nullable + assert users.c.primary_country.type.length == 2 + assert "ck_users_primary_country" in { + constraint.name for constraint in users.constraints + } + + +def test_run_outputs_reference_report_runs_not_base_reports() -> None: + for table_name in RUN_OUTPUT_TABLES: + table = V2_METADATA.tables[table_name] + assert "report_run_id" in table.c + assert not table.c.report_run_id.nullable + assert "report_id" not in table.c + foreign_key = next(iter(table.c.report_run_id.foreign_keys)) + assert foreign_key.target_fullname == "report_runs.id" + assert foreign_key.ondelete == "CASCADE" + + +def test_report_rerun_constraints_allow_same_versions_but_deduplicate_requests() -> ( + None +): + report_runs = V2_METADATA.tables["report_runs"] + unique_column_sets = { + tuple(column.name for column in constraint.columns) + for constraint in report_runs.constraints + if isinstance(constraint, sa.UniqueConstraint) + } + + assert ("report_id", "idempotency_key") in unique_column_sets + assert ( + "report_id", + "country_package_version", + "policyengine_version", + ) not in unique_column_sets + assert "ix_report_runs_current_output" in { + index.name for index in report_runs.indexes + } + + +def test_named_checks_and_required_indexes_cover_core_invariants() -> None: + constraint_names = { + constraint.name + for table in V2_METADATA.tables.values() + for constraint in table.constraints + } + index_names = { + index.name for table in V2_METADATA.tables.values() for index in table.indexes + } + + assert { + "ck_regions_required_filter_values", + "ck_simulations_type_input", + "ck_users_primary_country", + "ck_report_runs_idempotency_key_nonblank", + "ck_report_runs_terminal_completion", + "ck_parameter_values_single_owner", + }.issubset(constraint_names) + assert { + "ix_users_email", + "ix_simulations_status_created_at", + "ix_report_runs_current_output", + }.issubset(index_names) + + +def test_complete_metadata_compiles_for_postgres_without_mutation() -> None: + dialect = postgresql.dialect() + + statements = [ + str(CreateTable(table).compile(dialect=dialect)) + for table in V2_METADATA.sorted_tables + ] + + assert len(statements) == len(EXPECTED_V2_TABLES) + assert all("CREATE TABLE" in statement for statement in statements) diff --git a/tests/unit/v2/test_reference_data_autogenerate.py b/tests/unit/v2/test_reference_data_autogenerate.py new file mode 100644 index 000000000..899b0bbcc --- /dev/null +++ b/tests/unit/v2/test_reference_data_autogenerate.py @@ -0,0 +1,221 @@ +"""Tests for generated-only declarative application-data migrations.""" + +from types import SimpleNamespace + +from alembic.migration import MigrationContext +from alembic.operations import Operations, ops +import pytest +import sqlalchemy as sa + +from policyengine_api.data.v2.models import V2_METADATA +from policyengine_api.data.v2.reference_data import ( + REFERENCE_DATA, + REFERENCE_DATA_FORMAT_VERSION, + ReferenceDataDeclarationError, + ReferenceRow, + ReferenceTable, +) +from policyengine_api.data.v2.reference_data_autogenerate import ( + ReferenceDataMigrationError, + ReferenceRowChangeOp, + _render_reference_row_change, + compare_reference_data, + order_generated_operations, +) +from policyengine_api.data.v2.table_inventory import EXPECTED_V2_TABLES + + +def test_declaration_has_stable_scoped_natural_keys_and_wire_values() -> None: + assert REFERENCE_DATA_FORMAT_VERSION == 1 + assert [table.table_name for table in REFERENCE_DATA] == [ + "tax_benefit_models", + "tax_benefit_model_versions", + ] + for table in REFERENCE_DATA: + assert table.managed_prefix_column in table.key_columns + for row in table.rows: + assert tuple(row.key) == table.key_columns + assert row.key[table.managed_prefix_column].startswith(table.managed_prefix) + + +def test_declaration_rejects_unknown_tables_duplicate_keys_and_unsafe_values() -> None: + row = ReferenceRow.create(key={"name": "stage8-one"}, values={"value": 1}) + with pytest.raises(ReferenceDataDeclarationError, match="unreviewed"): + ReferenceTable( + table_name="runtime_bundles", + key_columns=("name",), + managed_prefix_column="name", + managed_prefix="stage8-", + rows=(row,), + ) + with pytest.raises(ReferenceDataDeclarationError, match="duplicate"): + ReferenceTable( + table_name="tax_benefit_models", + key_columns=("name",), + managed_prefix_column="name", + managed_prefix="stage8-", + rows=(row, row), + ) + with pytest.raises(ReferenceDataDeclarationError, match="unsupported"): + ReferenceRow.create(key={"name": "stage8-unsafe"}, values={"value": object()}) + + +def test_operation_requires_reviewed_table_key_and_reversible_state() -> None: + with pytest.raises(ReferenceDataMigrationError, match="unreviewed"): + ReferenceRowChangeOp( + "runtime_bundles", + key={"name": "stage8-test"}, + before=None, + after={"name": "stage8-test"}, + ) + with pytest.raises(ReferenceDataMigrationError, match="stable key"): + ReferenceRowChangeOp( + "tax_benefit_models", + key={}, + before=None, + after={"name": "stage8-test"}, + ) + + operation = ReferenceRowChangeOp( + "tax_benefit_models", + key={"name": "stage8-test"}, + before={"name": "stage8-test", "description": "before"}, + after={"name": "stage8-test", "description": "after"}, + ) + assert operation.reverse().before == operation.after + assert operation.reverse().after == operation.before + + +def test_renderer_is_deterministic_and_uses_the_registered_op_surface() -> None: + operation = ReferenceRowChangeOp( + "tax_benefit_models", + key={"name": "stage8-test"}, + before=None, + after={"name": "stage8-test", "description": "test"}, + ) + + first = _render_reference_row_change(None, operation) + second = _render_reference_row_change(None, operation) + + assert first == second + assert first.startswith("op.v2_reference_row_change(") + assert hasattr(Operations, "v2_reference_row_change") + assert operation.to_diff_tuple() == ( + "v2_reference_row_change", + "tax_benefit_models", + {"name": "stage8-test"}, + None, + {"description": "test", "name": "stage8-test"}, + ) + + +def test_generated_operation_executes_and_downgrades_against_reflected_table() -> None: + engine = sa.create_engine("sqlite://") + declaration = REFERENCE_DATA[0] + row = declaration.rows[0] + with engine.begin() as connection: + connection.exec_driver_sql("ATTACH DATABASE ':memory:' AS public") + table = V2_METADATA.tables[declaration.table_name].to_metadata( + sa.MetaData(schema="public") + ) + table.create(connection) + operations = Operations(MigrationContext.configure(connection)) + insert = ReferenceRowChangeOp( + declaration.table_name, + key=dict(row.key), + before=None, + after=row.complete_values, + ) + + operations.invoke(insert) + stored = connection.execute(sa.select(table)).mappings().one() + assert stored["name"] == "stage8-platform-validation" + + update_values = {**row.complete_values, "description": "updated"} + update = ReferenceRowChangeOp( + declaration.table_name, + key=dict(row.key), + before=row.complete_values, + after=update_values, + ) + operations.invoke(update) + assert connection.execute(sa.select(table.c.description)).scalar_one() == ( + "updated" + ) + operations.invoke(update.reverse()) + operations.invoke(insert.reverse()) + assert ( + connection.execute( + sa.select(sa.func.count()).select_from(table) + ).scalar_one() + == 0 + ) + engine.dispose() + + +def test_generated_data_is_ordered_between_constructive_and_destructive_schema() -> ( + None +): + metadata = sa.MetaData() + table = sa.Table("example", metadata, sa.Column("id", sa.Integer, primary_key=True)) + create = ops.CreateTableOp.from_table(table) + drop = ops.DropTableOp.from_table(table) + data = ReferenceRowChangeOp( + "tax_benefit_models", + key={"name": "stage8-test"}, + before=None, + after={"name": "stage8-test"}, + ) + script = SimpleNamespace( + upgrade_ops=SimpleNamespace(ops=[drop, data, create]), + downgrade_ops=SimpleNamespace(ops=[]), + ) + + order_generated_operations(None, None, [script]) + + assert script.upgrade_ops.ops == [create, data, drop] + assert isinstance(script.downgrade_ops.ops[0], ops.CreateTableOp) + assert isinstance(script.downgrade_ops.ops[1], ReferenceRowChangeOp) + assert isinstance(script.downgrade_ops.ops[2], ops.DropTableOp) + + +def test_comparator_deletes_children_first_and_upserts_parents_first( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import policyengine_api.data.v2.reference_data_autogenerate as module + + parent_remove = ReferenceRowChangeOp( + "tax_benefit_models", + key={"name": "stage8-parent"}, + before={"name": "stage8-parent"}, + after=None, + ) + parent_insert = parent_remove.reverse() + child_remove = ReferenceRowChangeOp( + "tax_benefit_model_versions", + key={"version": "stage8-child"}, + before={"version": "stage8-child"}, + after=None, + ) + child_insert = child_remove.reverse() + differences = iter( + [([parent_remove], [parent_insert]), ([child_remove], [child_insert])] + ) + monkeypatch.setattr(module, "_table_differences", lambda *_: next(differences)) + monkeypatch.setattr( + module.sa, + "inspect", + lambda _: SimpleNamespace( + get_table_names=lambda schema: list(EXPECTED_V2_TABLES) + ), + ) + upgrade = ops.UpgradeOps(ops=[]) + + compare_reference_data(SimpleNamespace(connection=object()), upgrade, {None}) + + assert upgrade.ops == [ + child_remove, + parent_remove, + parent_insert, + child_insert, + ] diff --git a/tests/unit/v2/test_report_runs.py b/tests/unit/v2/test_report_runs.py new file mode 100644 index 000000000..5b6ea226b --- /dev/null +++ b/tests/unit/v2/test_report_runs.py @@ -0,0 +1,415 @@ +"""Report definition, rerun, idempotency, worker, and selector tests.""" + +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timedelta, timezone +from pathlib import Path +import threading +from uuid import UUID, uuid4 + +import pytest +import sqlalchemy as sa +from sqlalchemy.pool import StaticPool +from sqlmodel import Session, create_engine, select + +from policyengine_api.data.v2.models import ( + AggregateOutput, + AggregateType, + Dataset, + Report, + ReportRun, + ReportRunStatus, + ReportRunTrigger, + Simulation, + SimulationType, + TaxBenefitModel, + TaxBenefitModelVersion, + V2_METADATA, +) +from policyengine_api.data.v2.report_runs import ( + ReportRunStateError, + ReportTypeImmutableError, + begin_report_run, + complete_report_run, + create_report_run, + fail_report_run, + select_current_report_run, + set_report_type, +) + + +DEPLOYED_COUNTRY_VERSIONS = {"us": "1.2.3"} +DEPLOYED_POLICYENGINE_VERSION = "4.5.6" +NOW = datetime(2026, 8, 14, 12, 0, tzinfo=timezone.utc) + + +@pytest.fixture +def engine(): + test_engine = create_engine( + "sqlite://", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + V2_METADATA.create_all(test_engine) + yield test_engine + test_engine.dispose() + + +def _create_report(session: Session, *, report_type: str | None = None) -> Report: + model = TaxBenefitModel(name=f"model-{uuid4()}") + report = Report( + label="Distributional report", + country="us", + type=report_type, + tax_benefit_model=model, + inputs={"reform": {"gov.example": 1}}, + ) + session.add(report) + session.flush() + return report + + +def _create_run( + session: Session, + report: Report, + *, + key: str, + country_version: str = "1.2.3", + policyengine_version: str = DEPLOYED_POLICYENGINE_VERSION, +) -> ReportRun: + return create_report_run( + session, + report_id=report.id, + country_package_version=country_version, + policyengine_version=policyengine_version, + trigger=ReportRunTrigger.MANUAL, + idempotency_key=key, + ) + + +def test_untyped_and_typed_report_definitions_store_no_execution_versions( + engine, +) -> None: + with Session(engine) as session: + untyped = _create_report(session) + typed = _create_report(session, report_type="marginal_tax_rate") + + assert untyped.type is None + assert typed.type == "marginal_tax_rate" + assert "country_package_version" not in Report.__table__.c + assert "policyengine_version" not in Report.__table__.c + + +def test_report_type_can_change_before_first_run_but_not_after(engine) -> None: + with Session(engine) as session: + report = _create_report(session) + set_report_type( + session, + report_id=report.id, + report_type="marginal_tax_rate", + ) + _create_run(session, report, key="first-run") + + unchanged = set_report_type( + session, + report_id=report.id, + report_type="marginal_tax_rate", + ) + assert unchanged.id == report.id + with pytest.raises(ReportTypeImmutableError): + set_report_type( + session, + report_id=report.id, + report_type="economy_comparison", + ) + + +def test_new_manual_keys_create_distinct_same_version_runs(engine) -> None: + with Session(engine) as session: + report = _create_report(session) + first = _create_run(session, report, key="manual-1") + second = _create_run(session, report, key="manual-2") + + assert first.id != second.id + assert first.country_package_version == second.country_package_version + assert first.policyengine_version == second.policyengine_version + assert len(session.exec(select(ReportRun)).all()) == 2 + + +def test_manual_rerun_rejects_a_whitespace_only_idempotency_key(engine) -> None: + with Session(engine) as session: + report = _create_report(session) + + with pytest.raises(ValueError, match="require an idempotency key"): + create_report_run( + session, + report_id=report.id, + country_package_version="1.2.3", + policyengine_version=DEPLOYED_POLICYENGINE_VERSION, + trigger=ReportRunTrigger.MANUAL, + idempotency_key=" \t\n", + ) + + +def test_database_rejects_non_null_blank_idempotency_keys(engine) -> None: + with Session(engine) as session: + report = _create_report(session) + session.add( + ReportRun( + report=report, + country_package_version="1.2.3", + policyengine_version=DEPLOYED_POLICYENGINE_VERSION, + trigger=ReportRunTrigger.SYSTEM, + idempotency_key=" ", + ) + ) + + with pytest.raises(sa.exc.IntegrityError): + session.flush() + + +def test_transport_retry_returns_the_existing_report_scoped_run(engine) -> None: + with Session(engine) as session: + report = _create_report(session) + first = _create_run(session, report, key="retry-me") + retried = _create_run(session, report, key="retry-me") + + assert retried.id == first.id + assert len(session.exec(select(ReportRun)).all()) == 1 + + +def test_concurrent_idempotent_requests_resolve_to_one_run(tmp_path: Path) -> None: + sqlite_path = tmp_path / "report-rerun-concurrency.db" + test_engine = create_engine( + f"sqlite:///{sqlite_path}", + connect_args={"check_same_thread": False, "timeout": 10}, + ) + + # BEGIN IMMEDIATE gives this explicit SQLite-only test fixture the same + # serialization point that SELECT FOR UPDATE provides in Postgres. + @sa.event.listens_for(test_engine, "connect") + def _set_sqlite_transaction_mode(dbapi_connection, _record) -> None: + dbapi_connection.isolation_level = None + + @sa.event.listens_for(test_engine, "begin") + def _begin_immediate(connection) -> None: + connection.exec_driver_sql("BEGIN IMMEDIATE") + + V2_METADATA.create_all(test_engine) + with Session(test_engine) as session: + report_id = _create_report(session).id + session.commit() + + barrier = threading.Barrier(2) + + def request_rerun() -> UUID: + with Session(test_engine) as session: + barrier.wait() + run = create_report_run( + session, + report_id=report_id, + country_package_version="1.2.3", + policyengine_version=DEPLOYED_POLICYENGINE_VERSION, + trigger=ReportRunTrigger.MANUAL, + idempotency_key="one-concurrent-request", + ) + run_id = run.id + session.commit() + return run_id + + with ThreadPoolExecutor(max_workers=2) as executor: + run_ids = set(executor.map(lambda _: request_rerun(), range(2))) + + with Session(test_engine) as session: + assert len(run_ids) == 1 + assert len(session.exec(select(ReportRun)).all()) == 1 + test_engine.dispose() + + +def test_worker_retry_resumes_the_same_run_and_terminal_runs_stay_terminal( + engine, +) -> None: + with Session(engine) as session: + report = _create_report(session) + run = _create_run(session, report, key="worker-retry") + started = begin_report_run(session, report_run_id=run.id, started_at=NOW) + resumed = begin_report_run(session, report_run_id=run.id) + + assert resumed.id == run.id + assert resumed.status is ReportRunStatus.RUNNING + assert resumed.started_at == started.started_at + assert len(session.exec(select(ReportRun)).all()) == 1 + + complete_report_run(session, report_run_id=run.id, completed_at=NOW) + with pytest.raises(ReportRunStateError): + begin_report_run(session, report_run_id=run.id) + + +def test_outputs_from_repeated_runs_are_preserved(engine) -> None: + with Session(engine) as session: + report = _create_report(session) + model = report.tax_benefit_model + version = TaxBenefitModelVersion(model=model, version="1.2.3") + dataset = Dataset( + name="dataset", + storage_path="datasets/test.h5", + year=2026, + tax_benefit_model=model, + ) + simulation = Simulation( + simulation_type=SimulationType.ECONOMY, + dataset=dataset, + tax_benefit_model_version=version, + ) + session.add(simulation) + session.flush() + first = _create_run(session, report, key="output-1") + second = _create_run(session, report, key="output-2") + session.add_all( + [ + AggregateOutput( + report_run=first, + simulation_id=simulation.id, + variable="household_net_income", + aggregate_type=AggregateType.SUM, + result=1.0, + ), + AggregateOutput( + report_run=second, + simulation_id=simulation.id, + variable="household_net_income", + aggregate_type=AggregateType.SUM, + result=2.0, + ), + ] + ) + session.flush() + + outputs = session.exec( + select(AggregateOutput).order_by(AggregateOutput.result) + ).all() + assert [output.result for output in outputs] == [1.0, 2.0] + assert outputs[0].report_run_id != outputs[1].report_run_id + + +def test_selector_uses_versions_success_completion_time_and_stable_id(engine) -> None: + with Session(engine) as session: + report = _create_report(session) + candidates = [ + ReportRun( + id=UUID(int=1), + report=report, + country_package_version="1.2.3", + policyengine_version=DEPLOYED_POLICYENGINE_VERSION, + status=ReportRunStatus.SUCCEEDED, + completed_at=NOW, + ), + ReportRun( + id=UUID(int=2), + report=report, + country_package_version="1.2.3", + policyengine_version=DEPLOYED_POLICYENGINE_VERSION, + status=ReportRunStatus.SUCCEEDED, + completed_at=NOW, + ), + ReportRun( + id=UUID(int=3), + report=report, + country_package_version="9.9.9", + policyengine_version=DEPLOYED_POLICYENGINE_VERSION, + status=ReportRunStatus.SUCCEEDED, + completed_at=NOW + timedelta(days=1), + ), + ReportRun( + id=UUID(int=4), + report=report, + country_package_version="1.2.3", + policyengine_version=DEPLOYED_POLICYENGINE_VERSION, + status=ReportRunStatus.RUNNING, + ), + ] + session.add_all(candidates) + session.flush() + + current = select_current_report_run( + session, + report_id=report.id, + country_package_versions=DEPLOYED_COUNTRY_VERSIONS, + policyengine_version=DEPLOYED_POLICYENGINE_VERSION, + ) + + assert current is not None + assert current.id == UUID(int=2) + + +def test_pending_and_failed_reruns_do_not_displace_success(engine) -> None: + with Session(engine) as session: + report = _create_report(session) + successful = _create_run(session, report, key="success") + complete_report_run(session, report_run_id=successful.id, completed_at=NOW) + pending = _create_run(session, report, key="pending") + failed = _create_run(session, report, key="failed") + fail_report_run( + session, + report_run_id=failed.id, + error_message="transient failure", + completed_at=NOW + timedelta(hours=1), + ) + + current = select_current_report_run( + session, + report_id=report.id, + country_package_versions=DEPLOYED_COUNTRY_VERSIONS, + policyengine_version=DEPLOYED_POLICYENGINE_VERSION, + ) + + assert pending.status is ReportRunStatus.PENDING + assert current is not None + assert current.id == successful.id + + +def test_new_successful_rerun_becomes_current_without_cache_state(engine) -> None: + with Session(engine) as session: + report = _create_report(session) + first = _create_run(session, report, key="first") + complete_report_run(session, report_run_id=first.id, completed_at=NOW) + rerun = _create_run(session, report, key="rerun") + complete_report_run( + session, + report_run_id=rerun.id, + completed_at=NOW + timedelta(minutes=1), + ) + + # No Redis/cache collaborator participates in durable selection; a + # flush or expiry therefore has no state to invalidate here. + current = select_current_report_run( + session, + report_id=report.id, + country_package_versions=DEPLOYED_COUNTRY_VERSIONS, + policyengine_version=DEPLOYED_POLICYENGINE_VERSION, + ) + + assert current is not None + assert current.id == rerun.id + assert len(session.exec(select(ReportRun)).all()) == 2 + + +def test_selector_returns_none_without_a_matching_success(engine) -> None: + with Session(engine) as session: + report = _create_report(session) + mismatched = _create_run( + session, + report, + key="old-version", + country_version="0.0.1", + ) + complete_report_run(session, report_run_id=mismatched.id, completed_at=NOW) + + assert ( + select_current_report_run( + session, + report_id=report.id, + country_package_versions=DEPLOYED_COUNTRY_VERSIONS, + policyengine_version=DEPLOYED_POLICYENGINE_VERSION, + ) + is None + ) diff --git a/tests/unit/v2/test_scaffolding_hygiene.py b/tests/unit/v2/test_scaffolding_hygiene.py new file mode 100644 index 000000000..71a2b4f93 --- /dev/null +++ b/tests/unit/v2/test_scaffolding_hygiene.py @@ -0,0 +1,40 @@ +"""Tests for the staged one-off Supabase artifact guard.""" + +from scripts.check_stage8_scaffolding_hygiene import prohibited_staged_paths + + +def test_rejects_one_off_supabase_and_secret_shaped_artifacts() -> None: + assert prohibited_staged_paths( + [ + "supabase/.temp/project-ref", + ".agent-artifacts/stage8/bootstrap.json", + "tmp/stage8-scratch.sql", + "config/.env", + "private/storage-admin.key", + "scripts/one-off-supabase.py", + ] + ) == [ + ".agent-artifacts/stage8/bootstrap.json", + "config/.env", + "private/storage-admin.key", + "scripts/one-off-supabase.py", + "supabase/.temp/project-ref", + "tmp/stage8-scratch.sql", + ] + + +def test_allows_durable_migrations_bootstrap_tests_and_docs() -> None: + assert ( + prohibited_staged_paths( + [ + "migrations/v2/versions/abc_generated.py", + "policyengine_api/data/v2/reference_data.py", + "scripts/bootstrap_v2_supabase_storage.py", + "scripts/check_stage8_scaffolding_hygiene.py", + "tests/unit/v2/test_storage_bootstrap.py", + "docs/migration/stage-8-supabase-bootstrap.md", + ".env.example", + ] + ) + == [] + ) diff --git a/tests/unit/v2/test_settings.py b/tests/unit/v2/test_settings.py new file mode 100644 index 000000000..837487f8c --- /dev/null +++ b/tests/unit/v2/test_settings.py @@ -0,0 +1,137 @@ +"""Tests for explicit and secret-safe v2 persistence configuration.""" + +import pytest + +from policyengine_api.data.v2.settings import ( + V2_MIGRATION_DATABASE_URL, + V2_RUNTIME_DATABASE_URL, + V2_SUPABASE_ENVIRONMENT, + V2_SUPABASE_PROJECT_REF, + V2_SUPABASE_STORAGE_ADMIN_KEY, + V2_SUPABASE_STORAGE_BUCKET, + V2_SUPABASE_STORAGE_URL, + V2ConfigurationError, + load_supabase_storage_settings, + load_v2_migration_database_settings, + load_v2_runtime_database_settings, +) + + +PROJECT_REF = "abcdefghijklmnopqrst" +TARGET_ENVIRONMENT = { + V2_SUPABASE_PROJECT_REF: PROJECT_REF, + V2_SUPABASE_ENVIRONMENT: "production-foundation", +} +RUNTIME_URL = ( + "postgresql+psycopg://runtime:test-runtime-password@db.example.com:5432/" + "postgres?sslmode=require" +) +MIGRATION_URL = ( + "postgresql+psycopg://migrator:test-migration-password@db.example.com:5432/" + "postgres?sslmode=verify-full" +) + + +def test_runtime_and_migration_urls_are_explicit_and_separate() -> None: + environment = { + **TARGET_ENVIRONMENT, + V2_RUNTIME_DATABASE_URL: RUNTIME_URL, + V2_MIGRATION_DATABASE_URL: MIGRATION_URL, + } + + runtime = load_v2_runtime_database_settings(environment) + migration = load_v2_migration_database_settings(environment) + + assert runtime.connection.url.username == "runtime" + assert migration.connection.url.username == "migrator" + assert runtime.target == migration.target + + +def test_postgres_password_is_hidden_from_string_and_repr() -> None: + settings = load_v2_runtime_database_settings( + {**TARGET_ENVIRONMENT, V2_RUNTIME_DATABASE_URL: RUNTIME_URL} + ) + + rendered = f"{settings!r} {settings.connection}" + + assert "test-runtime-password" not in rendered + assert "***" in rendered + + +@pytest.mark.parametrize( + "url", + [ + "sqlite+pysqlite:///policyengine.db", + "mysql+pymysql://user:password@db.example.com/policyengine", + "postgresql+psycopg://user:password@localhost/postgres?sslmode=require", + "postgresql+psycopg://user:password@127.0.0.1/postgres?sslmode=require", + "postgresql+psycopg://user:password@db.example.com/postgres", + ], +) +def test_runtime_rejects_non_persistent_postgres_targets(url: str) -> None: + with pytest.raises(V2ConfigurationError): + load_v2_runtime_database_settings( + {**TARGET_ENVIRONMENT, V2_RUNTIME_DATABASE_URL: url} + ) + + +def test_v1_and_debug_settings_never_supply_missing_v2_configuration() -> None: + environment = { + **TARGET_ENVIRONMENT, + "ALEMBIC_DATABASE_URL": "mysql+pymysql://v1:secret@db/v1", + "POLICYENGINE_DB_PASSWORD": "v1-password", + "FLASK_DEBUG": "1", + } + + with pytest.raises(V2ConfigurationError, match=V2_RUNTIME_DATABASE_URL): + load_v2_runtime_database_settings(environment) + with pytest.raises(V2ConfigurationError, match=V2_MIGRATION_DATABASE_URL): + load_v2_migration_database_settings(environment) + + +def test_storage_settings_require_the_recorded_https_project_origin() -> None: + settings = load_supabase_storage_settings( + { + **TARGET_ENVIRONMENT, + V2_SUPABASE_STORAGE_URL: f"https://{PROJECT_REF}.supabase.co/", + V2_SUPABASE_STORAGE_BUCKET: "policyengine-v2-alpha", + V2_SUPABASE_STORAGE_ADMIN_KEY: "test-storage-admin-key", + } + ) + + assert settings.api_url == f"https://{PROJECT_REF}.supabase.co" + assert settings.bucket == "policyengine-v2-alpha" + assert "test-storage-admin-key" not in repr(settings) + assert settings.admin_key.get_secret_value() == "test-storage-admin-key" + + +@pytest.mark.parametrize( + "api_url", + [ + f"http://{PROJECT_REF}.supabase.co", + "https://another-project.supabase.co", + f"https://{PROJECT_REF}.supabase.co/storage/v1", + f"https://user:password@{PROJECT_REF}.supabase.co", + ], +) +def test_storage_settings_reject_an_inexact_project_origin(api_url: str) -> None: + with pytest.raises(V2ConfigurationError, match=V2_SUPABASE_STORAGE_URL): + load_supabase_storage_settings( + { + **TARGET_ENVIRONMENT, + V2_SUPABASE_STORAGE_URL: api_url, + V2_SUPABASE_STORAGE_BUCKET: "policyengine-v2-alpha", + V2_SUPABASE_STORAGE_ADMIN_KEY: "test-storage-admin-key", + } + ) + + +def test_configuration_errors_do_not_echo_secret_values() -> None: + secret_url = "postgresql+psycopg://user:do-not-echo@localhost/postgres" + + with pytest.raises(V2ConfigurationError) as raised: + load_v2_runtime_database_settings( + {**TARGET_ENVIRONMENT, V2_RUNTIME_DATABASE_URL: secret_url} + ) + + assert "do-not-echo" not in str(raised.value) diff --git a/tests/unit/v2/test_stage8_activation.py b/tests/unit/v2/test_stage8_activation.py new file mode 100644 index 000000000..b3b056390 --- /dev/null +++ b/tests/unit/v2/test_stage8_activation.py @@ -0,0 +1,46 @@ +"""Guards that keep the dormant v2 report schema off production paths.""" + +import inspect + +from policyengine_api.migration_flags import ( + DEFAULT_DB_SOURCE, + DEFAULT_SIM_COMPUTE_BACKEND, + DEFAULT_SIM_ENTRYPOINT, + RouteImplementation, + get_migration_context, +) +from policyengine_api.routes import report_output_routes +from policyengine_api.services.report_output_service import ReportOutputService + + +def test_default_report_migration_context_keeps_existing_owners( + monkeypatch, +) -> None: + for name in ( + "DB_READ_REPORT", + "DB_WRITE_REPORT", + "ROUTE_IMPL_REPORT", + "SIM_COMPUTE_REPORT", + "SIM_ENTRYPOINT", + ): + monkeypatch.delenv(name, raising=False) + + context = get_migration_context("report") + + assert DEFAULT_DB_SOURCE == "cloud_sql" + assert context.db_read == "cloud_sql" + assert context.db_write == "cloud_sql" + assert context.route_impl is RouteImplementation.FLASK_FALLBACK + assert context.sim_compute == DEFAULT_SIM_COMPUTE_BACKEND == "old_gateway" + assert context.sim_entrypoint == DEFAULT_SIM_ENTRYPOINT == "old_gateway_direct" + + +def test_existing_report_route_and_service_import_only_v1_persistence() -> None: + route_source = inspect.getsource(report_output_routes) + service_source = inspect.getsource(ReportOutputService) + + assert "policyengine_api.data.v2" not in route_source + assert "policyengine_api.data.v2" not in service_source + assert "get_v1_session_factory" in service_source + assert "ReportOutput" in service_source + assert "ReportOutputRun" in service_source diff --git a/tests/unit/v2/test_storage_bootstrap.py b/tests/unit/v2/test_storage_bootstrap.py new file mode 100644 index 000000000..857a4a776 --- /dev/null +++ b/tests/unit/v2/test_storage_bootstrap.py @@ -0,0 +1,186 @@ +"""Isolated contract tests for explicit Supabase Storage initialization.""" + +import json +from pathlib import Path + +import httpx +from pydantic import SecretStr +import pytest + +from policyengine_api.constants import REPO +from policyengine_api.data.v2.settings import SupabaseStorageSettings +from policyengine_api.data.v2.storage_bootstrap import ( + StorageBootstrapError, + initialize_supabase_storage, +) + + +PROJECT_REF = "kvrifaviwhzjztcbrfpy" +BUCKET = "policyengine-v2-alpha" +ADMIN_KEY = "test-storage-admin-secret" + + +def _settings(**overrides) -> SupabaseStorageSettings: + values = { + "project_ref": PROJECT_REF, + "environment": "production-foundation", + "api_url": f"https://{PROJECT_REF}.supabase.co", + "bucket": BUCKET, + "admin_key": SecretStr(ADMIN_KEY), + } + values.update(overrides) + return SupabaseStorageSettings(**values) + + +def _bucket(**overrides) -> dict: + values = { + "id": BUCKET, + "name": BUCKET, + "public": False, + "file_size_limit": None, + "allowed_mime_types": None, + } + values.update(overrides) + return values + + +def _client(handler) -> httpx.Client: + return httpx.Client(transport=httpx.MockTransport(handler)) + + +def test_fresh_bootstrap_creates_then_verifies_private_bucket() -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if len(requests) == 1: + return httpx.Response( + 400, + json={"statusCode": "404", "code": "NoSuchBucket"}, + ) + if request.method == "POST": + return httpx.Response(200, json={"name": BUCKET}) + return httpx.Response(200, json=_bucket()) + + with _client(handler) as client: + result = initialize_supabase_storage(_settings(), client=client) + + assert result.created is True + assert result.public is False + assert [request.method for request in requests] == ["GET", "POST", "GET"] + payload = json.loads(requests[1].content) + assert payload == _bucket() + assert "authorization" not in requests[1].headers + assert requests[1].headers["apikey"] == ADMIN_KEY + + +def test_second_identical_bootstrap_is_a_read_only_success() -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_bucket()) + + with _client(handler) as client: + result = initialize_supabase_storage(_settings(), client=client) + + assert result.created is False + assert [request.method for request in requests] == ["GET"] + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("public", True), + ("name", "wrong-bucket"), + ("file_size_limit", 1024), + ("allowed_mime_types", ["image/png"]), + ], +) +def test_incompatible_bucket_fails_without_update_delete_or_recreate( + field: str, + value, +) -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json=_bucket(**{field: value})) + + with _client(handler) as client: + with pytest.raises(StorageBootstrapError, match=field): + initialize_supabase_storage(_settings(), client=client) + + assert [request.method for request in requests] == ["GET"] + + +def test_concurrent_creation_conflict_is_verified_without_overwrite() -> None: + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if len(requests) == 1: + return httpx.Response(404) + if request.method == "POST": + return httpx.Response( + 400, + json={"statusCode": "409", "code": "BucketAlreadyExists"}, + ) + return httpx.Response(200, json=_bucket()) + + with _client(handler) as client: + result = initialize_supabase_storage(_settings(), client=client) + + assert result.created is False + assert [request.method for request in requests] == ["GET", "POST", "GET"] + + +def test_target_mismatch_fails_before_any_storage_request() -> None: + def handler(_request: httpx.Request) -> httpx.Response: + raise AssertionError("target mismatch must make no request") + + with _client(handler) as client: + with pytest.raises(StorageBootstrapError, match="recorded Stage 8"): + initialize_supabase_storage( + _settings(project_ref="aaaaaaaaaaaaaaaaaaaa"), + client=client, + ) + + +def test_failures_and_results_never_expose_storage_credentials() -> None: + def handler(_request: httpx.Request) -> httpx.Response: + return httpx.Response(503, text=ADMIN_KEY) + + with _client(handler) as client: + with pytest.raises(StorageBootstrapError) as raised: + initialize_supabase_storage(_settings(), client=client) + + assert ADMIN_KEY not in str(raised.value) + assert ADMIN_KEY not in repr(_settings()) + + +def test_bootstrap_surface_cannot_mutate_application_schema_or_data() -> None: + implementation = (REPO / "policyengine_api/data/v2/storage_bootstrap.py").read_text( + encoding="utf-8" + ) + command = (REPO / "scripts/bootstrap_v2_supabase_storage.py").read_text( + encoding="utf-8" + ) + prohibited = { + "V2_MIGRATION_DATABASE_URL", + "create_all", + "drop_all", + "alembic", + "sqlalchemy", + "/rest/v1/", + "/storage/v1/object", + } + assert all(value not in implementation + command for value in prohibited) + assert "/storage/v1/bucket" in implementation + assert "policyengine_api.api" not in command + + +def test_bootstrap_script_is_durable_tooling_not_one_off_scaffolding() -> None: + path = Path("scripts/bootstrap_v2_supabase_storage.py") + assert (REPO / path).is_file() + assert "supabase/.temp" not in path.as_posix() diff --git a/tests/unit/v2/test_table_inventory.py b/tests/unit/v2/test_table_inventory.py new file mode 100644 index 000000000..55666ae27 --- /dev/null +++ b/tests/unit/v2/test_table_inventory.py @@ -0,0 +1,43 @@ +"""Tests for the reviewed Stage 8 table allowlist.""" + +import pytest + +from policyengine_api.data.v2.table_inventory import ( + EXPECTED_V2_TABLES, + PROHIBITED_V2_TABLES, + V1_ONLY_TABLES, + V2_TABLE_GROUPS, + V2TableInventoryError, + validate_v2_table_inventory, +) + + +def test_reviewed_table_groups_are_disjoint_and_complete() -> None: + grouped_tables = [ + table_name for _, table_names in V2_TABLE_GROUPS for table_name in table_names + ] + + assert len(grouped_tables) == len(set(grouped_tables)) + assert frozenset(grouped_tables) == EXPECTED_V2_TABLES + assert "reports" in EXPECTED_V2_TABLES + assert "report_runs" in EXPECTED_V2_TABLES + assert EXPECTED_V2_TABLES.isdisjoint(PROHIBITED_V2_TABLES) + assert EXPECTED_V2_TABLES.isdisjoint(V1_ONLY_TABLES) + + +def test_exact_reviewed_inventory_is_accepted() -> None: + validate_v2_table_inventory(EXPECTED_V2_TABLES) + + +@pytest.mark.parametrize( + "table_name", + ["runtime_bundles", "populations", "household", "unreviewed_predecessor"], +) +def test_unreviewed_tables_are_rejected(table_name: str) -> None: + with pytest.raises(V2TableInventoryError, match=table_name): + validate_v2_table_inventory(EXPECTED_V2_TABLES | {table_name}) + + +def test_missing_reviewed_table_is_rejected() -> None: + with pytest.raises(V2TableInventoryError, match="report_runs"): + validate_v2_table_inventory(EXPECTED_V2_TABLES - {"report_runs"}) diff --git a/uv.lock b/uv.lock index 0a548c102..d9d78904b 100644 --- a/uv.lock +++ b/uv.lock @@ -2642,7 +2642,7 @@ models = [ [[package]] name = "policyengine-api" -version = "3.48.0" +version = "3.48.2" source = { editable = "." } dependencies = [ { name = "a2wsgi" }, @@ -2667,12 +2667,14 @@ dependencies = [ { name = "policyengine-canada" }, { name = "policyengine-il" }, { name = "policyengine-ng" }, + { name = "psycopg", extra = ["binary"] }, { name = "pydantic" }, { name = "pymysql" }, { name = "python-dotenv" }, { name = "redis" }, { name = "rq" }, { name = "sqlalchemy" }, + { name = "sqlmodel" }, { name = "streamlit" }, { name = "uvicorn", extra = ["standard"] }, { name = "werkzeug" }, @@ -2715,6 +2717,7 @@ requires-dist = [ { name = "policyengine-canada", specifier = "==0.96.3" }, { name = "policyengine-il", specifier = "==0.1.0" }, { name = "policyengine-ng", specifier = "==0.5.1" }, + { name = "psycopg", extras = ["binary"], specifier = ">=3.3,<4" }, { name = "pydantic" }, { name = "pymysql" }, { name = "pytest", marker = "extra == 'dev'" }, @@ -2725,6 +2728,7 @@ requires-dist = [ { name = "rq" }, { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.9.0" }, { name = "sqlalchemy", specifier = ">=2,<3" }, + { name = "sqlmodel", specifier = ">=0.0.39,<0.1" }, { name = "streamlit" }, { name = "towncrier", marker = "extra == 'dev'", specifier = ">=24.8.0" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.32,<1" }, @@ -2998,6 +3002,75 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/d7/7831438e6c3ebbfa6e01a927127a6cb42ad3ab844247f3c5b96bea25d73d/psutil-6.1.1-cp37-abi3-win_amd64.whl", hash = "sha256:f35cfccb065fff93529d2afb4a2e89e363fe63ca1e4a5da22b603a85833c2649", size = 254444, upload-time = "2024-12-19T18:22:11.335Z" }, ] +[[package]] +name = "psycopg" +version = "3.3.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/2f/cb91e5502ec9de1de6f1b76cfbf69531932725361168bb06963620c77e2e/psycopg-3.3.4.tar.gz", hash = "sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc", size = 165799, upload-time = "2026-05-01T23:31:55.179Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/e0/7b3dee031daae7743609ce3c746565d4a3ed7c2c186479eb48e34e838c64/psycopg-3.3.4-py3-none-any.whl", hash = "sha256:b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a", size = 213001, upload-time = "2026-05-01T23:20:50.816Z" }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.3.4" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/82/df3312c0ca083d5b43b352f27d4dd8b1e614bd334473074715d9e0000da4/psycopg_binary-3.3.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:612a627d733f695b1de1f9b4bd511c15f999a5d8b915d444bbd7dd71cf3370da", size = 4609813, upload-time = "2026-05-01T23:26:30.612Z" }, + { url = "https://files.pythonhosted.org/packages/1f/b5/d74d542458d3e8ac0571d8a88f57ca369999b9a82f4fa528052d0d7d3e4c/psycopg_binary-3.3.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:13a7f380824c35896dcac7fe0f61440f7ca49d6dc73f3c13a9a4471e6a3b302e", size = 4676799, upload-time = "2026-05-01T23:26:38.475Z" }, + { url = "https://files.pythonhosted.org/packages/09/67/06bab9c60671999f4c6ceff1b334f3ac1f9fc5789eb467c714623ea21de9/psycopg_binary-3.3.4-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:276904e3452d6a23d474ef9a21eee19f20eed3d53ddd2576af033827e0ba0992", size = 5497050, upload-time = "2026-05-01T23:26:47.061Z" }, + { url = "https://files.pythonhosted.org/packages/72/9b/023433e2b20f970de1e22d29132a95281277646da0b2e2879dd4ee94b8c1/psycopg_binary-3.3.4-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab8cca8ef8fb1ccf5b048ae5bd78ba55b9e4b5d472e3ce5ca39ff4d2a9c249e4", size = 5172428, upload-time = "2026-05-01T23:26:56.708Z" }, + { url = "https://files.pythonhosted.org/packages/08/cd/ae16da8fde228a38b2fe9269bbc13cf89e0186173f2265600f02d6a71e64/psycopg_binary-3.3.4-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7465bfe6087d2d5b42d4c53b9b11ca9f218e477317a4a162a10e3c19e984ba8e", size = 6762746, upload-time = "2026-05-01T23:27:07.023Z" }, + { url = "https://files.pythonhosted.org/packages/4f/81/0ba09fa5f5f88779093a2541a8e02489825721f258ab88058b11d68b3eb5/psycopg_binary-3.3.4-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:22cdbf5f91ef7bb91fe0c5757e1962d3127a8010256eefd9c61fcaf441802097", size = 5006033, upload-time = "2026-05-01T23:27:12.221Z" }, + { url = "https://files.pythonhosted.org/packages/73/6a/629136040cc3497adb442a305710b5913f2a754d4630fc3d3717c4c0df65/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:e2631da29253a98bd496e6c4813b24e09a4fe3fb2a9e88513305d6f8747cce95", size = 4534175, upload-time = "2026-05-01T23:27:18.248Z" }, + { url = "https://files.pythonhosted.org/packages/7c/32/1027f843c6dc2d5d51960ee62cc0c2cf755a4c39455aff1371173edbef7d/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7f7668f30b9dd5163197e5cbf4e0efd54e00f0a859cc566ce56cfc31f4054839", size = 4224203, upload-time = "2026-05-01T23:27:24.3Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e1/380a724d9093c74adb14d4fce920ea8327838abb61f760b1448586b14a8e/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:cffc3408d77a27973f33e5d909b624cce683db5fc25964b02fe0aae7886c1007", size = 3954509, upload-time = "2026-05-01T23:27:30.815Z" }, + { url = "https://files.pythonhosted.org/packages/db/cd/895893ae575a09c97ccfd5def070d88993d955ef34df45a881fd5ff506d6/psycopg_binary-3.3.4-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0579252a1202cd73e4da137a1426e2dae993ae44e757605344282af3a082848c", size = 4259551, upload-time = "2026-05-01T23:27:38.828Z" }, + { url = "https://files.pythonhosted.org/packages/dd/c6/2330a20794e37a3ec609ef2fd8522919ec7a4395a1abf979a8e2d1775cd5/psycopg_binary-3.3.4-cp311-cp311-win_amd64.whl", hash = "sha256:41f2ec0fea529832982bcb6c9415de3c86264ebe562b77a467c0fbcd7efbba8d", size = 3572054, upload-time = "2026-05-01T23:27:45.455Z" }, + { url = "https://files.pythonhosted.org/packages/95/7d/03818e13ba7f36de93573c93ee3482006d3dfa8b0f8d28df511bad0a1a92/psycopg_binary-3.3.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5ab28a2a7649df3b72e6b674b4c190e448e8e77cf496a65bd846472048de2089", size = 4591122, upload-time = "2026-05-01T23:27:56.162Z" }, + { url = "https://files.pythonhosted.org/packages/a5/b9/11b341edf8d54e2694726b273fe9652b254d989f4f63e3ac6816ad6b55f4/psycopg_binary-3.3.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6402a9d8146cf4b3974ded3fd28a971e83dc6a0333eb7822524a3aa20b546578", size = 4669943, upload-time = "2026-05-01T23:28:04.522Z" }, + { url = "https://files.pythonhosted.org/packages/8b/18/4665bacd65e7865b4372fcd8abb8b9186ada4b0025f8c2ca691b364a556c/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:580ae30a5f95ccd90008ec697d3ed6a4a2047a516407ad904283fa42086936e9", size = 5469697, upload-time = "2026-05-01T23:28:11.337Z" }, + { url = "https://files.pythonhosted.org/packages/7c/b1/b83136c6e510593d9b0c759ba5384337bc4ad82d19fda675adc4b2703c84/psycopg_binary-3.3.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e7510c37550f91a187e3660a8cc50d4b760f8c3b8b2f89ebc5698cd2c7f2c85d", size = 5152995, upload-time = "2026-05-01T23:28:20.529Z" }, + { url = "https://files.pythonhosted.org/packages/67/8d/a9821e2a648afe6091989929982a3b0f00b2631a859cb81379728f08fb75/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:77df19583501ea288eaf15ac0fe7ad01e6d8091a91d5c41df5c718f307d8e31b", size = 6738180, upload-time = "2026-05-01T23:28:30.654Z" }, + { url = "https://files.pythonhosted.org/packages/7e/58/2e349e8d23905dc2317b80ac65f48fb6f821a4777a4e994a60da91c4850f/psycopg_binary-3.3.4-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:018fbed325936da502feb546642c982dcc4b9ffdea32dfef78dbf3b7f7ad4070", size = 4978828, upload-time = "2026-05-01T23:28:37.277Z" }, + { url = "https://files.pythonhosted.org/packages/45/48/57b00d03b4721878326122a1f1e6b0a90b85bcaec56b5b2f8ea6cfa45235/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:17a21953a9e5ff3a16dab692625a3676e2f101db5e40072f39dbee2250194d68", size = 4509757, upload-time = "2026-05-01T23:28:43.078Z" }, + { url = "https://files.pythonhosted.org/packages/25/37/33b47d8c007df69aec500df5889767c4d313748e8e9e27a2fef8a6dabcee/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:eb05ee1c2b817d27c537333224c9e83c7afb86fe7296ba970990068baf819b16", size = 4190546, upload-time = "2026-05-01T23:28:50.016Z" }, + { url = "https://files.pythonhosted.org/packages/ca/c6/32b0835dbc2122617902b649d76a91c1e75406e76bf3d595b0c3bb5ffad6/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:773d573e11f437ce0bdb95b7c18dc58390494f96d43f8b45b9760436114f7652", size = 3926197, upload-time = "2026-05-01T23:28:55.55Z" }, + { url = "https://files.pythonhosted.org/packages/cd/68/d190ef0c0c5b16ded07831dabc8ddd412f4cdab07ec6e30ed38d9bda0e1f/psycopg_binary-3.3.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:71e55ccbdfae79a2ed9c6369c3008a3025817ff9d7e27b32a2d84e2a4267e66e", size = 4236627, upload-time = "2026-05-01T23:29:05.336Z" }, + { url = "https://files.pythonhosted.org/packages/25/8f/81dcbc2e8454b74d14881275ea45f00791052dac531a9fa8be1730d1685b/psycopg_binary-3.3.4-cp312-cp312-win_amd64.whl", hash = "sha256:494ca54901be8cf9eb7e02c25b731f2317c378efa44f43e8f9bd0e1184ae7be4", size = 3560782, upload-time = "2026-05-01T23:29:11.967Z" }, + { url = "https://files.pythonhosted.org/packages/09/43/13e9c406fbbf354580476e248a16b64802a376873ebe6339e30bb655572d/psycopg_binary-3.3.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7", size = 4590377, upload-time = "2026-05-01T23:29:18.782Z" }, + { url = "https://files.pythonhosted.org/packages/22/be/2923cd7c3683e7afdecf4f10796a18de02f5c5ddc0969aa2ad0a8cdd3bbd/psycopg_binary-3.3.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:75a9067e236f9b9ae3535b66fe99bddb33d39c0de10112e49b9ab11eee53dc31", size = 4669023, upload-time = "2026-05-01T23:29:25.884Z" }, + { url = "https://files.pythonhosted.org/packages/96/a0/2c913d6fe13d6a8bd13597d36739bf47af063ad9399e402cfecab16f3c1e/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b56b603ebcea8aa10b46228b8410ba7f13e7c2ee54389d4d9be0927fd8ce2a70", size = 5467423, upload-time = "2026-05-01T23:29:33.416Z" }, + { url = "https://files.pythonhosted.org/packages/e7/38/205d10bc1ad0df4a21c5c51659126bd3ea0ef98fcad1e852f78c249bb9c3/psycopg_binary-3.3.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c677c4ad433cb7150c8cd304a0769ae3bcfbe5ea0676eb53faa7b1443b16d0d3", size = 5151137, upload-time = "2026-05-01T23:29:42.013Z" }, + { url = "https://files.pythonhosted.org/packages/36/fc/f0381ddcd45eff3bb70dbca6823a996048d7f507b2ec3fc92c6fabc0fe87/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:26df2717e59c0473e4465a97dfb1b7afebaa479277870fd5784d1436470db47c", size = 6736671, upload-time = "2026-05-01T23:29:51.626Z" }, + { url = "https://files.pythonhosted.org/packages/95/40/fa545ae152c24327651e5624e4902121e808270be36c10b12e9939be09bc/psycopg_binary-3.3.4-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:1dc1f79fd16bb1f3f4421417a514607539f17804d95c7ed617265369d1981cae", size = 4979601, upload-time = "2026-05-01T23:29:56.961Z" }, + { url = "https://files.pythonhosted.org/packages/86/e4/2f8a47ee97f90cd2b933d0463081d35631ff419de2b8c984a5f369857de0/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:136f199a407b5348b9b857c504aff60c77622a28482e7195839ce1b51238c4cc", size = 4510513, upload-time = "2026-05-01T23:30:07.243Z" }, + { url = "https://files.pythonhosted.org/packages/0e/0e/94e842ff4a7f98ed162580ca2e8b8864b28c1e0350f2443f8ee47f821167/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:b6f5a29e9c775b9f12a1a717aa7a2c80f9e1db6f27ba44a5b59c80ac61d2ffcf", size = 4187243, upload-time = "2026-05-01T23:30:15.352Z" }, + { url = "https://files.pythonhosted.org/packages/d0/83/fc6c174b672e29b7de996ea77b6cbddf46c891751c3355f6974292baa6b4/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:ee17a2cf4943cde261adfad1bbc5bf38d6b3776d7afff74c7cabcbeaeb08c260", size = 3927347, upload-time = "2026-05-01T23:30:21.186Z" }, + { url = "https://files.pythonhosted.org/packages/e9/65/768364d4a97a15b1a7f47ba52688c1686f22941d8332a8398cefc468e25f/psycopg_binary-3.3.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5c4ab71be17bdca30cb34c34c4e1496e2f5d6f20c199c12bad226070b22ef9bf", size = 4236393, upload-time = "2026-05-01T23:30:26.211Z" }, + { url = "https://files.pythonhosted.org/packages/bd/3b/218efbc9e645becd80cdf651acda05f85cfe546b7a9c0458c7cbc8fe1f74/psycopg_binary-3.3.4-cp313-cp313-win_amd64.whl", hash = "sha256:dbfdb9b6cc79f31104a7b162a2b921b765fcc62af6c00540a167a8de47e4ed38", size = 3564592, upload-time = "2026-05-01T23:30:31.764Z" }, + { url = "https://files.pythonhosted.org/packages/48/a6/828c9185701dab71b234c2a76c38a08b098ebfec5020716b4e93807492b5/psycopg_binary-3.3.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:28b7398fdd19db3232c884fb24550bdfe951221f510e195e233299e4c9b78f97", size = 4607292, upload-time = "2026-05-01T23:30:38.962Z" }, + { url = "https://files.pythonhosted.org/packages/92/58/5b40dbc9d839045c9dae956960e4fb6d20bcabe6c59a2aa34fc3a371913f/psycopg_binary-3.3.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1fbaa292a3c8bb61b45df1ad3da1908ccee7cb889db9425e3557d9e34e2a4829", size = 4687023, upload-time = "2026-05-01T23:30:47.227Z" }, + { url = "https://files.pythonhosted.org/packages/85/a9/793f0ac107a9003b48441d0d1f9f616d96e0f37458dd8dc12528ceff55fb/psycopg_binary-3.3.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:94596f9e7633ee3f6440711d43bb70aa31cc0a46a900ab8b4201a366ace5c9e7", size = 5486985, upload-time = "2026-05-01T23:30:55.517Z" }, + { url = "https://files.pythonhosted.org/packages/8f/26/42e8533497e2592334f68ec529cf5f840f7fa4e99575a4bb61aa184dbfbf/psycopg_binary-3.3.4-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8c0056529e68dbe9184cd4019a1f3d8f3a4ead2f6fc7a5afcf27d3314edd1277", size = 5168745, upload-time = "2026-05-01T23:31:01.904Z" }, + { url = "https://files.pythonhosted.org/packages/15/af/b7151776cc08d5935d45c833ec818a9beb417cf7c08239af1aafbdae78ee/psycopg_binary-3.3.4-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c09aad7051326e7603c14e50636db9c01f78272dc54b3accff03d46370461e6", size = 6761486, upload-time = "2026-05-01T23:31:14.511Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ed/c92533b9124712d592cbf1cd6c76da933a2e0acea81dfe1fbe7e735f0cff/psycopg_binary-3.3.4-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:514404ed543efd620c85602b747df2a23cf1241b4067199e1a66f2d2757aaa41", size = 4997427, upload-time = "2026-05-01T23:31:20.901Z" }, + { url = "https://files.pythonhosted.org/packages/a2/23/ccadfd0de416aa188356daa199453af24087b042e296088706d190ae0295/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:46893c26858be12cc49ca4226ed6a60b4bfccadd946b3bebb783a60b38788228", size = 4533549, upload-time = "2026-05-01T23:31:26.204Z" }, + { url = "https://files.pythonhosted.org/packages/fd/a0/c8f43cee36386f7bc891ab41a9d31ea07cf9826038e732da79f26b1e5f34/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:df1d567fc430f6df15c9fcf67d87685fc49bdb325adc0db5af1adfb2f44eb5c9", size = 4210256, upload-time = "2026-05-01T23:31:33.884Z" }, + { url = "https://files.pythonhosted.org/packages/4e/2c/c1547871be3790676e8868b38655496422f94f0978dfb66b74bdba2f1676/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:6b9016b1714da4dd5ecaaa75b82098aa5a0b87854ce9b092e21c27c4ae23e014", size = 3946204, upload-time = "2026-05-01T23:31:39.626Z" }, + { url = "https://files.pythonhosted.org/packages/c4/b1/f6670f00fa7ea601584623f6c11602ab92117d83eaff885e0210f6de7418/psycopg_binary-3.3.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:47c656a8a7ba6eb0cff1801a4caaa9c8bdc12d03080e273aff1c8ac39971a77e", size = 4255811, upload-time = "2026-05-01T23:31:44.986Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" }, +] + [[package]] name = "ptyprocess" version = "0.7.0" @@ -3760,6 +3833,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e5/30/8519fdde58a7bdf155b714359791ad1dc018b47d60269d5d160d311fdc36/sqlalchemy-2.0.49-py3-none-any.whl", hash = "sha256:ec44cfa7ef1a728e88ad41674de50f6db8cfdb3e2af84af86e0041aaf02d43d0", size = 1942158, upload-time = "2026-04-03T16:53:44.135Z" }, ] +[[package]] +name = "sqlmodel" +version = "0.0.39" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/68/ee/22a0559283c3cf6048678e787ed5d4959dcd00dedd8ba4567eeae684eeb1/sqlmodel-0.0.39.tar.gz", hash = "sha256:23d8e50a8d8ee936032ed79c55023a5d618dd6bc3c510bbf4909d1a7a605a570", size = 91057, upload-time = "2026-06-25T13:01:38.475Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/7d/b9813a582d4eb310be35e1fc7dfaae71207d7b62e9e53be314ebd251b53b/sqlmodel-0.0.39-py3-none-any.whl", hash = "sha256:90ebe92ce5cc11d7fff8dc7cb594790a102333c8fe7c14865254f6fc5c939795", size = 29680, upload-time = "2026-06-25T13:01:37.494Z" }, +] + [[package]] name = "stack-data" version = "0.6.3" From 06cb55527589fabc60ce17f26ac55fa9dbc2b4f8 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:39:14 +0300 Subject: [PATCH 02/18] Harden Stage 8 review follow-ups --- docs/engineering/skills/alembic-migrations.md | 13 ++ ...use_native_uuid_report_run_idempotency_.py | 59 +++++++ policyengine_api/data/v2/models/reports.py | 6 +- policyengine_api/data/v2/report_runs.py | 11 +- .../routes/reform_impact_routes.py | 46 ++++-- policyengine_api/runtime_cache/settings.py | 8 +- policyengine_api/services/economy_service.py | 58 +++++-- .../services/reform_impacts_service.py | 9 +- tests/fixtures/services/economy_service.py | 2 +- .../integration/test_alembic_v2_lifecycle.py | 98 +++++++++++- .../unit/routes/test_reform_impact_routes.py | 149 ++++++++++++------ tests/unit/runtime_cache/test_settings.py | 22 +++ tests/unit/services/test_economy_service.py | 65 +++++++- .../services/test_reform_impacts_service.py | 23 ++- tests/unit/v2/test_alembic_v2.py | 18 ++- tests/unit/v2/test_models.py | 3 +- tests/unit/v2/test_report_runs.py | 57 +++---- 17 files changed, 502 insertions(+), 145 deletions(-) create mode 100644 migrations/v2/versions/4faee127fa16_use_native_uuid_report_run_idempotency_.py diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 2f8ecac8b..2e7d806e8 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -203,3 +203,16 @@ last dependent table is dropped. Its only post-generation correction drops the nine generated `v2_*` enum types at the end of the baseline downgrade. The v2 Postgres lifecycle test covers empty upgrade, downgrade to base, and re-upgrade so stale enum types cannot make the generated baseline non-reversible. + +### API v2 report-run idempotency UUID conversion + +Revision `4faee127fa16` was autogenerated after changing the SQLModel +`report_runs.idempotency_key` field from bounded text to native UUID. Alembic +detected both the type change and removal of the text-only nonblank check, but +PostgreSQL cannot infer the required casts and cannot retain the text-only +check while changing the column type. The bounded post-generation correction +drops that check before the upgrade conversion, restores it after the +downgrade conversion, and supplies `postgresql_using` casts in both +directions. The v2 PostgreSQL lifecycle test covers the one-revision downgrade +and upgrade and verifies the live column type and check constraint at both +states. diff --git a/migrations/v2/versions/4faee127fa16_use_native_uuid_report_run_idempotency_.py b/migrations/v2/versions/4faee127fa16_use_native_uuid_report_run_idempotency_.py new file mode 100644 index 000000000..a096c8ee2 --- /dev/null +++ b/migrations/v2/versions/4faee127fa16_use_native_uuid_report_run_idempotency_.py @@ -0,0 +1,59 @@ +"""use native UUID report run idempotency keys + +Revision ID: 4faee127fa16 +Revises: 5f048586d8f1 +Create Date: 2026-08-18 16:20:46.343261 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = "4faee127fa16" +down_revision: Union[str, None] = "5f048586d8f1" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + # Post-generation correction: the text-only check must be removed before + # the type change, and PostgreSQL requires an explicit text-to-UUID cast. + op.drop_constraint( + op.f("ck_report_runs_idempotency_key_nonblank"), + "report_runs", + type_="check", + ) + op.alter_column( + "report_runs", + "idempotency_key", + existing_type=sa.VARCHAR(length=255), + type_=sa.Uuid(), + existing_nullable=True, + postgresql_using="idempotency_key::uuid", + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + # Post-generation correction: cast UUIDs back to text before restoring the + # text-only check constraint. + op.alter_column( + "report_runs", + "idempotency_key", + existing_type=sa.Uuid(), + type_=sa.VARCHAR(length=255), + existing_nullable=True, + postgresql_using="idempotency_key::text", + ) + op.create_check_constraint( + op.f("ck_report_runs_idempotency_key_nonblank"), + "report_runs", + "idempotency_key IS NULL OR length(TRIM(BOTH FROM idempotency_key)) > 0", + ) + # ### end Alembic commands ### diff --git a/policyengine_api/data/v2/models/reports.py b/policyengine_api/data/v2/models/reports.py index 568880e19..327132e98 100644 --- a/policyengine_api/data/v2/models/reports.py +++ b/policyengine_api/data/v2/models/reports.py @@ -155,10 +155,6 @@ class ReportRun(TimestampedModel, table=True): "status NOT IN ('succeeded', 'failed') OR completed_at IS NOT NULL", name="ck_report_runs_terminal_completion", ), - sa.CheckConstraint( - "idempotency_key IS NULL OR length(trim(idempotency_key)) > 0", - name="ck_report_runs_idempotency_key_nonblank", - ), sa.Index( "ix_report_runs_current_output", "report_id", @@ -184,7 +180,7 @@ class ReportRun(TimestampedModel, table=True): default=ReportRunTrigger.INITIAL, sa_type=enum_type(ReportRunTrigger, "v2_report_run_trigger"), ) - idempotency_key: str | None = Field(default=None, max_length=255) + idempotency_key: UUID | None = Field(default=None) started_at: datetime | None = Field( default=None, sa_type=sa.DateTime(timezone=True), diff --git a/policyengine_api/data/v2/report_runs.py b/policyengine_api/data/v2/report_runs.py index 7390c3b9a..b18851b5b 100644 --- a/policyengine_api/data/v2/report_runs.py +++ b/policyengine_api/data/v2/report_runs.py @@ -78,13 +78,12 @@ def create_report_run( country_package_version: str, policyengine_version: str, trigger: ReportRunTrigger, - idempotency_key: str | None = None, + idempotency_key: UUID | None = None, ) -> ReportRun: """Create one run, or return the run for a retried idempotent request.""" _locked_report(session, report_id) - normalized_key = idempotency_key.strip() if idempotency_key else None - if trigger is ReportRunTrigger.MANUAL and not normalized_key: + if trigger is ReportRunTrigger.MANUAL and idempotency_key is None: raise ValueError("manual report reruns require an idempotency key") if not country_package_version or not policyengine_version: raise ValueError("report run package versions must be non-empty") @@ -94,9 +93,9 @@ def create_report_run( country_package_version=country_package_version, policyengine_version=policyengine_version, trigger=trigger, - idempotency_key=normalized_key, + idempotency_key=idempotency_key, ) - if normalized_key is None: + if idempotency_key is None: session.add(run) session.flush() return run @@ -112,7 +111,7 @@ def create_report_run( existing = session.exec( select(ReportRun).where( ReportRun.report_id == report_id, - ReportRun.idempotency_key == normalized_key, + ReportRun.idempotency_key == idempotency_key, ) ).one_or_none() if existing is None: diff --git a/policyengine_api/routes/reform_impact_routes.py b/policyengine_api/routes/reform_impact_routes.py index e49076304..92c2a2d58 100644 --- a/policyengine_api/routes/reform_impact_routes.py +++ b/policyengine_api/routes/reform_impact_routes.py @@ -1,9 +1,10 @@ from datetime import datetime import json +from typing import Any from flask import Blueprint, Response, request -from policyengine_api.data.v1_models import ReformImpact +from policyengine_api.runtime_cache.repositories import CachedReformImpact from policyengine_api.services.reform_impacts_service import ReformImpactsService @@ -14,22 +15,35 @@ _DEFAULT_SIMULATION_RESULTS = 100 -def _serialize_v1_reform_impact(impact: ReformImpact) -> dict: - """Project canonical ORM values onto the historical v1 response shape.""" +def _serialize_v1_json(value: dict[str, Any] | None) -> str | None: + return None if value is None else json.dumps(value) - result = { - column.name: getattr(impact, column.name) - for column in ReformImpact.__table__.columns + +def _serialize_v1_datetime(value: datetime | None) -> str | None: + return None if value is None else str(value) + + +def _serialize_v1_reform_impact(impact: CachedReformImpact) -> dict[str, object]: + """Project a cached impact onto the explicit historical v1 response shape.""" + + return { + "reform_impact_id": impact.reform_impact_id, + "baseline_policy_id": impact.baseline_policy_id, + "reform_policy_id": impact.reform_policy_id, + "country_id": impact.country_id, + "region": impact.region, + "dataset": impact.dataset, + "time_period": impact.time_period, + "options_json": _serialize_v1_json(impact.options_json), + "options_hash": impact.options_hash, + "api_version": impact.api_version, + "reform_impact_json": _serialize_v1_json(impact.reform_impact_json), + "status": impact.status, + "message": impact.message, + "start_time": _serialize_v1_datetime(impact.start_time), + "end_time": _serialize_v1_datetime(impact.end_time), + "execution_id": impact.execution_id, } - for field in ("options_json", "reform_impact_json"): - value = result[field] - if value is not None and not isinstance(value, str): - result[field] = json.dumps(value) - for field in ("start_time", "end_time"): - value = result[field] - if isinstance(value, datetime): - result[field] = str(value) - return result def _parse_result_limit(value: str | None) -> int: @@ -44,7 +58,7 @@ def _parse_result_limit(value: str | None) -> int: @reform_impact_bp.route("/simulations", methods=["GET"]) def get_simulations() -> Response: - """Return recent reform impacts, bounded to protect the database query.""" + """Return recent reform impacts through a bounded cache-index lookup.""" result_limit = _parse_result_limit(request.args.get("max_results")) impacts = reform_impacts_service.get_recent_reform_impacts(result_limit) diff --git a/policyengine_api/runtime_cache/settings.py b/policyengine_api/runtime_cache/settings.py index 0cfb68c23..24d3bf25a 100644 --- a/policyengine_api/runtime_cache/settings.py +++ b/policyengine_api/runtime_cache/settings.py @@ -180,11 +180,17 @@ def load_runtime_cache_settings( values = os.environ if environ is None else environ raw_mode = values.get(RUNTIME_CACHE_MODE, "").strip() - mode = raw_mode or ("deployed" if _is_deployed(values) else "disabled") + deployed_runtime = _is_deployed(values) + mode = raw_mode or ("deployed" if deployed_runtime else "disabled") if mode not in CACHE_MODES: raise RuntimeCacheConfigurationError( f"{RUNTIME_CACHE_MODE} must be disabled, local, or deployed" ) + if deployed_runtime and mode != "deployed": + raise RuntimeCacheConfigurationError( + f"{RUNTIME_CACHE_MODE} must be deployed when K_SERVICE or GAE_ENV " + "identifies a deployed runtime" + ) if mode == "disabled": if values.get(RUNTIME_CACHE_URL) or values.get( RUNTIME_CACHE_URL_SECRET_RESOURCE diff --git a/policyengine_api/services/economy_service.py b/policyengine_api/services/economy_service.py index d1a6c6bcd..371bce166 100644 --- a/policyengine_api/services/economy_service.py +++ b/policyengine_api/services/economy_service.py @@ -29,6 +29,7 @@ from policyengine_api.services.budget_window_cache import BudgetWindowCache from policyengine_api.services.policy_service import PolicyService from policyengine_api.services.reform_impacts_service import ( + ReformImpactHandoffError, ReformImpactsService, ) from policyengine_api.utils import budget_window as budget_window_utils @@ -566,8 +567,11 @@ def _get_budget_window_result_from_batch_job_id( queued_years=batch_execution.queued_years or queued_years_on_submit, cache_status=cache_status, ) - self._budget_window_cache.set_completed_result(cache_key, result) - self._budget_window_cache.clear_batch_job_id(cache_key) + result_stored = self._budget_window_cache.set_completed_result( + cache_key, result + ) + if result_stored: + self._budget_window_cache.clear_batch_job_id(cache_key) return BudgetWindowEconomicImpactResult.completed( result, cache_status=cache_status, @@ -750,11 +754,19 @@ def _get_or_create_economic_impact( severity="INFO", ) try: - return self._handle_create_impact( + result = self._handle_create_impact( setup_options=setup_options, ) - finally: + except ReformImpactHandoffError: + # The upstream job exists, but its polling pointer was not + # durably handed off. Retain the claim until expiry so another + # request cannot immediately submit a duplicate job. + raise + except Exception: self._release_reform_impact_start(setup_options) + raise + self._release_reform_impact_start(setup_options) + return result raise ValueError(f"Unexpected impact action: {impact_action}") @@ -1063,22 +1075,34 @@ def _handle_create_impact( sim_params["time_period"] = str(sim_params["time_period"]) entrypoint_execution = self._simulation_gateway.run(sim_params) - execution_id = self._simulation_gateway.get_execution_id(entrypoint_execution) + try: + execution_id = self._simulation_gateway.get_execution_id( + entrypoint_execution + ) - run_id = getattr(entrypoint_execution, "run_id", None) or telemetry["run_id"] + run_id = ( + getattr(entrypoint_execution, "run_id", None) or telemetry["run_id"] + ) - progress_log = { - **setup_options.model_dump(), - "message": "Sim API job started", - "execution_id": execution_id, - "run_id": run_id, - } - logger.log_struct(progress_log, severity="INFO") + progress_log = { + **setup_options.model_dump(), + "message": "Sim API job started", + "execution_id": execution_id, + "run_id": run_id, + } + logger.log_struct(progress_log, severity="INFO") - self._set_reform_impact_computing( - setup_options=setup_options, - execution_id=execution_id, - ) + self._set_reform_impact_computing( + setup_options=setup_options, + execution_id=execution_id, + ) + except ReformImpactHandoffError: + raise + except Exception as error: + raise ReformImpactHandoffError( + "simulation was submitted but its execution state could not be " + "handed off" + ) from error return EconomicImpactResult.computing() diff --git a/policyengine_api/services/reform_impacts_service.py b/policyengine_api/services/reform_impacts_service.py index 790c830c5..9e4c5c6a8 100644 --- a/policyengine_api/services/reform_impacts_service.py +++ b/policyengine_api/services/reform_impacts_service.py @@ -12,6 +12,10 @@ ) +class ReformImpactHandoffError(CacheCoordinationError): + """Raised when an accepted simulation cannot be recorded for polling.""" + + class ReformImpactsService: """Preserve historical lookup contracts over an expiring cache repository.""" @@ -168,7 +172,10 @@ def set_reform_impact( end_time=None, execution_id=execution_id, ) - self._cache.set(impact) + if not self._cache.set(impact): + raise ReformImpactHandoffError( + "submitted reform-impact execution could not be stored" + ) return impact def delete_reform_impact( diff --git a/tests/fixtures/services/economy_service.py b/tests/fixtures/services/economy_service.py index 93f60ca90..8b22ef534 100644 --- a/tests/fixtures/services/economy_service.py +++ b/tests/fixtures/services/economy_service.py @@ -168,7 +168,7 @@ def mock_budget_window_cache(): mock_cache.claim_batch_start.return_value = True mock_cache.store_batch_job_id.return_value = None mock_cache.clear_starting_claim.return_value = None - mock_cache.set_completed_result.return_value = None + mock_cache.set_completed_result.return_value = True mock_cache.clear_batch_job_id.return_value = None with patch( diff --git a/tests/integration/test_alembic_v2_lifecycle.py b/tests/integration/test_alembic_v2_lifecycle.py index b424cb5cc..0eaff8dbb 100644 --- a/tests/integration/test_alembic_v2_lifecycle.py +++ b/tests/integration/test_alembic_v2_lifecycle.py @@ -1,13 +1,16 @@ """Exercise the isolated v2 Alembic lifecycle against disposable Postgres.""" import os +from uuid import UUID, uuid4 from alembic import command from alembic.autogenerate import compare_metadata from alembic.config import Config from alembic.migration import MigrationContext import pytest +import sqlalchemy as sa from sqlalchemy import create_engine, inspect, text +from sqlalchemy.dialects.postgresql import UUID as PostgresUUID from policyengine_api.constants import REPO from policyengine_api.data.v2.migration_target import ( @@ -21,8 +24,8 @@ BASELINE_REVISION = "47592781336f" -PREVIOUS_HEAD_REVISION = "b4c69674dd47" -HEAD_REVISION = "5f048586d8f1" +PREVIOUS_HEAD_REVISION = "5f048586d8f1" +HEAD_REVISION = "4faee127fa16" def _disposable_url() -> str: @@ -91,7 +94,14 @@ def test_empty_upgrade_check_boundary_downgrade_and_reupgrade() -> None: assert boundary_kinds.count("v2_reference_row_change") == 2 assert boundary_kinds.count("add_fk") == 4 assert boundary_kinds.count("add_column") == 1 - assert boundary_kinds.count("add_constraint") == 2 + assert boundary_kinds.count("add_constraint") == 1 + assert ( + sum( + isinstance(kind, tuple) and kind[0] == "modify_type" + for kind in boundary_kinds + ) + == 1 + ) assert len(boundary_kinds) == 9 model_count = connection.execute( text( @@ -133,3 +143,85 @@ def test_upgrade_to_head_validates_the_resulting_table_inventory() -> None: connection.execute(text("DROP TABLE IF EXISTS unreviewed_runtime_table")) command.upgrade(config, "head") engine.dispose() + + +def test_report_run_idempotency_uuid_revision_downgrades_and_reupgrades() -> None: + database_url = _disposable_url() + config = _config() + engine = create_engine(database_url) + + def idempotency_column_type(): + return next( + column["type"] + for column in inspect(engine).get_columns("report_runs") + if column["name"] == "idempotency_key" + ) + + def report_run_checks() -> set[str]: + return { + constraint["name"] + for constraint in inspect(engine).get_check_constraints("report_runs") + } + + try: + command.upgrade(config, "head") + assert isinstance(idempotency_column_type(), PostgresUUID) + assert "ck_report_runs_idempotency_key_nonblank" not in report_run_checks() + + command.downgrade(config, PREVIOUS_HEAD_REVISION) + assert isinstance(idempotency_column_type(), sa.String) + assert "ck_report_runs_idempotency_key_nonblank" in report_run_checks() + + model_id = uuid4() + report_id = uuid4() + report_run_id = uuid4() + request_key = uuid4() + with engine.begin() as connection: + connection.execute( + text("INSERT INTO tax_benefit_models (id, name) VALUES (:id, :name)"), + {"id": model_id, "name": f"uuid-cast-{model_id.hex[:8]}"}, + ) + connection.execute( + text( + "INSERT INTO reports " + "(id, label, country, tax_benefit_model_id, inputs) " + "VALUES (:id, 'UUID cast report', 'us', :model_id, '{}')" + ), + {"id": report_id, "model_id": model_id}, + ) + connection.execute( + text( + "INSERT INTO report_runs " + "(id, report_id, country_package_version, " + "policyengine_version, status, trigger, idempotency_key) " + "VALUES (:id, :report_id, '1.0', '1.0', 'pending', " + "'manual', :request_key)" + ), + { + "id": report_run_id, + "report_id": report_id, + "request_key": str(request_key), + }, + ) + + command.upgrade(config, "head") + assert isinstance(idempotency_column_type(), PostgresUUID) + assert "ck_report_runs_idempotency_key_nonblank" not in report_run_checks() + with engine.connect() as connection: + stored_key = connection.execute( + text("SELECT idempotency_key FROM report_runs WHERE id = :run_id"), + {"run_id": report_run_id}, + ).scalar_one() + assert stored_key == request_key + assert isinstance(stored_key, UUID) + + command.downgrade(config, PREVIOUS_HEAD_REVISION) + with engine.connect() as connection: + stored_key = connection.execute( + text("SELECT idempotency_key FROM report_runs WHERE id = :run_id"), + {"run_id": report_run_id}, + ).scalar_one() + assert stored_key == str(request_key) + finally: + command.upgrade(config, "head") + engine.dispose() diff --git a/tests/unit/routes/test_reform_impact_routes.py b/tests/unit/routes/test_reform_impact_routes.py index 4fe072ee4..bde5109c7 100644 --- a/tests/unit/routes/test_reform_impact_routes.py +++ b/tests/unit/routes/test_reform_impact_routes.py @@ -1,10 +1,8 @@ -"""Regression tests for issue #3451. +"""Regression tests for the bounded recent-simulations route. -get_simulations built its LIMIT via an f-string -(`f"DESC LIMIT {max_results}"`), which is a SQL injection vector -(max_results flows in from a caller) and had no cap, so a tall -integer could drop unbounded rows on a production MySQL. The fix: -always LIMIT, clamp to [1, 1000], and bind as a parameter. +The route originally built a SQL LIMIT through an f-string. Reform impacts now +use the shared runtime cache, but the public limit, input-safety, ordering, and +v1 response-shape contracts remain unchanged. """ from datetime import datetime @@ -12,11 +10,34 @@ from fastapi.testclient import TestClient from flask import Flask -from sqlalchemy import func, select +import pytest from policyengine_api.asgi_factory import create_asgi_app -from policyengine_api.data.v1_models import ReformImpact +from policyengine_api.routes import reform_impact_routes from policyengine_api.routes.reform_impact_routes import reform_impact_bp +from policyengine_api.runtime_cache.core import CacheNamespace +from policyengine_api.runtime_cache.fake import InMemoryCacheBackend +from policyengine_api.runtime_cache.repositories import ( + ReformImpactCache, + reform_impact_id, +) +from policyengine_api.services.reform_impacts_service import ReformImpactsService + + +COMPLETED_AT = datetime(2026, 1, 1, 1) + + +@pytest.fixture +def reform_impacts_service(monkeypatch: pytest.MonkeyPatch) -> ReformImpactsService: + service = ReformImpactsService( + ReformImpactCache( + InMemoryCacheBackend(), + CacheNamespace("test", "api"), + ) + ) + monkeypatch.setattr(service, "_now", lambda: COMPLETED_AT) + monkeypatch.setattr(reform_impact_routes, "reform_impacts_service", service) + return service def _get_simulations(max_results=100): @@ -32,80 +53,104 @@ def _create_app() -> Flask: return app -def _seed_reform_impacts(orm_session, n: int) -> None: +def _seed_reform_impacts(service: ReformImpactsService, n: int) -> None: for i in range(n): - orm_session.add( - ReformImpact( - baseline_policy_id=i + 1, - reform_policy_id=i + 2, - country_id="us", - region="us", - dataset="custom_dataset", - time_period="2025", - options_json={}, - options_hash=f"hash-{i}", - api_version="1.0.0", - reform_impact_json={}, - status="complete", - start_time=datetime(2026, 1, 1, 0, i // 60, i % 60), - end_time=datetime(2026, 1, 1, 1, i // 60, i % 60), - execution_id=f"exec-{i}", - ) + execution_id = f"exec-{i}" + service.set_reform_impact( + baseline_policy_id=i + 1, + policy_id=i + 2, + country_id="us", + region="us", + dataset="custom_dataset", + time_period="2025", + options={}, + options_hash=f"hash-{i}", + api_version="1.0.0", + reform_impact_json={}, + status="computing", + start_time=datetime(2026, 1, 1, 0, i // 60, i % 60), + execution_id=execution_id, + ) + service.set_complete_reform_impact( + country_id="us", + reform_policy_id=i + 2, + baseline_policy_id=i + 1, + region="us", + dataset="custom_dataset", + time_period="2025", + options_hash=f"hash-{i}", + reform_impact_json={}, + execution_id=execution_id, ) - orm_session.commit() -def test_get_simulations_default_limit_caps_at_100(orm_session): - _seed_reform_impacts(orm_session, 150) +def test_get_simulations_default_limit_caps_at_100(reform_impacts_service): + _seed_reform_impacts(reform_impacts_service, 150) result = _get_simulations() assert len(result["result"]) == 100 + assert result["result"][0]["execution_id"] == "exec-149" + assert result["result"][-1]["execution_id"] == "exec-50" -def test_get_simulations_clamps_huge_max_results(orm_session): - _seed_reform_impacts(orm_session, 50) +def test_get_simulations_clamps_huge_max_results(reform_impacts_service): + _seed_reform_impacts(reform_impacts_service, 50) # A caller passing an absurdly large value must not crash and # must not cause a full scan; the value is clamped at 1000. result = _get_simulations(max_results=10**9) assert len(result["result"]) == 50 # only 50 seeded -def test_get_simulations_clamps_negative_max_results(orm_session): - _seed_reform_impacts(orm_session, 5) +def test_get_simulations_clamps_negative_max_results(reform_impacts_service): + _seed_reform_impacts(reform_impacts_service, 5) # max_results of 0 or negative must still return something sane. result = _get_simulations(max_results=0) assert 1 <= len(result["result"]) <= 5 -def test_get_simulations_defaults_when_none(orm_session): - _seed_reform_impacts(orm_session, 10) +def test_get_simulations_defaults_when_none(reform_impacts_service): + _seed_reform_impacts(reform_impacts_service, 10) result = _get_simulations(max_results=None) assert len(result["result"]) == 10 # fewer than the default 100 -def test_get_simulations_rejects_non_integer_gracefully(orm_session): - _seed_reform_impacts(orm_session, 5) - # A string like "100; DROP TABLE reform_impact" must not reach - # the SQL statement; it falls back to the default. +def test_get_simulations_rejects_non_integer_gracefully(reform_impacts_service): + _seed_reform_impacts(reform_impacts_service, 5) + # An invalid value must not become a cache-index bound; it falls back to + # the default without altering the stored values. result = _get_simulations(max_results="100; DROP TABLE reform_impact") assert len(result["result"]) == 5 - - # And the table must still exist. - assert orm_session.scalar(select(func.count()).select_from(ReformImpact)) == 5 + assert len(reform_impacts_service.get_recent_reform_impacts(100)) == 5 -def test_get_simulations_preserves_v1_json_and_timestamp_fields(orm_session): - _seed_reform_impacts(orm_session, 1) +def test_get_simulations_preserves_complete_v1_response_shape( + reform_impacts_service, +): + _seed_reform_impacts(reform_impacts_service, 1) impact = _get_simulations(max_results=1)["result"][0] - assert json.loads(impact["options_json"]) == {} - assert json.loads(impact["reform_impact_json"]) == {} - assert impact["start_time"] == "2026-01-01 00:00:00" - assert impact["end_time"] == "2026-01-01 01:00:00" - - -def test_get_simulations_matches_through_fastapi_fallback(orm_session): - _seed_reform_impacts(orm_session, 1) + assert impact == { + "reform_impact_id": reform_impact_id("exec-0"), + "baseline_policy_id": 1, + "reform_policy_id": 2, + "country_id": "us", + "region": "us", + "dataset": "custom_dataset", + "time_period": "2025", + "options_json": json.dumps({}), + "options_hash": "hash-0", + "api_version": "1.0.0", + "reform_impact_json": json.dumps({}), + "status": "ok", + "message": "Completed", + "start_time": "2026-01-01 00:00:00", + "end_time": "2026-01-01 01:00:00", + "execution_id": "exec-0", + } + + +def test_get_simulations_matches_through_fastapi_fallback(reform_impacts_service): + _seed_reform_impacts(reform_impacts_service, 1) app = _create_app() flask_response = app.test_client().get("/simulations?max_results=1") diff --git a/tests/unit/runtime_cache/test_settings.py b/tests/unit/runtime_cache/test_settings.py index dad8012ec..9f940264b 100644 --- a/tests/unit/runtime_cache/test_settings.py +++ b/tests/unit/runtime_cache/test_settings.py @@ -31,6 +31,28 @@ def test_platform_marker_selects_deployed_mode_and_requires_configuration() -> N load_runtime_cache_settings({"K_SERVICE": "policyengine-api"}) +@pytest.mark.parametrize("platform_marker", ["K_SERVICE", "GAE_ENV"]) +@pytest.mark.parametrize("mode", ["disabled", "local"]) +def test_deployed_platform_marker_rejects_non_deployed_cache_mode( + platform_marker: str, + mode: str, +) -> None: + with pytest.raises(RuntimeCacheConfigurationError, match="must be deployed"): + load_runtime_cache_settings( + { + platform_marker: "deployed-runtime", + RUNTIME_CACHE_MODE: mode, + } + ) + + +def test_unknown_development_cache_mode_is_rejected() -> None: + with pytest.raises( + RuntimeCacheConfigurationError, match="disabled, local, or deployed" + ): + load_runtime_cache_settings({RUNTIME_CACHE_MODE: "dev"}) + + @pytest.mark.parametrize( "url", [ diff --git a/tests/unit/services/test_economy_service.py b/tests/unit/services/test_economy_service.py index 08448a458..cac48da2b 100644 --- a/tests/unit/services/test_economy_service.py +++ b/tests/unit/services/test_economy_service.py @@ -5,6 +5,9 @@ import httpx import pytest from policyengine_api.runtime_cache.core import CacheCoordinationError +from policyengine_api.services.reform_impacts_service import ( + ReformImpactHandoffError, +) from policyengine_api.services.economy_service import ( BUDGET_WINDOW_MAX_END_YEAR, BUDGET_WINDOW_MAX_YEARS, @@ -368,7 +371,7 @@ def test__given_start_claim_cache_failure__fails_before_submission( mock_simulation_entrypoint.run.assert_not_called() mock_reform_impacts_service.set_reform_impact.assert_not_called() - def test__given_simulation_submission_failure__releases_start_claim( + def test__given_gateway_raises_before_returning_execution__releases_start_claim( self, economy_service, base_params, @@ -392,6 +395,55 @@ def test__given_simulation_submission_failure__releases_start_claim( **mock_reform_impacts_service.claim_reform_impact_start.call_args.kwargs ) + def test__given_submitted_simulation_handoff_failure__retains_start_claim( + self, + economy_service, + base_params, + mock_country_package_versions, + mock_policyengine_version, + mock_policy_service, + mock_reform_impacts_service, + mock_simulation_entrypoint, + mock_logger, + mock_datetime, + mock_numpy_random, + ): + mock_reform_impacts_service.set_reform_impact.side_effect = ( + ReformImpactHandoffError("cache unavailable") + ) + + with pytest.raises(ReformImpactHandoffError, match="cache unavailable"): + economy_service.get_economic_impact(**base_params) + + mock_simulation_entrypoint.run.assert_called_once() + mock_reform_impacts_service.release_reform_impact_start.assert_not_called() + + def test__given_submitted_simulation_without_execution_id__retains_start_claim( + self, + economy_service, + base_params, + mock_country_package_versions, + mock_policyengine_version, + mock_policy_service, + mock_reform_impacts_service, + mock_simulation_entrypoint, + mock_logger, + mock_datetime, + mock_numpy_random, + ): + mock_simulation_entrypoint.get_execution_id.side_effect = RuntimeError( + "missing execution identifier" + ) + + with pytest.raises( + ReformImpactHandoffError, match="could not be handed off" + ): + economy_service.get_economic_impact(**base_params) + + mock_simulation_entrypoint.run.assert_called_once() + mock_reform_impacts_service.set_reform_impact.assert_not_called() + mock_reform_impacts_service.release_reform_impact_start.assert_not_called() + def test__given_policies_created_through_orm__submits_decoded_json( self, orm_session_factory, @@ -1053,9 +1105,7 @@ def test__given_completed_batch_cache_write_fails__does_not_clear_batch_id( "totals": {}, } mock_budget_window_cache.get_batch_job_id.return_value = "fc-budget-123" - mock_budget_window_cache.set_completed_result.side_effect = RuntimeError( - "redis unavailable" - ) + mock_budget_window_cache.set_completed_result.return_value = False mock_simulation_entrypoint.get_budget_window_batch_by_id.return_value = ( create_mock_budget_window_batch_execution( batch_job_id="fc-budget-123", @@ -1066,9 +1116,10 @@ def test__given_completed_batch_cache_write_fails__does_not_clear_batch_id( ) ) - with pytest.raises(RuntimeError, match="redis unavailable"): - economy_service.get_budget_window_economic_impact(**base_params) + result = economy_service.get_budget_window_economic_impact(**base_params) + assert result.status == ImpactStatus.OK + assert result.data == completed_result mock_budget_window_cache.clear_batch_job_id.assert_not_called() def test__given_failed_batch_poll__returns_failed( @@ -1121,7 +1172,7 @@ def test__given_existing_start_claim__does_not_submit_duplicate_batch( assert result.cache_status == "starting-claim-hit" mock_simulation_entrypoint.run_budget_window_batch.assert_not_called() - def test__given_batch_submission_fails__clears_start_claim( + def test__given_gateway_raises_before_returning_batch__clears_start_claim( self, economy_service, base_params, diff --git a/tests/unit/services/test_reform_impacts_service.py b/tests/unit/services/test_reform_impacts_service.py index 420dff9c0..ad26122a1 100644 --- a/tests/unit/services/test_reform_impacts_service.py +++ b/tests/unit/services/test_reform_impacts_service.py @@ -1,11 +1,15 @@ from datetime import datetime +from unittest.mock import MagicMock import pytest from policyengine_api.runtime_cache.core import CacheNamespace from policyengine_api.runtime_cache.fake import InMemoryCacheBackend from policyengine_api.runtime_cache.repositories import ReformImpactCache -from policyengine_api.services.reform_impacts_service import ReformImpactsService +from policyengine_api.services.reform_impacts_service import ( + ReformImpactHandoffError, + ReformImpactsService, +) @pytest.fixture @@ -42,6 +46,23 @@ def _create_impact( ) +def test_set_reform_impact_fails_closed_when_execution_pointer_is_not_stored(): + cache = MagicMock(spec=ReformImpactCache) + cache.set.return_value = False + service = ReformImpactsService(cache) + + with pytest.raises( + ReformImpactHandoffError, + match="execution could not be stored", + ): + _create_impact( + service, + execution_id="submitted-job", + options_hash="hash", + day=1, + ) + + def test_get_recent_reform_impacts_orders_and_limits_results(service): older = _create_impact( service, diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index be0aeb3b8..2d0ea2f88 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -212,8 +212,9 @@ def test_v2_files_are_mechanically_separate_from_v1() -> None: def test_v2_revision_chain_is_linear_generated_and_correction_bounded() -> None: config = Config(str(REPO / "alembic-v2.ini")) script = ScriptDirectory.from_config(config) - assert script.get_heads() == ["5f048586d8f1"] + assert script.get_heads() == ["4faee127fa16"] assert [revision.revision for revision in script.walk_revisions()] == [ + "4faee127fa16", "5f048586d8f1", "b4c69674dd47", "6ee725e0c563", @@ -236,10 +237,14 @@ def test_v2_revision_chain_is_linear_generated_and_correction_bounded() -> None: REPO / "migrations/v2/versions/5f048586d8f1_constrain_v2_user_country_and_report_.py" ).read_text(encoding="utf-8") - revisions = baseline + data + ownership + constraints + native_uuid = ( + REPO / "migrations/v2/versions/" + "4faee127fa16_use_native_uuid_report_run_idempotency_.py" + ).read_text(encoding="utf-8") + revisions = baseline + data + ownership + constraints + native_uuid assert all( "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" in source - for source in (baseline, data, ownership, constraints) + for source in (baseline, data, ownership, constraints, native_uuid) ) assert "op.execute(" not in revisions assert "op.bulk_insert(" not in revisions @@ -253,6 +258,13 @@ def test_v2_revision_chain_is_linear_generated_and_correction_bounded() -> None: assert constraints.count("op.create_check_constraint(") == 2 assert "ck_users_primary_country" in constraints assert "ck_report_runs_idempotency_key_nonblank" in constraints + assert native_uuid.count("op.alter_column(") == 2 + assert native_uuid.count("postgresql_using=") == 2 + assert 'postgresql_using="idempotency_key::uuid"' in native_uuid + assert 'postgresql_using="idempotency_key::text"' in native_uuid + assert native_uuid.index("op.drop_constraint(") < native_uuid.index( + "op.alter_column(" + ) corrected_enum_names = set( re.findall( diff --git a/tests/unit/v2/test_models.py b/tests/unit/v2/test_models.py index 0f5003ecb..648a6604c 100644 --- a/tests/unit/v2/test_models.py +++ b/tests/unit/v2/test_models.py @@ -152,6 +152,8 @@ def test_report_definition_and_run_columns_are_separated() -> None: }.isdisjoint(report.c.keys()) assert not report_run.c.country_package_version.nullable assert not report_run.c.policyengine_version.nullable + assert isinstance(report_run.c.idempotency_key.type, sa.Uuid) + assert report_run.c.idempotency_key.type.as_uuid def test_user_primary_country_is_required_and_limited_to_supported_values() -> None: @@ -210,7 +212,6 @@ def test_named_checks_and_required_indexes_cover_core_invariants() -> None: "ck_regions_required_filter_values", "ck_simulations_type_input", "ck_users_primary_country", - "ck_report_runs_idempotency_key_nonblank", "ck_report_runs_terminal_completion", "ck_parameter_values_single_owner", }.issubset(constraint_names) diff --git a/tests/unit/v2/test_report_runs.py b/tests/unit/v2/test_report_runs.py index 5b6ea226b..e8deda9c0 100644 --- a/tests/unit/v2/test_report_runs.py +++ b/tests/unit/v2/test_report_runs.py @@ -72,7 +72,7 @@ def _create_run( session: Session, report: Report, *, - key: str, + key: UUID | None = None, country_version: str = "1.2.3", policyengine_version: str = DEPLOYED_POLICYENGINE_VERSION, ) -> ReportRun: @@ -82,7 +82,7 @@ def _create_run( country_package_version=country_version, policyengine_version=policyengine_version, trigger=ReportRunTrigger.MANUAL, - idempotency_key=key, + idempotency_key=key if key is not None else uuid4(), ) @@ -107,7 +107,7 @@ def test_report_type_can_change_before_first_run_but_not_after(engine) -> None: report_id=report.id, report_type="marginal_tax_rate", ) - _create_run(session, report, key="first-run") + _create_run(session, report) unchanged = set_report_type( session, @@ -126,8 +126,8 @@ def test_report_type_can_change_before_first_run_but_not_after(engine) -> None: def test_new_manual_keys_create_distinct_same_version_runs(engine) -> None: with Session(engine) as session: report = _create_report(session) - first = _create_run(session, report, key="manual-1") - second = _create_run(session, report, key="manual-2") + first = _create_run(session, report) + second = _create_run(session, report) assert first.id != second.id assert first.country_package_version == second.country_package_version @@ -135,7 +135,7 @@ def test_new_manual_keys_create_distinct_same_version_runs(engine) -> None: assert len(session.exec(select(ReportRun)).all()) == 2 -def test_manual_rerun_rejects_a_whitespace_only_idempotency_key(engine) -> None: +def test_manual_rerun_requires_an_idempotency_key(engine) -> None: with Session(engine) as session: report = _create_report(session) @@ -146,32 +146,27 @@ def test_manual_rerun_rejects_a_whitespace_only_idempotency_key(engine) -> None: country_package_version="1.2.3", policyengine_version=DEPLOYED_POLICYENGINE_VERSION, trigger=ReportRunTrigger.MANUAL, - idempotency_key=" \t\n", ) -def test_database_rejects_non_null_blank_idempotency_keys(engine) -> None: +def test_idempotency_key_round_trips_as_a_python_uuid(engine) -> None: with Session(engine) as session: report = _create_report(session) - session.add( - ReportRun( - report=report, - country_package_version="1.2.3", - policyengine_version=DEPLOYED_POLICYENGINE_VERSION, - trigger=ReportRunTrigger.SYSTEM, - idempotency_key=" ", - ) - ) + request_key = uuid4() + run = _create_run(session, report, key=request_key) + session.flush() + session.expire(run) - with pytest.raises(sa.exc.IntegrityError): - session.flush() + assert run.idempotency_key == request_key + assert isinstance(run.idempotency_key, UUID) def test_transport_retry_returns_the_existing_report_scoped_run(engine) -> None: with Session(engine) as session: report = _create_report(session) - first = _create_run(session, report, key="retry-me") - retried = _create_run(session, report, key="retry-me") + request_key = uuid4() + first = _create_run(session, report, key=request_key) + retried = _create_run(session, report, key=request_key) assert retried.id == first.id assert len(session.exec(select(ReportRun)).all()) == 1 @@ -200,6 +195,7 @@ def _begin_immediate(connection) -> None: session.commit() barrier = threading.Barrier(2) + request_key = uuid4() def request_rerun() -> UUID: with Session(test_engine) as session: @@ -210,7 +206,7 @@ def request_rerun() -> UUID: country_package_version="1.2.3", policyengine_version=DEPLOYED_POLICYENGINE_VERSION, trigger=ReportRunTrigger.MANUAL, - idempotency_key="one-concurrent-request", + idempotency_key=request_key, ) run_id = run.id session.commit() @@ -230,7 +226,7 @@ def test_worker_retry_resumes_the_same_run_and_terminal_runs_stay_terminal( ) -> None: with Session(engine) as session: report = _create_report(session) - run = _create_run(session, report, key="worker-retry") + run = _create_run(session, report) started = begin_report_run(session, report_run_id=run.id, started_at=NOW) resumed = begin_report_run(session, report_run_id=run.id) @@ -262,8 +258,8 @@ def test_outputs_from_repeated_runs_are_preserved(engine) -> None: ) session.add(simulation) session.flush() - first = _create_run(session, report, key="output-1") - second = _create_run(session, report, key="output-2") + first = _create_run(session, report) + second = _create_run(session, report) session.add_all( [ AggregateOutput( @@ -344,10 +340,10 @@ def test_selector_uses_versions_success_completion_time_and_stable_id(engine) -> def test_pending_and_failed_reruns_do_not_displace_success(engine) -> None: with Session(engine) as session: report = _create_report(session) - successful = _create_run(session, report, key="success") + successful = _create_run(session, report) complete_report_run(session, report_run_id=successful.id, completed_at=NOW) - pending = _create_run(session, report, key="pending") - failed = _create_run(session, report, key="failed") + pending = _create_run(session, report) + failed = _create_run(session, report) fail_report_run( session, report_run_id=failed.id, @@ -370,9 +366,9 @@ def test_pending_and_failed_reruns_do_not_displace_success(engine) -> None: def test_new_successful_rerun_becomes_current_without_cache_state(engine) -> None: with Session(engine) as session: report = _create_report(session) - first = _create_run(session, report, key="first") + first = _create_run(session, report) complete_report_run(session, report_run_id=first.id, completed_at=NOW) - rerun = _create_run(session, report, key="rerun") + rerun = _create_run(session, report) complete_report_run( session, report_run_id=rerun.id, @@ -399,7 +395,6 @@ def test_selector_returns_none_without_a_matching_success(engine) -> None: mismatched = _create_run( session, report, - key="old-version", country_version="0.0.1", ) complete_report_run(session, report_run_id=mismatched.id, completed_at=NOW) From a8987cd4e92ad9dbe72d6fe56c9f19e3483c02de Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 18 Aug 2026 17:46:43 +0300 Subject: [PATCH 03/18] Add Stage 8 changelog fragment --- changelog.d/stage-8-v2-platform-foundation.added.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changelog.d/stage-8-v2-platform-foundation.added.md diff --git a/changelog.d/stage-8-v2-platform-foundation.added.md b/changelog.d/stage-8-v2-platform-foundation.added.md new file mode 100644 index 000000000..93e332c91 --- /dev/null +++ b/changelog.d/stage-8-v2-platform-foundation.added.md @@ -0,0 +1,3 @@ +Add the dormant API v2 Supabase and SQLModel foundation, and replace writable +runtime SQLite and embedded production Redis with managed shared caching while +preserving existing API behavior. From 659bbf2469e49ebe1f44908aa396799046fcde8a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:06:16 +0300 Subject: [PATCH 04/18] Fix legacy tests after SQLite removal --- tests/to_refactor/conftest.py | 52 +++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 tests/to_refactor/conftest.py diff --git a/tests/to_refactor/conftest.py b/tests/to_refactor/conftest.py new file mode 100644 index 000000000..fcd3b7580 --- /dev/null +++ b/tests/to_refactor/conftest.py @@ -0,0 +1,52 @@ +"""Explicit test-only persistence for the legacy route suite.""" + +from collections.abc import Iterator + +import pytest +from sqlalchemy import Engine, create_engine +from sqlalchemy.pool import StaticPool + +from policyengine_api.constants import COUNTRY_PACKAGE_VERSIONS +from policyengine_api.data import orm +from policyengine_api.data.v1_models import Policy +from policyengine_api.utils import hash_object +from tests.fixtures.local_v1_database import create_test_v1_schema + + +def _seed_current_law_policies(engine: Engine) -> None: + policies = [ + Policy( + id=policy_id, + country_id=country_id, + label="Current law", + api_version=COUNTRY_PACKAGE_VERSIONS[country_id], + policy_json={}, + policy_hash=hash_object({}), + ) + for policy_id, country_id in enumerate(COUNTRY_PACKAGE_VERSIONS, start=1) + ] + with orm.build_session_factory(engine).begin() as session: + session.add_all(policies) + + +@pytest.fixture(scope="session", autouse=True) +def isolated_legacy_v1_database() -> Iterator[None]: + """Bind legacy tests to explicit in-memory persistence, never Cloud SQL.""" + + engine = create_engine( + "sqlite+pysqlite:///:memory:", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, + ) + create_test_v1_schema(engine) + _seed_current_law_policies(engine) + + patcher = pytest.MonkeyPatch() + patcher.setattr(orm, "get_v1_engine", lambda: engine) + orm.clear_v1_session_factories() + try: + yield + finally: + orm.clear_v1_session_factories() + patcher.undo() + engine.dispose() From 59d16dfcde3a99af2ab7bc442ce9dad20a0181a0 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:18:51 +0300 Subject: [PATCH 05/18] Extract v2 lifecycle CI script --- .github/scripts/test_alembic_v2_lifecycle.sh | 8 ++++++++ .github/workflows/alembic-v2-check.yml | 5 +---- tests/unit/test_alembic_workflows.py | 8 +++++++- 3 files changed, 16 insertions(+), 5 deletions(-) create mode 100755 .github/scripts/test_alembic_v2_lifecycle.sh diff --git a/.github/scripts/test_alembic_v2_lifecycle.sh b/.github/scripts/test_alembic_v2_lifecycle.sh new file mode 100755 index 000000000..503bf87db --- /dev/null +++ b/.github/scripts/test_alembic_v2_lifecycle.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +uv run pytest -q \ + tests/unit/v2/test_alembic_v2.py \ + tests/unit/v2/test_reference_data_autogenerate.py \ + tests/integration/test_alembic_v2_lifecycle.py diff --git a/.github/workflows/alembic-v2-check.yml b/.github/workflows/alembic-v2-check.yml index ef80ba775..dd05786f9 100644 --- a/.github/workflows/alembic-v2-check.yml +++ b/.github/workflows/alembic-v2-check.yml @@ -47,10 +47,7 @@ jobs: - name: Install locked dependencies run: uv sync --frozen - name: Test generated-only v2 migration configuration and lifecycle - run: >- - uv run pytest -q tests/unit/v2/test_alembic_v2.py - tests/unit/v2/test_reference_data_autogenerate.py - tests/integration/test_alembic_v2_lifecycle.py + run: bash .github/scripts/test_alembic_v2_lifecycle.sh - name: Require database at the v2 head run: uv run alembic -c alembic-v2.ini current --check-heads - name: Require no ungenerated v2 schema or data operations diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index 8665e59e0..7b6959978 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -120,6 +120,9 @@ def test_reusable_alembic_check_uses_the_installed_python_environment(): def test_reusable_v2_check_uses_disposable_postgres_and_real_redis(): workflow = _workflow("alembic-v2-check.yml") + lifecycle_script = ( + REPO / ".github" / "scripts" / "test_alembic_v2_lifecycle.sh" + ).read_text(encoding="utf-8") assert "workflow_call:" in workflow assert "workflow_dispatch:" in workflow @@ -127,7 +130,10 @@ def test_reusable_v2_check_uses_disposable_postgres_and_real_redis(): assert "redis:7.2-alpine" in workflow assert "V2_ALEMBIC_DISPOSABLE_TEST" in workflow assert "alembic-v2.ini" in workflow - assert "test_alembic_v2_lifecycle.py" in workflow + assert "bash .github/scripts/test_alembic_v2_lifecycle.sh" in workflow + assert "test_alembic_v2.py" in lifecycle_script + assert "test_reference_data_autogenerate.py" in lifecycle_script + assert "test_alembic_v2_lifecycle.py" in lifecycle_script assert "test_runtime_cache_redis.py" in workflow assert "uv sync --frozen" in workflow From eefa7d394838b2b8d02a4673871659c9ca0b166a Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:25:21 +0300 Subject: [PATCH 06/18] Run v2 Alembic checks on every PR --- .github/workflows/pr.yml | 30 ---------------------------- tests/unit/test_alembic_workflows.py | 16 +++++++-------- 2 files changed, 7 insertions(+), 39 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 418f48395..fd2f50891 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -54,38 +54,8 @@ jobs: alembic-v1-check: name: Alembic v1 qualification uses: ./.github/workflows/alembic-v1-check.yml - detect-v2-platform-changes: - name: Detect v2 platform changes - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: read - outputs: - v2: ${{ steps.paths.outputs.v2 }} - steps: - - name: Checkout repo - uses: actions/checkout@v4 - - name: Detect v2 migration and cache paths - id: paths - uses: dorny/paths-filter@v3 - with: - filters: | - v2: - - 'alembic-v2.ini' - - 'migrations/v2/**' - - 'policyengine_api/data/v2/**' - - 'policyengine_api/runtime_cache/**' - - 'tests/unit/v2/**' - - 'tests/unit/runtime_cache/**' - - 'tests/integration/test_alembic_v2_lifecycle.py' - - 'tests/integration/test_runtime_cache_redis.py' - - '.github/workflows/alembic-v2-check.yml' - - 'pyproject.toml' - - 'uv.lock' alembic-v2-check: name: Alembic v2 and Redis qualification - needs: detect-v2-platform-changes - if: needs.detect-v2-platform-changes.outputs.v2 == 'true' uses: ./.github/workflows/alembic-v2-check.yml check-changelog: name: Check changelog fragment diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index 7b6959978..2069b9236 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -56,21 +56,19 @@ def test_workflows_do_not_inline_long_shell_programs(): def test_pr_always_runs_reusable_alembic_check(): workflow = _workflow("pr.yml") + v2_job = workflow[workflow.index(" alembic-v2-check:") :] + v2_job = v2_job[: v2_job.index("\n check-changelog:")] assert "alembic-v1-check:" in workflow assert "uses: ./.github/workflows/alembic-v1-check.yml" in workflow assert "detect-v1-alembic-changes:" not in workflow assert "needs.detect-v1-alembic-changes" not in workflow - assert "detect-v2-platform-changes:" in workflow + assert "detect-v2-platform-changes:" not in workflow + assert "dorny/paths-filter" not in workflow assert "alembic-v2-check:" in workflow - assert "uses: ./.github/workflows/alembic-v2-check.yml" in workflow - for path in ( - "alembic-v2.ini", - "migrations/v2/**", - "policyengine_api/data/v2/**", - "policyengine_api/runtime_cache/**", - ): - assert path in workflow + assert "uses: ./.github/workflows/alembic-v2-check.yml" in v2_job + assert "needs:" not in v2_job + assert "if:" not in v2_job def test_push_always_runs_lint_and_alembic_qualification_before_versioning(): From 421788a01fe221a18cfde8db22c371898b94f715 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:33:54 +0300 Subject: [PATCH 07/18] Sanitize Stage 8 migration documentation --- docs/migration/stage-8-managed-redis.md | 141 ++++++------ docs/migration/stage-8-platform-runbook.md | 77 ++++--- docs/migration/stage-8-supabase-bootstrap.md | 77 ++++--- docs/migration/stage-8-supabase-target.md | 213 +++++++++---------- 4 files changed, 258 insertions(+), 250 deletions(-) diff --git a/docs/migration/stage-8-managed-redis.md b/docs/migration/stage-8-managed-redis.md index 9b68a248f..9fd6a9c2a 100644 --- a/docs/migration/stage-8-managed-redis.md +++ b/docs/migration/stage-8-managed-redis.md @@ -1,86 +1,85 @@ # Stage 8 managed Redis topology +This document records the topology requirements without publishing concrete +project, region, network, instance, secret-resource, or service-account +identifiers. Operators must resolve those values from the approved deployment +configuration and secret-management surfaces. + ## Decision -Stage 8 will use **Google Cloud Memorystore for Redis** in `us-central1`, the -same region as the production and staging Cloud Run services. Each deployed -environment receives its own instance and credentials: - -| Environment | Instance ID | Tier | Capacity | Redis version | -| --- | --- | --- | --- | --- | -| Staging | `policyengine-api-cache-staging` | Basic | 1 GiB | 7.2 | -| Production | `policyengine-api-cache-prod` | Standard HA | 5 GiB | 7.2 | - -Production uses Standard Tier's cross-zone replica and automatic failover -because Redis coordinates expensive work in addition to caching completed -results. Staging uses Basic Tier to keep the validation environment -independent at lower cost. Neither environment enables read replicas. -Capacity is an initial allocation, not a durable-data commitment; metrics and -eviction pressure may justify resizing it later. - -Memorystore for Redis is preferred over Memorystore for Redis Cluster or -Memorystore for Valkey for this stage. The existing consumers use ordinary -Redis commands and require multi-key atomic operations. A single-instance -Redis-compatible endpoint preserves those semantics without introducing hash -slot constraints or a cluster-aware client during the SQLite-to-cache -migration. +Stage 8 uses Google Cloud Memorystore for Redis in the reviewed deployment +region. Staging and production use separate instances and credentials: + +| Environment | Topology requirement | +| --- | --- | +| Staging | Independent non-production instance sized for validation | +| Production | High-availability instance with automatic failover | + +Production uses a cross-zone replica because Redis coordinates expensive work +in addition to caching completed results. Staging uses a lower-cost independent +topology. Neither environment enables read replicas. Capacity is an initial +operational allocation, not a durable-data commitment; metrics and eviction +pressure determine later resizing. + +Memorystore for Redis is preferred over clustered alternatives for this stage. +The existing consumers use ordinary Redis commands and require multi-key atomic +operations. A single Redis-compatible endpoint preserves those semantics +without introducing hash-slot constraints or a cluster-aware client during the +SQLite-to-cache migration. Google documents [Memorystore for Redis tiers and pricing][pricing], the -[supported Redis versions and creation flags][create], and direct Cloud Run +[supported versions and creation flags][create], and direct Cloud Run [connectivity to Memorystore][cloud-run-redis]. ## Network and transport -Both instances use all of the following settings: +Both environments require all of the following: -- the existing `default` VPC in the `policyengine-api` Google Cloud project; -- the `policyengine-api-memorystore-psa` automatically allocated `/24` Private - Service Access range and - `PRIVATE_SERVICE_ACCESS` connection mode; -- Redis AUTH enabled; -- in-transit encryption set to `SERVER_AUTHENTICATION`; and +- a reviewed private VPC and subnet selected through deployment configuration; +- a non-overlapping private-service allocation maintained in the operator + infrastructure inventory; +- Redis authentication enabled; +- in-transit encryption with server authentication; and - no public endpoint, localhost fallback, or container-launched Redis server. -Cloud Run revisions will use [Direct VPC egress][direct-vpc] through the -`default` `us-central1` subnet with `private-ranges-only` routing. Direct VPC -egress avoids a permanently provisioned Serverless VPC Access connector while -keeping ordinary internet-bound traffic on its current path. The deployment -must account for Cloud Run subnet address consumption and connection resets; -the runtime client therefore needs bounded pools, timeouts, and reconnect -behavior. +Cloud Run revisions use [Direct VPC egress][direct-vpc]. The workflow resolves +the network, subnet, and egress policy from `CLOUD_RUN_VPC_NETWORK`, +`CLOUD_RUN_VPC_SUBNET`, and `CLOUD_RUN_VPC_EGRESS`; this document intentionally +does not record their environment-specific values. The runtime client uses +bounded pools, timeouts, and reconnect behavior to account for subnet address +consumption and connection resets. -Stage 8 enables the Memorystore and service-networking APIs, creates the -non-overlapping allocation and both instances, and validates -TLS-authenticated access from more than one Cloud Run instance before rollout. +Stage 8 enables the required managed-service APIs, creates the private +allocation and environment-specific instances, and validates TLS-authenticated +access from more than one Cloud Run instance before rollout. ## Authentication and secret boundaries -The authenticated URL and all currently downloadable instance CAs are stored -in separate Secret Manager secrets: +The authenticated URL and current server CA bundle are stored in separate +environment-specific secrets. Deployment resolves only their identifiers: -| Environment | URL secret | CA secret | +| Runtime | URL secret configuration | CA secret configuration | | --- | --- | --- | -| Staging | `policyengine-api-staging-runtime-cache-url` | `policyengine-api-staging-runtime-cache-ca` | -| Production | `policyengine-api-prod-runtime-cache-url` | `policyengine-api-prod-runtime-cache-ca` | - -They are exposed only to the Cloud Run and App Engine runtime identities for -that environment. Staging and -production must not share an endpoint or credential. Endpoint, port, expected -TLS mode, and environment are explicit deployed settings; application code -must not infer them from v1 database configuration or fall back to localhost. - -The client validates the Memorystore server certificate using in-memory -`ssl_ca_data` and Google's documented instance-specific certificate-authority -material. CA rotation requires storing every currently downloadable CA in the -CA secret before Google rotates the serving certificate. Secret-bearing URLs, AUTH values, -and certificate material must never be logged, committed, or placed in image -layers. +| Cloud Run | `CLOUD_RUN_RUNTIME_CACHE_URL_SECRET` | `CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET` | +| App Engine | `RUNTIME_CACHE_URL_SECRET_RESOURCE` | `RUNTIME_CACHE_CA_CERT_SECRET_RESOURCE` | + +Staging and production must not share an endpoint or credential. Endpoint, +port, expected TLS mode, and environment are explicit deployed settings; +application code must not infer them from v1 database configuration or fall +back to localhost. + +The client validates the server certificate using in-memory `ssl_ca_data`. +CA rotation requires storing every currently valid CA in the environment's CA +secret before the provider rotates the serving certificate. Secret-bearing +URLs, authentication values, certificate material, concrete secret-resource +names, and workload identities must never be logged or added to migration +documentation or image layers. ## Namespace and loss semantics -Every key is namespaced by environment, service, cache family, and cache -schema version. Result keys additionally include every result-affecting input -and relevant package version. This is defense in depth around the physically +Every key is namespaced by environment, service, cache family, and cache schema +version. Result keys additionally include every result-affecting input and +relevant package version. This is defense in depth around the physically separate instances and permits schema transitions without interpreting old payloads as current. @@ -104,20 +103,20 @@ record may exist only in Redis. ## Revision rollout and rollback Managed-cache configuration is revision-specific. A revision cannot receive -traffic unless its environment's endpoint, AUTH secret, TLS configuration, -and VPC attachment are valid. New Stage 8 revisions use managed Redis only; -they never start or select an embedded Redis process. +traffic unless its environment's endpoint, authentication secret, TLS +configuration, and VPC attachment are valid. New Stage 8 revisions use managed +Redis only; they never start or select an embedded Redis process. The immediately preceding application revision retains its own prior runtime -configuration. Rolling back means moving Cloud Run traffic to that known-good -revision. It does not copy, restore, or downgrade cache contents. The managed -cache may be retained or flushed because cache loss is handled as a miss, and -the dormant v2 Postgres schema remains in place unless an operator separately -invokes a reviewed migration downgrade. +configuration. Rolling back means moving traffic to that known-good revision. +It does not copy, restore, or downgrade cache contents. The managed cache may be +retained or flushed because cache loss is handled as a miss, and the dormant v2 +Postgres schema remains in place unless an operator separately invokes a +reviewed migration downgrade. If a rollout requires incompatible cache serialization, the new revision uses -a new cache-schema namespace. During a traffic split, old and new revisions -may therefore coexist without either revision reading the other's incompatible +a new cache-schema namespace. During a traffic split, old and new revisions may +therefore coexist without either revision reading the other's incompatible values. [cloud-run-redis]: https://cloud.google.com/memorystore/docs/redis/connect-redis-instance-cloud-run diff --git a/docs/migration/stage-8-platform-runbook.md b/docs/migration/stage-8-platform-runbook.md index 1a7fef7b3..db64aeb1c 100644 --- a/docs/migration/stage-8-platform-runbook.md +++ b/docs/migration/stage-8-platform-runbook.md @@ -1,5 +1,10 @@ # Stage 8 v2 platform runbook +This runbook intentionally omits concrete project references, organizations, +regions, hosts, network names, instance identifiers, secret-resource names, +service-account identities, and provisioning timestamps. Resolve those values +from the approved environment configuration and secret-management surfaces. + ## Scope and authority Stage 8 keeps Cloud SQL, existing routes, Simulation Entrypoint selection, and @@ -7,55 +12,57 @@ existing compute primary. The Supabase Postgres schema is dormant. Redis holds only recoverable completed results and expiring coordination state; it is never a durable domain store. -The recorded Supabase target is project `kvrifaviwhzjztcbrfpy`, organization -`PolicyEngine`, region `us-east-2`, environment `production-foundation`. Stop -if a supplied connection cannot be proven to resolve to that target. +Resolve the intended target through `V2_SUPABASE_PROJECT_REF` and +`V2_SUPABASE_ENVIRONMENT`. Stop if the supplied connection cannot be proven to +match the separately maintained approved target inventory. ## Persistent Supabase qualification and initialization -1. Confirm the target record in `stage-8-supabase-target.md` and its successful - fresh-state audit. A target with application tables, Alembic history, a - mismatched project reference, or ambiguous identity is not reset, adopted, - dropped, or stamped. -2. Retrieve only the migration database credential from Secret Manager and - supply `V2_MIGRATION_DATABASE_URL`, `V2_SUPABASE_PROJECT_REF`, and - `V2_SUPABASE_ENVIRONMENT` to the explicit operator process. +1. Confirm the approved target inventory and its successful fresh-state audit. + A target with application tables, Alembic history, a mismatched project + reference, or ambiguous identity is not reset, adopted, dropped, or stamped. +2. Retrieve only the migration database credential from the approved secret + manager and supply `V2_MIGRATION_DATABASE_URL`, + `V2_SUPABASE_PROJECT_REF`, and `V2_SUPABASE_ENVIRONMENT` to the explicit + operator process. 3. Run `uv run alembic -c alembic-v2.ini upgrade head` and then `uv run alembic -c alembic-v2.ini check`. Do not run either operation during application startup. 4. Only after migration succeeds, retrieve the separate Storage administration credential and run - `uv run python3 scripts/bootstrap_v2_supabase_storage.py`. - A second identical run must be a no-op; incompatible existing configuration - is an error, not permission to replace the bucket. + `uv run python3 scripts/bootstrap_v2_supabase_storage.py`. A second + identical run must be a no-op; incompatible existing configuration is an + error, not permission to replace the bucket. 5. Confirm no migration URL, password, Storage key, scratch SQL, generated - payload, dump, or one-off scaffolding file entered the repository or logs. + payload, dump, target identifier, or one-off scaffolding file entered the + repository or logs. -Application runtime receives the non-secret dormant project identity only. It -does not receive the migration password or Storage administration key. +Application runtime receives only the dormant target identity required by its +validated configuration. It does not receive the migration password or Storage +administration key. ## Managed-cache rollout -Staging uses `policyengine-api-cache-staging` (Basic, 1 GiB) and production uses -`policyengine-api-cache-prod` (Standard HA, 5 GiB). Both are Redis 7.2, -AUTH-enabled, TLS-only on port 6378, and private through the `default` VPC and -`policyengine-api-memorystore-psa` allocation. +Staging and production use separate managed Redis instances and credentials. +Production uses the reviewed high-availability topology; staging uses an +independent validation topology. Both require authentication, TLS, and private +network connectivity. Resolve all concrete infrastructure values from the +deployment configuration rather than this document. Before sending traffic to a candidate: -1. Verify the instance is `READY`, its AUTH and TLS modes are enabled, and all - current server CAs are present in the environment's CA secret. +1. Verify the selected instance is ready, authentication and TLS are enabled, + and all current server CAs are present in the environment's CA secret. 2. Verify the candidate has `RUNTIME_CACHE_MODE=deployed`, the correct - environment namespace, both Secret Manager bindings, and Direct VPC egress - through `default/default` with `private-ranges-only` routing. -3. Verify Cloud Run uses the dedicated runtime service account and App Engine - uses only non-secret Secret Manager resource names. Confirm the App Engine - staging and production service accounts have repository-scoped Artifact - Registry Reader access so each can pull the reviewed candidate image, and - those identities have `secretAccessor` only on the required database - password, GitHub microdata, Anthropic, OpenAI, Hugging Face, gateway-auth, - and environment-specific cache secrets. Neither runtime identity receives - v2 migration or Storage administration access. + `RUNTIME_CACHE_ENVIRONMENT`, both environment-specific Secret Manager + bindings, and the reviewed Direct VPC egress settings supplied through + `CLOUD_RUN_VPC_NETWORK`, `CLOUD_RUN_VPC_SUBNET`, and + `CLOUD_RUN_VPC_EGRESS`. +3. Verify Cloud Run and App Engine use their dedicated runtime identities and + only non-secret Secret Manager resource names. Confirm each identity has + artifact-read access and per-secret accessor rights only for its required + runtime secrets. Neither runtime identity receives v2 migration or Storage + administration access. 4. Send test traffic to at least two Cloud Run instances and verify one connection's value is visible to another. Confirm the container has no `redis-server` child and startup creates no SQLite database or lock file. @@ -69,11 +76,13 @@ Completed-result writes use subtract-only TTL jitter of up to ten percent to spread normal expirations; coordination and claim TTLs remain exact. Jitter does not spread misses after a full flush, so bounded recomputation and atomic claims remain the controls for complete cache loss. + Runtime cache operations emit the stable `runtime_cache_operations` structured metric with `metric_value=1`, `cache_family`, `cache_event`, optional `cache_operation`, and `latency_ms`. Use the value as a counter grouped by the -family and event fields and the latency field as the distribution source. -Logs must include no keys, values, URLs, AUTH strings, or CA payloads. +family and event fields and the latency field as the distribution source. Logs +must include no keys, values, URLs, authentication strings, CA payloads, or +concrete infrastructure identifiers. ## Rollback diff --git a/docs/migration/stage-8-supabase-bootstrap.md b/docs/migration/stage-8-supabase-bootstrap.md index 2cdfbca25..a4ad8b8ff 100644 --- a/docs/migration/stage-8-supabase-bootstrap.md +++ b/docs/migration/stage-8-supabase-bootstrap.md @@ -1,29 +1,33 @@ -# Stage 8 Supabase Migration and Storage Bootstrap +# Stage 8 Supabase migration and Storage bootstrap -This runbook operates only on the dedicated dormant API v2-alpha target -recorded in `docs/migration/stage-8-supabase-target.md`. It is not application -startup logic. Cloud SQL and all existing production routes and compute remain -primary throughout Stage 8. +This runbook operates only on the approved dormant API v2-alpha target. It is +not application startup logic. Cloud SQL and all existing production routes and +compute remain primary throughout Stage 8. + +Concrete organization, project, region, endpoint, bucket, credential, and +secret-resource identifiers are intentionally excluded. Resolve them from the +approved environment configuration and secret-management surfaces. ## Required identity and credential boundaries -The non-secret identity must be exactly: +The operator must resolve and validate all of the following without copying +their values into this repository: -- environment: `production-foundation` -- project reference: `kvrifaviwhzjztcbrfpy` -- Storage API origin: `https://kvrifaviwhzjztcbrfpy.supabase.co` -- private bucket: `policyengine-v2-alpha` +- `V2_SUPABASE_ENVIRONMENT` +- `V2_SUPABASE_PROJECT_REF` +- `V2_SUPABASE_STORAGE_URL` +- `V2_SUPABASE_STORAGE_BUCKET` -Inject the database migration password and the Storage administration key from -their separate GCP Secret Manager secrets at execution time. Do not echo them, -place them in repository files, reuse the migration URL as runtime -configuration, or expose the Storage key to the application service account. +Inject the database migration password and Storage administration key from +their separate approved secrets at execution time. Do not echo them, place them +in repository files, reuse the migration URL as runtime configuration, or +expose the Storage key to the application service account. ## Ordered explicit operations -1. Confirm the target record and its successful freshness audit. Stop on any - identity ambiguity or unexpected application state; never reset, adopt, or - stamp the database. +1. Confirm the separately maintained target inventory and its successful + freshness audit. Stop on any identity ambiguity or unexpected application + state; never reset, adopt, or stamp the database. 2. Supply `V2_MIGRATION_DATABASE_URL`, `V2_SUPABASE_ENVIRONMENT`, and `V2_SUPABASE_PROJECT_REF` to an explicit operator or CI migration step. 3. Run `uv run alembic -c alembic-v2.ini upgrade head`, followed by @@ -31,26 +35,28 @@ configuration, or expose the Storage key to the application service account. connection so it can qualify the persistent target and verify generated application-data before/after states. 4. Remove the migration credential from the execution environment. Supply the - separate `V2_SUPABASE_STORAGE_ADMIN_KEY` together with the recorded - identity, `V2_SUPABASE_STORAGE_URL`, and - `V2_SUPABASE_STORAGE_BUCKET=policyengine-v2-alpha`. + separate `V2_SUPABASE_STORAGE_ADMIN_KEY` together with the validated + `V2_SUPABASE_STORAGE_URL`, `V2_SUPABASE_STORAGE_BUCKET`, + `V2_SUPABASE_PROJECT_REF`, and `V2_SUPABASE_ENVIRONMENT` values. 5. Run `uv run python3 scripts/bootstrap_v2_supabase_storage.py`. A fresh run creates the reviewed private bucket. A repeat run verifies it and reports `created: false`. An incompatible existing bucket stops without update, deletion, recreation, or public exposure. -6. Remove the Storage administration credential from the execution - environment and run - `uv run python3 scripts/check_stage8_scaffolding_hygiene.py` before commit. +6. Remove the Storage administration credential from the execution environment + and run `uv run python3 scripts/check_stage8_scaffolding_hygiene.py` before + commit. The Storage initializer calls only Supabase's bucket-management endpoint. It does not run Alembic, import application startup, modify application tables or rows, initialize canonical metadata, upload an object, or create an access -policy. The dedicated `sb_secret_...` key is sent only in the `apikey` header; -it is not a JWT and must not be placed in `Authorization: Bearer`. The -initializer recognizes both structured current Storage errors such as -`NoSuchBucket` and legacy HTTP 404/409 responses without logging response -bodies. Supabase documents that buckets are private by default and that bucket -creation needs bucket insert permission but no object permission: +policy. The dedicated server-side Storage administration key is sent only in +the `apikey` header; it is not a JWT and must not be placed in +`Authorization: Bearer`. The initializer recognizes both structured current +Storage errors such as `NoSuchBucket` and legacy HTTP 404/409 responses without +logging response bodies. + +Supabase documents that buckets are private by default and that bucket creation +needs bucket insert permission but no object permission: and . The current key and Storage error contracts are documented at @@ -60,9 +66,10 @@ and . ## Repository hygiene One-off SQL, dumps, generated payloads, temporary environment files, Supabase -CLI state, and scratch scaffolding belong only in ignored local-artifact or -system-temporary locations and must be removed after use. If an operation is -needed again, promote it to tested idempotent tooling before committing it. -Generated Alembic revisions, declarative migration sources, this supported -initializer, tests, and durable documentation are reviewed project artifacts, -not disposable scaffolding. +CLI state, scratch scaffolding, target identifiers, and secret-resource names +belong only in approved ignored local-artifact or system-temporary locations +and must be removed after use. If an operation is needed again, promote it to +tested idempotent tooling before committing it. Generated Alembic revisions, +declarative migration sources, this supported initializer, tests, and sanitized +durable documentation are reviewed project artifacts, not disposable +scaffolding. diff --git a/docs/migration/stage-8-supabase-target.md b/docs/migration/stage-8-supabase-target.md index f534981d0..3120b2ded 100644 --- a/docs/migration/stage-8-supabase-target.md +++ b/docs/migration/stage-8-supabase-target.md @@ -1,25 +1,28 @@ -# Stage 8 Supabase Target +# Stage 8 Supabase target handling -This document is the durable, non-secret identity record for the Supabase -project introduced by Stage 8 of the unified API v2-alpha migration. It does -not contain credentials, connection URLs, API keys, or one-off provisioning -output. +This document records how the dedicated Supabase target is selected and +qualified without publishing its concrete identity. Organization IDs, project +names and references, regions, hosts, endpoints, usernames, secret-resource +names, network allowlists, provisioning timestamps, and service-account +identities must remain in the approved operator inventory, deployment +configuration, or secret-management surface—not migration documentation. -## Target identity +## Target identity resolution -| Field | Value | +| Required field | Approved source | | --- | --- | -| Supabase organization | `PolicyEngine` | -| Organization ID | `jygirqnhxzbevhozzrzi` | -| Project name | `policyengine-api-v2-alpha` | -| Project reference | `kvrifaviwhzjztcbrfpy` | -| Region | `us-east-2` | -| Environment classification | Production foundation; dormant during Stage 8 | -| Owning team | PolicyEngine engineering | -| Stage 8 authority | No production request reads, writes, routes, or compute | -| Database host identity | `db.kvrifaviwhzjztcbrfpy.supabase.co` | -| Postgres engine | PostgreSQL 17, Supabase GA channel | -| Provisioned | 2026-08-13; observed `ACTIVE_HEALTHY` | +| Supabase organization | Operator platform inventory | +| Project name and reference | Operator inventory and `V2_SUPABASE_PROJECT_REF` | +| Region | Operator platform inventory | +| Environment classification | `V2_SUPABASE_ENVIRONMENT` | +| Database host and pooler endpoint | Validated migration URL and provider console | +| Storage API origin | `V2_SUPABASE_STORAGE_URL` | +| Private bucket | `V2_SUPABASE_STORAGE_BUCKET` | +| Owning team | Internal ownership inventory | + +The runtime and migration configuration must fail closed if the supplied values +do not match the approved inventory. This repository records variable names and +validation behavior only. ## Purpose @@ -29,17 +32,18 @@ SQL and the existing API and compute paths remain primary. Later migration stages may populate and activate the target under their own reviewed cutover contracts. -## Selection record +## Selection requirements -- The authenticated Supabase account exposes one organization, `PolicyEngine`, - with organization ID `jygirqnhxzbevhozzrzi`. -- No existing project is named `policyengine-api-v2-alpha`. In particular, the - unrelated project named `database` is not reused. -- The deployed API and Cloud SQL defaults are in GCP `us-central1`. Supabase - currently offers `us-east-2`; it is selected as the nearby supported region - for this AWS-hosted project. -- A Supabase project's region is fixed at the infrastructure level, so a later - region change would require a new project and a reviewed migration. +- Use a newly provisioned project dedicated to the API v2-alpha migration; do + not adopt an unrelated existing project. +- Resolve the owning organization, unique project identity, environment + purpose, and supported region before creation. +- Select the reviewed region according to latency and operational requirements + recorded in the operator inventory. +- Treat a later region change as a new-project migration because the provider's + project region is fixed at the infrastructure level. +- Keep the project dormant during Stage 8: no production request reads, writes, + routes, or compute use it. ## Provisioning boundary @@ -50,107 +54,96 @@ bucket. Application schema and versioned application data remain exclusively owned by the generated v2 Alembic chain. Storage initialization is a separate, later idempotent operation. -The owner credential created with the project is stored in GCP Secret Manager -as `policyengine-api-v2-alpha-prod-db-owner-password`. It is a provisioning -credential, not the later migration or application-runtime identity, and its -value is never stored in this repository. - -Tasks 1.3 through 1.6 qualify connectivity, credential separation, database -freshness, and repository hygiene before any v2 baseline is generated. +Store the initial owner credential in the approved secret manager as a +provisioning-only credential. Its value and resource identifier are not +recorded here and must not become routine migration or application-runtime +configuration. ## Connectivity qualification -- External database SSL enforcement is enabled at the Supabase project level. -- The direct database identity remains - `db.kvrifaviwhzjztcbrfpy.supabase.co:5432`. Supabase direct endpoints require - IPv6 unless the IPv4 add-on is enabled, so it is not the qualified - `us-central1` Cloud Run path at this stage. -- Authenticated operator connectivity is qualified over the IPv4 Supavisor - session endpoint `aws-0-us-east-2.pooler.supabase.com:5432`, using database - `postgres` and the project-qualified owner username. A read-only connection - reported PostgreSQL 17.6 and confirmed TLS in `pg_stat_ssl`. -- Database network restrictions currently allow `0.0.0.0/0` and `::/0` because - the existing Cloud Run service has no reviewed static egress CIDR. Narrowing - the allowlist to an invented address would make connectivity unreliable. - Mandatory TLS and credential isolation are the active boundary; a later - network restriction requires a provisioned, tested egress range. -- No IPv4 add-on, custom Postgres override, database DDL, Alembic stamp, - application row, or Storage bucket was introduced during connectivity setup. +- Enforce external database TLS at the project level. +- Resolve direct and pooled database endpoints from the provider console and + validated operator configuration; do not copy them into repository docs. +- Qualify operator connectivity through the reviewed TLS endpoint using the + provisioning identity only for the bounded setup operation. +- Maintain the reviewed database network allowlist outside the repository. Do + not infer or invent an address range when the deployed service lacks a + qualified static egress range. +- Confirm the observed Postgres engine and TLS session meet the migration + requirements without recording connection strings, hosts, usernames, or + addresses in logs or documentation. +- Do not introduce an unreviewed networking add-on, custom Postgres override, + database DDL, Alembic stamp, application row, or Storage bucket during + connectivity setup. Supabase's connection-mode guidance is documented at . ## Credential boundaries -All secret values live in GCP Secret Manager in project `policyengine-api`. -The repository records names and intended use only. +All secret values live in the approved secret-management surface. The +repository records access classes and intended use only: -| Access path | Identity or key | Secret Manager secret | Effective boundary | -| --- | --- | --- | --- | -| Initial project ownership and emergency administration | Supabase `postgres` owner | `policyengine-api-v2-alpha-prod-db-owner-password` | Provisioning only; not application or routine migration configuration | -| Generated v2 Alembic chain | `policyengine_v2_migrator` | `policyengine-api-v2-alpha-prod-db-migration-password` | Login, database connect, and `USAGE`/`CREATE` on `public`; no superuser, role creation, database creation, replication, or RLS bypass | -| Future ordinary v2 persistence | `policyengine_v2_runtime` | `policyengine-api-v2-alpha-prod-db-runtime-password` | Login, database connect, and `USAGE` on `public`; no schema creation, superuser, role creation, database creation, replication, or RLS bypass | -| Explicit Storage bootstrap | Supabase secret key `stage_8_storage_bootstrap` | `policyengine-api-v2-alpha-prod-storage-admin-key` | Dedicated, independently rotatable server-side credential exposed only to the Storage bootstrap operation | +| Access path | Credential class | Effective boundary | +| --- | --- | --- | +| Initial ownership and emergency administration | Provisioning owner credential | Provisioning only; not application or routine migration configuration | +| Generated v2 Alembic chain | Dedicated migration credential | Database connect and reviewed schema creation; no platform administration | +| Future ordinary v2 persistence | Dedicated runtime credential | Ordinary application data access; no schema migration or platform administration | +| Explicit Storage bootstrap | Dedicated server-side Storage administration key | Independently rotatable and exposed only to the Storage bootstrap operation | -The migration role owns the default privileges for objects it later creates: +The migration role owns the default privileges for objects it later creates; ordinary table read/write and sequence use are granted to the runtime role. Those grants do not create an application object or row. -Supabase secret keys are elevated server-side credentials that bypass RLS; the -platform does not represent them as Storage-only keys. Least privilege is -therefore enforced by using a distinct named key, storing it separately, and -making it available only to the explicit Storage initializer. Neither the -runtime database identity nor the migration identity receives this key. +Supabase server-side secret keys are elevated credentials. Least privilege is +therefore enforced by using a distinct key, storing it separately, and making +it available only to the explicit Storage initializer. Neither the runtime +database identity nor the migration identity receives this key. -The Cloud Run runtime service account previously held project-wide Secret -Manager accessor rights. Before completing this credential split, that broad -binding was replaced with per-secret access to its six existing production -runtime secrets: the gateway client secret plus the database, microdata, -Anthropic, OpenAI, and Hugging Face secrets. It has no project-wide Secret -Manager role and no binding on any Stage 8 administrative secret. +Runtime service accounts must not hold project-wide secret-access rights. Give +each identity per-secret access only to the runtime values it needs, and grant +no Stage 8 migration or Storage-administration secret to an application runtime +identity. ## Freshness qualification -On 2026-08-13, the recorded project was audited through a PostgreSQL -`READ ONLY` transaction over the qualified TLS connection. - -- Connected project reference: `kvrifaviwhzjztcbrfpy`. -- Database and role: `postgres` as the provisioning owner. -- Application schema: `public` contains zero tables. -- Alembic history: no `alembic_version` table exists in any schema. -- Predecessor application state: no table matching the reviewed v2 model - groups, `runtime_bundles`, or a population table exists in `public`. -- Storage initialization: `storage.buckets` contains zero rows. -- Service-managed schemas observed: `auth`, `extensions`, `graphql`, - `graphql_public`, `pgbouncer`, `realtime`, `storage`, and `vault`. Their - platform-owned tables do not count as application state. - -The audit result is fresh. No reset, drop, stamp, reconciliation, or adoption -was required or performed. This qualification permits later v2 baseline +Before baseline generation, audit the approved project through a PostgreSQL +`READ ONLY` transaction over the qualified TLS connection. Retain the concrete +audit evidence only in the approved operator record. + +The audit must establish all of the following: + +- the connected project identity matches the approved target; +- the application schema contains zero application tables; +- no `alembic_version` table or revision history exists; +- no predecessor v2 model table, `runtime_bundles`, or population table exists; +- no application-owned Storage bucket or object has been initialized; and +- observed provider-managed schemas contain only platform-owned state. + +Any mismatch or ambiguity fails closed. Do not reset, drop, stamp, reconcile, +or adopt the target automatically. A successful audit permits later v2 baseline generation only when the migration workflow independently verifies the same -recorded target identity. +approved target identity. ## Provisioning hygiene -The provisioning review completed on 2026-08-14 with these results: - -- No application table, application row, Alembic stamp, or Storage bucket was - created. -- No secret value, secret-bearing URL, SQL dump, scratch SQL, generated - payload, or temporary configuration is tracked or staged. -- `supabase/.temp/` is ignored because the Supabase CLI writes ephemeral linked - project metadata there even for management operations. Its generated file - was removed after the project reference was recorded above. -- The local `.venv` used for read-only Postgres qualification is already an - ignored development artifact and is not part of the change. -- The repository secret-pattern scan found no credential value in the changed - files. -- The four Stage 8 GCP secrets have automatic replication and purpose labels; - none grants access to the Cloud Run runtime service account. -- The dedicated Supabase Storage key exists as - `stage_8_storage_bootstrap`; only its non-secret identifier and prefix are - recorded, while the complete value exists only in GCP Secret Manager. - -The dedicated Supabase foundation is therefore ready for the generated v2 -schema work, subject to the target-identity gate implemented later in this -change. +Before declaring the foundation ready, verify: + +- no application table, application row, Alembic stamp, or Storage bucket was + created during provisioning; +- no secret value, secret-bearing URL, target identifier, endpoint, SQL dump, + scratch SQL, generated payload, or temporary configuration is tracked or + staged; +- Supabase CLI state and other linked-project metadata remain ignored and are + removed after the bounded operator action; +- local qualification environments remain ignored development artifacts; +- the repository secret-pattern scan finds no credential or infrastructure + identity in changed migration documents; +- administrative secrets have purpose labels and no application runtime access; + and +- the dedicated Storage credential exists only in the approved secret manager + and explicit bootstrap environment. + +The dedicated Supabase foundation is ready for generated v2 schema work only +after these controls pass and the target-identity gate confirms the separately +maintained approved inventory. From 4a6980597fcaef846beade8739506a3d3b014dee Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 18 Aug 2026 19:54:03 +0300 Subject: [PATCH 08/18] Split v2 domain models by topic --- policyengine_api/data/v2/models/__init__.py | 12 +- .../data/v2/models/associations.py | 129 ++++++ policyengine_api/data/v2/models/domain.py | 396 ------------------ policyengine_api/data/v2/models/households.py | 87 ++++ policyengine_api/data/v2/models/metadata.py | 3 +- policyengine_api/data/v2/models/policies.py | 55 +++ policyengine_api/data/v2/models/reports.py | 12 +- .../data/v2/models/simulations.py | 142 +++++++ policyengine_api/data/v2/models/users.py | 51 +++ tests/unit/v2/test_models.py | 25 ++ 10 files changed, 505 insertions(+), 407 deletions(-) create mode 100644 policyengine_api/data/v2/models/associations.py delete mode 100644 policyengine_api/data/v2/models/domain.py create mode 100644 policyengine_api/data/v2/models/households.py create mode 100644 policyengine_api/data/v2/models/policies.py create mode 100644 policyengine_api/data/v2/models/simulations.py create mode 100644 policyengine_api/data/v2/models/users.py diff --git a/policyengine_api/data/v2/models/__init__.py b/policyengine_api/data/v2/models/__init__.py index 82cc4344f..5795c6ae9 100644 --- a/policyengine_api/data/v2/models/__init__.py +++ b/policyengine_api/data/v2/models/__init__.py @@ -29,16 +29,22 @@ TaxBenefitModelVersion, Variable, ) -from policyengine_api.data.v2.models.domain import ( # noqa: E402 +from policyengine_api.data.v2.models.users import User # noqa: E402 +from policyengine_api.data.v2.models.policies import ( # noqa: E402 Dynamic, + Policy, +) +from policyengine_api.data.v2.models.households import ( # noqa: E402 Household, HouseholdJob, HouseholdJobStatus, - Policy, +) +from policyengine_api.data.v2.models.simulations import ( # noqa: E402 Simulation, SimulationStatus, SimulationType, - User, +) +from policyengine_api.data.v2.models.associations import ( # noqa: E402 UserHouseholdAssociation, UserPolicy, UserReportAssociation, diff --git a/policyengine_api/data/v2/models/associations.py b/policyengine_api/data/v2/models/associations.py new file mode 100644 index 000000000..e15dd60e6 --- /dev/null +++ b/policyengine_api/data/v2/models/associations.py @@ -0,0 +1,129 @@ +"""Canonical SQLModel tables linking v2 users to their domain records.""" + +from datetime import datetime +from typing import TYPE_CHECKING +from uuid import UUID + +import sqlalchemy as sa +from sqlmodel import Field, Relationship + +from policyengine_api.data.v2.models.base import TimestampedModel +from policyengine_api.data.v2.models.households import Household +from policyengine_api.data.v2.models.policies import Policy +from policyengine_api.data.v2.models.simulations import Simulation +from policyengine_api.data.v2.models.users import User + +if TYPE_CHECKING: + from policyengine_api.data.v2.models.reports import Report + + +class UserHouseholdAssociation(TimestampedModel, table=True): + __tablename__ = "user_household_associations" + __table_args__ = ( + sa.UniqueConstraint( + "user_id", + "household_id", + name="uq_user_household_associations_user_household", + ), + ) + + user_id: UUID = Field( + foreign_key="users.id", + ondelete="CASCADE", + index=True, + ) + household_id: UUID = Field( + foreign_key="households.id", + ondelete="CASCADE", + index=True, + ) + country: str = Field(max_length=16) + label: str | None = Field(default=None, max_length=255) + + user: User = Relationship(back_populates="household_associations") + household: Household = Relationship(back_populates="user_associations") + + +class UserPolicy(TimestampedModel, table=True): + __tablename__ = "user_policies" + __table_args__ = ( + sa.UniqueConstraint( + "user_id", + "policy_id", + name="uq_user_policies_user_policy", + ), + ) + + user_id: UUID = Field( + foreign_key="users.id", + ondelete="CASCADE", + index=True, + ) + policy_id: UUID = Field( + foreign_key="policies.id", + ondelete="CASCADE", + index=True, + ) + country: str = Field(max_length=16) + label: str | None = Field(default=None, max_length=255) + + user: User = Relationship(back_populates="policy_associations") + policy: Policy = Relationship(back_populates="user_associations") + + +class UserSimulationAssociation(TimestampedModel, table=True): + __tablename__ = "user_simulation_associations" + __table_args__ = ( + sa.UniqueConstraint( + "user_id", + "simulation_id", + name="uq_user_simulation_associations_user_simulation", + ), + ) + + user_id: UUID = Field( + foreign_key="users.id", + ondelete="CASCADE", + index=True, + ) + simulation_id: UUID = Field( + foreign_key="simulations.id", + ondelete="CASCADE", + index=True, + ) + country: str = Field(max_length=16) + label: str | None = Field(default=None, max_length=255) + + user: User = Relationship(back_populates="simulation_associations") + simulation: Simulation = Relationship(back_populates="user_associations") + + +class UserReportAssociation(TimestampedModel, table=True): + __tablename__ = "user_report_associations" + __table_args__ = ( + sa.UniqueConstraint( + "user_id", + "report_id", + name="uq_user_report_associations_user_report", + ), + ) + + user_id: UUID = Field( + foreign_key="users.id", + ondelete="CASCADE", + index=True, + ) + report_id: UUID = Field( + foreign_key="reports.id", + ondelete="CASCADE", + index=True, + ) + country: str = Field(max_length=16) + label: str | None = Field(default=None, max_length=255) + last_run_at: datetime | None = Field( + default=None, + sa_type=sa.DateTime(timezone=True), + ) + + user: User = Relationship(back_populates="report_associations") + report: "Report" = Relationship(back_populates="user_associations") diff --git a/policyengine_api/data/v2/models/domain.py b/policyengine_api/data/v2/models/domain.py deleted file mode 100644 index 2051ceff6..000000000 --- a/policyengine_api/data/v2/models/domain.py +++ /dev/null @@ -1,396 +0,0 @@ -"""Canonical SQLModel tables for v2 policies, households, and simulations.""" - -from datetime import datetime -from enum import Enum -from typing import TYPE_CHECKING, Any -from uuid import UUID - -import sqlalchemy as sa -from sqlmodel import Field, Relationship - -from policyengine_api.data.v2.models.base import ( - IdentifiedModel, - TimestampedModel, - enum_type, -) -from policyengine_api.data.v2.models.metadata import ( - Dataset, - Region, - TaxBenefitModel, - TaxBenefitModelVersion, -) - -if TYPE_CHECKING: - from policyengine_api.data.v2.models.metadata import ParameterValue - from policyengine_api.data.v2.models.reports import Report - - -class HouseholdJobStatus(str, Enum): - PENDING = "pending" - RUNNING = "running" - SUCCEEDED = "succeeded" - FAILED = "failed" - - -class SimulationStatus(str, Enum): - PENDING = "pending" - RUNNING = "running" - SUCCEEDED = "succeeded" - FAILED = "failed" - - -class SimulationType(str, Enum): - HOUSEHOLD = "household" - ECONOMY = "economy" - - -class User(IdentifiedModel, table=True): - __tablename__ = "users" - __table_args__ = ( - sa.UniqueConstraint("email", name="uq_users_email"), - sa.CheckConstraint( - "primary_country IN ('us', 'uk')", - name="ck_users_primary_country", - ), - ) - - first_name: str = Field(max_length=255) - last_name: str = Field(max_length=255) - email: str = Field(max_length=320, index=True) - primary_country: str = Field(max_length=2) - - reports: list["Report"] = Relationship(back_populates="user") - household_associations: list["UserHouseholdAssociation"] = Relationship( - back_populates="user", - cascade_delete=True, - ) - policy_associations: list["UserPolicy"] = Relationship( - back_populates="user", - cascade_delete=True, - ) - simulation_associations: list["UserSimulationAssociation"] = Relationship( - back_populates="user", - cascade_delete=True, - ) - report_associations: list["UserReportAssociation"] = Relationship( - back_populates="user", - cascade_delete=True, - ) - - -class Policy(TimestampedModel, table=True): - __tablename__ = "policies" - - name: str = Field(max_length=255) - description: str | None = None - tax_benefit_model_id: UUID = Field( - foreign_key="tax_benefit_models.id", - ondelete="RESTRICT", - index=True, - ) - - tax_benefit_model: TaxBenefitModel = Relationship(back_populates="policies") - parameter_values: list["ParameterValue"] = Relationship( - back_populates="policy", - cascade_delete=True, - ) - simulations: list["Simulation"] = Relationship(back_populates="policy") - household_jobs: list["HouseholdJob"] = Relationship(back_populates="policy") - reports: list["Report"] = Relationship(back_populates="policy") - user_associations: list["UserPolicy"] = Relationship( - back_populates="policy", - cascade_delete=True, - ) - - -class Dynamic(TimestampedModel, table=True): - __tablename__ = "dynamics" - - name: str = Field(max_length=255) - description: str | None = None - - parameter_values: list["ParameterValue"] = Relationship( - back_populates="dynamic", - cascade_delete=True, - ) - simulations: list["Simulation"] = Relationship(back_populates="dynamic") - household_jobs: list["HouseholdJob"] = Relationship(back_populates="dynamic") - - -class Household(TimestampedModel, table=True): - __tablename__ = "households" - __table_args__ = ( - sa.CheckConstraint( - "year BETWEEN 1900 AND 2200", - name="ck_households_year", - ), - ) - - country: str = Field(max_length=16, index=True) - year: int - label: str | None = Field(default=None, max_length=255) - household_data: dict[str, Any] = Field(sa_type=sa.JSON) - - simulations: list["Simulation"] = Relationship(back_populates="household") - reports: list["Report"] = Relationship(back_populates="household") - user_associations: list["UserHouseholdAssociation"] = Relationship( - back_populates="household", - cascade_delete=True, - ) - - -class HouseholdJob(IdentifiedModel, table=True): - __tablename__ = "household_jobs" - __table_args__ = ( - sa.Index("ix_household_jobs_status_created_at", "status", "created_at"), - ) - - country: str = Field(max_length=16) - request_data: dict[str, Any] = Field(sa_type=sa.JSON) - policy_id: UUID | None = Field( - default=None, - foreign_key="policies.id", - ondelete="SET NULL", - ) - dynamic_id: UUID | None = Field( - default=None, - foreign_key="dynamics.id", - ondelete="SET NULL", - ) - status: HouseholdJobStatus = Field( - default=HouseholdJobStatus.PENDING, - sa_type=enum_type(HouseholdJobStatus, "v2_household_job_status"), - ) - error_message: str | None = None - result: dict[str, Any] | None = Field(default=None, sa_type=sa.JSON) - started_at: datetime | None = Field( - default=None, - sa_type=sa.DateTime(timezone=True), - ) - completed_at: datetime | None = Field( - default=None, - sa_type=sa.DateTime(timezone=True), - ) - - policy: Policy | None = Relationship(back_populates="household_jobs") - dynamic: Dynamic | None = Relationship(back_populates="household_jobs") - - -class Simulation(TimestampedModel, table=True): - __tablename__ = "simulations" - __table_args__ = ( - sa.CheckConstraint( - "(simulation_type = 'household' AND household_id IS NOT NULL " - "AND dataset_id IS NULL) OR " - "(simulation_type = 'economy' AND dataset_id IS NOT NULL " - "AND household_id IS NULL)", - name="ck_simulations_type_input", - ), - sa.CheckConstraint( - "(filter_field IS NULL) = (filter_value IS NULL)", - name="ck_simulations_filter_pair", - ), - sa.CheckConstraint( - "year IS NULL OR year BETWEEN 1900 AND 2200", - name="ck_simulations_year", - ), - sa.Index("ix_simulations_status_created_at", "status", "created_at"), - ) - - simulation_type: SimulationType = Field( - default=SimulationType.ECONOMY, - sa_type=enum_type(SimulationType, "v2_simulation_type"), - ) - dataset_id: UUID | None = Field( - default=None, - foreign_key="datasets.id", - ondelete="RESTRICT", - ) - household_id: UUID | None = Field( - default=None, - foreign_key="households.id", - ondelete="RESTRICT", - ) - policy_id: UUID | None = Field( - default=None, - foreign_key="policies.id", - ondelete="SET NULL", - ) - dynamic_id: UUID | None = Field( - default=None, - foreign_key="dynamics.id", - ondelete="SET NULL", - ) - tax_benefit_model_version_id: UUID = Field( - foreign_key="tax_benefit_model_versions.id", - ondelete="RESTRICT", - index=True, - ) - output_dataset_id: UUID | None = Field( - default=None, - foreign_key="datasets.id", - ondelete="SET NULL", - ) - region_id: UUID | None = Field( - default=None, - foreign_key="regions.id", - ondelete="SET NULL", - ) - status: SimulationStatus = Field( - default=SimulationStatus.PENDING, - sa_type=enum_type(SimulationStatus, "v2_simulation_status"), - ) - error_message: str | None = None - filter_field: str | None = Field(default=None, max_length=128) - filter_value: str | None = Field(default=None, max_length=255) - filter_strategy: str | None = Field(default=None, max_length=64) - year: int | None = None - started_at: datetime | None = Field( - default=None, - sa_type=sa.DateTime(timezone=True), - ) - completed_at: datetime | None = Field( - default=None, - sa_type=sa.DateTime(timezone=True), - ) - household_result: dict[str, Any] | None = Field(default=None, sa_type=sa.JSON) - - dataset: Dataset | None = Relationship( - back_populates="input_simulations", - sa_relationship_kwargs={"foreign_keys": "Simulation.dataset_id"}, - ) - output_dataset: Dataset | None = Relationship( - back_populates="output_simulations", - sa_relationship_kwargs={"foreign_keys": "Simulation.output_dataset_id"}, - ) - household: Household | None = Relationship(back_populates="simulations") - policy: Policy | None = Relationship(back_populates="simulations") - dynamic: Dynamic | None = Relationship(back_populates="simulations") - tax_benefit_model_version: TaxBenefitModelVersion = Relationship( - back_populates="simulations" - ) - region: Region | None = Relationship(back_populates="simulations") - baseline_reports: list["Report"] = Relationship( - back_populates="baseline_simulation", - sa_relationship_kwargs={"foreign_keys": "Report.baseline_simulation_id"}, - ) - reform_reports: list["Report"] = Relationship( - back_populates="reform_simulation", - sa_relationship_kwargs={"foreign_keys": "Report.reform_simulation_id"}, - ) - user_associations: list["UserSimulationAssociation"] = Relationship( - back_populates="simulation", - cascade_delete=True, - ) - - -class UserHouseholdAssociation(TimestampedModel, table=True): - __tablename__ = "user_household_associations" - __table_args__ = ( - sa.UniqueConstraint( - "user_id", - "household_id", - name="uq_user_household_associations_user_household", - ), - ) - - user_id: UUID = Field( - foreign_key="users.id", - ondelete="CASCADE", - index=True, - ) - household_id: UUID = Field( - foreign_key="households.id", - ondelete="CASCADE", - index=True, - ) - country: str = Field(max_length=16) - label: str | None = Field(default=None, max_length=255) - - user: User = Relationship(back_populates="household_associations") - household: Household = Relationship(back_populates="user_associations") - - -class UserPolicy(TimestampedModel, table=True): - __tablename__ = "user_policies" - __table_args__ = ( - sa.UniqueConstraint( - "user_id", - "policy_id", - name="uq_user_policies_user_policy", - ), - ) - - user_id: UUID = Field( - foreign_key="users.id", - ondelete="CASCADE", - index=True, - ) - policy_id: UUID = Field( - foreign_key="policies.id", - ondelete="CASCADE", - index=True, - ) - country: str = Field(max_length=16) - label: str | None = Field(default=None, max_length=255) - - user: User = Relationship(back_populates="policy_associations") - policy: Policy = Relationship(back_populates="user_associations") - - -class UserSimulationAssociation(TimestampedModel, table=True): - __tablename__ = "user_simulation_associations" - __table_args__ = ( - sa.UniqueConstraint( - "user_id", - "simulation_id", - name="uq_user_simulation_associations_user_simulation", - ), - ) - - user_id: UUID = Field( - foreign_key="users.id", - ondelete="CASCADE", - index=True, - ) - simulation_id: UUID = Field( - foreign_key="simulations.id", - ondelete="CASCADE", - index=True, - ) - country: str = Field(max_length=16) - label: str | None = Field(default=None, max_length=255) - - user: User = Relationship(back_populates="simulation_associations") - simulation: Simulation = Relationship(back_populates="user_associations") - - -class UserReportAssociation(TimestampedModel, table=True): - __tablename__ = "user_report_associations" - __table_args__ = ( - sa.UniqueConstraint( - "user_id", - "report_id", - name="uq_user_report_associations_user_report", - ), - ) - - user_id: UUID = Field( - foreign_key="users.id", - ondelete="CASCADE", - index=True, - ) - report_id: UUID = Field( - foreign_key="reports.id", - ondelete="CASCADE", - index=True, - ) - country: str = Field(max_length=16) - label: str | None = Field(default=None, max_length=255) - last_run_at: datetime | None = Field( - default=None, - sa_type=sa.DateTime(timezone=True), - ) - - user: User = Relationship(back_populates="report_associations") - report: "Report" = Relationship(back_populates="user_associations") diff --git a/policyengine_api/data/v2/models/households.py b/policyengine_api/data/v2/models/households.py new file mode 100644 index 000000000..b6c510c09 --- /dev/null +++ b/policyengine_api/data/v2/models/households.py @@ -0,0 +1,87 @@ +"""Canonical SQLModel tables for v2 households and household jobs.""" + +from datetime import datetime +from enum import Enum +from typing import TYPE_CHECKING, Any +from uuid import UUID + +import sqlalchemy as sa +from sqlmodel import Field, Relationship + +from policyengine_api.data.v2.models.base import ( + IdentifiedModel, + TimestampedModel, + enum_type, +) +from policyengine_api.data.v2.models.policies import Dynamic, Policy + +if TYPE_CHECKING: + from policyengine_api.data.v2.models.associations import UserHouseholdAssociation + from policyengine_api.data.v2.models.reports import Report + from policyengine_api.data.v2.models.simulations import Simulation + + +class HouseholdJobStatus(str, Enum): + PENDING = "pending" + RUNNING = "running" + SUCCEEDED = "succeeded" + FAILED = "failed" + + +class Household(TimestampedModel, table=True): + __tablename__ = "households" + __table_args__ = ( + sa.CheckConstraint( + "year BETWEEN 1900 AND 2200", + name="ck_households_year", + ), + ) + + country: str = Field(max_length=16, index=True) + year: int + label: str | None = Field(default=None, max_length=255) + household_data: dict[str, Any] = Field(sa_type=sa.JSON) + + simulations: list["Simulation"] = Relationship(back_populates="household") + reports: list["Report"] = Relationship(back_populates="household") + user_associations: list["UserHouseholdAssociation"] = Relationship( + back_populates="household", + cascade_delete=True, + ) + + +class HouseholdJob(IdentifiedModel, table=True): + __tablename__ = "household_jobs" + __table_args__ = ( + sa.Index("ix_household_jobs_status_created_at", "status", "created_at"), + ) + + country: str = Field(max_length=16) + request_data: dict[str, Any] = Field(sa_type=sa.JSON) + policy_id: UUID | None = Field( + default=None, + foreign_key="policies.id", + ondelete="SET NULL", + ) + dynamic_id: UUID | None = Field( + default=None, + foreign_key="dynamics.id", + ondelete="SET NULL", + ) + status: HouseholdJobStatus = Field( + default=HouseholdJobStatus.PENDING, + sa_type=enum_type(HouseholdJobStatus, "v2_household_job_status"), + ) + error_message: str | None = None + result: dict[str, Any] | None = Field(default=None, sa_type=sa.JSON) + started_at: datetime | None = Field( + default=None, + sa_type=sa.DateTime(timezone=True), + ) + completed_at: datetime | None = Field( + default=None, + sa_type=sa.DateTime(timezone=True), + ) + + policy: Policy | None = Relationship(back_populates="household_jobs") + dynamic: Dynamic | None = Relationship(back_populates="household_jobs") diff --git a/policyengine_api/data/v2/models/metadata.py b/policyengine_api/data/v2/models/metadata.py index b57a33edf..fe693d345 100644 --- a/policyengine_api/data/v2/models/metadata.py +++ b/policyengine_api/data/v2/models/metadata.py @@ -15,8 +15,9 @@ ) if TYPE_CHECKING: - from policyengine_api.data.v2.models.domain import Dynamic, Policy, Simulation + from policyengine_api.data.v2.models.policies import Dynamic, Policy from policyengine_api.data.v2.models.reports import Report + from policyengine_api.data.v2.models.simulations import Simulation class RegionType(str, Enum): diff --git a/policyengine_api/data/v2/models/policies.py b/policyengine_api/data/v2/models/policies.py new file mode 100644 index 000000000..ba632a240 --- /dev/null +++ b/policyengine_api/data/v2/models/policies.py @@ -0,0 +1,55 @@ +"""Canonical SQLModel tables for v2 policies and dynamics.""" + +from typing import TYPE_CHECKING +from uuid import UUID + +from sqlmodel import Field, Relationship + +from policyengine_api.data.v2.models.base import TimestampedModel +from policyengine_api.data.v2.models.metadata import TaxBenefitModel + +if TYPE_CHECKING: + from policyengine_api.data.v2.models.associations import UserPolicy + from policyengine_api.data.v2.models.households import HouseholdJob + from policyengine_api.data.v2.models.metadata import ParameterValue + from policyengine_api.data.v2.models.reports import Report + from policyengine_api.data.v2.models.simulations import Simulation + + +class Policy(TimestampedModel, table=True): + __tablename__ = "policies" + + name: str = Field(max_length=255) + description: str | None = None + tax_benefit_model_id: UUID = Field( + foreign_key="tax_benefit_models.id", + ondelete="RESTRICT", + index=True, + ) + + tax_benefit_model: TaxBenefitModel = Relationship(back_populates="policies") + parameter_values: list["ParameterValue"] = Relationship( + back_populates="policy", + cascade_delete=True, + ) + simulations: list["Simulation"] = Relationship(back_populates="policy") + household_jobs: list["HouseholdJob"] = Relationship(back_populates="policy") + reports: list["Report"] = Relationship(back_populates="policy") + user_associations: list["UserPolicy"] = Relationship( + back_populates="policy", + cascade_delete=True, + ) + + +class Dynamic(TimestampedModel, table=True): + __tablename__ = "dynamics" + + name: str = Field(max_length=255) + description: str | None = None + + parameter_values: list["ParameterValue"] = Relationship( + back_populates="dynamic", + cascade_delete=True, + ) + simulations: list["Simulation"] = Relationship(back_populates="dynamic") + household_jobs: list["HouseholdJob"] = Relationship(back_populates="dynamic") diff --git a/policyengine_api/data/v2/models/reports.py b/policyengine_api/data/v2/models/reports.py index 327132e98..4e1aa2feb 100644 --- a/policyengine_api/data/v2/models/reports.py +++ b/policyengine_api/data/v2/models/reports.py @@ -13,18 +13,16 @@ TimestampedModel, enum_type, ) -from policyengine_api.data.v2.models.domain import ( - Household, - Policy, - Simulation, - User, - UserReportAssociation, -) +from policyengine_api.data.v2.models.associations import UserReportAssociation +from policyengine_api.data.v2.models.households import Household from policyengine_api.data.v2.models.metadata import ( Dataset, Region, TaxBenefitModel, ) +from policyengine_api.data.v2.models.policies import Policy +from policyengine_api.data.v2.models.simulations import Simulation +from policyengine_api.data.v2.models.users import User class ReportRunStatus(str, Enum): diff --git a/policyengine_api/data/v2/models/simulations.py b/policyengine_api/data/v2/models/simulations.py new file mode 100644 index 000000000..d076ea4df --- /dev/null +++ b/policyengine_api/data/v2/models/simulations.py @@ -0,0 +1,142 @@ +"""Canonical SQLModel table for v2 simulations.""" + +from datetime import datetime +from enum import Enum +from typing import TYPE_CHECKING, Any +from uuid import UUID + +import sqlalchemy as sa +from sqlmodel import Field, Relationship + +from policyengine_api.data.v2.models.base import TimestampedModel, enum_type +from policyengine_api.data.v2.models.households import Household +from policyengine_api.data.v2.models.metadata import ( + Dataset, + Region, + TaxBenefitModelVersion, +) +from policyengine_api.data.v2.models.policies import Dynamic, Policy + +if TYPE_CHECKING: + from policyengine_api.data.v2.models.associations import UserSimulationAssociation + from policyengine_api.data.v2.models.reports import Report + + +class SimulationStatus(str, Enum): + PENDING = "pending" + RUNNING = "running" + SUCCEEDED = "succeeded" + FAILED = "failed" + + +class SimulationType(str, Enum): + HOUSEHOLD = "household" + ECONOMY = "economy" + + +class Simulation(TimestampedModel, table=True): + __tablename__ = "simulations" + __table_args__ = ( + sa.CheckConstraint( + "(simulation_type = 'household' AND household_id IS NOT NULL " + "AND dataset_id IS NULL) OR " + "(simulation_type = 'economy' AND dataset_id IS NOT NULL " + "AND household_id IS NULL)", + name="ck_simulations_type_input", + ), + sa.CheckConstraint( + "(filter_field IS NULL) = (filter_value IS NULL)", + name="ck_simulations_filter_pair", + ), + sa.CheckConstraint( + "year IS NULL OR year BETWEEN 1900 AND 2200", + name="ck_simulations_year", + ), + sa.Index("ix_simulations_status_created_at", "status", "created_at"), + ) + + simulation_type: SimulationType = Field( + default=SimulationType.ECONOMY, + sa_type=enum_type(SimulationType, "v2_simulation_type"), + ) + dataset_id: UUID | None = Field( + default=None, + foreign_key="datasets.id", + ondelete="RESTRICT", + ) + household_id: UUID | None = Field( + default=None, + foreign_key="households.id", + ondelete="RESTRICT", + ) + policy_id: UUID | None = Field( + default=None, + foreign_key="policies.id", + ondelete="SET NULL", + ) + dynamic_id: UUID | None = Field( + default=None, + foreign_key="dynamics.id", + ondelete="SET NULL", + ) + tax_benefit_model_version_id: UUID = Field( + foreign_key="tax_benefit_model_versions.id", + ondelete="RESTRICT", + index=True, + ) + output_dataset_id: UUID | None = Field( + default=None, + foreign_key="datasets.id", + ondelete="SET NULL", + ) + region_id: UUID | None = Field( + default=None, + foreign_key="regions.id", + ondelete="SET NULL", + ) + status: SimulationStatus = Field( + default=SimulationStatus.PENDING, + sa_type=enum_type(SimulationStatus, "v2_simulation_status"), + ) + error_message: str | None = None + filter_field: str | None = Field(default=None, max_length=128) + filter_value: str | None = Field(default=None, max_length=255) + filter_strategy: str | None = Field(default=None, max_length=64) + year: int | None = None + started_at: datetime | None = Field( + default=None, + sa_type=sa.DateTime(timezone=True), + ) + completed_at: datetime | None = Field( + default=None, + sa_type=sa.DateTime(timezone=True), + ) + household_result: dict[str, Any] | None = Field(default=None, sa_type=sa.JSON) + + dataset: Dataset | None = Relationship( + back_populates="input_simulations", + sa_relationship_kwargs={"foreign_keys": "Simulation.dataset_id"}, + ) + output_dataset: Dataset | None = Relationship( + back_populates="output_simulations", + sa_relationship_kwargs={"foreign_keys": "Simulation.output_dataset_id"}, + ) + household: Household | None = Relationship(back_populates="simulations") + policy: Policy | None = Relationship(back_populates="simulations") + dynamic: Dynamic | None = Relationship(back_populates="simulations") + tax_benefit_model_version: TaxBenefitModelVersion = Relationship( + back_populates="simulations" + ) + region: Region | None = Relationship(back_populates="simulations") + baseline_reports: list["Report"] = Relationship( + back_populates="baseline_simulation", + sa_relationship_kwargs={"foreign_keys": "Report.baseline_simulation_id"}, + ) + reform_reports: list["Report"] = Relationship( + back_populates="reform_simulation", + sa_relationship_kwargs={"foreign_keys": "Report.reform_simulation_id"}, + ) + user_associations: list["UserSimulationAssociation"] = Relationship( + back_populates="simulation", + cascade_delete=True, + ) diff --git a/policyengine_api/data/v2/models/users.py b/policyengine_api/data/v2/models/users.py new file mode 100644 index 000000000..fc7582a5a --- /dev/null +++ b/policyengine_api/data/v2/models/users.py @@ -0,0 +1,51 @@ +"""Canonical SQLModel table for v2 users.""" + +from typing import TYPE_CHECKING + +import sqlalchemy as sa +from sqlmodel import Field, Relationship + +from policyengine_api.data.v2.models.base import IdentifiedModel + +if TYPE_CHECKING: + from policyengine_api.data.v2.models.associations import ( + UserHouseholdAssociation, + UserPolicy, + UserReportAssociation, + UserSimulationAssociation, + ) + from policyengine_api.data.v2.models.reports import Report + + +class User(IdentifiedModel, table=True): + __tablename__ = "users" + __table_args__ = ( + sa.UniqueConstraint("email", name="uq_users_email"), + sa.CheckConstraint( + "primary_country IN ('us', 'uk')", + name="ck_users_primary_country", + ), + ) + + first_name: str = Field(max_length=255) + last_name: str = Field(max_length=255) + email: str = Field(max_length=320, index=True) + primary_country: str = Field(max_length=2) + + reports: list["Report"] = Relationship(back_populates="user") + household_associations: list["UserHouseholdAssociation"] = Relationship( + back_populates="user", + cascade_delete=True, + ) + policy_associations: list["UserPolicy"] = Relationship( + back_populates="user", + cascade_delete=True, + ) + simulation_associations: list["UserSimulationAssociation"] = Relationship( + back_populates="user", + cascade_delete=True, + ) + report_associations: list["UserReportAssociation"] = Relationship( + back_populates="user", + cascade_delete=True, + ) diff --git a/tests/unit/v2/test_models.py b/tests/unit/v2/test_models.py index 648a6604c..3c61d5e50 100644 --- a/tests/unit/v2/test_models.py +++ b/tests/unit/v2/test_models.py @@ -7,6 +7,11 @@ from policyengine_api.data.v1_models import V1Base from policyengine_api.data.v2.models import ( + Dynamic, + Household, + HouseholdJob, + Policy, + Simulation, User, UserHouseholdAssociation, UserPolicy, @@ -35,6 +40,26 @@ ) +def test_domain_models_are_grouped_into_topic_scoped_modules() -> None: + expected_modules = { + User: "users", + Policy: "policies", + Dynamic: "policies", + Household: "households", + HouseholdJob: "households", + Simulation: "simulations", + UserHouseholdAssociation: "associations", + UserPolicy: "associations", + UserReportAssociation: "associations", + UserSimulationAssociation: "associations", + } + + assert { + model: model.__module__.rsplit(".", maxsplit=1)[-1] + for model in expected_modules + } == expected_modules + + def test_controlled_models_match_the_exact_reviewed_inventory() -> None: model_table_names = {model.__table__.name for model in V2_TABLE_MODELS} From 8452abed3209ef01e6b8e4a82ab85d1db0490b54 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 18 Aug 2026 21:40:36 +0300 Subject: [PATCH 09/18] Align v2 region dataset defaults --- docs/engineering/skills/alembic-migrations.md | 10 ++ ...d_assign_one_default_dataset_per_region.py | 104 ++++++++++++ policyengine_api/data/v2/models/__init__.py | 3 - policyengine_api/data/v2/models/base.py | 9 +- policyengine_api/data/v2/models/metadata.py | 53 +++--- policyengine_api/data/v2/table_inventory.py | 1 - .../integration/test_alembic_v2_lifecycle.py | 158 ++++++++++++++++-- tests/unit/v2/test_alembic_v2.py | 27 ++- tests/unit/v2/test_import_side_effects.py | 2 +- tests/unit/v2/test_models.py | 45 +++++ tests/unit/v2/test_table_inventory.py | 1 + 11 files changed, 365 insertions(+), 48 deletions(-) create mode 100644 migrations/v2/versions/56dcd15a3afd_assign_one_default_dataset_per_region.py diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 2e7d806e8..d899791d0 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -137,6 +137,16 @@ bootstrap must never call `create_all`, create or stamp application tables, or mutate versioned application data. The Supabase CLI is not an application schema or seed migration authority. +Large canonical metadata catalogs derived from the exact installed country and +`policyengine` packages are not hand-authored migration data. A later-stage, +explicit deployment seeder may materialize those package-derived rows after +`alembic upgrade head` when its reviewed contract requires transactional, +idempotent row-only behavior, recorded source package versions, and fail-closed +handling of partial catalogs. Such a seeder must perform no DDL, must not run at +application startup, and must not transform or delete retained domain data; +those operations remain Alembic migrations. Small reviewed reference rows stay +in the declarative autogeneration workflow above. + Migration credentials remain separate from future runtime credentials. The migration identity may create and alter the v2 application schema; the runtime identity must not. Neither credential nor a secret-bearing URL belongs in an diff --git a/migrations/v2/versions/56dcd15a3afd_assign_one_default_dataset_per_region.py b/migrations/v2/versions/56dcd15a3afd_assign_one_default_dataset_per_region.py new file mode 100644 index 000000000..f6334742b --- /dev/null +++ b/migrations/v2/versions/56dcd15a3afd_assign_one_default_dataset_per_region.py @@ -0,0 +1,104 @@ +"""assign one default dataset per region + +Revision ID: 56dcd15a3afd +Revises: 4faee127fa16 +Create Date: 2026-08-18 21:28:53.385203 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = "56dcd15a3afd" +down_revision: Union[str, None] = "4faee127fa16" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column( + "datasets", "storage_path", existing_type=sa.VARCHAR(length=1024), nullable=True + ) + op.drop_constraint( + op.f("uq_datasets_model_name_year_output"), "datasets", type_="unique" + ) + op.create_unique_constraint( + "uq_datasets_id_model", "datasets", ["id", "tax_benefit_model_id"] + ) + op.create_unique_constraint( + "uq_datasets_model_name", "datasets", ["tax_benefit_model_id", "name"] + ) + op.create_check_constraint( + op.f("ck_datasets_output_storage_path"), + "datasets", + "NOT is_output_dataset OR storage_path IS NOT NULL", + ) + op.add_column("regions", sa.Column("default_dataset_id", sa.Uuid(), nullable=False)) + op.create_index( + op.f("ix_regions_default_dataset_id"), + "regions", + ["default_dataset_id"], + unique=False, + ) + op.create_foreign_key( + "fk_regions_default_dataset_model_datasets", + "regions", + "datasets", + ["default_dataset_id", "tax_benefit_model_id"], + ["id", "tax_benefit_model_id"], + ondelete="RESTRICT", + ) + op.drop_table("region_datasets") + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "region_datasets", + sa.Column("region_id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column("dataset_id", sa.UUID(), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint( + ["dataset_id"], + ["datasets.id"], + name=op.f("fk_region_datasets_dataset_id_datasets"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["region_id"], + ["regions.id"], + name=op.f("fk_region_datasets_region_id_regions"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint( + "region_id", "dataset_id", name=op.f("pk_region_datasets") + ), + ) + op.drop_constraint( + "fk_regions_default_dataset_model_datasets", "regions", type_="foreignkey" + ) + op.drop_index(op.f("ix_regions_default_dataset_id"), table_name="regions") + op.drop_column("regions", "default_dataset_id") + op.drop_constraint( + op.f("ck_datasets_output_storage_path"), "datasets", type_="check" + ) + op.drop_constraint("uq_datasets_model_name", "datasets", type_="unique") + op.drop_constraint("uq_datasets_id_model", "datasets", type_="unique") + op.create_unique_constraint( + op.f("uq_datasets_model_name_year_output"), + "datasets", + ["tax_benefit_model_id", "name", "year", "is_output_dataset"], + postgresql_nulls_not_distinct=False, + ) + op.alter_column( + "datasets", + "storage_path", + existing_type=sa.VARCHAR(length=1024), + nullable=False, + ) + # ### end Alembic commands ### diff --git a/policyengine_api/data/v2/models/__init__.py b/policyengine_api/data/v2/models/__init__.py index 5795c6ae9..285bf7f22 100644 --- a/policyengine_api/data/v2/models/__init__.py +++ b/policyengine_api/data/v2/models/__init__.py @@ -23,7 +23,6 @@ ParameterNode, ParameterValue, Region, - RegionDatasetLink, RegionType, TaxBenefitModel, TaxBenefitModelVersion, @@ -95,7 +94,6 @@ Poverty, ProgramStatistics, Region, - RegionDatasetLink, Report, ReportRun, Simulation, @@ -136,7 +134,6 @@ "Poverty", "ProgramStatistics", "Region", - "RegionDatasetLink", "RegionType", "Report", "ReportRun", diff --git a/policyengine_api/data/v2/models/base.py b/policyengine_api/data/v2/models/base.py index 30fdd6c5a..e612eff8b 100644 --- a/policyengine_api/data/v2/models/base.py +++ b/policyengine_api/data/v2/models/base.py @@ -28,12 +28,13 @@ "types while remaining SQLModel Fields." ), "named_constraints_and_indexes": ( - "Composite uniqueness, database checks, and multi-column indexes " - "are not expressible by one SQLModel Field." + "Composite uniqueness and foreign keys, database checks, and " + "multi-column indexes are not expressible by one SQLModel Field." ), "ambiguous_foreign_key_relationships": ( - "Dataset input/output and baseline/reform joins need SQLAlchemy " - "foreign_keys hints exposed by SQLModel Relationship." + "Dataset input/output, baseline/reform, and overlapping composite " + "region-default joins need SQLAlchemy relationship hints exposed " + "by SQLModel Relationship." ), "transaction_conflict_recovery": ( "Concurrent report idempotency requires a savepoint and bounded " diff --git a/policyengine_api/data/v2/models/metadata.py b/policyengine_api/data/v2/models/metadata.py index fe693d345..4eaa3d7bd 100644 --- a/policyengine_api/data/v2/models/metadata.py +++ b/policyengine_api/data/v2/models/metadata.py @@ -6,7 +6,7 @@ from uuid import UUID import sqlalchemy as sa -from sqlmodel import Field, Relationship, SQLModel +from sqlmodel import Field, Relationship from policyengine_api.data.v2.models.base import ( IdentifiedModel, @@ -31,21 +31,6 @@ class RegionType(str, Enum): PLACE = "place" -class RegionDatasetLink(SQLModel, table=True): - __tablename__ = "region_datasets" - - region_id: UUID = Field( - foreign_key="regions.id", - ondelete="CASCADE", - primary_key=True, - ) - dataset_id: UUID = Field( - foreign_key="datasets.id", - ondelete="CASCADE", - primary_key=True, - ) - - class TaxBenefitModel(TimestampedModel, table=True): __tablename__ = "tax_benefit_models" __table_args__ = (sa.UniqueConstraint("name", name="uq_tax_benefit_models_name"),) @@ -115,6 +100,14 @@ class Region(TimestampedModel, table=True): "(filter_field IS NOT NULL AND filter_value IS NOT NULL)", name="ck_regions_required_filter_values", ), + # SQLModel does not expose table-level composite foreign keys. This + # keeps a seeded region and its one default dataset in the same model. + sa.ForeignKeyConstraint( + ["default_dataset_id", "tax_benefit_model_id"], + ["datasets.id", "datasets.tax_benefit_model_id"], + name="fk_regions_default_dataset_model_datasets", + ondelete="RESTRICT", + ), ) code: str = Field(max_length=255) @@ -132,11 +125,12 @@ class Region(TimestampedModel, table=True): ondelete="RESTRICT", index=True, ) + default_dataset_id: UUID = Field(index=True) tax_benefit_model: TaxBenefitModel = Relationship(back_populates="regions") - datasets: list["Dataset"] = Relationship( - back_populates="regions", - link_model=RegionDatasetLink, + default_dataset: "Dataset" = Relationship( + back_populates="default_for_regions", + sa_relationship_kwargs={"overlaps": "regions,tax_benefit_model"}, ) simulations: list["Simulation"] = Relationship(back_populates="region") reports: list["Report"] = Relationship(back_populates="region") @@ -148,19 +142,26 @@ class Dataset(TimestampedModel, table=True): sa.UniqueConstraint( "tax_benefit_model_id", "name", - "year", - "is_output_dataset", - name="uq_datasets_model_name_year_output", + name="uq_datasets_model_name", + ), + sa.UniqueConstraint( + "id", + "tax_benefit_model_id", + name="uq_datasets_id_model", ), sa.CheckConstraint( "year BETWEEN 1900 AND 2200", name="ck_datasets_year", ), + sa.CheckConstraint( + "NOT is_output_dataset OR storage_path IS NOT NULL", + name="ck_datasets_output_storage_path", + ), ) name: str = Field(max_length=255) description: str | None = None - storage_path: str = Field(max_length=1024) + storage_path: str | None = Field(default=None, max_length=1024) year: int is_output_dataset: bool = False tax_benefit_model_id: UUID = Field( @@ -174,9 +175,9 @@ class Dataset(TimestampedModel, table=True): back_populates="dataset", cascade_delete=True, ) - regions: list[Region] = Relationship( - back_populates="datasets", - link_model=RegionDatasetLink, + default_for_regions: list[Region] = Relationship( + back_populates="default_dataset", + sa_relationship_kwargs={"overlaps": "regions,tax_benefit_model"}, ) input_simulations: list["Simulation"] = Relationship( back_populates="dataset", diff --git a/policyengine_api/data/v2/table_inventory.py b/policyengine_api/data/v2/table_inventory.py index f88340e3f..ddc80c08f 100644 --- a/policyengine_api/data/v2/table_inventory.py +++ b/policyengine_api/data/v2/table_inventory.py @@ -38,7 +38,6 @@ "regions", "datasets", "dataset_versions", - "region_datasets", } ), ), diff --git a/tests/integration/test_alembic_v2_lifecycle.py b/tests/integration/test_alembic_v2_lifecycle.py index 0eaff8dbb..a1fe26879 100644 --- a/tests/integration/test_alembic_v2_lifecycle.py +++ b/tests/integration/test_alembic_v2_lifecycle.py @@ -24,8 +24,9 @@ BASELINE_REVISION = "47592781336f" -PREVIOUS_HEAD_REVISION = "5f048586d8f1" -HEAD_REVISION = "4faee127fa16" +REPORT_UUID_PREVIOUS_REVISION = "5f048586d8f1" +REGION_DEFAULT_PREVIOUS_REVISION = "4faee127fa16" +HEAD_REVISION = "56dcd15a3afd" def _disposable_url() -> str: @@ -92,9 +93,19 @@ def test_empty_upgrade_check_boundary_downgrade_and_reupgrade() -> None: boundary_drift = compare_metadata(context, V2_METADATA) boundary_kinds = [difference[0] for difference in boundary_drift] assert boundary_kinds.count("v2_reference_row_change") == 2 - assert boundary_kinds.count("add_fk") == 4 - assert boundary_kinds.count("add_column") == 1 - assert boundary_kinds.count("add_constraint") == 1 + assert boundary_kinds.count("remove_table") == 1 + assert boundary_kinds.count("remove_constraint") == 1 + assert boundary_kinds.count("add_fk") == 5 + assert boundary_kinds.count("add_column") == 2 + assert boundary_kinds.count("add_index") == 1 + assert boundary_kinds.count("add_constraint") == 4 + assert ( + sum( + isinstance(kind, tuple) and kind[0] == "modify_nullable" + for kind in boundary_kinds + ) + == 1 + ) assert ( sum( isinstance(kind, tuple) and kind[0] == "modify_type" @@ -102,7 +113,7 @@ def test_empty_upgrade_check_boundary_downgrade_and_reupgrade() -> None: ) == 1 ) - assert len(boundary_kinds) == 9 + assert len(boundary_kinds) == 18 model_count = connection.execute( text( "SELECT count(*) FROM public.tax_benefit_models " @@ -125,7 +136,7 @@ def test_upgrade_to_head_validates_the_resulting_table_inventory() -> None: engine = create_engine(database_url) try: - command.downgrade(config, PREVIOUS_HEAD_REVISION) + command.downgrade(config, REGION_DEFAULT_PREVIOUS_REVISION) with engine.begin() as connection: connection.execute(text("CREATE TABLE unreviewed_runtime_table (id INT)")) @@ -137,7 +148,7 @@ def test_upgrade_to_head_validates_the_resulting_table_inventory() -> None: with engine.connect() as connection: context = MigrationContext.configure(connection) - assert context.get_current_revision() == PREVIOUS_HEAD_REVISION + assert context.get_current_revision() == REGION_DEFAULT_PREVIOUS_REVISION finally: with engine.begin() as connection: connection.execute(text("DROP TABLE IF EXISTS unreviewed_runtime_table")) @@ -168,7 +179,7 @@ def report_run_checks() -> set[str]: assert isinstance(idempotency_column_type(), PostgresUUID) assert "ck_report_runs_idempotency_key_nonblank" not in report_run_checks() - command.downgrade(config, PREVIOUS_HEAD_REVISION) + command.downgrade(config, REPORT_UUID_PREVIOUS_REVISION) assert isinstance(idempotency_column_type(), sa.String) assert "ck_report_runs_idempotency_key_nonblank" in report_run_checks() @@ -215,7 +226,7 @@ def report_run_checks() -> set[str]: assert stored_key == request_key assert isinstance(stored_key, UUID) - command.downgrade(config, PREVIOUS_HEAD_REVISION) + command.downgrade(config, REPORT_UUID_PREVIOUS_REVISION) with engine.connect() as connection: stored_key = connection.execute( text("SELECT idempotency_key FROM report_runs WHERE id = :run_id"), @@ -225,3 +236,130 @@ def report_run_checks() -> set[str]: finally: command.upgrade(config, "head") engine.dispose() + + +def test_region_default_revision_downgrades_reupgrades_and_enforces_model() -> None: + database_url = _disposable_url() + config = _config() + engine = create_engine(database_url) + first_model_id = uuid4() + second_model_id = uuid4() + first_dataset_id = uuid4() + second_dataset_id = uuid4() + + try: + command.upgrade(config, "head") + command.downgrade(config, REGION_DEFAULT_PREVIOUS_REVISION) + assert "region_datasets" in inspect(engine).get_table_names(schema="public") + assert "default_dataset_id" not in { + column["name"] for column in inspect(engine).get_columns("regions") + } + + command.upgrade(config, "head") + assert "region_datasets" not in inspect(engine).get_table_names(schema="public") + default_column = next( + column + for column in inspect(engine).get_columns("regions") + if column["name"] == "default_dataset_id" + ) + assert not default_column["nullable"] + default_constraint = next( + constraint + for constraint in inspect(engine).get_foreign_keys("regions") + if constraint["name"] == "fk_regions_default_dataset_model_datasets" + ) + assert default_constraint["constrained_columns"] == [ + "default_dataset_id", + "tax_benefit_model_id", + ] + assert default_constraint["referred_columns"] == [ + "id", + "tax_benefit_model_id", + ] + + with engine.begin() as connection: + connection.execute( + text( + "INSERT INTO tax_benefit_models (id, name) " + "VALUES (:first_id, :first_name), (:second_id, :second_name)" + ), + { + "first_id": first_model_id, + "first_name": f"region-default-{first_model_id.hex[:8]}", + "second_id": second_model_id, + "second_name": f"region-default-{second_model_id.hex[:8]}", + }, + ) + connection.execute( + text( + "INSERT INTO datasets " + "(id, name, year, is_output_dataset, tax_benefit_model_id) " + "VALUES (:first_id, 'logical-input', 2024, false, :first_model), " + "(:second_id, 'logical-input', 2024, false, :second_model)" + ), + { + "first_id": first_dataset_id, + "first_model": first_model_id, + "second_id": second_dataset_id, + "second_model": second_model_id, + }, + ) + connection.execute( + text( + "INSERT INTO regions " + "(id, code, label, region_type, requires_filter, " + "tax_benefit_model_id, default_dataset_id) " + "VALUES (:id, 'us', 'United States', 'national', false, " + ":model_id, :dataset_id)" + ), + { + "id": uuid4(), + "model_id": first_model_id, + "dataset_id": first_dataset_id, + }, + ) + + with pytest.raises(sa.exc.IntegrityError): + with engine.begin() as connection: + connection.execute( + text( + "INSERT INTO regions " + "(id, code, label, region_type, requires_filter, " + "tax_benefit_model_id, default_dataset_id) " + "VALUES (:id, 'state/ca', 'California', 'state', false, " + ":model_id, :dataset_id)" + ), + { + "id": uuid4(), + "model_id": first_model_id, + "dataset_id": second_dataset_id, + }, + ) + + with pytest.raises(sa.exc.IntegrityError): + with engine.begin() as connection: + connection.execute( + text( + "INSERT INTO datasets " + "(id, name, year, is_output_dataset, " + "tax_benefit_model_id) " + "VALUES (:id, 'missing-output-path', 2024, true, :model_id)" + ), + {"id": uuid4(), "model_id": first_model_id}, + ) + finally: + with engine.begin() as connection: + connection.execute( + text("DELETE FROM regions WHERE tax_benefit_model_id IN (:a, :b)"), + {"a": first_model_id, "b": second_model_id}, + ) + connection.execute( + text("DELETE FROM datasets WHERE tax_benefit_model_id IN (:a, :b)"), + {"a": first_model_id, "b": second_model_id}, + ) + connection.execute( + text("DELETE FROM tax_benefit_models WHERE id IN (:a, :b)"), + {"a": first_model_id, "b": second_model_id}, + ) + command.upgrade(config, "head") + engine.dispose() diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index 2d0ea2f88..fc4c412a5 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -212,8 +212,9 @@ def test_v2_files_are_mechanically_separate_from_v1() -> None: def test_v2_revision_chain_is_linear_generated_and_correction_bounded() -> None: config = Config(str(REPO / "alembic-v2.ini")) script = ScriptDirectory.from_config(config) - assert script.get_heads() == ["4faee127fa16"] + assert script.get_heads() == ["56dcd15a3afd"] assert [revision.revision for revision in script.walk_revisions()] == [ + "56dcd15a3afd", "4faee127fa16", "5f048586d8f1", "b4c69674dd47", @@ -241,10 +242,23 @@ def test_v2_revision_chain_is_linear_generated_and_correction_bounded() -> None: REPO / "migrations/v2/versions/" "4faee127fa16_use_native_uuid_report_run_idempotency_.py" ).read_text(encoding="utf-8") - revisions = baseline + data + ownership + constraints + native_uuid + region_defaults = ( + REPO / "migrations/v2/versions/" + "56dcd15a3afd_assign_one_default_dataset_per_region.py" + ).read_text(encoding="utf-8") + revisions = ( + baseline + data + ownership + constraints + native_uuid + region_defaults + ) assert all( "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" in source - for source in (baseline, data, ownership, constraints, native_uuid) + for source in ( + baseline, + data, + ownership, + constraints, + native_uuid, + region_defaults, + ) ) assert "op.execute(" not in revisions assert "op.bulk_insert(" not in revisions @@ -265,6 +279,13 @@ def test_v2_revision_chain_is_linear_generated_and_correction_bounded() -> None: assert native_uuid.index("op.drop_constraint(") < native_uuid.index( "op.alter_column(" ) + assert 'op.drop_table("region_datasets")' in region_defaults + assert 'sa.Column("default_dataset_id", sa.Uuid(), nullable=False)' in ( + region_defaults + ) + assert "fk_regions_default_dataset_model_datasets" in region_defaults + assert "uq_datasets_model_name" in region_defaults + assert "ck_datasets_output_storage_path" in region_defaults corrected_enum_names = set( re.findall( diff --git a/tests/unit/v2/test_import_side_effects.py b/tests/unit/v2/test_import_side_effects.py index 947d1e324..33e5047b5 100644 --- a/tests/unit/v2/test_import_side_effects.py +++ b/tests/unit/v2/test_import_side_effects.py @@ -65,7 +65,7 @@ def reject_connect(*args, **kwargs): from policyengine_api.data.v2.models import V2_METADATA after = set(pathlib.Path.cwd().iterdir()) assert before == after -assert len(V2_METADATA.tables) == 33 +assert len(V2_METADATA.tables) == 32 """ result = subprocess.run( diff --git a/tests/unit/v2/test_models.py b/tests/unit/v2/test_models.py index 3c61d5e50..01f430a70 100644 --- a/tests/unit/v2/test_models.py +++ b/tests/unit/v2/test_models.py @@ -191,6 +191,51 @@ def test_user_primary_country_is_required_and_limited_to_supported_values() -> N } +def test_regions_have_one_same_model_default_logical_dataset() -> None: + regions = V2_METADATA.tables["regions"] + datasets = V2_METADATA.tables["datasets"] + + assert "region_datasets" not in V2_METADATA.tables + assert not regions.c.default_dataset_id.nullable + assert regions.c.default_dataset_id.index + default_constraint = next( + constraint + for constraint in regions.foreign_key_constraints + if constraint.name == "fk_regions_default_dataset_model_datasets" + ) + assert [element.parent.name for element in default_constraint.elements] == [ + "default_dataset_id", + "tax_benefit_model_id", + ] + assert [element.target_fullname for element in default_constraint.elements] == [ + "datasets.id", + "datasets.tax_benefit_model_id", + ] + assert default_constraint.ondelete == "RESTRICT" + + unique_column_sets = { + tuple(column.name for column in constraint.columns) + for constraint in datasets.constraints + if isinstance(constraint, sa.UniqueConstraint) + } + assert ("tax_benefit_model_id", "name") in unique_column_sets + assert ("id", "tax_benefit_model_id") in unique_column_sets + assert datasets.c.storage_path.nullable + assert "ck_datasets_output_storage_path" in { + constraint.name for constraint in datasets.constraints + } + + +def test_reports_and_simulations_snapshot_selected_datasets() -> None: + reports = V2_METADATA.tables["reports"] + simulations = V2_METADATA.tables["simulations"] + + for table in (reports, simulations): + dataset_foreign_key = next(iter(table.c.dataset_id.foreign_keys)) + assert dataset_foreign_key.target_fullname == "datasets.id" + assert dataset_foreign_key.ondelete in {"RESTRICT", "SET NULL"} + + def test_run_outputs_reference_report_runs_not_base_reports() -> None: for table_name in RUN_OUTPUT_TABLES: table = V2_METADATA.tables[table_name] diff --git a/tests/unit/v2/test_table_inventory.py b/tests/unit/v2/test_table_inventory.py index 55666ae27..6f82afc47 100644 --- a/tests/unit/v2/test_table_inventory.py +++ b/tests/unit/v2/test_table_inventory.py @@ -21,6 +21,7 @@ def test_reviewed_table_groups_are_disjoint_and_complete() -> None: assert frozenset(grouped_tables) == EXPECTED_V2_TABLES assert "reports" in EXPECTED_V2_TABLES assert "report_runs" in EXPECTED_V2_TABLES + assert "region_datasets" not in EXPECTED_V2_TABLES assert EXPECTED_V2_TABLES.isdisjoint(PROHIBITED_V2_TABLES) assert EXPECTED_V2_TABLES.isdisjoint(V1_ONLY_TABLES) From 7619d5eb2a195f8516f30c336ea63afc589f8c67 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 18 Aug 2026 22:36:52 +0300 Subject: [PATCH 10/18] Remove v2 reference data registry --- .github/scripts/test_alembic_v2_lifecycle.sh | 1 - docs/engineering/skills/alembic-migrations.md | 41 +-- docs/engineering/skills/testing.md | 6 +- migrations/v2/env.py | 6 +- ...e7fe26bb_remove_stage_8_validation_data.py | 83 +++++ .../historical_reference_data_operations.py | 192 ++++++++++ policyengine_api/data/v2/reference_data.py | 144 -------- .../data/v2/reference_data_autogenerate.py | 344 ------------------ .../integration/test_alembic_v2_lifecycle.py | 43 ++- tests/unit/test_alembic_workflows.py | 1 - tests/unit/v2/test_alembic_v2.py | 26 +- .../v2/test_reference_data_autogenerate.py | 221 ----------- tests/unit/v2/test_scaffolding_hygiene.py | 1 - 13 files changed, 363 insertions(+), 746 deletions(-) create mode 100644 migrations/v2/versions/8b8ee7fe26bb_remove_stage_8_validation_data.py create mode 100644 policyengine_api/data/v2/historical_reference_data_operations.py delete mode 100644 policyengine_api/data/v2/reference_data.py delete mode 100644 policyengine_api/data/v2/reference_data_autogenerate.py delete mode 100644 tests/unit/v2/test_reference_data_autogenerate.py diff --git a/.github/scripts/test_alembic_v2_lifecycle.sh b/.github/scripts/test_alembic_v2_lifecycle.sh index 503bf87db..1a3d58b87 100755 --- a/.github/scripts/test_alembic_v2_lifecycle.sh +++ b/.github/scripts/test_alembic_v2_lifecycle.sh @@ -4,5 +4,4 @@ set -euo pipefail uv run pytest -q \ tests/unit/v2/test_alembic_v2.py \ - tests/unit/v2/test_reference_data_autogenerate.py \ tests/integration/test_alembic_v2_lifecycle.py diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index d899791d0..97c2160dc 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -36,10 +36,9 @@ those narrow review corrections, stop and request a human migration decision. Before committing a migration, run these checks against the matching domain: 1. Run `uv run alembic -c check` and review the generated - schema and declared-data operations. + schema operations. 2. Upgrade a fresh database to `head`. -3. Compare the database to the complete ORM metadata and, for v2, the declared - application-data source before release. +3. Compare the database to the complete ORM metadata before release. 4. Downgrade one revision and upgrade to `head` again in an isolated database. 5. Confirm application startup performs no implicit DDL. @@ -110,42 +109,42 @@ the reviewed inventory and fail before generation or execution if expected tables are absent or v1, predecessor, `runtime_bundles`, population, or other unreviewed tables are registered. -## Generated v2 application-data migrations +## V2 application-data migrations -Alembic is also the sole authority for versioned v2 application-data changes. -Small reference data belongs in a versioned declarative source with stable -natural identifiers and deterministic before-and-after values. The bounded v2 -autogeneration comparator and renderer compare that declaration with the -current target and emit ordered, reversible operations through the same -command used for schema revisions: +Alembic is also the sole authority for intentional migrations of retained v2 +application data. Every such revision must originate from the same mandatory +autogeneration command used for schema revisions: ```bash uv run alembic -c alembic-v2.ini revision --autogenerate -m "" ``` Do not create a blank revision and add `bulk_insert`, SQL strings, ORM calls, -or other data operations by hand. Correct the declaration or generator and -regenerate when output is wrong. Generated data additions run only after their -required schema exists, and generated removals run before destructive schema -operations. An unsafe identifier, unknown prior value, non-deterministic -ordering, or non-reversible change must fail generation and invoke the human -decision rule above. - -`alembic check` for v2 must detect both schema drift and declared-data drift. +or other data operations by hand. A concrete data-migration requirement must +provide a reviewed generation source capable of producing deterministic and +reversible operations. If safe autogeneration is unavailable, invoke the human +decision rule above instead of maintaining speculative migration machinery. + +Stage 8 has no active desired-state reference-data declaration or row +comparator. Historical revisions `6ee725e0c563` and `8b8ee7fe26bb` contain the +autogenerated insertion and removal of two synthetic validation rows. Their +minimal operation executor remains importable only so immutable migration +history can be replayed; the current v2 head contains neither validation row. + +`alembic check` for v2 must detect schema drift. Application startup, model import, project provisioning, and Supabase Storage bootstrap must never call `create_all`, create or stamp application tables, or mutate versioned application data. The Supabase CLI is not an application schema or seed migration authority. -Large canonical metadata catalogs derived from the exact installed country and +Canonical metadata catalogs derived from the exact installed country and `policyengine` packages are not hand-authored migration data. A later-stage, explicit deployment seeder may materialize those package-derived rows after `alembic upgrade head` when its reviewed contract requires transactional, idempotent row-only behavior, recorded source package versions, and fail-closed handling of partial catalogs. Such a seeder must perform no DDL, must not run at application startup, and must not transform or delete retained domain data; -those operations remain Alembic migrations. Small reviewed reference rows stay -in the declarative autogeneration workflow above. +those operations remain Alembic migrations. Migration credentials remain separate from future runtime credentials. The migration identity may create and alter the v2 application schema; the runtime diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index a6d6421ca..9717ca117 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -61,16 +61,16 @@ SQLModel schema, report/run behavior, lazy configuration, and import effects: uv run pytest tests/unit/v2/test_models.py tests/unit/v2/test_model_persistence.py tests/unit/v2/test_report_runs.py tests/unit/v2/test_settings.py tests/unit/v2/test_import_side_effects.py -q ``` -The generated-only v2 Alembic extension and disposable Postgres lifecycle: +The generated-only v2 Alembic chain and disposable Postgres lifecycle: ```bash -uv run pytest tests/unit/v2/test_alembic_v2.py tests/unit/v2/test_reference_data_autogenerate.py tests/unit/test_alembic_workflows.py -q +uv run pytest tests/unit/v2/test_alembic_v2.py tests/unit/test_alembic_workflows.py -q V2_ALEMBIC_DISPOSABLE_TEST=1 V2_MIGRATION_DATABASE_URL="postgresql+psycopg://.../policyengine_v2_alembic_test" \ uv run pytest tests/integration/test_alembic_v2_lifecycle.py -q ``` The disposable lifecycle must start from empty Postgres, upgrade to `head`, -check schema and declared-data drift, compare the live schema with the exact +check schema drift, compare the live schema with the exact SQLModel inventory, downgrade the reviewed boundary, and upgrade again. It must not use the persistent Supabase qualification bypass outside explicit disposable-test mode. Continue running the existing isolated v1 MySQL diff --git a/migrations/v2/env.py b/migrations/v2/env.py index d376699ba..84b0f38a4 100644 --- a/migrations/v2/env.py +++ b/migrations/v2/env.py @@ -6,15 +6,14 @@ from alembic.script import ScriptDirectory from sqlalchemy import create_engine, pool +# Registers the custom operation embedded in immutable historical revisions. +import policyengine_api.data.v2.historical_reference_data_operations # noqa: F401 from policyengine_api.data.v2.migration_target import ( load_v2_alembic_settings, qualify_v2_connection, validate_v2_head_table_inventory, ) from policyengine_api.data.v2.models import V2_METADATA -from policyengine_api.data.v2.reference_data_autogenerate import ( - order_generated_operations, -) from policyengine_api.data.v2.table_inventory import validate_v2_table_inventory @@ -45,7 +44,6 @@ def _configure(connection) -> None: include_object=_include_application_object, version_table="alembic_version", version_table_schema="public", - process_revision_directives=order_generated_operations, ) migration_context = context.get_context() previous_heads = frozenset(migration_context.get_current_heads()) diff --git a/migrations/v2/versions/8b8ee7fe26bb_remove_stage_8_validation_data.py b/migrations/v2/versions/8b8ee7fe26bb_remove_stage_8_validation_data.py new file mode 100644 index 000000000..f16723b93 --- /dev/null +++ b/migrations/v2/versions/8b8ee7fe26bb_remove_stage_8_validation_data.py @@ -0,0 +1,83 @@ +"""remove stage 8 validation data + +Revision ID: 8b8ee7fe26bb +Revises: 56dcd15a3afd +Create Date: 2026-08-18 22:30:18.333006 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = "8b8ee7fe26bb" +down_revision: Union[str, None] = "56dcd15a3afd" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.v2_reference_row_change( + "tax_benefit_model_versions", + key={ + "model_id": "80000000-0000-4000-8000-000000000001", + "version": "stage8-platform-validation", + }, + before={ + "created_at": "2026-08-14T00:00:00+00:00", + "description": "Stage 8 generated data-migration validation row.", + "id": "80000000-0000-4000-8000-000000000002", + "model_id": "80000000-0000-4000-8000-000000000001", + "version": "stage8-platform-validation", + }, + after=None, + ) + op.v2_reference_row_change( + "tax_benefit_models", + key={"name": "stage8-platform-validation"}, + before={ + "created_at": "2026-08-14T00:00:00+00:00", + "description": "Stage 8 migration lifecycle validation; canonical metadata is introduced in Stage 9.", + "id": "80000000-0000-4000-8000-000000000001", + "name": "stage8-platform-validation", + "updated_at": "2026-08-14T00:00:00+00:00", + }, + after=None, + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.v2_reference_row_change( + "tax_benefit_models", + key={"name": "stage8-platform-validation"}, + before=None, + after={ + "created_at": "2026-08-14T00:00:00+00:00", + "description": "Stage 8 migration lifecycle validation; canonical metadata is introduced in Stage 9.", + "id": "80000000-0000-4000-8000-000000000001", + "name": "stage8-platform-validation", + "updated_at": "2026-08-14T00:00:00+00:00", + }, + ) + op.v2_reference_row_change( + "tax_benefit_model_versions", + key={ + "model_id": "80000000-0000-4000-8000-000000000001", + "version": "stage8-platform-validation", + }, + before=None, + after={ + "created_at": "2026-08-14T00:00:00+00:00", + "description": "Stage 8 generated data-migration validation row.", + "id": "80000000-0000-4000-8000-000000000002", + "model_id": "80000000-0000-4000-8000-000000000001", + "version": "stage8-platform-validation", + }, + ) + # ### end Alembic commands ### diff --git a/policyengine_api/data/v2/historical_reference_data_operations.py b/policyengine_api/data/v2/historical_reference_data_operations.py new file mode 100644 index 000000000..02c207b18 --- /dev/null +++ b/policyengine_api/data/v2/historical_reference_data_operations.py @@ -0,0 +1,192 @@ +"""Execute row operations already embedded in immutable v2 Alembic revisions.""" + +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any +from uuid import UUID + +from alembic.operations import MigrateOperation, Operations +import sqlalchemy as sa + + +HISTORICAL_REFERENCE_TABLES = frozenset( + {"tax_benefit_models", "tax_benefit_model_versions"} +) + + +class HistoricalDataMigrationError(RuntimeError): + """Raised when an immutable row transition cannot be replayed safely.""" + + +def _ordered(values: dict[str, Any] | None) -> dict[str, Any] | None: + if values is None: + return None + return {key: values[key] for key in sorted(values)} + + +@Operations.register_operation("v2_reference_row_change") +class HistoricalReferenceRowChangeOp(MigrateOperation): + """One guarded row transition embedded in the historical v2 chain.""" + + def __init__( + self, + table_name: str, + *, + key: dict[str, Any], + before: dict[str, Any] | None, + after: dict[str, Any] | None, + ) -> None: + if table_name not in HISTORICAL_REFERENCE_TABLES: + raise HistoricalDataMigrationError( + f"historical data operation targets unreviewed table {table_name}" + ) + if not key or (before is None and after is None): + raise HistoricalDataMigrationError( + "historical data operations need a stable key and one row state" + ) + self.table_name = table_name + self.key = _ordered(key) or {} + self.before = _ordered(before) + self.after = _ordered(after) + + @classmethod + def v2_reference_row_change( + cls, + operations: Operations, + table_name: str, + *, + key: dict[str, Any], + before: dict[str, Any] | None, + after: dict[str, Any] | None, + ) -> Any: + return operations.invoke(cls(table_name, key=key, before=before, after=after)) + + +def _normalize(value: Any) -> Any: + if isinstance(value, UUID): + return str(value) + if isinstance(value, datetime): + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat() + if isinstance(value, dict): + return {key: _normalize(item) for key, item in sorted(value.items())} + if isinstance(value, list | tuple): + return [_normalize(item) for item in value] + return value + + +def _coerce(column: sa.Column, value: Any) -> Any: + if value is None: + return None + if isinstance(column.type, sa.Uuid) and not isinstance(value, UUID): + return UUID(str(value)) + if isinstance(column.type, sa.DateTime) and not isinstance(value, datetime): + parsed = datetime.fromisoformat(str(value)) + return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) + return value + + +def _predicate(table: sa.Table, key: dict[str, Any]) -> sa.ColumnElement: + return sa.and_( + *( + table.c[column] == _coerce(table.c[column], value) + for column, value in key.items() + ) + ) + + +def _current_row( + bind: sa.Connection, + table: sa.Table, + key: dict[str, Any], +) -> dict[str, Any] | None: + row = ( + bind.execute(sa.select(table).where(_predicate(table, key))) + .mappings() + .one_or_none() + ) + if row is None: + return None + return {column: _normalize(value) for column, value in row.items()} + + +def _assert_before( + current: dict[str, Any] | None, + expected: dict[str, Any] | None, + *, + table_name: str, +) -> None: + if expected is None: + if current is not None: + raise HistoricalDataMigrationError( + f"{table_name} insert found an existing historical row" + ) + return + if current is None or any( + current.get(column) != _normalize(value) for column, value in expected.items() + ): + raise HistoricalDataMigrationError( + f"{table_name} row differs from the historical before state" + ) + + +def _assert_after( + current: dict[str, Any] | None, + expected: dict[str, Any] | None, + *, + table_name: str, +) -> None: + if expected is None: + if current is not None: + raise HistoricalDataMigrationError( + f"{table_name} historical delete left the row present" + ) + return + if current is None or any( + current.get(column) != _normalize(value) for column, value in expected.items() + ): + raise HistoricalDataMigrationError( + f"{table_name} row differs from the historical after state" + ) + + +@Operations.implementation_for(HistoricalReferenceRowChangeOp) +def _apply_historical_reference_row_change( + operations: Operations, + operation: HistoricalReferenceRowChangeOp, +) -> None: + bind = operations.get_bind() + table = sa.Table( + operation.table_name, + sa.MetaData(), + schema="public", + autoload_with=bind, + ) + current = _current_row(bind, table, operation.key) + _assert_before(current, operation.before, table_name=operation.table_name) + + if operation.after is None: + bind.execute(table.delete().where(_predicate(table, operation.key))) + elif operation.before is None: + values = { + column: _coerce(table.c[column], value) + for column, value in operation.after.items() + } + bind.execute(table.insert().values(**values)) + else: + values = { + column: _coerce(table.c[column], value) + for column, value in operation.after.items() + if column not in operation.key + } + bind.execute( + table.update().where(_predicate(table, operation.key)).values(**values) + ) + + _assert_after( + _current_row(bind, table, operation.key), + operation.after, + table_name=operation.table_name, + ) diff --git a/policyengine_api/data/v2/reference_data.py b/policyengine_api/data/v2/reference_data.py deleted file mode 100644 index 47b237c01..000000000 --- a/policyengine_api/data/v2/reference_data.py +++ /dev/null @@ -1,144 +0,0 @@ -"""Versioned declarative Stage 8 application data for Alembic autogeneration.""" - -from __future__ import annotations - -from dataclasses import dataclass -from types import MappingProxyType -from typing import Any - -from policyengine_api.data.v2.table_inventory import EXPECTED_V2_TABLES - - -REFERENCE_DATA_FORMAT_VERSION = 1 -VALIDATION_MODEL_ID = "80000000-0000-4000-8000-000000000001" -VALIDATION_MODEL_VERSION_ID = "80000000-0000-4000-8000-000000000002" -VALIDATION_TIMESTAMP = "2026-08-14T00:00:00+00:00" - - -class ReferenceDataDeclarationError(ValueError): - """Raised when declarative data is not safe for deterministic generation.""" - - -@dataclass(frozen=True) -class ReferenceRow: - key: MappingProxyType - values: MappingProxyType - - @classmethod - def create(cls, *, key: dict[str, Any], values: dict[str, Any]) -> "ReferenceRow": - if not key or any(value is None for value in key.values()): - raise ReferenceDataDeclarationError( - "reference rows require non-null stable key values" - ) - overlap = set(key) & set(values) - if overlap: - raise ReferenceDataDeclarationError( - f"reference row key/value columns overlap: {sorted(overlap)}" - ) - _validate_wire_values({**key, **values}) - return cls(MappingProxyType(dict(key)), MappingProxyType(dict(values))) - - @property - def complete_values(self) -> dict[str, Any]: - return {**self.key, **self.values} - - -@dataclass(frozen=True) -class ReferenceTable: - table_name: str - key_columns: tuple[str, ...] - managed_prefix_column: str - managed_prefix: str - rows: tuple[ReferenceRow, ...] - - def __post_init__(self) -> None: - if self.table_name not in EXPECTED_V2_TABLES: - raise ReferenceDataDeclarationError( - f"unreviewed reference-data table: {self.table_name}" - ) - if not self.key_columns or self.managed_prefix_column not in self.key_columns: - raise ReferenceDataDeclarationError( - "managed prefix column must be part of the stable key" - ) - seen_keys: set[tuple[Any, ...]] = set() - for row in self.rows: - if tuple(row.key) != self.key_columns: - raise ReferenceDataDeclarationError( - f"{self.table_name} row key columns must be {self.key_columns}" - ) - prefix_value = row.key[self.managed_prefix_column] - if not isinstance(prefix_value, str) or not prefix_value.startswith( - self.managed_prefix - ): - raise ReferenceDataDeclarationError( - f"{self.table_name} managed key is outside its declared scope" - ) - stable_key = tuple(row.key[column] for column in self.key_columns) - if stable_key in seen_keys: - raise ReferenceDataDeclarationError( - f"duplicate reference-data key for {self.table_name}: {stable_key}" - ) - seen_keys.add(stable_key) - - -def _validate_wire_values(values: Any) -> None: - if values is None or isinstance(values, str | int | float | bool): - return - if isinstance(values, list | tuple): - for value in values: - _validate_wire_values(value) - return - if isinstance(values, dict): - if not all(isinstance(key, str) for key in values): - raise ReferenceDataDeclarationError( - "reference-data object keys must be strings" - ) - for value in values.values(): - _validate_wire_values(value) - return - raise ReferenceDataDeclarationError( - f"unsupported reference-data value type: {type(values).__name__}" - ) - - -REFERENCE_DATA = ( - ReferenceTable( - table_name="tax_benefit_models", - key_columns=("name",), - managed_prefix_column="name", - managed_prefix="stage8-", - rows=( - ReferenceRow.create( - key={"name": "stage8-platform-validation"}, - values={ - "id": VALIDATION_MODEL_ID, - "description": ( - "Stage 8 migration lifecycle validation; canonical " - "metadata is introduced in Stage 9." - ), - "created_at": VALIDATION_TIMESTAMP, - "updated_at": VALIDATION_TIMESTAMP, - }, - ), - ), - ), - ReferenceTable( - table_name="tax_benefit_model_versions", - key_columns=("model_id", "version"), - managed_prefix_column="version", - managed_prefix="stage8-", - rows=( - ReferenceRow.create( - key={ - "model_id": VALIDATION_MODEL_ID, - "version": "stage8-platform-validation", - }, - values={ - "id": VALIDATION_MODEL_VERSION_ID, - "description": ("Stage 8 generated data-migration validation row."), - "created_at": VALIDATION_TIMESTAMP, - }, - ), - ), - ), -) diff --git a/policyengine_api/data/v2/reference_data_autogenerate.py b/policyengine_api/data/v2/reference_data_autogenerate.py deleted file mode 100644 index a3efb0676..000000000 --- a/policyengine_api/data/v2/reference_data_autogenerate.py +++ /dev/null @@ -1,344 +0,0 @@ -"""Bounded Alembic operations and comparators for declared v2 reference rows.""" - -from __future__ import annotations - -from datetime import datetime, timezone -from typing import Any -from uuid import UUID - -from alembic.autogenerate import comparators, renderers -from alembic.operations import MigrateOperation, Operations, ops -import sqlalchemy as sa - -from policyengine_api.data.v2.reference_data import REFERENCE_DATA, ReferenceTable -from policyengine_api.data.v2.table_inventory import EXPECTED_V2_TABLES - - -class ReferenceDataMigrationError(RuntimeError): - """Raised when a declared row change cannot be applied or reversed safely.""" - - -def _ordered(values: dict[str, Any] | None) -> dict[str, Any] | None: - if values is None: - return None - return {key: values[key] for key in sorted(values)} - - -@Operations.register_operation("v2_reference_row_change") -class ReferenceRowChangeOp(MigrateOperation): - """One deterministic and reversible declared-row transition.""" - - def __init__( - self, - table_name: str, - *, - key: dict[str, Any], - before: dict[str, Any] | None, - after: dict[str, Any] | None, - ) -> None: - if table_name not in EXPECTED_V2_TABLES: - raise ReferenceDataMigrationError( - f"reference-data operation targets unreviewed table {table_name}" - ) - if not key or (before is None and after is None): - raise ReferenceDataMigrationError( - "reference-data operations need a stable key and one row state" - ) - self.table_name = table_name - self.key = _ordered(key) or {} - self.before = _ordered(before) - self.after = _ordered(after) - - @classmethod - def v2_reference_row_change( - cls, - operations: Operations, - table_name: str, - *, - key: dict[str, Any], - before: dict[str, Any] | None, - after: dict[str, Any] | None, - ) -> Any: - return operations.invoke(cls(table_name, key=key, before=before, after=after)) - - def reverse(self) -> "ReferenceRowChangeOp": - return ReferenceRowChangeOp( - self.table_name, - key=self.key, - before=self.after, - after=self.before, - ) - - def to_diff_tuple(self) -> tuple[Any, ...]: - """Expose deterministic drift details to ``alembic check``.""" - - return ( - "v2_reference_row_change", - self.table_name, - self.key, - self.before, - self.after, - ) - - -def _normalize(value: Any) -> Any: - if isinstance(value, UUID): - return str(value) - if isinstance(value, datetime): - if value.tzinfo is None: - value = value.replace(tzinfo=timezone.utc) - return value.astimezone(timezone.utc).isoformat() - if isinstance(value, dict): - return {key: _normalize(item) for key, item in sorted(value.items())} - if isinstance(value, list | tuple): - return [_normalize(item) for item in value] - return value - - -def _coerce(column: sa.Column, value: Any) -> Any: - if value is None: - return None - if isinstance(column.type, sa.Uuid) and not isinstance(value, UUID): - return UUID(str(value)) - if isinstance(column.type, sa.DateTime) and not isinstance(value, datetime): - parsed = datetime.fromisoformat(str(value)) - return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) - return value - - -def _predicate(table: sa.Table, key: dict[str, Any]) -> sa.ColumnElement: - return sa.and_( - *( - table.c[column] == _coerce(table.c[column], value) - for column, value in key.items() - ) - ) - - -def _current_row( - bind: sa.Connection, - table: sa.Table, - key: dict[str, Any], -) -> dict[str, Any] | None: - row = ( - bind.execute(sa.select(table).where(_predicate(table, key))) - .mappings() - .one_or_none() - ) - if row is None: - return None - return {column: _normalize(value) for column, value in row.items()} - - -def _assert_before( - current: dict[str, Any] | None, - expected: dict[str, Any] | None, - *, - table_name: str, -) -> None: - if expected is None: - if current is not None: - raise ReferenceDataMigrationError( - f"{table_name} insert found an existing managed row" - ) - return - if current is None or any( - current.get(column) != _normalize(value) for column, value in expected.items() - ): - raise ReferenceDataMigrationError( - f"{table_name} row differs from the generated before state" - ) - - -def _assert_after( - current: dict[str, Any] | None, - expected: dict[str, Any] | None, - *, - table_name: str, -) -> None: - if expected is None: - if current is not None: - raise ReferenceDataMigrationError( - f"{table_name} generated delete left the managed row present" - ) - return - if current is None or any( - current.get(column) != _normalize(value) for column, value in expected.items() - ): - raise ReferenceDataMigrationError( - f"{table_name} row differs from the generated after state" - ) - - -@Operations.implementation_for(ReferenceRowChangeOp) -def _apply_reference_row_change( - operations: Operations, - operation: ReferenceRowChangeOp, -) -> None: - bind = operations.get_bind() - table = sa.Table( - operation.table_name, - sa.MetaData(), - schema="public", - autoload_with=bind, - ) - current = _current_row(bind, table, operation.key) - _assert_before(current, operation.before, table_name=operation.table_name) - - if operation.after is None: - bind.execute(table.delete().where(_predicate(table, operation.key))) - elif operation.before is None: - values = { - column: _coerce(table.c[column], value) - for column, value in operation.after.items() - } - bind.execute(table.insert().values(**values)) - else: - values = { - column: _coerce(table.c[column], value) - for column, value in operation.after.items() - if column not in operation.key - } - bind.execute( - table.update().where(_predicate(table, operation.key)).values(**values) - ) - _assert_after( - _current_row(bind, table, operation.key), - operation.after, - table_name=operation.table_name, - ) - - -@renderers.dispatch_for(ReferenceRowChangeOp) -def _render_reference_row_change( - autogen_context, operation: ReferenceRowChangeOp -) -> str: - return ( - "op.v2_reference_row_change(" - f"{operation.table_name!r}, key={operation.key!r}, " - f"before={operation.before!r}, after={operation.after!r})" - ) - - -def _managed_rows( - connection: sa.Connection, - declaration: ReferenceTable, -) -> dict[tuple[Any, ...], dict[str, Any]]: - table = sa.Table( - declaration.table_name, - sa.MetaData(), - schema="public", - autoload_with=connection, - ) - prefix_column = table.c[declaration.managed_prefix_column] - rows = connection.execute( - sa.select(table).where(prefix_column.startswith(declaration.managed_prefix)) - ).mappings() - return { - tuple(_normalize(row[column]) for column in declaration.key_columns): { - column: _normalize(value) for column, value in row.items() - } - for row in rows - } - - -def _table_differences( - connection: sa.Connection, - declaration: ReferenceTable, -) -> tuple[list[ReferenceRowChangeOp], list[ReferenceRowChangeOp]]: - live = _managed_rows(connection, declaration) - desired = { - tuple(row.key[column] for column in declaration.key_columns): row - for row in declaration.rows - } - removals: list[ReferenceRowChangeOp] = [] - upserts: list[ReferenceRowChangeOp] = [] - - for stable_key in sorted(live): - if stable_key not in desired: - key = dict(zip(declaration.key_columns, stable_key)) - removals.append( - ReferenceRowChangeOp( - declaration.table_name, - key=key, - before=live[stable_key], - after=None, - ) - ) - - for stable_key in sorted(desired): - row = desired[stable_key] - desired_values = row.complete_values - current = live.get(stable_key) - if current is None: - upserts.append( - ReferenceRowChangeOp( - declaration.table_name, - key=dict(row.key), - before=None, - after=desired_values, - ) - ) - continue - tracked_current = {column: current.get(column) for column in desired_values} - if tracked_current != desired_values: - upserts.append( - ReferenceRowChangeOp( - declaration.table_name, - key=dict(row.key), - before=tracked_current, - after=desired_values, - ) - ) - return removals, upserts - - -@comparators.dispatch_for("schema") -def compare_reference_data(autogen_context, upgrade_ops, _schemas) -> None: - """Append declared row drift after the complete schema already exists.""" - - connection = autogen_context.connection - if connection is None: - return - live_tables = set(sa.inspect(connection).get_table_names(schema="public")) - # Keep the clean schema baseline separate. The next autogeneration, after - # baseline upgrade, observes all tables and emits the data-only revision. - if not EXPECTED_V2_TABLES.issubset(live_tables): - return - - differences = [ - _table_differences(connection, declaration) for declaration in REFERENCE_DATA - ] - # Delete children before parents; insert/update parents before children. - removals = [ - operation - for table_removals, _ in reversed(differences) - for operation in table_removals - ] - upserts = [ - operation for _, table_upserts in differences for operation in table_upserts - ] - upgrade_ops.ops.extend([*removals, *upserts]) - - -def _is_destructive(operation: MigrateOperation) -> bool: - if isinstance(operation, (ops.DropTableOp, ops.DropColumnOp)): - return True - if isinstance(operation, ops.ModifyTableOps): - return any(_is_destructive(child) for child in operation.ops) - return False - - -def order_generated_operations(_context, _revision, directives) -> None: - """Place data changes after schema additions and before destructive DDL.""" - - for script in directives: - operations = script.upgrade_ops.ops - data = [op for op in operations if isinstance(op, ReferenceRowChangeOp)] - non_data = [op for op in operations if not isinstance(op, ReferenceRowChangeOp)] - constructive = [op for op in non_data if not _is_destructive(op)] - destructive = [op for op in non_data if _is_destructive(op)] - script.upgrade_ops.ops = [*constructive, *data, *destructive] - script.downgrade_ops.ops = [ - operation.reverse() for operation in reversed(script.upgrade_ops.ops) - ] diff --git a/tests/integration/test_alembic_v2_lifecycle.py b/tests/integration/test_alembic_v2_lifecycle.py index a1fe26879..460382ad4 100644 --- a/tests/integration/test_alembic_v2_lifecycle.py +++ b/tests/integration/test_alembic_v2_lifecycle.py @@ -26,7 +26,8 @@ BASELINE_REVISION = "47592781336f" REPORT_UUID_PREVIOUS_REVISION = "5f048586d8f1" REGION_DEFAULT_PREVIOUS_REVISION = "4faee127fa16" -HEAD_REVISION = "56dcd15a3afd" +VALIDATION_DATA_REVISION = "56dcd15a3afd" +HEAD_REVISION = "8b8ee7fe26bb" def _disposable_url() -> str: @@ -68,7 +69,7 @@ def _assert_head(engine) -> None: "WHERE version = 'stage8-platform-validation'" ) ).scalar_one() - assert (model_count, version_count) == (1, 1) + assert (model_count, version_count) == (0, 0) def test_empty_upgrade_check_boundary_downgrade_and_reupgrade() -> None: @@ -92,7 +93,6 @@ def test_empty_upgrade_check_boundary_downgrade_and_reupgrade() -> None: assert context.get_current_revision() == BASELINE_REVISION boundary_drift = compare_metadata(context, V2_METADATA) boundary_kinds = [difference[0] for difference in boundary_drift] - assert boundary_kinds.count("v2_reference_row_change") == 2 assert boundary_kinds.count("remove_table") == 1 assert boundary_kinds.count("remove_constraint") == 1 assert boundary_kinds.count("add_fk") == 5 @@ -113,7 +113,7 @@ def test_empty_upgrade_check_boundary_downgrade_and_reupgrade() -> None: ) == 1 ) - assert len(boundary_kinds) == 18 + assert len(boundary_kinds) == 16 model_count = connection.execute( text( "SELECT count(*) FROM public.tax_benefit_models " @@ -130,6 +130,41 @@ def test_empty_upgrade_check_boundary_downgrade_and_reupgrade() -> None: engine.dispose() +def test_validation_cleanup_downgrades_and_reupgrades() -> None: + database_url = _disposable_url() + config = _config() + engine = create_engine(database_url) + + def validation_counts() -> tuple[int, int]: + with engine.connect() as connection: + model_count = connection.execute( + text( + "SELECT count(*) FROM public.tax_benefit_models " + "WHERE name = 'stage8-platform-validation'" + ) + ).scalar_one() + version_count = connection.execute( + text( + "SELECT count(*) FROM public.tax_benefit_model_versions " + "WHERE version = 'stage8-platform-validation'" + ) + ).scalar_one() + return model_count, version_count + + try: + command.upgrade(config, "head") + assert validation_counts() == (0, 0) + + command.downgrade(config, VALIDATION_DATA_REVISION) + assert validation_counts() == (1, 1) + + command.upgrade(config, "head") + assert validation_counts() == (0, 0) + finally: + command.upgrade(config, "head") + engine.dispose() + + def test_upgrade_to_head_validates_the_resulting_table_inventory() -> None: database_url = _disposable_url() config = _config() diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index 2069b9236..d077934e2 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -130,7 +130,6 @@ def test_reusable_v2_check_uses_disposable_postgres_and_real_redis(): assert "alembic-v2.ini" in workflow assert "bash .github/scripts/test_alembic_v2_lifecycle.sh" in workflow assert "test_alembic_v2.py" in lifecycle_script - assert "test_reference_data_autogenerate.py" in lifecycle_script assert "test_alembic_v2_lifecycle.py" in lifecycle_script assert "test_runtime_cache_redis.py" in workflow assert "uv sync --frozen" in workflow diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index fc4c412a5..c4859e19b 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -195,6 +195,12 @@ def test_v2_environment_loads_only_the_exact_sqlmodel_inventory() -> None: assert "V1Base" not in env_source assert "migrations/v1" not in env_source assert "validate_v2_table_inventory" in env_source + assert "historical_reference_data_operations" in env_source + assert "reference_data_autogenerate" not in env_source + assert not (REPO / "policyengine_api/data/v2/reference_data.py").exists() + assert not ( + REPO / "policyengine_api/data/v2/reference_data_autogenerate.py" + ).exists() def test_v2_files_are_mechanically_separate_from_v1() -> None: @@ -212,8 +218,9 @@ def test_v2_files_are_mechanically_separate_from_v1() -> None: def test_v2_revision_chain_is_linear_generated_and_correction_bounded() -> None: config = Config(str(REPO / "alembic-v2.ini")) script = ScriptDirectory.from_config(config) - assert script.get_heads() == ["56dcd15a3afd"] + assert script.get_heads() == ["8b8ee7fe26bb"] assert [revision.revision for revision in script.walk_revisions()] == [ + "8b8ee7fe26bb", "56dcd15a3afd", "4faee127fa16", "5f048586d8f1", @@ -246,8 +253,17 @@ def test_v2_revision_chain_is_linear_generated_and_correction_bounded() -> None: REPO / "migrations/v2/versions/" "56dcd15a3afd_assign_one_default_dataset_per_region.py" ).read_text(encoding="utf-8") + validation_cleanup = ( + REPO / "migrations/v2/versions/8b8ee7fe26bb_remove_stage_8_validation_data.py" + ).read_text(encoding="utf-8") revisions = ( - baseline + data + ownership + constraints + native_uuid + region_defaults + baseline + + data + + ownership + + constraints + + native_uuid + + region_defaults + + validation_cleanup ) assert all( "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" in source @@ -258,11 +274,17 @@ def test_v2_revision_chain_is_linear_generated_and_correction_bounded() -> None: constraints, native_uuid, region_defaults, + validation_cleanup, ) ) assert "op.execute(" not in revisions assert "op.bulk_insert(" not in revisions assert data.count("op.v2_reference_row_change(") == 4 + assert validation_cleanup.count("op.v2_reference_row_change(") == 4 + assert validation_cleanup.count("after=None") == 2 + assert validation_cleanup.index("tax_benefit_model_versions") < ( + validation_cleanup.index("tax_benefit_models") + ) assert "op.create_table(" not in data assert "op.drop_table(" not in data assert ownership.count("op.create_foreign_key(") == 4 diff --git a/tests/unit/v2/test_reference_data_autogenerate.py b/tests/unit/v2/test_reference_data_autogenerate.py deleted file mode 100644 index 899b0bbcc..000000000 --- a/tests/unit/v2/test_reference_data_autogenerate.py +++ /dev/null @@ -1,221 +0,0 @@ -"""Tests for generated-only declarative application-data migrations.""" - -from types import SimpleNamespace - -from alembic.migration import MigrationContext -from alembic.operations import Operations, ops -import pytest -import sqlalchemy as sa - -from policyengine_api.data.v2.models import V2_METADATA -from policyengine_api.data.v2.reference_data import ( - REFERENCE_DATA, - REFERENCE_DATA_FORMAT_VERSION, - ReferenceDataDeclarationError, - ReferenceRow, - ReferenceTable, -) -from policyengine_api.data.v2.reference_data_autogenerate import ( - ReferenceDataMigrationError, - ReferenceRowChangeOp, - _render_reference_row_change, - compare_reference_data, - order_generated_operations, -) -from policyengine_api.data.v2.table_inventory import EXPECTED_V2_TABLES - - -def test_declaration_has_stable_scoped_natural_keys_and_wire_values() -> None: - assert REFERENCE_DATA_FORMAT_VERSION == 1 - assert [table.table_name for table in REFERENCE_DATA] == [ - "tax_benefit_models", - "tax_benefit_model_versions", - ] - for table in REFERENCE_DATA: - assert table.managed_prefix_column in table.key_columns - for row in table.rows: - assert tuple(row.key) == table.key_columns - assert row.key[table.managed_prefix_column].startswith(table.managed_prefix) - - -def test_declaration_rejects_unknown_tables_duplicate_keys_and_unsafe_values() -> None: - row = ReferenceRow.create(key={"name": "stage8-one"}, values={"value": 1}) - with pytest.raises(ReferenceDataDeclarationError, match="unreviewed"): - ReferenceTable( - table_name="runtime_bundles", - key_columns=("name",), - managed_prefix_column="name", - managed_prefix="stage8-", - rows=(row,), - ) - with pytest.raises(ReferenceDataDeclarationError, match="duplicate"): - ReferenceTable( - table_name="tax_benefit_models", - key_columns=("name",), - managed_prefix_column="name", - managed_prefix="stage8-", - rows=(row, row), - ) - with pytest.raises(ReferenceDataDeclarationError, match="unsupported"): - ReferenceRow.create(key={"name": "stage8-unsafe"}, values={"value": object()}) - - -def test_operation_requires_reviewed_table_key_and_reversible_state() -> None: - with pytest.raises(ReferenceDataMigrationError, match="unreviewed"): - ReferenceRowChangeOp( - "runtime_bundles", - key={"name": "stage8-test"}, - before=None, - after={"name": "stage8-test"}, - ) - with pytest.raises(ReferenceDataMigrationError, match="stable key"): - ReferenceRowChangeOp( - "tax_benefit_models", - key={}, - before=None, - after={"name": "stage8-test"}, - ) - - operation = ReferenceRowChangeOp( - "tax_benefit_models", - key={"name": "stage8-test"}, - before={"name": "stage8-test", "description": "before"}, - after={"name": "stage8-test", "description": "after"}, - ) - assert operation.reverse().before == operation.after - assert operation.reverse().after == operation.before - - -def test_renderer_is_deterministic_and_uses_the_registered_op_surface() -> None: - operation = ReferenceRowChangeOp( - "tax_benefit_models", - key={"name": "stage8-test"}, - before=None, - after={"name": "stage8-test", "description": "test"}, - ) - - first = _render_reference_row_change(None, operation) - second = _render_reference_row_change(None, operation) - - assert first == second - assert first.startswith("op.v2_reference_row_change(") - assert hasattr(Operations, "v2_reference_row_change") - assert operation.to_diff_tuple() == ( - "v2_reference_row_change", - "tax_benefit_models", - {"name": "stage8-test"}, - None, - {"description": "test", "name": "stage8-test"}, - ) - - -def test_generated_operation_executes_and_downgrades_against_reflected_table() -> None: - engine = sa.create_engine("sqlite://") - declaration = REFERENCE_DATA[0] - row = declaration.rows[0] - with engine.begin() as connection: - connection.exec_driver_sql("ATTACH DATABASE ':memory:' AS public") - table = V2_METADATA.tables[declaration.table_name].to_metadata( - sa.MetaData(schema="public") - ) - table.create(connection) - operations = Operations(MigrationContext.configure(connection)) - insert = ReferenceRowChangeOp( - declaration.table_name, - key=dict(row.key), - before=None, - after=row.complete_values, - ) - - operations.invoke(insert) - stored = connection.execute(sa.select(table)).mappings().one() - assert stored["name"] == "stage8-platform-validation" - - update_values = {**row.complete_values, "description": "updated"} - update = ReferenceRowChangeOp( - declaration.table_name, - key=dict(row.key), - before=row.complete_values, - after=update_values, - ) - operations.invoke(update) - assert connection.execute(sa.select(table.c.description)).scalar_one() == ( - "updated" - ) - operations.invoke(update.reverse()) - operations.invoke(insert.reverse()) - assert ( - connection.execute( - sa.select(sa.func.count()).select_from(table) - ).scalar_one() - == 0 - ) - engine.dispose() - - -def test_generated_data_is_ordered_between_constructive_and_destructive_schema() -> ( - None -): - metadata = sa.MetaData() - table = sa.Table("example", metadata, sa.Column("id", sa.Integer, primary_key=True)) - create = ops.CreateTableOp.from_table(table) - drop = ops.DropTableOp.from_table(table) - data = ReferenceRowChangeOp( - "tax_benefit_models", - key={"name": "stage8-test"}, - before=None, - after={"name": "stage8-test"}, - ) - script = SimpleNamespace( - upgrade_ops=SimpleNamespace(ops=[drop, data, create]), - downgrade_ops=SimpleNamespace(ops=[]), - ) - - order_generated_operations(None, None, [script]) - - assert script.upgrade_ops.ops == [create, data, drop] - assert isinstance(script.downgrade_ops.ops[0], ops.CreateTableOp) - assert isinstance(script.downgrade_ops.ops[1], ReferenceRowChangeOp) - assert isinstance(script.downgrade_ops.ops[2], ops.DropTableOp) - - -def test_comparator_deletes_children_first_and_upserts_parents_first( - monkeypatch: pytest.MonkeyPatch, -) -> None: - import policyengine_api.data.v2.reference_data_autogenerate as module - - parent_remove = ReferenceRowChangeOp( - "tax_benefit_models", - key={"name": "stage8-parent"}, - before={"name": "stage8-parent"}, - after=None, - ) - parent_insert = parent_remove.reverse() - child_remove = ReferenceRowChangeOp( - "tax_benefit_model_versions", - key={"version": "stage8-child"}, - before={"version": "stage8-child"}, - after=None, - ) - child_insert = child_remove.reverse() - differences = iter( - [([parent_remove], [parent_insert]), ([child_remove], [child_insert])] - ) - monkeypatch.setattr(module, "_table_differences", lambda *_: next(differences)) - monkeypatch.setattr( - module.sa, - "inspect", - lambda _: SimpleNamespace( - get_table_names=lambda schema: list(EXPECTED_V2_TABLES) - ), - ) - upgrade = ops.UpgradeOps(ops=[]) - - compare_reference_data(SimpleNamespace(connection=object()), upgrade, {None}) - - assert upgrade.ops == [ - child_remove, - parent_remove, - parent_insert, - child_insert, - ] diff --git a/tests/unit/v2/test_scaffolding_hygiene.py b/tests/unit/v2/test_scaffolding_hygiene.py index 71a2b4f93..90585f270 100644 --- a/tests/unit/v2/test_scaffolding_hygiene.py +++ b/tests/unit/v2/test_scaffolding_hygiene.py @@ -28,7 +28,6 @@ def test_allows_durable_migrations_bootstrap_tests_and_docs() -> None: prohibited_staged_paths( [ "migrations/v2/versions/abc_generated.py", - "policyengine_api/data/v2/reference_data.py", "scripts/bootstrap_v2_supabase_storage.py", "scripts/check_stage8_scaffolding_hygiene.py", "tests/unit/v2/test_storage_bootstrap.py", From 2c44c50bcb473960e3fb313d5f61cbbfcb99f21c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 18 Aug 2026 22:54:45 +0300 Subject: [PATCH 11/18] Generalize v2 testing guidance --- docs/engineering/skills/testing.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index 9717ca117..9ac7b4c53 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -52,8 +52,8 @@ FLASK_DEBUG=1 python -m pytest tests/unit/test_migration_flags.py tests/unit/tes python -m pytest tests/unit/test_cloud_run_deploy_scripts.py tests/unit/test_capture_migration_baseline.py tests/unit/test_compare_migration_baseline.py -q ``` -For Stage 8 v2 platform foundation work, run the smallest applicable group -while iterating, then all groups before qualification. +For v2 platform foundation work, run the smallest applicable group while +iterating, then all groups before qualification. SQLModel schema, report/run behavior, lazy configuration, and import effects: @@ -125,9 +125,9 @@ docker build -f gcp/cloud_run/Dockerfile -t policyengine-api-cloud-run:test . If the Cloud Run container startup script changes, keep the script syntax and child-process supervision assertions in `tests/unit/test_cloud_run_deploy_scripts.py` -updated. Stage 8 removes the tier 1 container-local Redis process, so the -tests must assert that only the API server is supervised and that deployed -configuration selects managed Redis without a localhost fallback. +updated. The tests must assert that only the API server is supervised, no +container-local Redis process is launched, and deployed configuration selects +managed Redis without a localhost fallback. Staging deployment checks should run the same live integration suite against both the App Engine staging URL and the tagged Cloud Run staging URL before From 5965b74554cfc0f1a6fd4022efa04466ed7de366 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 18 Aug 2026 23:55:09 +0300 Subject: [PATCH 12/18] Compact dormant v2 Alembic baseline --- docs/engineering/skills/alembic-migrations.md | 41 +- docs/engineering/skills/testing.md | 3 +- docs/migration/stage-8-platform-runbook.md | 9 + docs/migration/stage-8-supabase-bootstrap.md | 18 + migrations/v2/env.py | 2 - ...use_native_uuid_report_run_idempotency_.py | 59 --- ...d_assign_one_default_dataset_per_region.py | 104 ----- ...1_constrain_v2_user_country_and_report_.py | 50 --- ...63_add_stage_8_platform_validation_data.py | 83 ---- ...e7fe26bb_remove_stage_8_validation_data.py | 83 ---- ...7_enforce_v2_user_association_ownership.py | 79 ---- ...347cb2a_establish_v2_platform_baseline.py} | 409 +++++++++--------- .../historical_reference_data_operations.py | 192 -------- .../integration/test_alembic_v2_lifecycle.py | 124 ++---- tests/unit/v2/test_alembic_v2.py | 125 ++---- 15 files changed, 326 insertions(+), 1055 deletions(-) delete mode 100644 migrations/v2/versions/4faee127fa16_use_native_uuid_report_run_idempotency_.py delete mode 100644 migrations/v2/versions/56dcd15a3afd_assign_one_default_dataset_per_region.py delete mode 100644 migrations/v2/versions/5f048586d8f1_constrain_v2_user_country_and_report_.py delete mode 100644 migrations/v2/versions/6ee725e0c563_add_stage_8_platform_validation_data.py delete mode 100644 migrations/v2/versions/8b8ee7fe26bb_remove_stage_8_validation_data.py delete mode 100644 migrations/v2/versions/b4c69674dd47_enforce_v2_user_association_ownership.py rename migrations/v2/versions/{47592781336f_establish_v2_core_schema_baseline.py => f5ef4347cb2a_establish_v2_platform_baseline.py} (96%) delete mode 100644 policyengine_api/data/v2/historical_reference_data_operations.py diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 97c2160dc..94ded8de3 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -98,6 +98,17 @@ tables, Alembic history, or predecessor data. Missing, mismatched, ambiguous, or non-fresh qualification stops the command; never drop, reset, reconcile, adopt, or stamp the target automatically. +The sole exception is the completed pre-activation v2 baseline compaction. The +dedicated v2 target was still dormant, its complete row audit found only the +two known synthetic validation rows, and a recoverable ignored backup was +created. The old v2 chain then downgraded its own application schema to +`base` before any old revision file was removed. A new parentless baseline was +autogenerated from the complete current SQLModel metadata and qualified before +being applied. This is not permission to rewrite later applied history: never +repeat it once v2 contains retained domain data or serves production traffic, +never stamp across a compaction, and never apply it to v1 or Supabase-managed +schemas and Storage. + Only a separate, explicit disposable-test mode may omit Supabase identity. It is limited to an isolated Postgres database created for local or CI migration lifecycle tests and must be rejected for staging, production, or any other @@ -125,11 +136,11 @@ provide a reviewed generation source capable of producing deterministic and reversible operations. If safe autogeneration is unavailable, invoke the human decision rule above instead of maintaining speculative migration machinery. -Stage 8 has no active desired-state reference-data declaration or row -comparator. Historical revisions `6ee725e0c563` and `8b8ee7fe26bb` contain the -autogenerated insertion and removal of two synthetic validation rows. Their -minimal operation executor remains importable only so immutable migration -history can be replayed; the current v2 head contains neither validation row. +The v2 baseline contains no reference-data operation, desired-state row +declaration, comparator, or synthetic validation row. Future retained-data +migrations still require a concrete reviewed generation source; do not add a +speculative general-purpose data-migration framework merely to exercise the +policy. `alembic check` for v2 must detect schema drift. Application startup, model import, project provisioning, and Supabase Storage @@ -203,25 +214,15 @@ fresh databases built from the original baseline contain the table, while deployed MySQL databases do not. Integration tests cover both fresh and production-shaped schemas and verify that `execution_id` becomes non-nullable. -### API v2 baseline native-enum cleanup +### API v2 compact baseline native-enum cleanup -Revision `47592781336f` was autogenerated from the reviewed SQLModel metadata. +Revision `f5ef4347cb2a` was autogenerated from the complete reviewed SQLModel +metadata after the audited pre-activation v2-only reset. It has no parent and +directly represents the current schema; the superseded development revisions +and historical reference-row executor are not part of the active chain. PostgreSQL native enum types are created as part of the generated table DDL, but Alembic does not autogenerate removal of those schema-level types after the last dependent table is dropped. Its only post-generation correction drops the nine generated `v2_*` enum types at the end of the baseline downgrade. The v2 Postgres lifecycle test covers empty upgrade, downgrade to base, and re-upgrade so stale enum types cannot make the generated baseline non-reversible. - -### API v2 report-run idempotency UUID conversion - -Revision `4faee127fa16` was autogenerated after changing the SQLModel -`report_runs.idempotency_key` field from bounded text to native UUID. Alembic -detected both the type change and removal of the text-only nonblank check, but -PostgreSQL cannot infer the required casts and cannot retain the text-only -check while changing the column type. The bounded post-generation correction -drops that check before the upgrade conversion, restores it after the -downgrade conversion, and supplies `postgresql_using` casts in both -directions. The v2 PostgreSQL lifecycle test covers the one-revision downgrade -and upgrade and verifies the live column type and check constraint at both -states. diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index 9ac7b4c53..b5c9ee46d 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -71,7 +71,8 @@ V2_ALEMBIC_DISPOSABLE_TEST=1 V2_MIGRATION_DATABASE_URL="postgresql+psycopg://... The disposable lifecycle must start from empty Postgres, upgrade to `head`, check schema drift, compare the live schema with the exact -SQLModel inventory, downgrade the reviewed boundary, and upgrade again. It +SQLModel inventory, downgrade the compact baseline to `base`, verify its native +Postgres enum types are removed, and upgrade again. It must not use the persistent Supabase qualification bypass outside explicit disposable-test mode. Continue running the existing isolated v1 MySQL lifecycle whenever either Alembic domain changes. diff --git a/docs/migration/stage-8-platform-runbook.md b/docs/migration/stage-8-platform-runbook.md index db64aeb1c..be8a41772 100644 --- a/docs/migration/stage-8-platform-runbook.md +++ b/docs/migration/stage-8-platform-runbook.md @@ -41,6 +41,15 @@ Application runtime receives only the dormant target identity required by its validated configuration. It does not receive the migration password or Storage administration key. +The current v2 chain begins with one parentless baseline autogenerated from the +complete reviewed SQLModel metadata. Its superseded development history was +removed only after a one-time, audited, backed-up, pre-activation downgrade of +the empty v2 application schema to `base`. Do not use that completed operation +as a rollback technique: future databases with retained v2 data must advance +through ordinary generated revisions and fail closed on unknown history. The +v1 MySQL chain and Supabase-managed schemas and Storage were never part of the +compaction boundary. + ## Managed-cache rollout Staging and production use separate managed Redis instances and credentials. diff --git a/docs/migration/stage-8-supabase-bootstrap.md b/docs/migration/stage-8-supabase-bootstrap.md index a4ad8b8ff..387ccb3b3 100644 --- a/docs/migration/stage-8-supabase-bootstrap.md +++ b/docs/migration/stage-8-supabase-bootstrap.md @@ -46,6 +46,24 @@ expose the Storage key to the application service account. and run `uv run python3 scripts/check_stage8_scaffolding_hygiene.py` before commit. +## Completed pre-activation v2 baseline compaction + +Before v2 activation, the dedicated application schema underwent one bounded +rebaseline. The exact target and migration role were re-qualified; all public +application-table row counts were audited; the only rows were the two known +synthetic validation records; and a custom-format backup of `public` was +created in an ignored operator-artifact location and verified as readable. +With the old revisions still present, `alembic-v2.ini downgrade base` removed +their application objects. Only then were those revisions and their historical +custom-operation executor replaced by one newly autogenerated parentless +baseline. + +This sequence was v2-only. It used no Alembic stamp or raw schema reset and did +not target Cloud SQL, the v1 history, Supabase-managed schemas, or Storage. It +must not be repeated after the v2 database contains retained domain data or +serves production traffic. A later revision-history problem requires explicit +manual recovery rather than another compaction. + The Storage initializer calls only Supabase's bucket-management endpoint. It does not run Alembic, import application startup, modify application tables or rows, initialize canonical metadata, upload an object, or create an access diff --git a/migrations/v2/env.py b/migrations/v2/env.py index 84b0f38a4..2f1a30a40 100644 --- a/migrations/v2/env.py +++ b/migrations/v2/env.py @@ -6,8 +6,6 @@ from alembic.script import ScriptDirectory from sqlalchemy import create_engine, pool -# Registers the custom operation embedded in immutable historical revisions. -import policyengine_api.data.v2.historical_reference_data_operations # noqa: F401 from policyengine_api.data.v2.migration_target import ( load_v2_alembic_settings, qualify_v2_connection, diff --git a/migrations/v2/versions/4faee127fa16_use_native_uuid_report_run_idempotency_.py b/migrations/v2/versions/4faee127fa16_use_native_uuid_report_run_idempotency_.py deleted file mode 100644 index a096c8ee2..000000000 --- a/migrations/v2/versions/4faee127fa16_use_native_uuid_report_run_idempotency_.py +++ /dev/null @@ -1,59 +0,0 @@ -"""use native UUID report run idempotency keys - -Revision ID: 4faee127fa16 -Revises: 5f048586d8f1 -Create Date: 2026-08-18 16:20:46.343261 -Generation: uv run alembic -c alembic-v2.ini revision --autogenerate -""" - -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -import sqlmodel - - -revision: str = "4faee127fa16" -down_revision: Union[str, None] = "5f048586d8f1" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - # Post-generation correction: the text-only check must be removed before - # the type change, and PostgreSQL requires an explicit text-to-UUID cast. - op.drop_constraint( - op.f("ck_report_runs_idempotency_key_nonblank"), - "report_runs", - type_="check", - ) - op.alter_column( - "report_runs", - "idempotency_key", - existing_type=sa.VARCHAR(length=255), - type_=sa.Uuid(), - existing_nullable=True, - postgresql_using="idempotency_key::uuid", - ) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - # Post-generation correction: cast UUIDs back to text before restoring the - # text-only check constraint. - op.alter_column( - "report_runs", - "idempotency_key", - existing_type=sa.Uuid(), - type_=sa.VARCHAR(length=255), - existing_nullable=True, - postgresql_using="idempotency_key::text", - ) - op.create_check_constraint( - op.f("ck_report_runs_idempotency_key_nonblank"), - "report_runs", - "idempotency_key IS NULL OR length(TRIM(BOTH FROM idempotency_key)) > 0", - ) - # ### end Alembic commands ### diff --git a/migrations/v2/versions/56dcd15a3afd_assign_one_default_dataset_per_region.py b/migrations/v2/versions/56dcd15a3afd_assign_one_default_dataset_per_region.py deleted file mode 100644 index f6334742b..000000000 --- a/migrations/v2/versions/56dcd15a3afd_assign_one_default_dataset_per_region.py +++ /dev/null @@ -1,104 +0,0 @@ -"""assign one default dataset per region - -Revision ID: 56dcd15a3afd -Revises: 4faee127fa16 -Create Date: 2026-08-18 21:28:53.385203 -Generation: uv run alembic -c alembic-v2.ini revision --autogenerate -""" - -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -import sqlmodel - - -revision: str = "56dcd15a3afd" -down_revision: Union[str, None] = "4faee127fa16" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.alter_column( - "datasets", "storage_path", existing_type=sa.VARCHAR(length=1024), nullable=True - ) - op.drop_constraint( - op.f("uq_datasets_model_name_year_output"), "datasets", type_="unique" - ) - op.create_unique_constraint( - "uq_datasets_id_model", "datasets", ["id", "tax_benefit_model_id"] - ) - op.create_unique_constraint( - "uq_datasets_model_name", "datasets", ["tax_benefit_model_id", "name"] - ) - op.create_check_constraint( - op.f("ck_datasets_output_storage_path"), - "datasets", - "NOT is_output_dataset OR storage_path IS NOT NULL", - ) - op.add_column("regions", sa.Column("default_dataset_id", sa.Uuid(), nullable=False)) - op.create_index( - op.f("ix_regions_default_dataset_id"), - "regions", - ["default_dataset_id"], - unique=False, - ) - op.create_foreign_key( - "fk_regions_default_dataset_model_datasets", - "regions", - "datasets", - ["default_dataset_id", "tax_benefit_model_id"], - ["id", "tax_benefit_model_id"], - ondelete="RESTRICT", - ) - op.drop_table("region_datasets") - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "region_datasets", - sa.Column("region_id", sa.UUID(), autoincrement=False, nullable=False), - sa.Column("dataset_id", sa.UUID(), autoincrement=False, nullable=False), - sa.ForeignKeyConstraint( - ["dataset_id"], - ["datasets.id"], - name=op.f("fk_region_datasets_dataset_id_datasets"), - ondelete="CASCADE", - ), - sa.ForeignKeyConstraint( - ["region_id"], - ["regions.id"], - name=op.f("fk_region_datasets_region_id_regions"), - ondelete="CASCADE", - ), - sa.PrimaryKeyConstraint( - "region_id", "dataset_id", name=op.f("pk_region_datasets") - ), - ) - op.drop_constraint( - "fk_regions_default_dataset_model_datasets", "regions", type_="foreignkey" - ) - op.drop_index(op.f("ix_regions_default_dataset_id"), table_name="regions") - op.drop_column("regions", "default_dataset_id") - op.drop_constraint( - op.f("ck_datasets_output_storage_path"), "datasets", type_="check" - ) - op.drop_constraint("uq_datasets_model_name", "datasets", type_="unique") - op.drop_constraint("uq_datasets_id_model", "datasets", type_="unique") - op.create_unique_constraint( - op.f("uq_datasets_model_name_year_output"), - "datasets", - ["tax_benefit_model_id", "name", "year", "is_output_dataset"], - postgresql_nulls_not_distinct=False, - ) - op.alter_column( - "datasets", - "storage_path", - existing_type=sa.VARCHAR(length=1024), - nullable=False, - ) - # ### end Alembic commands ### diff --git a/migrations/v2/versions/5f048586d8f1_constrain_v2_user_country_and_report_.py b/migrations/v2/versions/5f048586d8f1_constrain_v2_user_country_and_report_.py deleted file mode 100644 index 0f69adb3c..000000000 --- a/migrations/v2/versions/5f048586d8f1_constrain_v2_user_country_and_report_.py +++ /dev/null @@ -1,50 +0,0 @@ -"""constrain v2 user country and report idempotency - -Revision ID: 5f048586d8f1 -Revises: b4c69674dd47 -Create Date: 2026-08-18 12:54:00.963782 -Generation: uv run alembic -c alembic-v2.ini revision --autogenerate -""" - -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -import sqlmodel - - -revision: str = "5f048586d8f1" -down_revision: Union[str, None] = "b4c69674dd47" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.create_check_constraint( - op.f("ck_report_runs_idempotency_key_nonblank"), - "report_runs", - "idempotency_key IS NULL OR length(trim(idempotency_key)) > 0", - ) - op.add_column( - "users", - sa.Column( - "primary_country", - sqlmodel.sql.sqltypes.AutoString(length=2), - nullable=False, - ), - ) - op.create_check_constraint( - op.f("ck_users_primary_country"), "users", "primary_country IN ('us', 'uk')" - ) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.drop_constraint(op.f("ck_users_primary_country"), "users", type_="check") - op.drop_column("users", "primary_country") - op.drop_constraint( - op.f("ck_report_runs_idempotency_key_nonblank"), "report_runs", type_="check" - ) - # ### end Alembic commands ### diff --git a/migrations/v2/versions/6ee725e0c563_add_stage_8_platform_validation_data.py b/migrations/v2/versions/6ee725e0c563_add_stage_8_platform_validation_data.py deleted file mode 100644 index 3206aa1f7..000000000 --- a/migrations/v2/versions/6ee725e0c563_add_stage_8_platform_validation_data.py +++ /dev/null @@ -1,83 +0,0 @@ -"""add stage 8 platform validation data - -Revision ID: 6ee725e0c563 -Revises: 47592781336f -Create Date: 2026-08-14 14:29:59.265364 -Generation: uv run alembic -c alembic-v2.ini revision --autogenerate -""" - -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -import sqlmodel - - -revision: str = "6ee725e0c563" -down_revision: Union[str, None] = "47592781336f" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.v2_reference_row_change( - "tax_benefit_models", - key={"name": "stage8-platform-validation"}, - before=None, - after={ - "created_at": "2026-08-14T00:00:00+00:00", - "description": "Stage 8 migration lifecycle validation; canonical metadata is introduced in Stage 9.", - "id": "80000000-0000-4000-8000-000000000001", - "name": "stage8-platform-validation", - "updated_at": "2026-08-14T00:00:00+00:00", - }, - ) - op.v2_reference_row_change( - "tax_benefit_model_versions", - key={ - "model_id": "80000000-0000-4000-8000-000000000001", - "version": "stage8-platform-validation", - }, - before=None, - after={ - "created_at": "2026-08-14T00:00:00+00:00", - "description": "Stage 8 generated data-migration validation row.", - "id": "80000000-0000-4000-8000-000000000002", - "model_id": "80000000-0000-4000-8000-000000000001", - "version": "stage8-platform-validation", - }, - ) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.v2_reference_row_change( - "tax_benefit_model_versions", - key={ - "model_id": "80000000-0000-4000-8000-000000000001", - "version": "stage8-platform-validation", - }, - before={ - "created_at": "2026-08-14T00:00:00+00:00", - "description": "Stage 8 generated data-migration validation row.", - "id": "80000000-0000-4000-8000-000000000002", - "model_id": "80000000-0000-4000-8000-000000000001", - "version": "stage8-platform-validation", - }, - after=None, - ) - op.v2_reference_row_change( - "tax_benefit_models", - key={"name": "stage8-platform-validation"}, - before={ - "created_at": "2026-08-14T00:00:00+00:00", - "description": "Stage 8 migration lifecycle validation; canonical metadata is introduced in Stage 9.", - "id": "80000000-0000-4000-8000-000000000001", - "name": "stage8-platform-validation", - "updated_at": "2026-08-14T00:00:00+00:00", - }, - after=None, - ) - # ### end Alembic commands ### diff --git a/migrations/v2/versions/8b8ee7fe26bb_remove_stage_8_validation_data.py b/migrations/v2/versions/8b8ee7fe26bb_remove_stage_8_validation_data.py deleted file mode 100644 index f16723b93..000000000 --- a/migrations/v2/versions/8b8ee7fe26bb_remove_stage_8_validation_data.py +++ /dev/null @@ -1,83 +0,0 @@ -"""remove stage 8 validation data - -Revision ID: 8b8ee7fe26bb -Revises: 56dcd15a3afd -Create Date: 2026-08-18 22:30:18.333006 -Generation: uv run alembic -c alembic-v2.ini revision --autogenerate -""" - -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -import sqlmodel - - -revision: str = "8b8ee7fe26bb" -down_revision: Union[str, None] = "56dcd15a3afd" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.v2_reference_row_change( - "tax_benefit_model_versions", - key={ - "model_id": "80000000-0000-4000-8000-000000000001", - "version": "stage8-platform-validation", - }, - before={ - "created_at": "2026-08-14T00:00:00+00:00", - "description": "Stage 8 generated data-migration validation row.", - "id": "80000000-0000-4000-8000-000000000002", - "model_id": "80000000-0000-4000-8000-000000000001", - "version": "stage8-platform-validation", - }, - after=None, - ) - op.v2_reference_row_change( - "tax_benefit_models", - key={"name": "stage8-platform-validation"}, - before={ - "created_at": "2026-08-14T00:00:00+00:00", - "description": "Stage 8 migration lifecycle validation; canonical metadata is introduced in Stage 9.", - "id": "80000000-0000-4000-8000-000000000001", - "name": "stage8-platform-validation", - "updated_at": "2026-08-14T00:00:00+00:00", - }, - after=None, - ) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.v2_reference_row_change( - "tax_benefit_models", - key={"name": "stage8-platform-validation"}, - before=None, - after={ - "created_at": "2026-08-14T00:00:00+00:00", - "description": "Stage 8 migration lifecycle validation; canonical metadata is introduced in Stage 9.", - "id": "80000000-0000-4000-8000-000000000001", - "name": "stage8-platform-validation", - "updated_at": "2026-08-14T00:00:00+00:00", - }, - ) - op.v2_reference_row_change( - "tax_benefit_model_versions", - key={ - "model_id": "80000000-0000-4000-8000-000000000001", - "version": "stage8-platform-validation", - }, - before=None, - after={ - "created_at": "2026-08-14T00:00:00+00:00", - "description": "Stage 8 generated data-migration validation row.", - "id": "80000000-0000-4000-8000-000000000002", - "model_id": "80000000-0000-4000-8000-000000000001", - "version": "stage8-platform-validation", - }, - ) - # ### end Alembic commands ### diff --git a/migrations/v2/versions/b4c69674dd47_enforce_v2_user_association_ownership.py b/migrations/v2/versions/b4c69674dd47_enforce_v2_user_association_ownership.py deleted file mode 100644 index a23925e7a..000000000 --- a/migrations/v2/versions/b4c69674dd47_enforce_v2_user_association_ownership.py +++ /dev/null @@ -1,79 +0,0 @@ -"""enforce v2 user association ownership - -Revision ID: b4c69674dd47 -Revises: 6ee725e0c563 -Create Date: 2026-08-18 12:25:45.926354 -Generation: uv run alembic -c alembic-v2.ini revision --autogenerate -""" - -from typing import Sequence, Union - -from alembic import op -import sqlalchemy as sa -import sqlmodel - - -revision: str = "b4c69674dd47" -down_revision: Union[str, None] = "6ee725e0c563" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.create_foreign_key( - op.f("fk_user_household_associations_user_id_users"), - "user_household_associations", - "users", - ["user_id"], - ["id"], - ondelete="CASCADE", - ) - op.create_foreign_key( - op.f("fk_user_policies_user_id_users"), - "user_policies", - "users", - ["user_id"], - ["id"], - ondelete="CASCADE", - ) - op.create_foreign_key( - op.f("fk_user_report_associations_user_id_users"), - "user_report_associations", - "users", - ["user_id"], - ["id"], - ondelete="CASCADE", - ) - op.create_foreign_key( - op.f("fk_user_simulation_associations_user_id_users"), - "user_simulation_associations", - "users", - ["user_id"], - ["id"], - ondelete="CASCADE", - ) - # ### end Alembic commands ### - - -def downgrade() -> None: - # ### commands auto generated by Alembic - please adjust! ### - op.drop_constraint( - op.f("fk_user_simulation_associations_user_id_users"), - "user_simulation_associations", - type_="foreignkey", - ) - op.drop_constraint( - op.f("fk_user_report_associations_user_id_users"), - "user_report_associations", - type_="foreignkey", - ) - op.drop_constraint( - op.f("fk_user_policies_user_id_users"), "user_policies", type_="foreignkey" - ) - op.drop_constraint( - op.f("fk_user_household_associations_user_id_users"), - "user_household_associations", - type_="foreignkey", - ) - # ### end Alembic commands ### diff --git a/migrations/v2/versions/47592781336f_establish_v2_core_schema_baseline.py b/migrations/v2/versions/f5ef4347cb2a_establish_v2_platform_baseline.py similarity index 96% rename from migrations/v2/versions/47592781336f_establish_v2_core_schema_baseline.py rename to migrations/v2/versions/f5ef4347cb2a_establish_v2_platform_baseline.py index 01deb6e81..56ac2080c 100644 --- a/migrations/v2/versions/47592781336f_establish_v2_core_schema_baseline.py +++ b/migrations/v2/versions/f5ef4347cb2a_establish_v2_platform_baseline.py @@ -1,8 +1,8 @@ -"""establish v2 core schema baseline +"""establish v2 platform baseline -Revision ID: 47592781336f +Revision ID: f5ef4347cb2a Revises: -Create Date: 2026-08-14 14:22:39.384555 +Create Date: 2026-08-18 23:42:23.783821 Generation: uv run alembic -c alembic-v2.ini revision --autogenerate """ @@ -13,7 +13,7 @@ import sqlmodel -revision: str = "47592781336f" +revision: str = "f5ef4347cb2a" down_revision: Union[str, None] = None branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None @@ -107,6 +107,14 @@ def upgrade() -> None: sa.Column( "email", sqlmodel.sql.sqltypes.AutoString(length=320), nullable=False ), + sa.Column( + "primary_country", + sqlmodel.sql.sqltypes.AutoString(length=2), + nullable=False, + ), + sa.CheckConstraint( + "primary_country IN ('us', 'uk')", name=op.f("ck_users_primary_country") + ), sa.PrimaryKeyConstraint("id", name=op.f("pk_users")), sa.UniqueConstraint("email", name="uq_users_email"), ) @@ -129,13 +137,15 @@ def upgrade() -> None: sa.Column("name", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False), sa.Column("description", sqlmodel.sql.sqltypes.AutoString(), nullable=True), sa.Column( - "storage_path", - sqlmodel.sql.sqltypes.AutoString(length=1024), - nullable=False, + "storage_path", sqlmodel.sql.sqltypes.AutoString(length=1024), nullable=True ), sa.Column("year", sa.Integer(), nullable=False), sa.Column("is_output_dataset", sa.Boolean(), nullable=False), sa.Column("tax_benefit_model_id", sa.Uuid(), nullable=False), + sa.CheckConstraint( + "NOT is_output_dataset OR storage_path IS NOT NULL", + name=op.f("ck_datasets_output_storage_path"), + ), sa.CheckConstraint("year BETWEEN 1900 AND 2200", name=op.f("ck_datasets_year")), sa.ForeignKeyConstraint( ["tax_benefit_model_id"], @@ -144,12 +154,9 @@ def upgrade() -> None: ondelete="RESTRICT", ), sa.PrimaryKeyConstraint("id", name=op.f("pk_datasets")), + sa.UniqueConstraint("id", "tax_benefit_model_id", name="uq_datasets_id_model"), sa.UniqueConstraint( - "tax_benefit_model_id", - "name", - "year", - "is_output_dataset", - name="uq_datasets_model_name_year_output", + "tax_benefit_model_id", "name", name="uq_datasets_model_name" ), ) op.create_index( @@ -190,83 +197,6 @@ def upgrade() -> None: ["tax_benefit_model_id"], unique=False, ) - op.create_table( - "regions", - sa.Column("id", sa.Uuid(), nullable=False), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=False, - ), - sa.Column("code", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False), - sa.Column( - "label", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False - ), - sa.Column( - "region_type", - sa.Enum( - "national", - "country", - "state", - "congressional_district", - "constituency", - "local_authority", - "city", - "place", - name="v2_region_type", - ), - nullable=False, - ), - sa.Column("requires_filter", sa.Boolean(), nullable=False), - sa.Column( - "filter_field", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=True - ), - sa.Column( - "filter_value", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=True - ), - sa.Column( - "filter_strategy", - sqlmodel.sql.sqltypes.AutoString(length=64), - nullable=True, - ), - sa.Column( - "parent_code", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=True - ), - sa.Column( - "state_code", sqlmodel.sql.sqltypes.AutoString(length=16), nullable=True - ), - sa.Column( - "state_name", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=True - ), - sa.Column("tax_benefit_model_id", sa.Uuid(), nullable=False), - sa.CheckConstraint( - "NOT requires_filter OR (filter_field IS NOT NULL AND filter_value IS NOT NULL)", - name=op.f("ck_regions_required_filter_values"), - ), - sa.ForeignKeyConstraint( - ["tax_benefit_model_id"], - ["tax_benefit_models.id"], - name=op.f("fk_regions_tax_benefit_model_id_tax_benefit_models"), - ondelete="RESTRICT", - ), - sa.PrimaryKeyConstraint("id", name=op.f("pk_regions")), - sa.UniqueConstraint( - "tax_benefit_model_id", "code", name="uq_regions_model_code" - ), - ) - op.create_index( - op.f("ix_regions_tax_benefit_model_id"), - "regions", - ["tax_benefit_model_id"], - unique=False, - ) op.create_table( "tax_benefit_model_versions", sa.Column("id", sa.Uuid(), nullable=False), @@ -325,6 +255,12 @@ def upgrade() -> None: name=op.f("fk_user_household_associations_household_id_households"), ondelete="CASCADE", ), + sa.ForeignKeyConstraint( + ["user_id"], + ["users.id"], + name=op.f("fk_user_household_associations_user_id_users"), + ondelete="CASCADE", + ), sa.PrimaryKeyConstraint("id", name=op.f("pk_user_household_associations")), sa.UniqueConstraint( "user_id", @@ -509,27 +445,7 @@ def upgrade() -> None: unique=False, ) op.create_table( - "region_datasets", - sa.Column("region_id", sa.Uuid(), nullable=False), - sa.Column("dataset_id", sa.Uuid(), nullable=False), - sa.ForeignKeyConstraint( - ["dataset_id"], - ["datasets.id"], - name=op.f("fk_region_datasets_dataset_id_datasets"), - ondelete="CASCADE", - ), - sa.ForeignKeyConstraint( - ["region_id"], - ["regions.id"], - name=op.f("fk_region_datasets_region_id_regions"), - ondelete="CASCADE", - ), - sa.PrimaryKeyConstraint( - "region_id", "dataset_id", name=op.f("pk_region_datasets") - ), - ) - op.create_table( - "simulations", + "regions", sa.Column("id", sa.Uuid(), nullable=False), sa.Column( "created_at", @@ -543,26 +459,26 @@ def upgrade() -> None: server_default=sa.text("now()"), nullable=False, ), + sa.Column("code", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False), sa.Column( - "simulation_type", - sa.Enum("household", "economy", name="v2_simulation_type"), - nullable=False, + "label", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=False ), - sa.Column("dataset_id", sa.Uuid(), nullable=True), - sa.Column("household_id", sa.Uuid(), nullable=True), - sa.Column("policy_id", sa.Uuid(), nullable=True), - sa.Column("dynamic_id", sa.Uuid(), nullable=True), - sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False), - sa.Column("output_dataset_id", sa.Uuid(), nullable=True), - sa.Column("region_id", sa.Uuid(), nullable=True), sa.Column( - "status", + "region_type", sa.Enum( - "pending", "running", "succeeded", "failed", name="v2_simulation_status" + "national", + "country", + "state", + "congressional_district", + "constituency", + "local_authority", + "city", + "place", + name="v2_region_type", ), nullable=False, ), - sa.Column("error_message", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column("requires_filter", sa.Boolean(), nullable=False), sa.Column( "filter_field", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=True ), @@ -574,78 +490,48 @@ def upgrade() -> None: sqlmodel.sql.sqltypes.AutoString(length=64), nullable=True, ), - sa.Column("year", sa.Integer(), nullable=True), - sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("household_result", sa.JSON(), nullable=True), - sa.CheckConstraint( - "(simulation_type = 'household' AND household_id IS NOT NULL AND dataset_id IS NULL) OR (simulation_type = 'economy' AND dataset_id IS NOT NULL AND household_id IS NULL)", - name=op.f("ck_simulations_type_input"), + sa.Column( + "parent_code", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=True ), - sa.CheckConstraint( - "(filter_field IS NULL) = (filter_value IS NULL)", - name=op.f("ck_simulations_filter_pair"), + sa.Column( + "state_code", sqlmodel.sql.sqltypes.AutoString(length=16), nullable=True ), + sa.Column( + "state_name", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=True + ), + sa.Column("tax_benefit_model_id", sa.Uuid(), nullable=False), + sa.Column("default_dataset_id", sa.Uuid(), nullable=False), sa.CheckConstraint( - "year IS NULL OR year BETWEEN 1900 AND 2200", - name=op.f("ck_simulations_year"), + "NOT requires_filter OR (filter_field IS NOT NULL AND filter_value IS NOT NULL)", + name=op.f("ck_regions_required_filter_values"), ), sa.ForeignKeyConstraint( - ["dataset_id"], - ["datasets.id"], - name=op.f("fk_simulations_dataset_id_datasets"), + ["default_dataset_id", "tax_benefit_model_id"], + ["datasets.id", "datasets.tax_benefit_model_id"], + name="fk_regions_default_dataset_model_datasets", ondelete="RESTRICT", ), sa.ForeignKeyConstraint( - ["dynamic_id"], - ["dynamics.id"], - name=op.f("fk_simulations_dynamic_id_dynamics"), - ondelete="SET NULL", - ), - sa.ForeignKeyConstraint( - ["household_id"], - ["households.id"], - name=op.f("fk_simulations_household_id_households"), + ["tax_benefit_model_id"], + ["tax_benefit_models.id"], + name=op.f("fk_regions_tax_benefit_model_id_tax_benefit_models"), ondelete="RESTRICT", ), - sa.ForeignKeyConstraint( - ["output_dataset_id"], - ["datasets.id"], - name=op.f("fk_simulations_output_dataset_id_datasets"), - ondelete="SET NULL", - ), - sa.ForeignKeyConstraint( - ["policy_id"], - ["policies.id"], - name=op.f("fk_simulations_policy_id_policies"), - ondelete="SET NULL", - ), - sa.ForeignKeyConstraint( - ["region_id"], - ["regions.id"], - name=op.f("fk_simulations_region_id_regions"), - ondelete="SET NULL", - ), - sa.ForeignKeyConstraint( - ["tax_benefit_model_version_id"], - ["tax_benefit_model_versions.id"], - name=op.f( - "fk_simulations_tax_benefit_model_version_id_tax_benefit_model_versions" - ), - ondelete="RESTRICT", + sa.PrimaryKeyConstraint("id", name=op.f("pk_regions")), + sa.UniqueConstraint( + "tax_benefit_model_id", "code", name="uq_regions_model_code" ), - sa.PrimaryKeyConstraint("id", name=op.f("pk_simulations")), ) op.create_index( - "ix_simulations_status_created_at", - "simulations", - ["status", "created_at"], + op.f("ix_regions_default_dataset_id"), + "regions", + ["default_dataset_id"], unique=False, ) op.create_index( - op.f("ix_simulations_tax_benefit_model_version_id"), - "simulations", - ["tax_benefit_model_version_id"], + op.f("ix_regions_tax_benefit_model_id"), + "regions", + ["tax_benefit_model_id"], unique=False, ) op.create_table( @@ -675,6 +561,12 @@ def upgrade() -> None: name=op.f("fk_user_policies_policy_id_policies"), ondelete="CASCADE", ), + sa.ForeignKeyConstraint( + ["user_id"], + ["users.id"], + name=op.f("fk_user_policies_user_id_users"), + ondelete="CASCADE", + ), sa.PrimaryKeyConstraint("id", name=op.f("pk_user_policies")), sa.UniqueConstraint( "user_id", "policy_id", name="uq_user_policies_user_policy" @@ -775,6 +667,126 @@ def upgrade() -> None: ["parameter_id", "start_date", "end_date"], unique=False, ) + op.create_table( + "simulations", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "simulation_type", + sa.Enum("household", "economy", name="v2_simulation_type"), + nullable=False, + ), + sa.Column("dataset_id", sa.Uuid(), nullable=True), + sa.Column("household_id", sa.Uuid(), nullable=True), + sa.Column("policy_id", sa.Uuid(), nullable=True), + sa.Column("dynamic_id", sa.Uuid(), nullable=True), + sa.Column("tax_benefit_model_version_id", sa.Uuid(), nullable=False), + sa.Column("output_dataset_id", sa.Uuid(), nullable=True), + sa.Column("region_id", sa.Uuid(), nullable=True), + sa.Column( + "status", + sa.Enum( + "pending", "running", "succeeded", "failed", name="v2_simulation_status" + ), + nullable=False, + ), + sa.Column("error_message", sqlmodel.sql.sqltypes.AutoString(), nullable=True), + sa.Column( + "filter_field", sqlmodel.sql.sqltypes.AutoString(length=128), nullable=True + ), + sa.Column( + "filter_value", sqlmodel.sql.sqltypes.AutoString(length=255), nullable=True + ), + sa.Column( + "filter_strategy", + sqlmodel.sql.sqltypes.AutoString(length=64), + nullable=True, + ), + sa.Column("year", sa.Integer(), nullable=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("household_result", sa.JSON(), nullable=True), + sa.CheckConstraint( + "(simulation_type = 'household' AND household_id IS NOT NULL AND dataset_id IS NULL) OR (simulation_type = 'economy' AND dataset_id IS NOT NULL AND household_id IS NULL)", + name=op.f("ck_simulations_type_input"), + ), + sa.CheckConstraint( + "(filter_field IS NULL) = (filter_value IS NULL)", + name=op.f("ck_simulations_filter_pair"), + ), + sa.CheckConstraint( + "year IS NULL OR year BETWEEN 1900 AND 2200", + name=op.f("ck_simulations_year"), + ), + sa.ForeignKeyConstraint( + ["dataset_id"], + ["datasets.id"], + name=op.f("fk_simulations_dataset_id_datasets"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["dynamic_id"], + ["dynamics.id"], + name=op.f("fk_simulations_dynamic_id_dynamics"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["household_id"], + ["households.id"], + name=op.f("fk_simulations_household_id_households"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["output_dataset_id"], + ["datasets.id"], + name=op.f("fk_simulations_output_dataset_id_datasets"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["policy_id"], + ["policies.id"], + name=op.f("fk_simulations_policy_id_policies"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["region_id"], + ["regions.id"], + name=op.f("fk_simulations_region_id_regions"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["tax_benefit_model_version_id"], + ["tax_benefit_model_versions.id"], + name=op.f( + "fk_simulations_tax_benefit_model_version_id_tax_benefit_model_versions" + ), + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_simulations")), + ) + op.create_index( + "ix_simulations_status_created_at", + "simulations", + ["status", "created_at"], + unique=False, + ) + op.create_index( + op.f("ix_simulations_tax_benefit_model_version_id"), + "simulations", + ["tax_benefit_model_version_id"], + unique=False, + ) op.create_table( "reports", sa.Column("id", sa.Uuid(), nullable=False), @@ -900,6 +912,12 @@ def upgrade() -> None: name=op.f("fk_user_simulation_associations_simulation_id_simulations"), ondelete="CASCADE", ), + sa.ForeignKeyConstraint( + ["user_id"], + ["users.id"], + name=op.f("fk_user_simulation_associations_user_id_users"), + ondelete="CASCADE", + ), sa.PrimaryKeyConstraint("id", name=op.f("pk_user_simulation_associations")), sa.UniqueConstraint( "user_id", @@ -957,11 +975,7 @@ def upgrade() -> None: sa.Enum("initial", "manual", "system", name="v2_report_run_trigger"), nullable=False, ), - sa.Column( - "idempotency_key", - sqlmodel.sql.sqltypes.AutoString(length=255), - nullable=True, - ), + sa.Column("idempotency_key", sa.Uuid(), nullable=True), sa.Column("started_at", sa.DateTime(timezone=True), nullable=True), sa.Column("completed_at", sa.DateTime(timezone=True), nullable=True), sa.Column("error_message", sqlmodel.sql.sqltypes.AutoString(), nullable=True), @@ -1022,6 +1036,12 @@ def upgrade() -> None: name=op.f("fk_user_report_associations_report_id_reports"), ondelete="CASCADE", ), + sa.ForeignKeyConstraint( + ["user_id"], + ["users.id"], + name=op.f("fk_user_report_associations_user_id_users"), + ondelete="CASCADE", + ), sa.PrimaryKeyConstraint("id", name=op.f("pk_user_report_associations")), sa.UniqueConstraint( "user_id", "report_id", name="uq_user_report_associations_user_report" @@ -1650,6 +1670,11 @@ def downgrade() -> None: op.drop_index(op.f("ix_reports_tax_benefit_model_id"), table_name="reports") op.drop_index("ix_reports_country_type_created_at", table_name="reports") op.drop_table("reports") + op.drop_index( + op.f("ix_simulations_tax_benefit_model_version_id"), table_name="simulations" + ) + op.drop_index("ix_simulations_status_created_at", table_name="simulations") + op.drop_table("simulations") op.drop_index("ix_parameter_values_parameter_period", table_name="parameter_values") op.drop_table("parameter_values") op.drop_index( @@ -1659,12 +1684,9 @@ def downgrade() -> None: op.drop_index(op.f("ix_user_policies_user_id"), table_name="user_policies") op.drop_index(op.f("ix_user_policies_policy_id"), table_name="user_policies") op.drop_table("user_policies") - op.drop_index( - op.f("ix_simulations_tax_benefit_model_version_id"), table_name="simulations" - ) - op.drop_index("ix_simulations_status_created_at", table_name="simulations") - op.drop_table("simulations") - op.drop_table("region_datasets") + op.drop_index(op.f("ix_regions_tax_benefit_model_id"), table_name="regions") + op.drop_index(op.f("ix_regions_default_dataset_id"), table_name="regions") + op.drop_table("regions") op.drop_index( op.f("ix_parameters_tax_benefit_model_version_id"), table_name="parameters" ) @@ -1695,8 +1717,6 @@ def downgrade() -> None: table_name="tax_benefit_model_versions", ) op.drop_table("tax_benefit_model_versions") - op.drop_index(op.f("ix_regions_tax_benefit_model_id"), table_name="regions") - op.drop_table("regions") op.drop_index(op.f("ix_policies_tax_benefit_model_id"), table_name="policies") op.drop_table("policies") op.drop_index(op.f("ix_datasets_tax_benefit_model_id"), table_name="datasets") @@ -1707,9 +1727,8 @@ def downgrade() -> None: op.drop_index(op.f("ix_households_country"), table_name="households") op.drop_table("households") op.drop_table("dynamics") - # PostgreSQL-native enum types outlive their final table. Alembic generated - # their creation but not their removal, so the reviewed reversible-dialect - # correction drops only those generated v2 enum types. + # Post-generation reversibility correction: Alembic drops the columns that + # use native PostgreSQL enums, but does not remove the schema-level types. sa.Enum(name="v2_aggregate_type").drop(op.get_bind()) sa.Enum(name="v2_decile_type").drop(op.get_bind()) sa.Enum(name="v2_household_job_status").drop(op.get_bind()) diff --git a/policyengine_api/data/v2/historical_reference_data_operations.py b/policyengine_api/data/v2/historical_reference_data_operations.py deleted file mode 100644 index 02c207b18..000000000 --- a/policyengine_api/data/v2/historical_reference_data_operations.py +++ /dev/null @@ -1,192 +0,0 @@ -"""Execute row operations already embedded in immutable v2 Alembic revisions.""" - -from __future__ import annotations - -from datetime import datetime, timezone -from typing import Any -from uuid import UUID - -from alembic.operations import MigrateOperation, Operations -import sqlalchemy as sa - - -HISTORICAL_REFERENCE_TABLES = frozenset( - {"tax_benefit_models", "tax_benefit_model_versions"} -) - - -class HistoricalDataMigrationError(RuntimeError): - """Raised when an immutable row transition cannot be replayed safely.""" - - -def _ordered(values: dict[str, Any] | None) -> dict[str, Any] | None: - if values is None: - return None - return {key: values[key] for key in sorted(values)} - - -@Operations.register_operation("v2_reference_row_change") -class HistoricalReferenceRowChangeOp(MigrateOperation): - """One guarded row transition embedded in the historical v2 chain.""" - - def __init__( - self, - table_name: str, - *, - key: dict[str, Any], - before: dict[str, Any] | None, - after: dict[str, Any] | None, - ) -> None: - if table_name not in HISTORICAL_REFERENCE_TABLES: - raise HistoricalDataMigrationError( - f"historical data operation targets unreviewed table {table_name}" - ) - if not key or (before is None and after is None): - raise HistoricalDataMigrationError( - "historical data operations need a stable key and one row state" - ) - self.table_name = table_name - self.key = _ordered(key) or {} - self.before = _ordered(before) - self.after = _ordered(after) - - @classmethod - def v2_reference_row_change( - cls, - operations: Operations, - table_name: str, - *, - key: dict[str, Any], - before: dict[str, Any] | None, - after: dict[str, Any] | None, - ) -> Any: - return operations.invoke(cls(table_name, key=key, before=before, after=after)) - - -def _normalize(value: Any) -> Any: - if isinstance(value, UUID): - return str(value) - if isinstance(value, datetime): - if value.tzinfo is None: - value = value.replace(tzinfo=timezone.utc) - return value.astimezone(timezone.utc).isoformat() - if isinstance(value, dict): - return {key: _normalize(item) for key, item in sorted(value.items())} - if isinstance(value, list | tuple): - return [_normalize(item) for item in value] - return value - - -def _coerce(column: sa.Column, value: Any) -> Any: - if value is None: - return None - if isinstance(column.type, sa.Uuid) and not isinstance(value, UUID): - return UUID(str(value)) - if isinstance(column.type, sa.DateTime) and not isinstance(value, datetime): - parsed = datetime.fromisoformat(str(value)) - return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) - return value - - -def _predicate(table: sa.Table, key: dict[str, Any]) -> sa.ColumnElement: - return sa.and_( - *( - table.c[column] == _coerce(table.c[column], value) - for column, value in key.items() - ) - ) - - -def _current_row( - bind: sa.Connection, - table: sa.Table, - key: dict[str, Any], -) -> dict[str, Any] | None: - row = ( - bind.execute(sa.select(table).where(_predicate(table, key))) - .mappings() - .one_or_none() - ) - if row is None: - return None - return {column: _normalize(value) for column, value in row.items()} - - -def _assert_before( - current: dict[str, Any] | None, - expected: dict[str, Any] | None, - *, - table_name: str, -) -> None: - if expected is None: - if current is not None: - raise HistoricalDataMigrationError( - f"{table_name} insert found an existing historical row" - ) - return - if current is None or any( - current.get(column) != _normalize(value) for column, value in expected.items() - ): - raise HistoricalDataMigrationError( - f"{table_name} row differs from the historical before state" - ) - - -def _assert_after( - current: dict[str, Any] | None, - expected: dict[str, Any] | None, - *, - table_name: str, -) -> None: - if expected is None: - if current is not None: - raise HistoricalDataMigrationError( - f"{table_name} historical delete left the row present" - ) - return - if current is None or any( - current.get(column) != _normalize(value) for column, value in expected.items() - ): - raise HistoricalDataMigrationError( - f"{table_name} row differs from the historical after state" - ) - - -@Operations.implementation_for(HistoricalReferenceRowChangeOp) -def _apply_historical_reference_row_change( - operations: Operations, - operation: HistoricalReferenceRowChangeOp, -) -> None: - bind = operations.get_bind() - table = sa.Table( - operation.table_name, - sa.MetaData(), - schema="public", - autoload_with=bind, - ) - current = _current_row(bind, table, operation.key) - _assert_before(current, operation.before, table_name=operation.table_name) - - if operation.after is None: - bind.execute(table.delete().where(_predicate(table, operation.key))) - elif operation.before is None: - values = { - column: _coerce(table.c[column], value) - for column, value in operation.after.items() - } - bind.execute(table.insert().values(**values)) - else: - values = { - column: _coerce(table.c[column], value) - for column, value in operation.after.items() - if column not in operation.key - } - bind.execute( - table.update().where(_predicate(table, operation.key)).values(**values) - ) - - _assert_after( - _current_row(bind, table, operation.key), - operation.after, - table_name=operation.table_name, - ) diff --git a/tests/integration/test_alembic_v2_lifecycle.py b/tests/integration/test_alembic_v2_lifecycle.py index 460382ad4..1fe4868f0 100644 --- a/tests/integration/test_alembic_v2_lifecycle.py +++ b/tests/integration/test_alembic_v2_lifecycle.py @@ -23,11 +23,7 @@ from policyengine_api.data.v2.table_inventory import EXPECTED_V2_TABLES -BASELINE_REVISION = "47592781336f" -REPORT_UUID_PREVIOUS_REVISION = "5f048586d8f1" -REGION_DEFAULT_PREVIOUS_REVISION = "4faee127fa16" -VALIDATION_DATA_REVISION = "56dcd15a3afd" -HEAD_REVISION = "8b8ee7fe26bb" +HEAD_REVISION = "f5ef4347cb2a" def _disposable_url() -> str: @@ -72,7 +68,7 @@ def _assert_head(engine) -> None: assert (model_count, version_count) == (0, 0) -def test_empty_upgrade_check_boundary_downgrade_and_reupgrade() -> None: +def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: database_url = _disposable_url() config = _config() engine = create_engine(database_url) @@ -87,40 +83,20 @@ def test_empty_upgrade_check_boundary_downgrade_and_reupgrade() -> None: command.check(config) _assert_head(engine) - command.downgrade(config, BASELINE_REVISION) + command.downgrade(config, "base") + assert set(inspect(engine).get_table_names(schema="public")) <= { + "alembic_version" + } with engine.connect() as connection: context = MigrationContext.configure(connection) - assert context.get_current_revision() == BASELINE_REVISION - boundary_drift = compare_metadata(context, V2_METADATA) - boundary_kinds = [difference[0] for difference in boundary_drift] - assert boundary_kinds.count("remove_table") == 1 - assert boundary_kinds.count("remove_constraint") == 1 - assert boundary_kinds.count("add_fk") == 5 - assert boundary_kinds.count("add_column") == 2 - assert boundary_kinds.count("add_index") == 1 - assert boundary_kinds.count("add_constraint") == 4 - assert ( - sum( - isinstance(kind, tuple) and kind[0] == "modify_nullable" - for kind in boundary_kinds - ) - == 1 - ) - assert ( - sum( - isinstance(kind, tuple) and kind[0] == "modify_type" - for kind in boundary_kinds - ) - == 1 - ) - assert len(boundary_kinds) == 16 - model_count = connection.execute( + assert context.get_current_revision() is None + remaining_enum_count = connection.execute( text( - "SELECT count(*) FROM public.tax_benefit_models " - "WHERE name = 'stage8-platform-validation'" + "SELECT count(*) FROM pg_type " + "WHERE typname LIKE 'v2_%' AND typtype = 'e'" ) ).scalar_one() - assert model_count == 0 + assert remaining_enum_count == 0 command.upgrade(config, "head") command.check(config) @@ -130,48 +106,13 @@ def test_empty_upgrade_check_boundary_downgrade_and_reupgrade() -> None: engine.dispose() -def test_validation_cleanup_downgrades_and_reupgrades() -> None: - database_url = _disposable_url() - config = _config() - engine = create_engine(database_url) - - def validation_counts() -> tuple[int, int]: - with engine.connect() as connection: - model_count = connection.execute( - text( - "SELECT count(*) FROM public.tax_benefit_models " - "WHERE name = 'stage8-platform-validation'" - ) - ).scalar_one() - version_count = connection.execute( - text( - "SELECT count(*) FROM public.tax_benefit_model_versions " - "WHERE version = 'stage8-platform-validation'" - ) - ).scalar_one() - return model_count, version_count - - try: - command.upgrade(config, "head") - assert validation_counts() == (0, 0) - - command.downgrade(config, VALIDATION_DATA_REVISION) - assert validation_counts() == (1, 1) - - command.upgrade(config, "head") - assert validation_counts() == (0, 0) - finally: - command.upgrade(config, "head") - engine.dispose() - - def test_upgrade_to_head_validates_the_resulting_table_inventory() -> None: database_url = _disposable_url() config = _config() engine = create_engine(database_url) try: - command.downgrade(config, REGION_DEFAULT_PREVIOUS_REVISION) + command.downgrade(config, "base") with engine.begin() as connection: connection.execute(text("CREATE TABLE unreviewed_runtime_table (id INT)")) @@ -183,7 +124,7 @@ def test_upgrade_to_head_validates_the_resulting_table_inventory() -> None: with engine.connect() as connection: context = MigrationContext.configure(connection) - assert context.get_current_revision() == REGION_DEFAULT_PREVIOUS_REVISION + assert context.get_current_revision() is None finally: with engine.begin() as connection: connection.execute(text("DROP TABLE IF EXISTS unreviewed_runtime_table")) @@ -191,7 +132,7 @@ def test_upgrade_to_head_validates_the_resulting_table_inventory() -> None: engine.dispose() -def test_report_run_idempotency_uuid_revision_downgrades_and_reupgrades() -> None: +def test_baseline_uses_native_uuid_report_run_idempotency() -> None: database_url = _disposable_url() config = _config() engine = create_engine(database_url) @@ -214,10 +155,6 @@ def report_run_checks() -> set[str]: assert isinstance(idempotency_column_type(), PostgresUUID) assert "ck_report_runs_idempotency_key_nonblank" not in report_run_checks() - command.downgrade(config, REPORT_UUID_PREVIOUS_REVISION) - assert isinstance(idempotency_column_type(), sa.String) - assert "ck_report_runs_idempotency_key_nonblank" in report_run_checks() - model_id = uuid4() report_id = uuid4() report_run_id = uuid4() @@ -250,9 +187,6 @@ def report_run_checks() -> set[str]: }, ) - command.upgrade(config, "head") - assert isinstance(idempotency_column_type(), PostgresUUID) - assert "ck_report_runs_idempotency_key_nonblank" not in report_run_checks() with engine.connect() as connection: stored_key = connection.execute( text("SELECT idempotency_key FROM report_runs WHERE id = :run_id"), @@ -260,20 +194,25 @@ def report_run_checks() -> set[str]: ).scalar_one() assert stored_key == request_key assert isinstance(stored_key, UUID) - - command.downgrade(config, REPORT_UUID_PREVIOUS_REVISION) - with engine.connect() as connection: - stored_key = connection.execute( - text("SELECT idempotency_key FROM report_runs WHERE id = :run_id"), - {"run_id": report_run_id}, - ).scalar_one() - assert stored_key == str(request_key) finally: + with engine.begin() as connection: + connection.execute( + text("DELETE FROM report_runs WHERE id = :run_id"), + {"run_id": report_run_id}, + ) + connection.execute( + text("DELETE FROM reports WHERE id = :report_id"), + {"report_id": report_id}, + ) + connection.execute( + text("DELETE FROM tax_benefit_models WHERE id = :model_id"), + {"model_id": model_id}, + ) command.upgrade(config, "head") engine.dispose() -def test_region_default_revision_downgrades_reupgrades_and_enforces_model() -> None: +def test_baseline_region_default_enforces_same_model_dataset() -> None: database_url = _disposable_url() config = _config() engine = create_engine(database_url) @@ -283,13 +222,6 @@ def test_region_default_revision_downgrades_reupgrades_and_enforces_model() -> N second_dataset_id = uuid4() try: - command.upgrade(config, "head") - command.downgrade(config, REGION_DEFAULT_PREVIOUS_REVISION) - assert "region_datasets" in inspect(engine).get_table_names(schema="public") - assert "default_dataset_id" not in { - column["name"] for column in inspect(engine).get_columns("regions") - } - command.upgrade(config, "head") assert "region_datasets" not in inspect(engine).get_table_names(schema="public") default_column = next( diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index c4859e19b..d69495c53 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -195,8 +195,11 @@ def test_v2_environment_loads_only_the_exact_sqlmodel_inventory() -> None: assert "V1Base" not in env_source assert "migrations/v1" not in env_source assert "validate_v2_table_inventory" in env_source - assert "historical_reference_data_operations" in env_source + assert "historical_reference_data_operations" not in env_source assert "reference_data_autogenerate" not in env_source + assert not ( + REPO / "policyengine_api/data/v2/historical_reference_data_operations.py" + ).exists() assert not (REPO / "policyengine_api/data/v2/reference_data.py").exists() assert not ( REPO / "policyengine_api/data/v2/reference_data_autogenerate.py" @@ -215,104 +218,43 @@ def test_v2_files_are_mechanically_separate_from_v1() -> None: assert all("migrations/v1" not in str(path) for path in v2_files) -def test_v2_revision_chain_is_linear_generated_and_correction_bounded() -> None: +def test_v2_revision_chain_is_one_generated_correction_bounded_baseline() -> None: config = Config(str(REPO / "alembic-v2.ini")) script = ScriptDirectory.from_config(config) - assert script.get_heads() == ["8b8ee7fe26bb"] + assert script.get_heads() == ["f5ef4347cb2a"] assert [revision.revision for revision in script.walk_revisions()] == [ - "8b8ee7fe26bb", - "56dcd15a3afd", - "4faee127fa16", - "5f048586d8f1", - "b4c69674dd47", - "6ee725e0c563", - "47592781336f", + "f5ef4347cb2a", ] baseline = ( - REPO - / "migrations/v2/versions/47592781336f_establish_v2_core_schema_baseline.py" - ).read_text(encoding="utf-8") - data = ( - REPO - / "migrations/v2/versions/6ee725e0c563_add_stage_8_platform_validation_data.py" - ).read_text(encoding="utf-8") - ownership = ( - REPO / "migrations/v2/versions/" - "b4c69674dd47_enforce_v2_user_association_ownership.py" - ).read_text(encoding="utf-8") - constraints = ( - REPO - / "migrations/v2/versions/5f048586d8f1_constrain_v2_user_country_and_report_.py" - ).read_text(encoding="utf-8") - native_uuid = ( - REPO / "migrations/v2/versions/" - "4faee127fa16_use_native_uuid_report_run_idempotency_.py" - ).read_text(encoding="utf-8") - region_defaults = ( - REPO / "migrations/v2/versions/" - "56dcd15a3afd_assign_one_default_dataset_per_region.py" - ).read_text(encoding="utf-8") - validation_cleanup = ( - REPO / "migrations/v2/versions/8b8ee7fe26bb_remove_stage_8_validation_data.py" + REPO / "migrations/v2/versions/f5ef4347cb2a_establish_v2_platform_baseline.py" ).read_text(encoding="utf-8") - revisions = ( - baseline - + data - + ownership - + constraints - + native_uuid - + region_defaults - + validation_cleanup + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in baseline ) - assert all( - "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" in source - for source in ( - baseline, - data, - ownership, - constraints, - native_uuid, - region_defaults, - validation_cleanup, - ) - ) - assert "op.execute(" not in revisions - assert "op.bulk_insert(" not in revisions - assert data.count("op.v2_reference_row_change(") == 4 - assert validation_cleanup.count("op.v2_reference_row_change(") == 4 - assert validation_cleanup.count("after=None") == 2 - assert validation_cleanup.index("tax_benefit_model_versions") < ( - validation_cleanup.index("tax_benefit_models") - ) - assert "op.create_table(" not in data - assert "op.drop_table(" not in data - assert ownership.count("op.create_foreign_key(") == 4 - assert ownership.count("op.drop_constraint(") == 4 - assert 'ondelete="CASCADE"' in ownership - assert constraints.count("op.add_column(") == 1 - assert constraints.count("op.create_check_constraint(") == 2 - assert "ck_users_primary_country" in constraints - assert "ck_report_runs_idempotency_key_nonblank" in constraints - assert native_uuid.count("op.alter_column(") == 2 - assert native_uuid.count("postgresql_using=") == 2 - assert 'postgresql_using="idempotency_key::uuid"' in native_uuid - assert 'postgresql_using="idempotency_key::text"' in native_uuid - assert native_uuid.index("op.drop_constraint(") < native_uuid.index( - "op.alter_column(" + assert "down_revision: Union[str, None] = None" in baseline + assert "op.execute(" not in baseline + assert "op.bulk_insert(" not in baseline + assert "op.v2_reference_row_change(" not in baseline + assert "region_datasets" not in baseline + assert "historical_reference_data_operations" not in baseline + assert "ck_users_primary_country" in baseline + assert "ck_report_runs_idempotency_key_nonblank" not in baseline + assert re.search( + r'sa\.Column\(\s*"idempotency_key",\s*sa\.Uuid\(\)', + baseline, ) - assert 'op.drop_table("region_datasets")' in region_defaults - assert 'sa.Column("default_dataset_id", sa.Uuid(), nullable=False)' in ( - region_defaults - ) - assert "fk_regions_default_dataset_model_datasets" in region_defaults - assert "uq_datasets_model_name" in region_defaults - assert "ck_datasets_output_storage_path" in region_defaults + assert "fk_regions_default_dataset_model_datasets" in baseline + assert "uq_datasets_model_name" in baseline + assert "ck_datasets_output_storage_path" in baseline + assert baseline.count("op.create_table(") == len(EXPECTED_V2_TABLES) + assert baseline.count("op.drop_table(") == len(EXPECTED_V2_TABLES) corrected_enum_names = set( re.findall( r"sa\.Enum\(name=[\"']([^\"']+)[\"']\)\.drop\(op\.get_bind\(\)\)", - revisions, + baseline, ) ) assert corrected_enum_names == { @@ -332,17 +274,18 @@ def test_alembic_rejects_unknown_missing_and_divergent_history(tmp_path: Path) - original = REPO / "migrations/v2" missing = tmp_path / "missing" shutil.copytree(original, missing) - (missing / "versions/47592781336f_establish_v2_core_schema_baseline.py").unlink() + (missing / "versions/f5ef4347cb2a_establish_v2_platform_baseline.py").unlink() missing_config = Config() missing_config.set_main_option("script_location", str(missing)) - with pytest.raises((KeyError, ResolutionError)): - list(ScriptDirectory.from_config(missing_config).walk_revisions()) + missing_script = ScriptDirectory.from_config(missing_config) + with pytest.raises((CommandError, ResolutionError)): + missing_script.get_revision("f5ef4347cb2a") divergent = tmp_path / "divergent" shutil.copytree(original, divergent) - source = divergent / "versions/6ee725e0c563_add_stage_8_platform_validation_data.py" + source = divergent / "versions/f5ef4347cb2a_establish_v2_platform_baseline.py" duplicate = source.read_text(encoding="utf-8").replace( - "6ee725e0c563", "aaaaaaaaaaaa" + "f5ef4347cb2a", "aaaaaaaaaaaa" ) (divergent / "versions/aaaaaaaaaaaa_divergent.py").write_text( duplicate, From 89a48af1f3b5289b01e97c96c0b8bf5e9d16475c Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:28:39 +0300 Subject: [PATCH 13/18] Externalize Supabase target configuration --- .github/scripts/cloud_run_env.sh | 3 - .../scripts/validate_app_engine_deploy_env.sh | 2 + .github/workflows/push.yml | 8 +++ docs/engineering/skills/alembic-migrations.md | 6 +- docs/migration/stage-8-platform-runbook.md | 4 +- docs/migration/stage-8-supabase-bootstrap.md | 4 +- docs/migration/stage-8-supabase-target.md | 4 +- gcp/export.py | 2 + gcp/policyengine_api/app.yaml | 4 +- policyengine_api/data/v2/migration_target.py | 53 +++++++-------- policyengine_api/data/v2/settings.py | 23 +++++-- policyengine_api/data/v2/storage_bootstrap.py | 22 ++++--- tests/unit/test_cloud_run_deploy_scripts.py | 65 +++++++++++++++++-- tests/unit/v2/test_alembic_v2.py | 54 +++++++-------- tests/unit/v2/test_database.py | 2 +- tests/unit/v2/test_import_side_effects.py | 2 +- tests/unit/v2/test_settings.py | 2 +- tests/unit/v2/test_storage_bootstrap.py | 9 +-- 18 files changed, 176 insertions(+), 93 deletions(-) diff --git a/.github/scripts/cloud_run_env.sh b/.github/scripts/cloud_run_env.sh index 13d131094..80cc7361d 100755 --- a/.github/scripts/cloud_run_env.sh +++ b/.github/scripts/cloud_run_env.sh @@ -47,9 +47,6 @@ cloud_run_set_defaults() { CLOUD_RUN_VPC_NETWORK="${CLOUD_RUN_VPC_NETWORK:-default}" CLOUD_RUN_VPC_SUBNET="${CLOUD_RUN_VPC_SUBNET:-default}" CLOUD_RUN_VPC_EGRESS="${CLOUD_RUN_VPC_EGRESS:-private-ranges-only}" - V2_SUPABASE_PROJECT_REF="${V2_SUPABASE_PROJECT_REF:-kvrifaviwhzjztcbrfpy}" - V2_SUPABASE_ENVIRONMENT="${V2_SUPABASE_ENVIRONMENT:-production-foundation}" - local sha sha="${GITHUB_SHA:-local}" CLOUD_RUN_IMAGE_TAG="${CLOUD_RUN_IMAGE_TAG:-${sha}}" diff --git a/.github/scripts/validate_app_engine_deploy_env.sh b/.github/scripts/validate_app_engine_deploy_env.sh index 396496e43..41ca1f6e4 100644 --- a/.github/scripts/validate_app_engine_deploy_env.sh +++ b/.github/scripts/validate_app_engine_deploy_env.sh @@ -25,6 +25,8 @@ required=( RUNTIME_CACHE_ENVIRONMENT RUNTIME_CACHE_URL_SECRET_RESOURCE RUNTIME_CACHE_CA_CERT_SECRET_RESOURCE + V2_SUPABASE_PROJECT_REF + V2_SUPABASE_ENVIRONMENT ) missing=() diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 790fb05c8..a9bb3c897 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -181,6 +181,8 @@ jobs: RUNTIME_CACHE_ENVIRONMENT: staging RUNTIME_CACHE_URL_SECRET_RESOURCE: projects/policyengine-api/secrets/policyengine-api-staging-runtime-cache-url/versions/latest RUNTIME_CACHE_CA_CERT_SECRET_RESOURCE: projects/policyengine-api/secrets/policyengine-api-staging-runtime-cache-ca/versions/latest + V2_SUPABASE_PROJECT_REF: ${{ vars.V2_SUPABASE_PROJECT_REF }} + V2_SUPABASE_ENVIRONMENT: ${{ vars.V2_SUPABASE_ENVIRONMENT }} permissions: contents: read id-token: write @@ -264,6 +266,8 @@ jobs: CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT: staging CLOUD_RUN_RUNTIME_CACHE_URL_SECRET: policyengine-api-staging-runtime-cache-url:latest CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET: policyengine-api-staging-runtime-cache-ca:latest + V2_SUPABASE_PROJECT_REF: ${{ vars.V2_SUPABASE_PROJECT_REF }} + V2_SUPABASE_ENVIRONMENT: ${{ vars.V2_SUPABASE_ENVIRONMENT }} # Staging stays scale-to-zero, single instance: it exists for per-push # validation, not capacity. Both the revision-level (--min-instances) and # service-level (--min) floors are 0. @@ -508,6 +512,8 @@ jobs: RUNTIME_CACHE_ENVIRONMENT: production RUNTIME_CACHE_URL_SECRET_RESOURCE: projects/policyengine-api/secrets/policyengine-api-prod-runtime-cache-url/versions/latest RUNTIME_CACHE_CA_CERT_SECRET_RESOURCE: projects/policyengine-api/secrets/policyengine-api-prod-runtime-cache-ca/versions/latest + V2_SUPABASE_PROJECT_REF: ${{ vars.V2_SUPABASE_PROJECT_REF }} + V2_SUPABASE_ENVIRONMENT: ${{ vars.V2_SUPABASE_ENVIRONMENT }} permissions: contents: read id-token: write @@ -627,6 +633,8 @@ jobs: CLOUD_RUN_RUNTIME_CACHE_ENVIRONMENT: production CLOUD_RUN_RUNTIME_CACHE_URL_SECRET: policyengine-api-prod-runtime-cache-url:latest CLOUD_RUN_RUNTIME_CACHE_CA_CERT_SECRET: policyengine-api-prod-runtime-cache-ca:latest + V2_SUPABASE_PROJECT_REF: ${{ vars.V2_SUPABASE_PROJECT_REF }} + V2_SUPABASE_ENVIRONMENT: ${{ vars.V2_SUPABASE_ENVIRONMENT }} # Sized by the Stage 2 qualification and the PR 4 host cutover — rationale # and numbers in docs/migration/cloud-run-operations.md ("Runtime shape and # scaling"). Warm capacity is expressed service-level (--min); the diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 94ded8de3..50cf52d5f 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -91,7 +91,11 @@ otherwise non-Postgres targets fail before any migration operation and without printing the URL. A command against a persistent Supabase environment must also verify its -declared environment and project reference against the durable target record. +declared environment and project reference against the approved target +configuration. Deployment obtains both non-secret values from controlled +GitHub Environment variables; local administrative commands must obtain and +supply the same values from the approved operator inventory. Concrete target +identifiers must not be committed as defaults or application target maps. Before baseline generation or the first upgrade, it must require a successful freshness qualification proving that the target contains no application tables, Alembic history, or predecessor data. Missing, mismatched, ambiguous, diff --git a/docs/migration/stage-8-platform-runbook.md b/docs/migration/stage-8-platform-runbook.md index be8a41772..9d9f820d3 100644 --- a/docs/migration/stage-8-platform-runbook.md +++ b/docs/migration/stage-8-platform-runbook.md @@ -14,7 +14,9 @@ a durable domain store. Resolve the intended target through `V2_SUPABASE_PROJECT_REF` and `V2_SUPABASE_ENVIRONMENT`. Stop if the supplied connection cannot be proven to -match the separately maintained approved target inventory. +match the separately maintained approved target inventory. Deployment reads +both values from the selected GitHub Environment and has no tracked fallback; +explicit local administrative commands must supply them separately. ## Persistent Supabase qualification and initialization diff --git a/docs/migration/stage-8-supabase-bootstrap.md b/docs/migration/stage-8-supabase-bootstrap.md index 387ccb3b3..8dac3b891 100644 --- a/docs/migration/stage-8-supabase-bootstrap.md +++ b/docs/migration/stage-8-supabase-bootstrap.md @@ -11,7 +11,9 @@ approved environment configuration and secret-management surfaces. ## Required identity and credential boundaries The operator must resolve and validate all of the following without copying -their values into this repository: +their values into this repository. Deployment reads the first two values from +the selected GitHub Environment; explicit local administrative commands obtain +and supply them from the approved operator inventory: - `V2_SUPABASE_ENVIRONMENT` - `V2_SUPABASE_PROJECT_REF` diff --git a/docs/migration/stage-8-supabase-target.md b/docs/migration/stage-8-supabase-target.md index 3120b2ded..d8d53e800 100644 --- a/docs/migration/stage-8-supabase-target.md +++ b/docs/migration/stage-8-supabase-target.md @@ -12,9 +12,9 @@ configuration, or secret-management surface—not migration documentation. | Required field | Approved source | | --- | --- | | Supabase organization | Operator platform inventory | -| Project name and reference | Operator inventory and `V2_SUPABASE_PROJECT_REF` | +| Project name and reference | Operator inventory; `V2_SUPABASE_PROJECT_REF` GitHub Environment variable for deployment | | Region | Operator platform inventory | -| Environment classification | `V2_SUPABASE_ENVIRONMENT` | +| Environment classification | Operator inventory; `V2_SUPABASE_ENVIRONMENT` GitHub Environment variable for deployment | | Database host and pooler endpoint | Validated migration URL and provider console | | Storage API origin | `V2_SUPABASE_STORAGE_URL` | | Private bucket | `V2_SUPABASE_STORAGE_BUCKET` | diff --git a/gcp/export.py b/gcp/export.py index e42b859e4..b061be11a 100644 --- a/gcp/export.py +++ b/gcp/export.py @@ -73,6 +73,8 @@ def _render_app_config() -> str: ".runtime_cache_ca_cert_secret_resource": _required( "RUNTIME_CACHE_CA_CERT_SECRET_RESOURCE" ), + ".v2_supabase_project_ref": _required("V2_SUPABASE_PROJECT_REF"), + ".v2_supabase_environment": _required("V2_SUPABASE_ENVIRONMENT"), } template = Path("gcp/policyengine_api/app.yaml").read_text(encoding="utf-8") diff --git a/gcp/policyengine_api/app.yaml b/gcp/policyengine_api/app.yaml index f84e4c2d2..0a3dc0251 100644 --- a/gcp/policyengine_api/app.yaml +++ b/gcp/policyengine_api/app.yaml @@ -42,8 +42,8 @@ env_variables: RUNTIME_CACHE_SERVICE: "api" RUNTIME_CACHE_URL_SECRET_RESOURCE: ".runtime_cache_url_secret_resource" RUNTIME_CACHE_CA_CERT_SECRET_RESOURCE: ".runtime_cache_ca_cert_secret_resource" - V2_SUPABASE_PROJECT_REF: "kvrifaviwhzjztcbrfpy" - V2_SUPABASE_ENVIRONMENT: "production-foundation" + V2_SUPABASE_PROJECT_REF: ".v2_supabase_project_ref" + V2_SUPABASE_ENVIRONMENT: ".v2_supabase_environment" readiness_check: path: "/readiness-check" check_interval_sec: 30 diff --git a/policyengine_api/data/v2/migration_target.py b/policyengine_api/data/v2/migration_target.py index 4a527e727..247b75be7 100644 --- a/policyengine_api/data/v2/migration_target.py +++ b/policyengine_api/data/v2/migration_target.py @@ -14,9 +14,8 @@ from policyengine_api.data.v2.settings import ( POSTGRES_DRIVER, V2_MIGRATION_DATABASE_URL, - V2_SUPABASE_ENVIRONMENT, - V2_SUPABASE_PROJECT_REF, V2ConfigurationError, + load_supabase_target_settings, parse_persistent_postgres_url, ) from policyengine_api.data.v2.table_inventory import EXPECTED_V2_TABLES @@ -33,7 +32,7 @@ class V2MigrationTargetError(V2ConfigurationError): @dataclass(frozen=True) -class RecordedSupabaseTarget: +class ConfiguredSupabaseTarget: environment: str project_ref: str database_name: str @@ -42,23 +41,11 @@ class RecordedSupabaseTarget: freshness_audit_passed: bool -RECORDED_SUPABASE_TARGETS = { - "production-foundation": RecordedSupabaseTarget( - environment="production-foundation", - project_ref="kvrifaviwhzjztcbrfpy", - database_name="postgres", - migration_role=MIGRATION_ROLE, - freshness_audited_on=date(2026, 8, 13), - freshness_audit_passed=True, - ) -} - - @dataclass(frozen=True) class V2AlembicSettings: url: URL disposable_test: bool - target: RecordedSupabaseTarget | None + target: ConfiguredSupabaseTarget | None def _required(environ: Mapping[str, str], name: str) -> str: @@ -96,7 +83,7 @@ def _validate_disposable_url(url: URL) -> None: def _validate_persistent_url_identity( url: URL, - target: RecordedSupabaseTarget, + target: ConfiguredSupabaseTarget, ) -> None: direct_host = f"db.{target.project_ref}.supabase.co" is_direct = url.host == direct_host @@ -108,18 +95,18 @@ def _validate_persistent_url_identity( ) if not (is_direct or is_pooler): raise V2MigrationTargetError( - "the v2 migration URL does not identify the recorded Supabase project" + "the v2 migration URL does not identify the configured Supabase project" ) if url.database != target.database_name: raise V2MigrationTargetError( - "the v2 migration URL does not identify the recorded database" + "the v2 migration URL does not identify the configured database" ) def load_v2_alembic_settings( environ: Mapping[str, str] | None = None, ) -> V2AlembicSettings: - """Select either the recorded persistent target or isolated test Postgres.""" + """Select either the configured persistent target or isolated test Postgres.""" values = os.environ if environ is None else environ raw_url = _required(values, V2_MIGRATION_DATABASE_URL) @@ -137,14 +124,18 @@ def load_v2_alembic_settings( raw_url, setting_name=V2_MIGRATION_DATABASE_URL, ) - environment = _required(values, V2_SUPABASE_ENVIRONMENT) - project_ref = _required(values, V2_SUPABASE_PROJECT_REF) - target = RECORDED_SUPABASE_TARGETS.get(environment) - if target is None or target.project_ref != project_ref: - raise V2MigrationTargetError( - "the requested environment and project reference do not match a " - "recorded v2 migration target" - ) + try: + configured_identity = load_supabase_target_settings(values) + except V2ConfigurationError as error: + raise V2MigrationTargetError(str(error)) from error + target = ConfiguredSupabaseTarget( + environment=configured_identity.environment, + project_ref=configured_identity.project_ref, + database_name="postgres", + migration_role=MIGRATION_ROLE, + freshness_audited_on=date(2026, 8, 13), + freshness_audit_passed=True, + ) _validate_persistent_url_identity(persistent.url, target) return V2AlembicSettings( url=persistent.url, @@ -170,7 +161,7 @@ def qualify_v2_connection( identity = connection.execute(text("SELECT current_database(), current_user")).one() if identity[0] != target.database_name or identity[1] != target.migration_role: raise V2MigrationTargetError( - "the live database or migration identity does not match the recorded " + "the live database or migration identity does not match the configured " "v2 target" ) can_create = connection.execute( @@ -178,14 +169,14 @@ def qualify_v2_connection( ).scalar_one() if can_create is not True: raise V2MigrationTargetError( - "the recorded v2 migration identity lacks public schema CREATE" + "the configured v2 migration identity lacks public schema CREATE" ) public_tables = set(inspect(connection).get_table_names(schema="public")) if "alembic_version" not in public_tables: if not target.freshness_audit_passed: raise V2MigrationTargetError( - "the recorded first-use freshness audit has not passed" + "the configured first-use freshness audit has not passed" ) if public_tables: raise V2MigrationTargetError( diff --git a/policyengine_api/data/v2/settings.py b/policyengine_api/data/v2/settings.py index 6c565282d..6fdf14f15 100644 --- a/policyengine_api/data/v2/settings.py +++ b/policyengine_api/data/v2/settings.py @@ -98,9 +98,14 @@ def _required(environ: Mapping[str, str], name: str) -> str: return value.strip() -def _load_target(environ: Mapping[str, str]) -> SupabaseTargetSettings: - project_ref = _required(environ, V2_SUPABASE_PROJECT_REF) - environment = _required(environ, V2_SUPABASE_ENVIRONMENT) +def load_supabase_target_settings( + environ: Mapping[str, str] | None = None, +) -> SupabaseTargetSettings: + """Load the externally configured non-secret Supabase target identity.""" + + values = _environment(environ) + project_ref = _required(values, V2_SUPABASE_PROJECT_REF) + environment = _required(values, V2_SUPABASE_ENVIRONMENT) if PROJECT_REF_PATTERN.fullmatch(project_ref) is None: raise V2ConfigurationError( @@ -160,7 +165,10 @@ def load_v2_runtime_database_settings( _required(values, V2_RUNTIME_DATABASE_URL), setting_name=V2_RUNTIME_DATABASE_URL, ) - return V2DatabaseSettings(connection=connection, target=_load_target(values)) + return V2DatabaseSettings( + connection=connection, + target=load_supabase_target_settings(values), + ) def load_v2_migration_database_settings( @@ -173,7 +181,10 @@ def load_v2_migration_database_settings( _required(values, V2_MIGRATION_DATABASE_URL), setting_name=V2_MIGRATION_DATABASE_URL, ) - return V2DatabaseSettings(connection=connection, target=_load_target(values)) + return V2DatabaseSettings( + connection=connection, + target=load_supabase_target_settings(values), + ) def load_supabase_storage_settings( @@ -182,7 +193,7 @@ def load_supabase_storage_settings( """Load the separately authorized Supabase Storage administration surface.""" values = _environment(environ) - target = _load_target(values) + target = load_supabase_target_settings(values) api_url = _required(values, V2_SUPABASE_STORAGE_URL) bucket = _required(values, V2_SUPABASE_STORAGE_BUCKET) admin_key = _required(values, V2_SUPABASE_STORAGE_ADMIN_KEY) diff --git a/policyengine_api/data/v2/storage_bootstrap.py b/policyengine_api/data/v2/storage_bootstrap.py index bcd3e1929..f4cd156e1 100644 --- a/policyengine_api/data/v2/storage_bootstrap.py +++ b/policyengine_api/data/v2/storage_bootstrap.py @@ -8,12 +8,13 @@ import httpx -from policyengine_api.data.v2.settings import SupabaseStorageSettings +from policyengine_api.data.v2.settings import ( + ENVIRONMENT_PATTERN, + PROJECT_REF_PATTERN, + SupabaseStorageSettings, +) -RECORDED_STORAGE_TARGETS = { - "production-foundation": "kvrifaviwhzjztcbrfpy", -} STORAGE_REQUEST_TIMEOUT_SECONDS = 10.0 @@ -65,11 +66,14 @@ class StorageBootstrapResult: def _qualify_target(settings: SupabaseStorageSettings) -> None: - recorded_ref = RECORDED_STORAGE_TARGETS.get(settings.environment) - if recorded_ref != settings.project_ref: + expected_api_url = f"https://{settings.project_ref}.supabase.co" + if ( + PROJECT_REF_PATTERN.fullmatch(settings.project_ref) is None + or ENVIRONMENT_PATTERN.fullmatch(settings.environment) is None + or settings.api_url.rstrip("/") != expected_api_url + ): raise StorageBootstrapError( - "Storage environment and project reference do not match the " - "recorded Stage 8 target" + "Storage API origin does not match the configured Supabase target" ) @@ -158,7 +162,7 @@ def initialize_supabase_storage( *, client: StorageHTTPClient | None = None, ) -> StorageBootstrapResult: - """Create or verify the recorded private bucket without replacing it.""" + """Create or verify the configured private bucket without replacing it.""" _qualify_target(settings) expected = StorageBucketConfiguration( diff --git a/tests/unit/test_cloud_run_deploy_scripts.py b/tests/unit/test_cloud_run_deploy_scripts.py index 202f837f5..1216e4914 100644 --- a/tests/unit/test_cloud_run_deploy_scripts.py +++ b/tests/unit/test_cloud_run_deploy_scripts.py @@ -15,6 +15,8 @@ PRODUCTION_CLOUD_SQL_INSTANCE = "policyengine-api:us-central1:policyengine-api-data" PRODUCTION_CLOUD_RUN_SERVICE = "policyengine-api" STAGING_CLOUD_RUN_SERVICE = "policyengine-api-staging" +TEST_V2_PROJECT_REF = "abcdefghijklmnopqrst" +TEST_V2_ENVIRONMENT = "test-foundation" CLOUD_RUN_SERVICE_SCRIPTS = ( "scripts/deploy_cloud_run_candidate.sh", "scripts/capture_cloud_run_service_state.sh", @@ -106,6 +108,13 @@ def _app_engine_secret_resource_env() -> dict[str, str]: return dict(APP_ENGINE_SECRET_RESOURCES) +def _v2_target_env() -> dict[str, str]: + return { + "V2_SUPABASE_PROJECT_REF": TEST_V2_PROJECT_REF, + "V2_SUPABASE_ENVIRONMENT": TEST_V2_ENVIRONMENT, + } + + def _required_runtime_env() -> dict[str, str]: return { "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME": PRODUCTION_CLOUD_SQL_INSTANCE, @@ -122,6 +131,7 @@ def _required_runtime_env() -> dict[str, str]: "ROUTE_IMPL_METADATA": "fastapi_native", **_app_engine_secret_resource_env(), **_runtime_cache_resource_env(), + **_v2_target_env(), **_gateway_auth_env(), } @@ -602,6 +612,7 @@ def test_validate_cloud_run_deploy_env_accepts_direct_mode_from_environment(): POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=PRODUCTION_CLOUD_SQL_INSTANCE, **_app_engine_secret_resource_env(), **_runtime_cache_resource_env(), + **_v2_target_env(), **_gateway_auth_env(), ), ) @@ -684,6 +695,7 @@ def test_validate_cloud_run_deploy_env_requires_only_selected_url( POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=PRODUCTION_CLOUD_SQL_INSTANCE, **_app_engine_secret_resource_env(), **_runtime_cache_resource_env(), + **_v2_target_env(), **_gateway_auth_env(), ) missing_result = _run_script( @@ -722,6 +734,7 @@ def test_validate_app_engine_deploy_env_accepts_direct_mode_from_environment(): POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=PRODUCTION_CLOUD_SQL_INSTANCE, **_app_engine_secret_resource_env(), **_runtime_cache_resource_env(), + **_v2_target_env(), **_gateway_auth_env(), ), ) @@ -729,6 +742,30 @@ def test_validate_app_engine_deploy_env_accepts_direct_mode_from_environment(): assert result.returncode == 0, result.stderr +@pytest.mark.parametrize( + "validation_script", + [ + ".github/scripts/validate_app_engine_deploy_env.sh", + ".github/scripts/validate_cloud_run_deploy_env.sh", + ], +) +@pytest.mark.parametrize( + "missing_name", + ["V2_SUPABASE_PROJECT_REF", "V2_SUPABASE_ENVIRONMENT"], +) +def test_deployment_validation_requires_supabase_target_variables( + validation_script, + missing_name, +): + env = _script_env(**_required_runtime_env()) + env.pop(missing_name) + + result = _run_script(validation_script, env) + + assert result.returncode == 1 + assert missing_name in result.stderr + + @pytest.mark.parametrize( ("entrypoint", "selected_url_env", "selected_url"), [ @@ -754,6 +791,7 @@ def test_validate_app_engine_deploy_env_requires_only_selected_url( POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=PRODUCTION_CLOUD_SQL_INSTANCE, **_app_engine_secret_resource_env(), **_runtime_cache_resource_env(), + **_v2_target_env(), **_gateway_auth_env(), ) missing_result = _run_script( @@ -808,8 +846,10 @@ def test_app_engine_bundle_contains_runtime_environment_placeholders(): is None ) assert 'RUNTIME_CACHE_MODE: "deployed"' in app_config - assert 'V2_SUPABASE_PROJECT_REF: "kvrifaviwhzjztcbrfpy"' in app_config - assert 'V2_SUPABASE_ENVIRONMENT: "production-foundation"' in app_config + assert 'V2_SUPABASE_PROJECT_REF: ".v2_supabase_project_ref"' in app_config + assert 'V2_SUPABASE_ENVIRONMENT: ".v2_supabase_environment"' in app_config + assert '"V2_SUPABASE_PROJECT_REF"' in export_script + assert '"V2_SUPABASE_ENVIRONMENT"' in export_script assert ( 'RUNTIME_CACHE_URL_SECRET_RESOURCE: ".runtime_cache_url_secret_resource"' in app_config @@ -822,6 +862,23 @@ def test_app_engine_bundle_contains_runtime_environment_placeholders(): assert "python3 gcp/export.py" in bundle_script +def test_deployment_jobs_read_supabase_identity_from_github_environment_variables(): + workflow = _push_workflow() + + for job_name in ( + "deploy-staging", + "deploy-cloud-run-staging", + "deploy-production-candidate", + "deploy-cloud-run-candidate", + ): + job = _workflow_job_block(workflow, job_name) + assert "V2_SUPABASE_PROJECT_REF: ${{ vars.V2_SUPABASE_PROJECT_REF }}" in job + assert "V2_SUPABASE_ENVIRONMENT: ${{ vars.V2_SUPABASE_ENVIRONMENT }}" in job + + assert TEST_V2_PROJECT_REF not in workflow + assert TEST_V2_ENVIRONMENT not in workflow + + @pytest.mark.parametrize( ("ignore_file", "required_rules"), [ @@ -1046,8 +1103,8 @@ def test_deploy_cloud_run_candidate_dry_run_never_shifts_traffic(): "RUNTIME_CACHE_CA_CERT=policyengine-api-prod-runtime-cache-ca:latest" in result.stdout ) - assert "V2_SUPABASE_PROJECT_REF=kvrifaviwhzjztcbrfpy" in result.stdout - assert "V2_SUPABASE_ENVIRONMENT=production-foundation" in result.stdout + assert f"V2_SUPABASE_PROJECT_REF={TEST_V2_PROJECT_REF}" in result.stdout + assert f"V2_SUPABASE_ENVIRONMENT={TEST_V2_ENVIRONMENT}" in result.stdout assert "V2_DATABASE_URL" not in result.stdout assert "V2_STORAGE_ADMIN_KEY" not in result.stdout for env_name, secret_ref in CLOUD_RUN_SECRET_MAPPINGS.items(): diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index d69495c53..9dee7598f 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -17,9 +17,9 @@ from policyengine_api.constants import REPO from policyengine_api.data.v1_models import V1Base from policyengine_api.data.v2.migration_target import ( + ConfiguredSupabaseTarget, DISPOSABLE_DATABASE_NAME, MIGRATION_ROLE, - RECORDED_SUPABASE_TARGETS, V2_ALEMBIC_DISPOSABLE_TEST, V2AlembicSettings, V2MigrationTargetError, @@ -36,7 +36,8 @@ from policyengine_api.data.v2.table_inventory import EXPECTED_V2_TABLES -PROJECT_REF = "kvrifaviwhzjztcbrfpy" +PROJECT_REF = "abcdefghijklmnopqrst" +TARGET_ENVIRONMENT = "test-foundation" POOLER_URL = ( "postgresql+psycopg://policyengine_v2_migrator." f"{PROJECT_REF}:test-password@aws-0-us-east-2.pooler.supabase.com:5432/" @@ -132,22 +133,22 @@ def test_v2_alembic_rejects_offline_execution_even_in_disposable_mode( command.upgrade(config, "head", sql=True) -def test_persistent_target_requires_the_recorded_environment_and_project() -> None: - with pytest.raises(V2MigrationTargetError, match="recorded"): +def test_persistent_target_requires_the_configured_project_to_match_the_url() -> None: + with pytest.raises(V2MigrationTargetError, match="configured Supabase project"): load_v2_alembic_settings( { V2_MIGRATION_DATABASE_URL: POOLER_URL, - V2_SUPABASE_ENVIRONMENT: "production-foundation", + V2_SUPABASE_ENVIRONMENT: TARGET_ENVIRONMENT, V2_SUPABASE_PROJECT_REF: "aaaaaaaaaaaaaaaaaaaa", } ) -def test_pooler_identity_resolves_the_recorded_persistent_target() -> None: +def test_pooler_identity_resolves_the_configured_persistent_target() -> None: settings = load_v2_alembic_settings( { V2_MIGRATION_DATABASE_URL: POOLER_URL, - V2_SUPABASE_ENVIRONMENT: "production-foundation", + V2_SUPABASE_ENVIRONMENT: TARGET_ENVIRONMENT, V2_SUPABASE_PROJECT_REF: PROJECT_REF, } ) @@ -159,14 +160,14 @@ def test_pooler_identity_resolves_the_recorded_persistent_target() -> None: def test_persistent_mode_rejects_an_ambiguous_non_supabase_host() -> None: - with pytest.raises(V2MigrationTargetError, match="recorded Supabase project"): + with pytest.raises(V2MigrationTargetError, match="configured Supabase project"): load_v2_alembic_settings( { V2_MIGRATION_DATABASE_URL: ( "postgresql+psycopg://policyengine_v2_migrator:password@" "db.example.com/postgres?sslmode=require" ), - V2_SUPABASE_ENVIRONMENT: "production-foundation", + V2_SUPABASE_ENVIRONMENT: TARGET_ENVIRONMENT, V2_SUPABASE_PROJECT_REF: PROJECT_REF, } ) @@ -321,26 +322,27 @@ def _persistent_connection( return connection -def test_persistent_first_use_requires_recorded_successful_freshness_audit( +def test_persistent_first_use_requires_configured_successful_freshness_audit( monkeypatch: pytest.MonkeyPatch, ) -> None: - target = RECORDED_SUPABASE_TARGETS["production-foundation"] - unaudited = target.__class__( - environment=target.environment, - project_ref=target.project_ref, - database_name=target.database_name, - migration_role=target.migration_role, - freshness_audited_on=target.freshness_audited_on, + configured = load_v2_alembic_settings( + { + V2_MIGRATION_DATABASE_URL: POOLER_URL, + V2_SUPABASE_ENVIRONMENT: TARGET_ENVIRONMENT, + V2_SUPABASE_PROJECT_REF: PROJECT_REF, + } + ) + assert configured.target is not None + unaudited = ConfiguredSupabaseTarget( + environment=configured.target.environment, + project_ref=configured.target.project_ref, + database_name=configured.target.database_name, + migration_role=configured.target.migration_role, + freshness_audited_on=configured.target.freshness_audited_on, freshness_audit_passed=False, ) settings = V2AlembicSettings( - url=load_v2_alembic_settings( - { - V2_MIGRATION_DATABASE_URL: POOLER_URL, - V2_SUPABASE_ENVIRONMENT: "production-foundation", - V2_SUPABASE_PROJECT_REF: PROJECT_REF, - } - ).url, + url=configured.url, disposable_test=False, target=unaudited, ) @@ -358,7 +360,7 @@ def test_persistent_target_rejects_unstamped_nonfresh_inventory( settings = load_v2_alembic_settings( { V2_MIGRATION_DATABASE_URL: POOLER_URL, - V2_SUPABASE_ENVIRONMENT: "production-foundation", + V2_SUPABASE_ENVIRONMENT: TARGET_ENVIRONMENT, V2_SUPABASE_PROJECT_REF: PROJECT_REF, } ) @@ -379,7 +381,7 @@ def test_persistent_target_allows_stamped_previous_revision_inventory( settings = load_v2_alembic_settings( { V2_MIGRATION_DATABASE_URL: POOLER_URL, - V2_SUPABASE_ENVIRONMENT: "production-foundation", + V2_SUPABASE_ENVIRONMENT: TARGET_ENVIRONMENT, V2_SUPABASE_PROJECT_REF: PROJECT_REF, } ) diff --git a/tests/unit/v2/test_database.py b/tests/unit/v2/test_database.py index 53b0fe864..de94a1d13 100644 --- a/tests/unit/v2/test_database.py +++ b/tests/unit/v2/test_database.py @@ -20,7 +20,7 @@ def _environment(*, username: str = "runtime") -> dict[str, str]: return { V2_SUPABASE_PROJECT_REF: "abcdefghijklmnopqrst", - V2_SUPABASE_ENVIRONMENT: "production-foundation", + V2_SUPABASE_ENVIRONMENT: "test-foundation", V2_RUNTIME_DATABASE_URL: ( f"postgresql+psycopg://{username}:test-password@db.example.com:5432/" "postgres?sslmode=require" diff --git a/tests/unit/v2/test_import_side_effects.py b/tests/unit/v2/test_import_side_effects.py index 33e5047b5..c9810f89c 100644 --- a/tests/unit/v2/test_import_side_effects.py +++ b/tests/unit/v2/test_import_side_effects.py @@ -139,7 +139,7 @@ def test_selected_v2_runtime_without_its_url_fails_closed() -> None: load_v2_runtime_database_settings( { V2_SUPABASE_PROJECT_REF: "abcdefghijklmnopqrst", - V2_SUPABASE_ENVIRONMENT: "production-foundation", + V2_SUPABASE_ENVIRONMENT: "test-foundation", "ALEMBIC_DATABASE_URL": "mysql+pymysql://v1:secret@db/v1", "FLASK_DEBUG": "1", } diff --git a/tests/unit/v2/test_settings.py b/tests/unit/v2/test_settings.py index 837487f8c..5a444e0c4 100644 --- a/tests/unit/v2/test_settings.py +++ b/tests/unit/v2/test_settings.py @@ -20,7 +20,7 @@ PROJECT_REF = "abcdefghijklmnopqrst" TARGET_ENVIRONMENT = { V2_SUPABASE_PROJECT_REF: PROJECT_REF, - V2_SUPABASE_ENVIRONMENT: "production-foundation", + V2_SUPABASE_ENVIRONMENT: "test-foundation", } RUNTIME_URL = ( "postgresql+psycopg://runtime:test-runtime-password@db.example.com:5432/" diff --git a/tests/unit/v2/test_storage_bootstrap.py b/tests/unit/v2/test_storage_bootstrap.py index 857a4a776..338d7ba02 100644 --- a/tests/unit/v2/test_storage_bootstrap.py +++ b/tests/unit/v2/test_storage_bootstrap.py @@ -15,7 +15,8 @@ ) -PROJECT_REF = "kvrifaviwhzjztcbrfpy" +PROJECT_REF = "abcdefghijklmnopqrst" +TARGET_ENVIRONMENT = "test-foundation" BUCKET = "policyengine-v2-alpha" ADMIN_KEY = "test-storage-admin-secret" @@ -23,7 +24,7 @@ def _settings(**overrides) -> SupabaseStorageSettings: values = { "project_ref": PROJECT_REF, - "environment": "production-foundation", + "environment": TARGET_ENVIRONMENT, "api_url": f"https://{PROJECT_REF}.supabase.co", "bucket": BUCKET, "admin_key": SecretStr(ADMIN_KEY), @@ -140,9 +141,9 @@ def handler(_request: httpx.Request) -> httpx.Response: raise AssertionError("target mismatch must make no request") with _client(handler) as client: - with pytest.raises(StorageBootstrapError, match="recorded Stage 8"): + with pytest.raises(StorageBootstrapError, match="configured Supabase target"): initialize_supabase_storage( - _settings(project_ref="aaaaaaaaaaaaaaaaaaaa"), + _settings(api_url="https://aaaaaaaaaaaaaaaaaaaa.supabase.co"), client=client, ) From f9416568a79c31ec3b53410b839e50d08a5e61bb Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:44:02 +0300 Subject: [PATCH 14/18] Derive v2 schema validation from metadata --- docs/engineering/skills/alembic-migrations.md | 10 +- docs/engineering/skills/testing.md | 4 +- migrations/v2/env.py | 6 +- .../data/v2/metadata_validation.py | 53 ++++++ policyengine_api/data/v2/migration_target.py | 17 +- policyengine_api/data/v2/models/__init__.py | 41 +---- policyengine_api/data/v2/table_inventory.py | 159 ------------------ .../integration/test_alembic_v2_lifecycle.py | 8 +- tests/unit/v2/test_alembic_v2.py | 30 ++-- tests/unit/v2/test_metadata_validation.py | 35 ++++ tests/unit/v2/test_models.py | 30 ++-- tests/unit/v2/test_table_inventory.py | 44 ----- 12 files changed, 146 insertions(+), 291 deletions(-) create mode 100644 policyengine_api/data/v2/metadata_validation.py delete mode 100644 policyengine_api/data/v2/table_inventory.py create mode 100644 tests/unit/v2/test_metadata_validation.py delete mode 100644 tests/unit/v2/test_table_inventory.py diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 50cf52d5f..e70a69d75 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -119,10 +119,12 @@ lifecycle tests and must be rejected for staging, production, or any other persistent target. The v2 environment imports the controlled v2 table-model package before -exposing `SQLModel.metadata`. It must compare the resulting table names with -the reviewed inventory and fail before generation or execution if expected -tables are absent or v1, predecessor, `runtime_bundles`, population, or other -unreviewed tables are registered. +exposing `SQLModel.metadata`. That metadata is the authoritative application +table set; do not maintain a second exact table-name allowlist. Targeted +validation must reject known v1-only names, `runtime_bundles`, and standalone +population tables. Review every autogenerated table change, and compare the +live schema directly with the complete SQLModel metadata during the migration +lifecycle. ## V2 application-data migrations diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index b5c9ee46d..9d812b12c 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -70,8 +70,8 @@ V2_ALEMBIC_DISPOSABLE_TEST=1 V2_MIGRATION_DATABASE_URL="postgresql+psycopg://... ``` The disposable lifecycle must start from empty Postgres, upgrade to `head`, -check schema drift, compare the live schema with the exact -SQLModel inventory, downgrade the compact baseline to `base`, verify its native +check schema drift, compare the live schema directly with the authoritative +SQLModel metadata, downgrade the compact baseline to `base`, verify its native Postgres enum types are removed, and upgrade again. It must not use the persistent Supabase qualification bypass outside explicit disposable-test mode. Continue running the existing isolated v1 MySQL diff --git a/migrations/v2/env.py b/migrations/v2/env.py index 2f1a30a40..7a65ce625 100644 --- a/migrations/v2/env.py +++ b/migrations/v2/env.py @@ -9,10 +9,9 @@ from policyengine_api.data.v2.migration_target import ( load_v2_alembic_settings, qualify_v2_connection, - validate_v2_head_table_inventory, + validate_v2_head_schema, ) from policyengine_api.data.v2.models import V2_METADATA -from policyengine_api.data.v2.table_inventory import validate_v2_table_inventory config = context.config @@ -21,7 +20,6 @@ settings = load_v2_alembic_settings() target_metadata = V2_METADATA -validate_v2_table_inventory(target_metadata.tables) script = ScriptDirectory.from_config(config) @@ -50,7 +48,7 @@ def _configure(connection) -> None: current_heads = frozenset(migration_context.get_current_heads()) script_heads = frozenset(script.get_heads()) if current_heads != previous_heads and current_heads == script_heads: - validate_v2_head_table_inventory(connection) + validate_v2_head_schema(connection, target_metadata) def run_migrations_offline() -> None: diff --git a/policyengine_api/data/v2/metadata_validation.py b/policyengine_api/data/v2/metadata_validation.py new file mode 100644 index 000000000..20b2efa0b --- /dev/null +++ b/policyengine_api/data/v2/metadata_validation.py @@ -0,0 +1,53 @@ +"""Targeted validation for tables that must not enter v2 metadata.""" + +from collections.abc import Iterable + + +# These names belong only to the existing v1 database domain. Names that are +# intentionally shared between v1 and v2, such as `simulations` and +# `user_policies`, are excluded from this targeted rejection set. +V1_ONLY_TABLES = frozenset( + { + "analysis", + "computed_household", + "economy", + "household", + "legacy_report_output_aliases", + "policy", + "reform_impact", + "report_output_runs", + "report_outputs", + "simulation_runs", + "tracers", + "user_profiles", + } +) + +# The v2 design explicitly rejects the former runtime-bundle indirection and a +# standalone population model. Population-valued columns on impact tables are +# unrelated to these table names. +PROHIBITED_V2_TABLES = frozenset( + { + "population", + "populations", + "runtime_bundle", + "runtime_bundles", + } +) + + +class V2MetadataValidationError(RuntimeError): + """Raised when v2 metadata contains a specifically rejected table.""" + + +def validate_v2_metadata_table_names(table_names: Iterable[str]) -> None: + """Reject known v1-only and explicitly prohibited v2 table names.""" + + actual = frozenset(table_names) + prohibited = sorted(actual & PROHIBITED_V2_TABLES) + v1_only = sorted(actual & V1_ONLY_TABLES) + if prohibited or v1_only: + raise V2MetadataValidationError( + "API v2-alpha metadata contains rejected tables: " + f"prohibited={prohibited}, v1_only={v1_only}" + ) diff --git a/policyengine_api/data/v2/migration_target.py b/policyengine_api/data/v2/migration_target.py index 247b75be7..0fd95722d 100644 --- a/policyengine_api/data/v2/migration_target.py +++ b/policyengine_api/data/v2/migration_target.py @@ -7,7 +7,7 @@ from datetime import date import os -from sqlalchemy import Connection, inspect, text +from sqlalchemy import Connection, MetaData, inspect, text from sqlalchemy.engine import URL, make_url from sqlalchemy.exc import ArgumentError @@ -18,8 +18,6 @@ load_supabase_target_settings, parse_persistent_postgres_url, ) -from policyengine_api.data.v2.table_inventory import EXPECTED_V2_TABLES - V2_ALEMBIC_DISPOSABLE_TEST = "V2_ALEMBIC_DISPOSABLE_TEST" DISPOSABLE_DATABASE_NAME = "policyengine_v2_alembic_test" @@ -185,15 +183,16 @@ def qualify_v2_connection( ) -def validate_v2_head_table_inventory(connection: Connection) -> None: - """Require the exact reviewed table inventory after upgrading to v2 head.""" +def validate_v2_head_schema(connection: Connection, metadata: MetaData) -> None: + """Require live application tables to match authoritative ORM metadata.""" public_tables = set(inspect(connection).get_table_names(schema="public")) application_tables = public_tables - {"alembic_version"} - unexpected = application_tables - EXPECTED_V2_TABLES - missing = EXPECTED_V2_TABLES - application_tables + metadata_tables = {table.name for table in metadata.tables.values()} + unexpected = application_tables - metadata_tables + missing = metadata_tables - application_tables if unexpected or missing: raise V2MigrationTargetError( - "the v2 head table inventory differs from the reviewed " - f"schema: missing={sorted(missing)}, unexpected={sorted(unexpected)}" + "the v2 head schema differs from SQLModel metadata: " + f"missing={sorted(missing)}, unexpected={sorted(unexpected)}" ) diff --git a/policyengine_api/data/v2/models/__init__.py b/policyengine_api/data/v2/models/__init__.py index 285bf7f22..491207ef4 100644 --- a/policyengine_api/data/v2/models/__init__.py +++ b/policyengine_api/data/v2/models/__init__.py @@ -2,7 +2,9 @@ from sqlmodel import SQLModel -from policyengine_api.data.v2.table_inventory import validate_v2_table_inventory +from policyengine_api.data.v2.metadata_validation import ( + validate_v2_metadata_table_names, +) # Apply deterministic names before any v2 table is declared. This is a local @@ -72,41 +74,7 @@ V2_METADATA = SQLModel.metadata -V2_TABLE_MODELS = ( - AggregateOutput, - BudgetSummary, - ChangeAggregate, - CongressionalDistrictImpact, - ConstituencyImpact, - Dataset, - DatasetVersion, - DecileImpact, - Dynamic, - Household, - HouseholdJob, - Inequality, - IntraDecileImpact, - LocalAuthorityImpact, - Parameter, - ParameterNode, - ParameterValue, - Policy, - Poverty, - ProgramStatistics, - Region, - Report, - ReportRun, - Simulation, - TaxBenefitModel, - TaxBenefitModelVersion, - User, - UserHouseholdAssociation, - UserPolicy, - UserReportAssociation, - UserSimulationAssociation, - Variable, -) -validate_v2_table_inventory(V2_METADATA.tables) +validate_v2_metadata_table_names(V2_METADATA.tables) __all__ = [ "AggregateOutput", @@ -150,6 +118,5 @@ "UserReportAssociation", "UserSimulationAssociation", "V2_METADATA", - "V2_TABLE_MODELS", "Variable", ] diff --git a/policyengine_api/data/v2/table_inventory.py b/policyengine_api/data/v2/table_inventory.py deleted file mode 100644 index ddc80c08f..000000000 --- a/policyengine_api/data/v2/table_inventory.py +++ /dev/null @@ -1,159 +0,0 @@ -"""Reviewed Stage 8 API v2-alpha application-table inventory. - -This module is deliberately independent of ORM imports. SQLModel metadata, -Alembic generation, lifecycle tests, and live-schema comparisons all use this -single allowlist. Supabase-managed tables and Alembic's version table are not -application tables and are outside this inventory. -""" - -from collections.abc import Iterable - - -V2_TABLE_GROUPS: tuple[tuple[str, frozenset[str]], ...] = ( - ( - "identity", - frozenset( - { - "users", - "user_household_associations", - "user_policies", - "user_report_associations", - "user_simulation_associations", - } - ), - ), - ( - "model_metadata", - frozenset( - { - "tax_benefit_models", - "tax_benefit_model_versions", - } - ), - ), - ( - "regions_and_datasets", - frozenset( - { - "regions", - "datasets", - "dataset_versions", - } - ), - ), - ( - "variables_and_parameters", - frozenset( - { - "variables", - "parameter_nodes", - "parameters", - "parameter_values", - } - ), - ), - ( - "policies", - frozenset( - { - "policies", - "dynamics", - } - ), - ), - ( - "households_and_simulations", - frozenset( - { - "households", - "household_jobs", - "simulations", - } - ), - ), - ( - "reports_and_outputs", - frozenset( - { - "reports", - "report_runs", - "aggregates", - "change_aggregates", - } - ), - ), - ( - "impact_results", - frozenset( - { - "budget_summary", - "congressional_district_impacts", - "constituency_impacts", - "decile_impacts", - "inequality", - "intra_decile_impacts", - "local_authority_impacts", - "poverty", - "program_statistics", - } - ), - ), -) - -EXPECTED_V2_TABLES = frozenset( - table_name for _, table_names in V2_TABLE_GROUPS for table_name in table_names -) - -# These v1-only names make accidental V1Base metadata registration especially -# clear. Names intentionally reviewed for both domains, such as `simulations` -# and `user_policies`, remain valid because the exact allowlist is authoritative. -V1_ONLY_TABLES = frozenset( - { - "analysis", - "computed_household", - "economy", - "household", - "legacy_report_output_aliases", - "policy", - "reform_impact", - "report_output_runs", - "report_outputs", - "simulation_runs", - "tracers", - "user_profiles", - } -) - -# Stage 8 explicitly rejects the former runtime-bundle indirection and any -# standalone population model. Population-valued columns on reviewed impact -# tables are unrelated to these prohibited table names. -PROHIBITED_V2_TABLES = frozenset( - { - "population", - "populations", - "runtime_bundle", - "runtime_bundles", - } -) - - -class V2TableInventoryError(RuntimeError): - """Raised when table metadata differs from the reviewed Stage 8 schema.""" - - -def validate_v2_table_inventory(table_names: Iterable[str]) -> None: - """Fail closed unless *table_names* exactly match the reviewed allowlist.""" - - actual = frozenset(table_names) - if actual == EXPECTED_V2_TABLES: - return - - missing = sorted(EXPECTED_V2_TABLES - actual) - unexpected = sorted(actual - EXPECTED_V2_TABLES) - prohibited = sorted(actual & PROHIBITED_V2_TABLES) - v1_only = sorted(actual & V1_ONLY_TABLES) - raise V2TableInventoryError( - "API v2-alpha table inventory mismatch: " - f"missing={missing}, unexpected={unexpected}, " - f"prohibited={prohibited}, v1_only={v1_only}" - ) diff --git a/tests/integration/test_alembic_v2_lifecycle.py b/tests/integration/test_alembic_v2_lifecycle.py index 1fe4868f0..69ab2e69f 100644 --- a/tests/integration/test_alembic_v2_lifecycle.py +++ b/tests/integration/test_alembic_v2_lifecycle.py @@ -20,10 +20,10 @@ ) from policyengine_api.data.v2.models import V2_METADATA from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL -from policyengine_api.data.v2.table_inventory import EXPECTED_V2_TABLES HEAD_REVISION = "f5ef4347cb2a" +V2_TABLE_NAMES = frozenset(table.name for table in V2_METADATA.tables.values()) def _disposable_url() -> str: @@ -47,7 +47,7 @@ def _config() -> Config: def _assert_head(engine) -> None: assert set(inspect(engine).get_table_names(schema="public")) == ( - EXPECTED_V2_TABLES | {"alembic_version"} + V2_TABLE_NAMES | {"alembic_version"} ) with engine.connect() as connection: context = MigrationContext.configure(connection) @@ -106,7 +106,7 @@ def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: engine.dispose() -def test_upgrade_to_head_validates_the_resulting_table_inventory() -> None: +def test_upgrade_to_head_validates_the_resulting_schema_against_metadata() -> None: database_url = _disposable_url() config = _config() engine = create_engine(database_url) @@ -118,7 +118,7 @@ def test_upgrade_to_head_validates_the_resulting_table_inventory() -> None: with pytest.raises( V2MigrationTargetError, - match="v2 head table inventory.*unreviewed_runtime_table", + match="v2 head schema.*unreviewed_runtime_table", ): command.upgrade(config, "head") diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index 9dee7598f..dfb0f0dc9 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -25,7 +25,7 @@ V2MigrationTargetError, load_v2_alembic_settings, qualify_v2_connection, - validate_v2_head_table_inventory, + validate_v2_head_schema, ) from policyengine_api.data.v2.models import V2_METADATA from policyengine_api.data.v2.settings import ( @@ -33,11 +33,10 @@ V2_SUPABASE_ENVIRONMENT, V2_SUPABASE_PROJECT_REF, ) -from policyengine_api.data.v2.table_inventory import EXPECTED_V2_TABLES - PROJECT_REF = "abcdefghijklmnopqrst" TARGET_ENVIRONMENT = "test-foundation" +V2_TABLE_NAMES = frozenset(table.name for table in V2_METADATA.tables.values()) POOLER_URL = ( "postgresql+psycopg://policyengine_v2_migrator." f"{PROJECT_REF}:test-password@aws-0-us-east-2.pooler.supabase.com:5432/" @@ -188,14 +187,13 @@ def test_target_errors_never_echo_url_passwords() -> None: assert secret not in str(raised.value) -def test_v2_environment_loads_only_the_exact_sqlmodel_inventory() -> None: +def test_v2_environment_uses_only_sqlmodel_metadata() -> None: env_source = (REPO / "migrations" / "v2" / "env.py").read_text(encoding="utf-8") assert V2_METADATA is not V1Base.metadata - assert set(V2_METADATA.tables) == EXPECTED_V2_TABLES + assert V2_TABLE_NAMES assert "V1Base" not in env_source assert "migrations/v1" not in env_source - assert "validate_v2_table_inventory" in env_source assert "historical_reference_data_operations" not in env_source assert "reference_data_autogenerate" not in env_source assert not ( @@ -249,8 +247,8 @@ def test_v2_revision_chain_is_one_generated_correction_bounded_baseline() -> Non assert "fk_regions_default_dataset_model_datasets" in baseline assert "uq_datasets_model_name" in baseline assert "ck_datasets_output_storage_path" in baseline - assert baseline.count("op.create_table(") == len(EXPECTED_V2_TABLES) - assert baseline.count("op.drop_table(") == len(EXPECTED_V2_TABLES) + assert baseline.count("op.create_table(") == len(V2_TABLE_NAMES) + assert baseline.count("op.drop_table(") == len(V2_TABLE_NAMES) corrected_enum_names = set( re.findall( @@ -399,10 +397,10 @@ def test_persistent_target_allows_stamped_previous_revision_inventory( "public_tables", [ {"alembic_version", "reports"}, - {"alembic_version", *EXPECTED_V2_TABLES, "runtime_bundles"}, + {"alembic_version", *V2_TABLE_NAMES, "runtime_bundles"}, ], ) -def test_v2_head_rejects_divergent_table_inventory( +def test_v2_head_rejects_schema_divergent_from_metadata( monkeypatch: pytest.MonkeyPatch, public_tables: set[str], ) -> None: @@ -411,19 +409,19 @@ def test_v2_head_rejects_divergent_table_inventory( public_tables=public_tables, ) - with pytest.raises(V2MigrationTargetError, match="v2 head table inventory"): - validate_v2_head_table_inventory(connection) + with pytest.raises(V2MigrationTargetError, match="v2 head schema"): + validate_v2_head_schema(connection, V2_METADATA) -def test_v2_head_accepts_exact_table_inventory( +def test_v2_head_accepts_schema_matching_metadata( monkeypatch: pytest.MonkeyPatch, ) -> None: connection = _persistent_connection( monkeypatch, - public_tables={"alembic_version", *EXPECTED_V2_TABLES}, + public_tables={"alembic_version", *V2_TABLE_NAMES}, ) - validate_v2_head_table_inventory(connection) + validate_v2_head_schema(connection, V2_METADATA) def test_v2_environment_contains_no_reset_adopt_or_restamp_path() -> None: @@ -441,4 +439,4 @@ def test_v2_environment_commits_after_persistent_target_qualification() -> None: assert "with engine.connect() as connection:" not in env_source assert "current_heads != previous_heads" in env_source assert "current_heads == script_heads" in env_source - assert "validate_v2_head_table_inventory(connection)" in env_source + assert "validate_v2_head_schema(connection, target_metadata)" in env_source diff --git a/tests/unit/v2/test_metadata_validation.py b/tests/unit/v2/test_metadata_validation.py new file mode 100644 index 000000000..935e4ed9b --- /dev/null +++ b/tests/unit/v2/test_metadata_validation.py @@ -0,0 +1,35 @@ +"""Tests for targeted v2 metadata rejection rules.""" + +import pytest + +from policyengine_api.data.v2.metadata_validation import ( + PROHIBITED_V2_TABLES, + V1_ONLY_TABLES, + V2MetadataValidationError, + validate_v2_metadata_table_names, +) +from policyengine_api.data.v2.models import V2_METADATA + + +def test_current_metadata_contains_no_rejected_tables() -> None: + validate_v2_metadata_table_names(V2_METADATA.tables) + + +@pytest.mark.parametrize( + "table_name", + ["runtime_bundles", "populations", "household", "user_profiles"], +) +def test_known_rejected_tables_are_identified(table_name: str) -> None: + with pytest.raises(V2MetadataValidationError, match=table_name): + validate_v2_metadata_table_names({*V2_METADATA.tables, table_name}) + + +def test_rejection_sets_do_not_block_current_metadata() -> None: + table_names = set(V2_METADATA.tables) + + assert table_names.isdisjoint(PROHIBITED_V2_TABLES) + assert table_names.isdisjoint(V1_ONLY_TABLES) + + +def test_new_table_name_requires_no_allowlist_update() -> None: + validate_v2_metadata_table_names({*V2_METADATA.tables, "future_domain_table"}) diff --git a/tests/unit/v2/test_models.py b/tests/unit/v2/test_models.py index 01f430a70..3fd8cd1f5 100644 --- a/tests/unit/v2/test_models.py +++ b/tests/unit/v2/test_models.py @@ -18,9 +18,7 @@ UserReportAssociation, UserSimulationAssociation, V2_METADATA, - V2_TABLE_MODELS, ) -from policyengine_api.data.v2.table_inventory import EXPECTED_V2_TABLES RUN_OUTPUT_TABLES = frozenset( @@ -60,13 +58,22 @@ def test_domain_models_are_grouped_into_topic_scoped_modules() -> None: } == expected_modules -def test_controlled_models_match_the_exact_reviewed_inventory() -> None: - model_table_names = {model.__table__.name for model in V2_TABLE_MODELS} +def _v2_mappers(): + configure_mappers() + return tuple( + mapper + for mapper in sa.inspect(User).registry.mappers + if mapper.local_table is not None and mapper.local_table.metadata is V2_METADATA + ) + + +def test_every_metadata_table_has_one_mapped_model() -> None: + metadata_table_names = set(V2_METADATA.tables) + model_table_names = {mapper.local_table.name for mapper in _v2_mappers()} assert V2_METADATA is not V1Base.metadata - assert set(V2_METADATA.tables) == EXPECTED_V2_TABLES - assert model_table_names == EXPECTED_V2_TABLES - assert len(V2_TABLE_MODELS) == len(EXPECTED_V2_TABLES) + assert model_table_names == metadata_table_names + assert len(_v2_mappers()) == len(metadata_table_names) def test_every_table_has_named_primary_foreign_and_relational_constraints() -> None: @@ -80,17 +87,16 @@ def test_every_table_has_named_primary_foreign_and_relational_constraints() -> N for foreign_key in table.foreign_keys: assert foreign_key.constraint.name assert foreign_key.ondelete in {"CASCADE", "RESTRICT", "SET NULL"} - assert foreign_key.column.table.name in EXPECTED_V2_TABLES + assert foreign_key.column.table.name in V2_METADATA.tables def test_every_declared_relationship_has_a_complete_back_populates_pair() -> None: configure_mappers() - for model in V2_TABLE_MODELS: - mapper = sa.inspect(model) + for mapper in _v2_mappers(): for relationship in mapper.relationships: assert relationship.back_populates, ( - f"{model.__name__}.{relationship.key} lacks back_populates" + f"{mapper.class_.__name__}.{relationship.key} lacks back_populates" ) inverse = relationship.mapper.relationships[relationship.back_populates] assert inverse.back_populates == relationship.key @@ -300,5 +306,5 @@ def test_complete_metadata_compiles_for_postgres_without_mutation() -> None: for table in V2_METADATA.sorted_tables ] - assert len(statements) == len(EXPECTED_V2_TABLES) + assert len(statements) == len(V2_METADATA.tables) assert all("CREATE TABLE" in statement for statement in statements) diff --git a/tests/unit/v2/test_table_inventory.py b/tests/unit/v2/test_table_inventory.py deleted file mode 100644 index 6f82afc47..000000000 --- a/tests/unit/v2/test_table_inventory.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Tests for the reviewed Stage 8 table allowlist.""" - -import pytest - -from policyengine_api.data.v2.table_inventory import ( - EXPECTED_V2_TABLES, - PROHIBITED_V2_TABLES, - V1_ONLY_TABLES, - V2_TABLE_GROUPS, - V2TableInventoryError, - validate_v2_table_inventory, -) - - -def test_reviewed_table_groups_are_disjoint_and_complete() -> None: - grouped_tables = [ - table_name for _, table_names in V2_TABLE_GROUPS for table_name in table_names - ] - - assert len(grouped_tables) == len(set(grouped_tables)) - assert frozenset(grouped_tables) == EXPECTED_V2_TABLES - assert "reports" in EXPECTED_V2_TABLES - assert "report_runs" in EXPECTED_V2_TABLES - assert "region_datasets" not in EXPECTED_V2_TABLES - assert EXPECTED_V2_TABLES.isdisjoint(PROHIBITED_V2_TABLES) - assert EXPECTED_V2_TABLES.isdisjoint(V1_ONLY_TABLES) - - -def test_exact_reviewed_inventory_is_accepted() -> None: - validate_v2_table_inventory(EXPECTED_V2_TABLES) - - -@pytest.mark.parametrize( - "table_name", - ["runtime_bundles", "populations", "household", "unreviewed_predecessor"], -) -def test_unreviewed_tables_are_rejected(table_name: str) -> None: - with pytest.raises(V2TableInventoryError, match=table_name): - validate_v2_table_inventory(EXPECTED_V2_TABLES | {table_name}) - - -def test_missing_reviewed_table_is_rejected() -> None: - with pytest.raises(V2TableInventoryError, match="report_runs"): - validate_v2_table_inventory(EXPECTED_V2_TABLES - {"report_runs"}) From fbe22f140f748db8c1be439558f84bd92ddfb861 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 19 Aug 2026 16:27:58 +0300 Subject: [PATCH 15/18] Split runtime cache domains --- .../routes/reform_impact_routes.py | 2 +- policyengine_api/runtime_cache/ai_analyses.py | 52 +++++++++ .../runtime_cache/household_traces.py | 69 ++++++++++++ .../{repositories.py => reform_impacts.py} | 103 +----------------- .../services/ai_analysis_service.py | 2 +- .../services/household_calculation_service.py | 2 +- .../services/reform_impacts_service.py | 2 +- .../services/tracer_analysis_service.py | 4 +- .../services/tracer_analysis_service.py | 2 +- .../services/tracer_fixture_service.py | 2 +- tests/integration/test_runtime_cache_redis.py | 8 +- ...st_household_and_user_policy_orm_routes.py | 2 +- .../unit/routes/test_reform_impact_routes.py | 2 +- tests/unit/runtime_cache/test_ai_analyses.py | 19 ++++ .../runtime_cache/test_household_traces.py | 47 ++++++++ ...repositories.py => test_reform_impacts.py} | 59 +--------- .../unit/services/test_ai_analysis_service.py | 2 +- .../test_direct_orm_local_analysis.py | 4 +- .../test_household_calculation_service.py | 2 +- .../services/test_reform_impacts_service.py | 2 +- 20 files changed, 214 insertions(+), 173 deletions(-) create mode 100644 policyengine_api/runtime_cache/ai_analyses.py create mode 100644 policyengine_api/runtime_cache/household_traces.py rename policyengine_api/runtime_cache/{repositories.py => reform_impacts.py} (81%) create mode 100644 tests/unit/runtime_cache/test_ai_analyses.py create mode 100644 tests/unit/runtime_cache/test_household_traces.py rename tests/unit/runtime_cache/{test_repositories.py => test_reform_impacts.py} (70%) diff --git a/policyengine_api/routes/reform_impact_routes.py b/policyengine_api/routes/reform_impact_routes.py index 92c2a2d58..a44f6be56 100644 --- a/policyengine_api/routes/reform_impact_routes.py +++ b/policyengine_api/routes/reform_impact_routes.py @@ -4,7 +4,7 @@ from flask import Blueprint, Response, request -from policyengine_api.runtime_cache.repositories import CachedReformImpact +from policyengine_api.runtime_cache.reform_impacts import CachedReformImpact from policyengine_api.services.reform_impacts_service import ReformImpactsService diff --git a/policyengine_api/runtime_cache/ai_analyses.py b/policyengine_api/runtime_cache/ai_analyses.py new file mode 100644 index 000000000..6e75e5fd2 --- /dev/null +++ b/policyengine_api/runtime_cache/ai_analyses.py @@ -0,0 +1,52 @@ +"""Recoverable AI-analysis caching.""" + +from dataclasses import asdict, dataclass + +from policyengine_api.runtime_cache.core import ( + CacheBackend, + CacheNamespace, + RecoverableJSONCache, +) + + +AI_ANALYSIS_SCHEMA_VERSION = 1 +AI_ANALYSIS_TTL_SECONDS = 604_800 + + +@dataclass(frozen=True) +class CachedAnalysis: + prompt: str + analysis: str + status: str = "ok" + + +class AIAnalysisCache: + def __init__(self, client: CacheBackend, namespace: CacheNamespace) -> None: + self._cache = RecoverableJSONCache( + client, + namespace, + family="ai-analysis", + schema_version=AI_ANALYSIS_SCHEMA_VERSION, + ttl_seconds=AI_ANALYSIS_TTL_SECONDS, + ) + + @staticmethod + def _inputs(prompt: str, model: str) -> dict[str, str]: + return {"model": model, "prompt": prompt} + + def get(self, prompt: str, *, model: str) -> CachedAnalysis | None: + payload = self._cache.get(self._inputs(prompt, model)) + if not isinstance(payload, dict): + return None + if payload.get("prompt") != prompt or not isinstance( + payload.get("analysis"), str + ): + return None + return CachedAnalysis( + prompt=prompt, + analysis=payload["analysis"], + status=str(payload.get("status", "ok")), + ) + + def set(self, value: CachedAnalysis, *, model: str) -> bool: + return self._cache.set(self._inputs(value.prompt, model), asdict(value)) diff --git a/policyengine_api/runtime_cache/household_traces.py b/policyengine_api/runtime_cache/household_traces.py new file mode 100644 index 000000000..539ff4463 --- /dev/null +++ b/policyengine_api/runtime_cache/household_traces.py @@ -0,0 +1,69 @@ +"""Recoverable computed-household and tracer caching.""" + +from dataclasses import asdict, dataclass +from typing import Any + +from policyengine_api.runtime_cache.core import ( + CacheBackend, + CacheNamespace, + RecoverableJSONCache, +) + + +HOUSEHOLD_TRACE_SCHEMA_VERSION = 1 +HOUSEHOLD_TRACE_TTL_SECONDS = 86_400 + + +@dataclass(frozen=True) +class HouseholdTraceIdentity: + country_id: str + household_id: int + policy_id: int + household_hash: str + policy_hash: str + country_package_version: str + policyengine_version: str + + +@dataclass(frozen=True) +class HouseholdTraceValue: + household: dict[str, Any] + tracer_output: list[str] + + +class HouseholdTraceCache: + """One atomic value for a computed household and its matching tracer.""" + + def __init__(self, client: CacheBackend, namespace: CacheNamespace) -> None: + self._cache = RecoverableJSONCache( + client, + namespace, + family="household-trace", + schema_version=HOUSEHOLD_TRACE_SCHEMA_VERSION, + ttl_seconds=HOUSEHOLD_TRACE_TTL_SECONDS, + ) + + def cache_key(self, identity: HouseholdTraceIdentity) -> str: + return self._cache.key(asdict(identity)) + + def get(self, identity: HouseholdTraceIdentity) -> HouseholdTraceValue | None: + payload = self._cache.get(asdict(identity)) + if not isinstance(payload, dict): + return None + household = payload.get("household") + tracer_output = payload.get("tracer_output") + if not isinstance(household, dict) or not isinstance(tracer_output, list): + return None + if not all(isinstance(line, str) for line in tracer_output): + return None + return HouseholdTraceValue( + household=household, + tracer_output=tracer_output, + ) + + def set( + self, + identity: HouseholdTraceIdentity, + value: HouseholdTraceValue, + ) -> bool: + return self._cache.set(asdict(identity), asdict(value)) diff --git a/policyengine_api/runtime_cache/repositories.py b/policyengine_api/runtime_cache/reform_impacts.py similarity index 81% rename from policyengine_api/runtime_cache/repositories.py rename to policyengine_api/runtime_cache/reform_impacts.py index 0352b7c2e..0a5e89062 100644 --- a/policyengine_api/runtime_cache/repositories.py +++ b/policyengine_api/runtime_cache/reform_impacts.py @@ -1,6 +1,4 @@ -"""Typed repositories for recoverable API runtime state.""" - -from __future__ import annotations +"""Recoverable reform-impact caching and submission coordination.""" from dataclasses import asdict, dataclass, replace from datetime import datetime, timezone @@ -13,7 +11,6 @@ from policyengine_api.runtime_cache.core import ( CacheBackend, CacheNamespace, - RecoverableJSONCache, decode_envelope, encode_envelope, jittered_ttl, @@ -21,110 +18,12 @@ ) -HOUSEHOLD_TRACE_SCHEMA_VERSION = 1 -HOUSEHOLD_TRACE_TTL_SECONDS = 86_400 -AI_ANALYSIS_SCHEMA_VERSION = 1 -AI_ANALYSIS_TTL_SECONDS = 604_800 REFORM_IMPACT_SCHEMA_VERSION = 1 REFORM_IMPACT_TTL_SECONDS = 2_592_000 REFORM_IMPACT_INDEX_LIMIT = 1_000 REFORM_IMPACT_START_CLAIM_TTL_SECONDS = 300 -@dataclass(frozen=True) -class HouseholdTraceIdentity: - country_id: str - household_id: int - policy_id: int - household_hash: str - policy_hash: str - country_package_version: str - policyengine_version: str - - -@dataclass(frozen=True) -class HouseholdTraceValue: - household: dict[str, Any] - tracer_output: list[str] - - -class HouseholdTraceCache: - """One atomic value for a computed household and its matching tracer.""" - - def __init__(self, client: CacheBackend, namespace: CacheNamespace) -> None: - self._cache = RecoverableJSONCache( - client, - namespace, - family="household-trace", - schema_version=HOUSEHOLD_TRACE_SCHEMA_VERSION, - ttl_seconds=HOUSEHOLD_TRACE_TTL_SECONDS, - ) - - def cache_key(self, identity: HouseholdTraceIdentity) -> str: - return self._cache.key(asdict(identity)) - - def get(self, identity: HouseholdTraceIdentity) -> HouseholdTraceValue | None: - payload = self._cache.get(asdict(identity)) - if not isinstance(payload, dict): - return None - household = payload.get("household") - tracer_output = payload.get("tracer_output") - if not isinstance(household, dict) or not isinstance(tracer_output, list): - return None - if not all(isinstance(line, str) for line in tracer_output): - return None - return HouseholdTraceValue( - household=household, - tracer_output=tracer_output, - ) - - def set( - self, - identity: HouseholdTraceIdentity, - value: HouseholdTraceValue, - ) -> bool: - return self._cache.set(asdict(identity), asdict(value)) - - -@dataclass(frozen=True) -class CachedAnalysis: - prompt: str - analysis: str - status: str = "ok" - - -class AIAnalysisCache: - def __init__(self, client: CacheBackend, namespace: CacheNamespace) -> None: - self._cache = RecoverableJSONCache( - client, - namespace, - family="ai-analysis", - schema_version=AI_ANALYSIS_SCHEMA_VERSION, - ttl_seconds=AI_ANALYSIS_TTL_SECONDS, - ) - - @staticmethod - def _inputs(prompt: str, model: str) -> dict[str, str]: - return {"model": model, "prompt": prompt} - - def get(self, prompt: str, *, model: str) -> CachedAnalysis | None: - payload = self._cache.get(self._inputs(prompt, model)) - if not isinstance(payload, dict): - return None - if payload.get("prompt") != prompt or not isinstance( - payload.get("analysis"), str - ): - return None - return CachedAnalysis( - prompt=prompt, - analysis=payload["analysis"], - status=str(payload.get("status", "ok")), - ) - - def set(self, value: CachedAnalysis, *, model: str) -> bool: - return self._cache.set(self._inputs(value.prompt, model), asdict(value)) - - @dataclass(frozen=True) class CachedReformImpact: reform_impact_id: int diff --git a/policyengine_api/services/ai_analysis_service.py b/policyengine_api/services/ai_analysis_service.py index d0114ef7b..315d4998e 100644 --- a/policyengine_api/services/ai_analysis_service.py +++ b/policyengine_api/services/ai_analysis_service.py @@ -9,7 +9,7 @@ from policyengine_api.runtime_cache.dependencies import get_runtime_cache_context from policyengine_api.runtime_cache.core import record_cache_event -from policyengine_api.runtime_cache.repositories import ( +from policyengine_api.runtime_cache.ai_analyses import ( AIAnalysisCache, CachedAnalysis, ) diff --git a/policyengine_api/services/household_calculation_service.py b/policyengine_api/services/household_calculation_service.py index ddd004862..466d5fb9f 100644 --- a/policyengine_api/services/household_calculation_service.py +++ b/policyengine_api/services/household_calculation_service.py @@ -17,7 +17,7 @@ ) from policyengine_api.runtime_cache.dependencies import get_runtime_cache_context from policyengine_api.runtime_cache.core import record_cache_event -from policyengine_api.runtime_cache.repositories import ( +from policyengine_api.runtime_cache.household_traces import ( HouseholdTraceCache, HouseholdTraceIdentity, HouseholdTraceValue, diff --git a/policyengine_api/services/reform_impacts_service.py b/policyengine_api/services/reform_impacts_service.py index 9e4c5c6a8..7ced6709b 100644 --- a/policyengine_api/services/reform_impacts_service.py +++ b/policyengine_api/services/reform_impacts_service.py @@ -5,7 +5,7 @@ from policyengine_api.runtime_cache.core import CacheCoordinationError from policyengine_api.runtime_cache.dependencies import get_runtime_cache_context -from policyengine_api.runtime_cache.repositories import ( +from policyengine_api.runtime_cache.reform_impacts import ( CachedReformImpact, ReformImpactCache, reform_impact_id, diff --git a/policyengine_api/services/tracer_analysis_service.py b/policyengine_api/services/tracer_analysis_service.py index 7e3ff361e..22e605f74 100644 --- a/policyengine_api/services/tracer_analysis_service.py +++ b/policyengine_api/services/tracer_analysis_service.py @@ -11,8 +11,8 @@ from policyengine_api.data.orm import get_v1_session_factory from policyengine_api.data.v1_models import Household, Policy from policyengine_api.runtime_cache.dependencies import get_runtime_cache_context -from policyengine_api.runtime_cache.repositories import ( - AIAnalysisCache, +from policyengine_api.runtime_cache.ai_analyses import AIAnalysisCache +from policyengine_api.runtime_cache.household_traces import ( HouseholdTraceCache, HouseholdTraceIdentity, ) diff --git a/tests/fixtures/services/tracer_analysis_service.py b/tests/fixtures/services/tracer_analysis_service.py index e93634aff..b8894720c 100644 --- a/tests/fixtures/services/tracer_analysis_service.py +++ b/tests/fixtures/services/tracer_analysis_service.py @@ -3,7 +3,7 @@ TracerAnalysisService, ) from unittest.mock import patch -from policyengine_api.runtime_cache.repositories import CachedAnalysis +from policyengine_api.runtime_cache.ai_analyses import CachedAnalysis valid_tracer_output = [ " snap<2027, (default)> = [6769.799]", diff --git a/tests/fixtures/services/tracer_fixture_service.py b/tests/fixtures/services/tracer_fixture_service.py index 67a871fb1..c1c31d251 100644 --- a/tests/fixtures/services/tracer_fixture_service.py +++ b/tests/fixtures/services/tracer_fixture_service.py @@ -6,7 +6,7 @@ from policyengine_api.data.v1_models import Household, Policy from policyengine_api.runtime_cache.core import CacheNamespace from policyengine_api.runtime_cache.fake import InMemoryCacheBackend -from policyengine_api.runtime_cache.repositories import ( +from policyengine_api.runtime_cache.household_traces import ( HouseholdTraceCache, HouseholdTraceIdentity, HouseholdTraceValue, diff --git a/tests/integration/test_runtime_cache_redis.py b/tests/integration/test_runtime_cache_redis.py index 04bbc3a30..4352daaf0 100644 --- a/tests/integration/test_runtime_cache_redis.py +++ b/tests/integration/test_runtime_cache_redis.py @@ -11,11 +11,13 @@ from policyengine_api.runtime_cache.claims import ExpiringClaimStore from policyengine_api.runtime_cache.core import CacheNamespace, RecoverableJSONCache -from policyengine_api.runtime_cache.repositories import ( - CachedReformImpact, +from policyengine_api.runtime_cache.household_traces import ( HouseholdTraceCache, HouseholdTraceIdentity, HouseholdTraceValue, +) +from policyengine_api.runtime_cache.reform_impacts import ( + CachedReformImpact, ReformImpactCache, reform_impact_id, ) @@ -132,7 +134,7 @@ def test_real_reform_indexes_are_cross_connection_bounded_and_expiring( redis_pair, monkeypatch, ) -> None: - import policyengine_api.runtime_cache.repositories as module + import policyengine_api.runtime_cache.reform_impacts as module first, second, namespace = redis_pair monkeypatch.setattr(module, "REFORM_IMPACT_INDEX_LIMIT", 2) diff --git a/tests/unit/routes/test_household_and_user_policy_orm_routes.py b/tests/unit/routes/test_household_and_user_policy_orm_routes.py index 889944e16..69fc6167e 100644 --- a/tests/unit/routes/test_household_and_user_policy_orm_routes.py +++ b/tests/unit/routes/test_household_and_user_policy_orm_routes.py @@ -13,7 +13,7 @@ ) from policyengine_api.runtime_cache.core import CacheNamespace from policyengine_api.runtime_cache.fake import InMemoryCacheBackend -from policyengine_api.runtime_cache.repositories import ( +from policyengine_api.runtime_cache.household_traces import ( HouseholdTraceCache, HouseholdTraceIdentity, HouseholdTraceValue, diff --git a/tests/unit/routes/test_reform_impact_routes.py b/tests/unit/routes/test_reform_impact_routes.py index bde5109c7..422db041a 100644 --- a/tests/unit/routes/test_reform_impact_routes.py +++ b/tests/unit/routes/test_reform_impact_routes.py @@ -17,7 +17,7 @@ from policyengine_api.routes.reform_impact_routes import reform_impact_bp from policyengine_api.runtime_cache.core import CacheNamespace from policyengine_api.runtime_cache.fake import InMemoryCacheBackend -from policyengine_api.runtime_cache.repositories import ( +from policyengine_api.runtime_cache.reform_impacts import ( ReformImpactCache, reform_impact_id, ) diff --git a/tests/unit/runtime_cache/test_ai_analyses.py b/tests/unit/runtime_cache/test_ai_analyses.py new file mode 100644 index 000000000..288fdfd40 --- /dev/null +++ b/tests/unit/runtime_cache/test_ai_analyses.py @@ -0,0 +1,19 @@ +"""AI-analysis cache tests.""" + +from policyengine_api.runtime_cache.ai_analyses import ( + AIAnalysisCache, + CachedAnalysis, +) +from policyengine_api.runtime_cache.core import CacheNamespace +from policyengine_api.runtime_cache.fake import InMemoryCacheBackend + + +def test_analysis_cache_is_model_and_prompt_specific_and_expiring() -> None: + backend = InMemoryCacheBackend() + cache = AIAnalysisCache(backend, CacheNamespace("test", "api")) + value = CachedAnalysis(prompt="explain", analysis="answer") + + assert cache.set(value, model="model-a") is True + assert cache.get("explain", model="model-a") == value + assert cache.get("explain", model="model-b") is None + assert cache.get("different", model="model-a") is None diff --git a/tests/unit/runtime_cache/test_household_traces.py b/tests/unit/runtime_cache/test_household_traces.py new file mode 100644 index 000000000..18af73e9f --- /dev/null +++ b/tests/unit/runtime_cache/test_household_traces.py @@ -0,0 +1,47 @@ +"""Computed-household and tracer cache tests.""" + +from policyengine_api.runtime_cache.core import CacheNamespace +from policyengine_api.runtime_cache.fake import InMemoryCacheBackend +from policyengine_api.runtime_cache.household_traces import ( + HouseholdTraceCache, + HouseholdTraceIdentity, + HouseholdTraceValue, +) + + +def _namespace() -> CacheNamespace: + return CacheNamespace("test", "api") + + +def _identity(**changes) -> HouseholdTraceIdentity: + values = { + "country_id": "us", + "household_id": 1, + "policy_id": 2, + "household_hash": "household-a", + "policy_hash": "policy-a", + "country_package_version": "1.2.3", + "policyengine_version": "4.5.6", + } + values.update(changes) + return HouseholdTraceIdentity(**values) + + +def test_household_and_tracer_share_one_atomic_versioned_value() -> None: + backend = InMemoryCacheBackend() + cache = HouseholdTraceCache(backend, _namespace()) + value = HouseholdTraceValue( + household={"people": {"you": {"income": {"2026": 42}}}}, + tracer_output=["income <2026>"], + ) + identity = _identity() + + assert cache.set(identity, value) is True + assert cache.get(identity) == value + assert list(backend._values) == [cache.cache_key(identity)] + assert cache.cache_key(identity) != cache.cache_key( + _identity(household_hash="household-b") + ) + assert cache.cache_key(identity) != cache.cache_key( + _identity(country_package_version="9.9.9") + ) diff --git a/tests/unit/runtime_cache/test_repositories.py b/tests/unit/runtime_cache/test_reform_impacts.py similarity index 70% rename from tests/unit/runtime_cache/test_repositories.py rename to tests/unit/runtime_cache/test_reform_impacts.py index 2a2a973f1..cb5a1e5ed 100644 --- a/tests/unit/runtime_cache/test_repositories.py +++ b/tests/unit/runtime_cache/test_reform_impacts.py @@ -1,18 +1,13 @@ -"""Typed runtime-cache repository tests.""" +"""Reform-impact cache tests.""" from datetime import datetime from policyengine_api.runtime_cache.core import CacheNamespace from policyengine_api.runtime_cache.fake import InMemoryCacheBackend -from policyengine_api.runtime_cache.repositories import ( - AIAnalysisCache, - CachedAnalysis, +from policyengine_api.runtime_cache.reform_impacts import ( + REFORM_IMPACT_START_CLAIM_TTL_SECONDS, CachedReformImpact, - HouseholdTraceCache, - HouseholdTraceIdentity, - HouseholdTraceValue, ReformImpactCache, - REFORM_IMPACT_START_CLAIM_TTL_SECONDS, reform_impact_id, ) @@ -21,50 +16,6 @@ def _namespace() -> CacheNamespace: return CacheNamespace("test", "api") -def _identity(**changes) -> HouseholdTraceIdentity: - values = { - "country_id": "us", - "household_id": 1, - "policy_id": 2, - "household_hash": "household-a", - "policy_hash": "policy-a", - "country_package_version": "1.2.3", - "policyengine_version": "4.5.6", - } - values.update(changes) - return HouseholdTraceIdentity(**values) - - -def test_household_and_tracer_share_one_atomic_versioned_value() -> None: - backend = InMemoryCacheBackend() - cache = HouseholdTraceCache(backend, _namespace()) - value = HouseholdTraceValue( - household={"people": {"you": {"income": {"2026": 42}}}}, - tracer_output=["income <2026>"], - ) - identity = _identity() - - assert cache.set(identity, value) is True - assert cache.get(identity) == value - assert list(backend._values) == [cache.cache_key(identity)] - assert cache.cache_key(identity) != cache.cache_key( - _identity(household_hash="household-b") - ) - assert cache.cache_key(identity) != cache.cache_key( - _identity(country_package_version="9.9.9") - ) - - -def test_analysis_cache_is_model_and_prompt_specific_and_expiring() -> None: - backend = InMemoryCacheBackend() - cache = AIAnalysisCache(backend, _namespace()) - value = CachedAnalysis(prompt="explain", analysis="answer") - assert cache.set(value, model="model-a") is True - assert cache.get("explain", model="model-a") == value - assert cache.get("explain", model="model-b") is None - assert cache.get("different", model="model-a") is None - - def _impact(execution_id: str, options_hash: str, day: int) -> CachedReformImpact: return CachedReformImpact( reform_impact_id=reform_impact_id(execution_id), @@ -126,7 +77,7 @@ def test_reform_impact_start_claim_is_atomic_exact_ttl_and_token_safe() -> None: def test_reform_impact_indexes_are_bounded_expiring_and_query_compatible( monkeypatch, ) -> None: - import policyengine_api.runtime_cache.repositories as module + import policyengine_api.runtime_cache.reform_impacts as module monkeypatch.setattr(module, "REFORM_IMPACT_INDEX_LIMIT", 2) backend = InMemoryCacheBackend() @@ -164,7 +115,7 @@ def test_reform_impact_indexes_are_bounded_expiring_and_query_compatible( def test_reform_impact_record_and_indexes_share_one_jittered_ttl( monkeypatch, ) -> None: - import policyengine_api.runtime_cache.repositories as module + import policyengine_api.runtime_cache.reform_impacts as module monkeypatch.setattr(module, "jittered_ttl", lambda _ttl: 123) backend = InMemoryCacheBackend() diff --git a/tests/unit/services/test_ai_analysis_service.py b/tests/unit/services/test_ai_analysis_service.py index 27d9949b7..6f2fb8de0 100644 --- a/tests/unit/services/test_ai_analysis_service.py +++ b/tests/unit/services/test_ai_analysis_service.py @@ -5,7 +5,7 @@ from policyengine_api.runtime_cache.core import CacheNamespace from policyengine_api.runtime_cache.fake import InMemoryCacheBackend -from policyengine_api.runtime_cache.repositories import AIAnalysisCache +from policyengine_api.runtime_cache.ai_analyses import AIAnalysisCache from policyengine_api.services.ai_analysis_service import ( AI_ANALYSIS_MODEL, AIAnalysisService, diff --git a/tests/unit/services/test_direct_orm_local_analysis.py b/tests/unit/services/test_direct_orm_local_analysis.py index 3f3ed165b..ee8612641 100644 --- a/tests/unit/services/test_direct_orm_local_analysis.py +++ b/tests/unit/services/test_direct_orm_local_analysis.py @@ -4,9 +4,11 @@ from policyengine_api.runtime_cache.core import CacheNamespace from policyengine_api.runtime_cache.fake import InMemoryCacheBackend -from policyengine_api.runtime_cache.repositories import ( +from policyengine_api.runtime_cache.ai_analyses import ( AIAnalysisCache, CachedAnalysis, +) +from policyengine_api.runtime_cache.reform_impacts import ( CachedReformImpact, ReformImpactCache, ) diff --git a/tests/unit/services/test_household_calculation_service.py b/tests/unit/services/test_household_calculation_service.py index 32db86a18..4c5d331e9 100644 --- a/tests/unit/services/test_household_calculation_service.py +++ b/tests/unit/services/test_household_calculation_service.py @@ -10,7 +10,7 @@ ) from policyengine_api.runtime_cache.core import CacheNamespace from policyengine_api.runtime_cache.fake import InMemoryCacheBackend -from policyengine_api.runtime_cache.repositories import ( +from policyengine_api.runtime_cache.household_traces import ( HouseholdTraceCache, HouseholdTraceIdentity, HouseholdTraceValue, diff --git a/tests/unit/services/test_reform_impacts_service.py b/tests/unit/services/test_reform_impacts_service.py index ad26122a1..16a6bc943 100644 --- a/tests/unit/services/test_reform_impacts_service.py +++ b/tests/unit/services/test_reform_impacts_service.py @@ -5,7 +5,7 @@ from policyengine_api.runtime_cache.core import CacheNamespace from policyengine_api.runtime_cache.fake import InMemoryCacheBackend -from policyengine_api.runtime_cache.repositories import ReformImpactCache +from policyengine_api.runtime_cache.reform_impacts import ReformImpactCache from policyengine_api.services.reform_impacts_service import ( ReformImpactHandoffError, ReformImpactsService, From 4cffb7404a8bbc524a4b091cc056e3e41acbf2dd Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 19 Aug 2026 16:45:41 +0300 Subject: [PATCH 16/18] Remove one-time Supabase storage scaffolding --- docs/engineering/skills/alembic-migrations.md | 8 +- docs/engineering/skills/testing.md | 4 +- docs/migration/stage-8-platform-runbook.md | 17 +- docs/migration/stage-8-supabase-bootstrap.md | 95 -------- docs/migration/stage-8-supabase-target.md | 32 +-- policyengine_api/data/v2/settings.py | 58 ----- policyengine_api/data/v2/storage_bootstrap.py | 217 ------------------ scripts/bootstrap_v2_supabase_storage.py | 27 --- scripts/check_stage8_scaffolding_hygiene.py | 8 + tests/unit/v2/test_import_side_effects.py | 3 - tests/unit/v2/test_scaffolding_hygiene.py | 15 +- tests/unit/v2/test_settings.py | 41 ---- tests/unit/v2/test_storage_bootstrap.py | 187 --------------- 13 files changed, 41 insertions(+), 671 deletions(-) delete mode 100644 docs/migration/stage-8-supabase-bootstrap.md delete mode 100644 policyengine_api/data/v2/storage_bootstrap.py delete mode 100644 scripts/bootstrap_v2_supabase_storage.py delete mode 100644 tests/unit/v2/test_storage_bootstrap.py diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index e70a69d75..b8c79484b 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -149,10 +149,10 @@ speculative general-purpose data-migration framework merely to exercise the policy. `alembic check` for v2 must detect schema drift. -Application startup, model import, project provisioning, and Supabase Storage -bootstrap must never call `create_all`, create or stamp application tables, or -mutate versioned application data. The Supabase CLI is not an application -schema or seed migration authority. +Application startup, model import, project provisioning, and external +infrastructure provisioning must never call `create_all`, create or stamp +application tables, or mutate versioned application data. The Supabase CLI is +not an application schema or seed migration authority. Canonical metadata catalogs derived from the exact installed country and `policyengine` packages are not hand-authored migration data. A later-stage, diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index 9d812b12c..44564526a 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -77,10 +77,10 @@ must not use the persistent Supabase qualification bypass outside explicit disposable-test mode. Continue running the existing isolated v1 MySQL lifecycle whenever either Alembic domain changes. -Supabase Storage bootstrap and repository hygiene: +Repository hygiene for one-time Supabase scaffolding: ```bash -uv run pytest tests/unit/v2/test_storage_bootstrap.py tests/unit/v2/test_scaffolding_hygiene.py -q +uv run pytest tests/unit/v2/test_scaffolding_hygiene.py -q ``` Shared-cache unit behavior and real Redis-compatible integration semantics: diff --git a/docs/migration/stage-8-platform-runbook.md b/docs/migration/stage-8-platform-runbook.md index 9d9f820d3..2a42171ff 100644 --- a/docs/migration/stage-8-platform-runbook.md +++ b/docs/migration/stage-8-platform-runbook.md @@ -30,18 +30,12 @@ explicit local administrative commands must supply them separately. 3. Run `uv run alembic -c alembic-v2.ini upgrade head` and then `uv run alembic -c alembic-v2.ini check`. Do not run either operation during application startup. -4. Only after migration succeeds, retrieve the separate Storage administration - credential and run - `uv run python3 scripts/bootstrap_v2_supabase_storage.py`. A second - identical run must be a no-op; incompatible existing configuration is an - error, not permission to replace the bucket. -5. Confirm no migration URL, password, Storage key, scratch SQL, generated - payload, dump, target identifier, or one-off scaffolding file entered the - repository or logs. +4. Confirm no migration URL, password, scratch SQL, generated payload, dump, + target identifier, or one-time scaffolding file entered the repository or + logs. Application runtime receives only the dormant target identity required by its -validated configuration. It does not receive the migration password or Storage -administration key. +validated configuration. It does not receive the migration password. The current v2 chain begins with one parentless baseline autogenerated from the complete reviewed SQLModel metadata. Its superseded development history was @@ -72,8 +66,7 @@ Before sending traffic to a candidate: 3. Verify Cloud Run and App Engine use their dedicated runtime identities and only non-secret Secret Manager resource names. Confirm each identity has artifact-read access and per-secret accessor rights only for its required - runtime secrets. Neither runtime identity receives v2 migration or Storage - administration access. + runtime secrets. Neither runtime identity receives v2 migration access. 4. Send test traffic to at least two Cloud Run instances and verify one connection's value is visible to another. Confirm the container has no `redis-server` child and startup creates no SQLite database or lock file. diff --git a/docs/migration/stage-8-supabase-bootstrap.md b/docs/migration/stage-8-supabase-bootstrap.md deleted file mode 100644 index 8dac3b891..000000000 --- a/docs/migration/stage-8-supabase-bootstrap.md +++ /dev/null @@ -1,95 +0,0 @@ -# Stage 8 Supabase migration and Storage bootstrap - -This runbook operates only on the approved dormant API v2-alpha target. It is -not application startup logic. Cloud SQL and all existing production routes and -compute remain primary throughout Stage 8. - -Concrete organization, project, region, endpoint, bucket, credential, and -secret-resource identifiers are intentionally excluded. Resolve them from the -approved environment configuration and secret-management surfaces. - -## Required identity and credential boundaries - -The operator must resolve and validate all of the following without copying -their values into this repository. Deployment reads the first two values from -the selected GitHub Environment; explicit local administrative commands obtain -and supply them from the approved operator inventory: - -- `V2_SUPABASE_ENVIRONMENT` -- `V2_SUPABASE_PROJECT_REF` -- `V2_SUPABASE_STORAGE_URL` -- `V2_SUPABASE_STORAGE_BUCKET` - -Inject the database migration password and Storage administration key from -their separate approved secrets at execution time. Do not echo them, place them -in repository files, reuse the migration URL as runtime configuration, or -expose the Storage key to the application service account. - -## Ordered explicit operations - -1. Confirm the separately maintained target inventory and its successful - freshness audit. Stop on any identity ambiguity or unexpected application - state; never reset, adopt, or stamp the database. -2. Supply `V2_MIGRATION_DATABASE_URL`, `V2_SUPABASE_ENVIRONMENT`, and - `V2_SUPABASE_PROJECT_REF` to an explicit operator or CI migration step. -3. Run `uv run alembic -c alembic-v2.ini upgrade head`, followed by - `uv run alembic -c alembic-v2.ini check`. The v2 chain requires an online - connection so it can qualify the persistent target and verify generated - application-data before/after states. -4. Remove the migration credential from the execution environment. Supply the - separate `V2_SUPABASE_STORAGE_ADMIN_KEY` together with the validated - `V2_SUPABASE_STORAGE_URL`, `V2_SUPABASE_STORAGE_BUCKET`, - `V2_SUPABASE_PROJECT_REF`, and `V2_SUPABASE_ENVIRONMENT` values. -5. Run `uv run python3 scripts/bootstrap_v2_supabase_storage.py`. A fresh run - creates the reviewed private bucket. A repeat run verifies it and reports - `created: false`. An incompatible existing bucket stops without update, - deletion, recreation, or public exposure. -6. Remove the Storage administration credential from the execution environment - and run `uv run python3 scripts/check_stage8_scaffolding_hygiene.py` before - commit. - -## Completed pre-activation v2 baseline compaction - -Before v2 activation, the dedicated application schema underwent one bounded -rebaseline. The exact target and migration role were re-qualified; all public -application-table row counts were audited; the only rows were the two known -synthetic validation records; and a custom-format backup of `public` was -created in an ignored operator-artifact location and verified as readable. -With the old revisions still present, `alembic-v2.ini downgrade base` removed -their application objects. Only then were those revisions and their historical -custom-operation executor replaced by one newly autogenerated parentless -baseline. - -This sequence was v2-only. It used no Alembic stamp or raw schema reset and did -not target Cloud SQL, the v1 history, Supabase-managed schemas, or Storage. It -must not be repeated after the v2 database contains retained domain data or -serves production traffic. A later revision-history problem requires explicit -manual recovery rather than another compaction. - -The Storage initializer calls only Supabase's bucket-management endpoint. It -does not run Alembic, import application startup, modify application tables or -rows, initialize canonical metadata, upload an object, or create an access -policy. The dedicated server-side Storage administration key is sent only in -the `apikey` header; it is not a JWT and must not be placed in -`Authorization: Bearer`. The initializer recognizes both structured current -Storage errors such as `NoSuchBucket` and legacy HTTP 404/409 responses without -logging response bodies. - -Supabase documents that buckets are private by default and that bucket creation -needs bucket insert permission but no object permission: - and -. -The current key and Storage error contracts are documented at - -and . - -## Repository hygiene - -One-off SQL, dumps, generated payloads, temporary environment files, Supabase -CLI state, scratch scaffolding, target identifiers, and secret-resource names -belong only in approved ignored local-artifact or system-temporary locations -and must be removed after use. If an operation is needed again, promote it to -tested idempotent tooling before committing it. Generated Alembic revisions, -declarative migration sources, this supported initializer, tests, and sanitized -durable documentation are reviewed project artifacts, not disposable -scaffolding. diff --git a/docs/migration/stage-8-supabase-target.md b/docs/migration/stage-8-supabase-target.md index d8d53e800..2ae9877cb 100644 --- a/docs/migration/stage-8-supabase-target.md +++ b/docs/migration/stage-8-supabase-target.md @@ -16,8 +16,6 @@ configuration, or secret-management surface—not migration documentation. | Region | Operator platform inventory | | Environment classification | Operator inventory; `V2_SUPABASE_ENVIRONMENT` GitHub Environment variable for deployment | | Database host and pooler endpoint | Validated migration URL and provider console | -| Storage API origin | `V2_SUPABASE_STORAGE_URL` | -| Private bucket | `V2_SUPABASE_STORAGE_BUCKET` | | Owning team | Internal ownership inventory | The runtime and migration configuration must fail closed if the supplied values @@ -49,10 +47,11 @@ contracts. Project creation is an explicit operator action. It may establish the project, database service, ownership, networking, and secret placement, but it must not -create application tables or rows, stamp Alembic, or initialize a Storage -bucket. Application schema and versioned application data remain exclusively -owned by the generated v2 Alembic chain. Storage initialization is a separate, -later idempotent operation. +create application tables or rows or stamp Alembic. Application schema and +versioned application data remain exclusively owned by the generated v2 +Alembic chain. The existing Storage bucket is outside this migration's tracked +application tooling. If recurring Storage provisioning becomes necessary, it +belongs in the infrastructure-management system rather than this repository. Store the initial owner credential in the approved secret manager as a provisioning-only credential. Its value and resource identifier are not @@ -73,8 +72,7 @@ configuration. requirements without recording connection strings, hosts, usernames, or addresses in logs or documentation. - Do not introduce an unreviewed networking add-on, custom Postgres override, - database DDL, Alembic stamp, application row, or Storage bucket during - connectivity setup. + database DDL, Alembic stamp, or application row during connectivity setup. Supabase's connection-mode guidance is documented at . @@ -89,21 +87,14 @@ repository records access classes and intended use only: | Initial ownership and emergency administration | Provisioning owner credential | Provisioning only; not application or routine migration configuration | | Generated v2 Alembic chain | Dedicated migration credential | Database connect and reviewed schema creation; no platform administration | | Future ordinary v2 persistence | Dedicated runtime credential | Ordinary application data access; no schema migration or platform administration | -| Explicit Storage bootstrap | Dedicated server-side Storage administration key | Independently rotatable and exposed only to the Storage bootstrap operation | The migration role owns the default privileges for objects it later creates; ordinary table read/write and sequence use are granted to the runtime role. Those grants do not create an application object or row. -Supabase server-side secret keys are elevated credentials. Least privilege is -therefore enforced by using a distinct key, storing it separately, and making -it available only to the explicit Storage initializer. Neither the runtime -database identity nor the migration identity receives this key. - Runtime service accounts must not hold project-wide secret-access rights. Give each identity per-secret access only to the runtime values it needs, and grant -no Stage 8 migration or Storage-administration secret to an application runtime -identity. +no Stage 8 migration secret to an application runtime identity. ## Freshness qualification @@ -117,7 +108,6 @@ The audit must establish all of the following: - the application schema contains zero application tables; - no `alembic_version` table or revision history exists; - no predecessor v2 model table, `runtime_bundles`, or population table exists; -- no application-owned Storage bucket or object has been initialized; and - observed provider-managed schemas contain only platform-owned state. Any mismatch or ambiguity fails closed. Do not reset, drop, stamp, reconcile, @@ -129,8 +119,8 @@ approved target identity. Before declaring the foundation ready, verify: -- no application table, application row, Alembic stamp, or Storage bucket was - created during provisioning; +- no application table, application row, or Alembic stamp was created during + provisioning; - no secret value, secret-bearing URL, target identifier, endpoint, SQL dump, scratch SQL, generated payload, or temporary configuration is tracked or staged; @@ -141,8 +131,8 @@ Before declaring the foundation ready, verify: identity in changed migration documents; - administrative secrets have purpose labels and no application runtime access; and -- the dedicated Storage credential exists only in the approved secret manager - and explicit bootstrap environment. +- one-time Storage bucket scaffolding remains external to tracked application + code, tests, and operator documentation. The dedicated Supabase foundation is ready for generated v2 schema work only after these controls pass and the target-identity gate confirms the separately diff --git a/policyengine_api/data/v2/settings.py b/policyengine_api/data/v2/settings.py index 6fdf14f15..0cfca4548 100644 --- a/policyengine_api/data/v2/settings.py +++ b/policyengine_api/data/v2/settings.py @@ -11,9 +11,7 @@ from dataclasses import dataclass, field import os import re -from urllib.parse import urlsplit -from pydantic import SecretStr from sqlalchemy.engine import URL, make_url from sqlalchemy.exc import ArgumentError @@ -22,16 +20,12 @@ V2_MIGRATION_DATABASE_URL = "V2_MIGRATION_DATABASE_URL" V2_SUPABASE_PROJECT_REF = "V2_SUPABASE_PROJECT_REF" V2_SUPABASE_ENVIRONMENT = "V2_SUPABASE_ENVIRONMENT" -V2_SUPABASE_STORAGE_URL = "V2_SUPABASE_STORAGE_URL" -V2_SUPABASE_STORAGE_ADMIN_KEY = "V2_SUPABASE_STORAGE_ADMIN_KEY" -V2_SUPABASE_STORAGE_BUCKET = "V2_SUPABASE_STORAGE_BUCKET" POSTGRES_DRIVER = "postgresql+psycopg" PERSISTENT_SSL_MODES = frozenset({"require", "verify-ca", "verify-full"}) LOCAL_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"}) PROJECT_REF_PATTERN = re.compile(r"^[a-z0-9]{20}$") ENVIRONMENT_PATTERN = re.compile(r"^[a-z][a-z0-9-]{1,31}$") -BUCKET_PATTERN = re.compile(r"^[a-z0-9][a-z0-9._-]{1,61}[a-z0-9]$") class V2ConfigurationError(RuntimeError): @@ -76,17 +70,6 @@ class V2DatabaseSettings: target: SupabaseTargetSettings -@dataclass(frozen=True) -class SupabaseStorageSettings: - """Storage-only administration settings, separate from database access.""" - - project_ref: str - environment: str - api_url: str - bucket: str - admin_key: SecretStr = field(repr=False) - - def _environment(environ: Mapping[str, str] | None) -> Mapping[str, str]: return os.environ if environ is None else environ @@ -185,44 +168,3 @@ def load_v2_migration_database_settings( connection=connection, target=load_supabase_target_settings(values), ) - - -def load_supabase_storage_settings( - environ: Mapping[str, str] | None = None, -) -> SupabaseStorageSettings: - """Load the separately authorized Supabase Storage administration surface.""" - - values = _environment(environ) - target = load_supabase_target_settings(values) - api_url = _required(values, V2_SUPABASE_STORAGE_URL) - bucket = _required(values, V2_SUPABASE_STORAGE_BUCKET) - admin_key = _required(values, V2_SUPABASE_STORAGE_ADMIN_KEY) - - parsed_url = urlsplit(api_url) - expected_host = f"{target.project_ref}.supabase.co" - if ( - parsed_url.scheme != "https" - or parsed_url.hostname != expected_host - or parsed_url.username is not None - or parsed_url.password is not None - or parsed_url.port is not None - or parsed_url.path.rstrip("/") - or parsed_url.query - or parsed_url.fragment - ): - raise V2ConfigurationError( - f"{V2_SUPABASE_STORAGE_URL} must be the HTTPS API origin for the " - "recorded project reference" - ) - if BUCKET_PATTERN.fullmatch(bucket) is None: - raise V2ConfigurationError( - f"{V2_SUPABASE_STORAGE_BUCKET} is not a valid bucket name" - ) - - return SupabaseStorageSettings( - project_ref=target.project_ref, - environment=target.environment, - api_url=api_url.rstrip("/"), - bucket=bucket, - admin_key=SecretStr(admin_key), - ) diff --git a/policyengine_api/data/v2/storage_bootstrap.py b/policyengine_api/data/v2/storage_bootstrap.py deleted file mode 100644 index f4cd156e1..000000000 --- a/policyengine_api/data/v2/storage_bootstrap.py +++ /dev/null @@ -1,217 +0,0 @@ -"""Explicit, idempotent bootstrap for the Stage 8 private Storage bucket.""" - -from __future__ import annotations - -from dataclasses import dataclass -from typing import Any, Protocol -from urllib.parse import quote - -import httpx - -from policyengine_api.data.v2.settings import ( - ENVIRONMENT_PATTERN, - PROJECT_REF_PATTERN, - SupabaseStorageSettings, -) - - -STORAGE_REQUEST_TIMEOUT_SECONDS = 10.0 - - -class StorageBootstrapError(RuntimeError): - """Raised without response bodies or credentials when bootstrap is unsafe.""" - - -class StorageHTTPClient(Protocol): - """Narrow HTTP surface needed by the Storage initializer.""" - - def get(self, url: str, **kwargs: Any) -> httpx.Response: ... - - def post(self, url: str, **kwargs: Any) -> httpx.Response: ... - - -@dataclass(frozen=True) -class StorageBucketConfiguration: - """Reviewed Stage 8 Storage bucket properties.""" - - id: str - name: str - public: bool = False - file_size_limit: int | None = None - allowed_mime_types: tuple[str, ...] | None = None - - def create_payload(self) -> dict[str, Any]: - return { - "id": self.id, - "name": self.name, - "public": self.public, - "file_size_limit": self.file_size_limit, - "allowed_mime_types": ( - list(self.allowed_mime_types) - if self.allowed_mime_types is not None - else None - ), - } - - -@dataclass(frozen=True) -class StorageBootstrapResult: - """Secret-free result suitable for operator output.""" - - bucket: str - created: bool - public: bool - environment: str - project_ref: str - - -def _qualify_target(settings: SupabaseStorageSettings) -> None: - expected_api_url = f"https://{settings.project_ref}.supabase.co" - if ( - PROJECT_REF_PATTERN.fullmatch(settings.project_ref) is None - or ENVIRONMENT_PATTERN.fullmatch(settings.environment) is None - or settings.api_url.rstrip("/") != expected_api_url - ): - raise StorageBootstrapError( - "Storage API origin does not match the configured Supabase target" - ) - - -def _headers(settings: SupabaseStorageSettings) -> dict[str, str]: - key = settings.admin_key.get_secret_value() - return { - "apikey": key, - "Content-Type": "application/json", - } - - -def _decode_bucket(response: httpx.Response) -> dict[str, Any]: - try: - value = response.json() - except ValueError as error: - raise StorageBootstrapError( - "Supabase Storage returned an invalid bucket response" - ) from error - if not isinstance(value, dict): - raise StorageBootstrapError( - "Supabase Storage returned an invalid bucket response" - ) - return value - - -def _storage_error(response: httpx.Response) -> tuple[str | None, str | None]: - """Return only stable, non-secret Storage error identifiers.""" - - try: - value = response.json() - except ValueError: - return None, None - if not isinstance(value, dict): - return None, None - code = value.get("code") - status_code = value.get("statusCode", value.get("httpStatusCode")) - return ( - code if isinstance(code, str) else None, - str(status_code) if status_code is not None else None, - ) - - -def _is_missing_bucket(response: httpx.Response) -> bool: - code, status_code = _storage_error(response) - return response.status_code == 404 or code == "NoSuchBucket" or status_code == "404" - - -def _is_creation_conflict(response: httpx.Response) -> bool: - code, status_code = _storage_error(response) - return ( - response.status_code == 409 - or code in {"BucketAlreadyExists", "ResourceAlreadyExists"} - or status_code == "409" - ) - - -def _verify_bucket( - observed: dict[str, Any], - expected: StorageBucketConfiguration, -) -> None: - expected_values = { - "id": expected.id, - "name": expected.name, - "public": expected.public, - "file_size_limit": expected.file_size_limit, - "allowed_mime_types": ( - list(expected.allowed_mime_types) - if expected.allowed_mime_types is not None - else None - ), - } - incompatible = { - field: {"expected": expected_value, "observed": observed.get(field)} - for field, expected_value in expected_values.items() - if observed.get(field) != expected_value - } - if incompatible: - fields = ", ".join(sorted(incompatible)) - raise StorageBootstrapError( - f"existing Storage bucket has incompatible fields: {fields}" - ) - - -def initialize_supabase_storage( - settings: SupabaseStorageSettings, - *, - client: StorageHTTPClient | None = None, -) -> StorageBootstrapResult: - """Create or verify the configured private bucket without replacing it.""" - - _qualify_target(settings) - expected = StorageBucketConfiguration( - id=settings.bucket, - name=settings.bucket, - ) - headers = _headers(settings) - bucket_url = ( - f"{settings.api_url}/storage/v1/bucket/{quote(settings.bucket, safe='')}" - ) - collection_url = f"{settings.api_url}/storage/v1/bucket" - owns_client = client is None - active_client = client or httpx.Client( - timeout=STORAGE_REQUEST_TIMEOUT_SECONDS, - follow_redirects=False, - ) - created = False - try: - try: - response = active_client.get(bucket_url, headers=headers) - if _is_missing_bucket(response): - response = active_client.post( - collection_url, - headers=headers, - json=expected.create_payload(), - ) - if _is_creation_conflict(response): - response = active_client.get(bucket_url, headers=headers) - elif 200 <= response.status_code < 300: - created = True - response = active_client.get(bucket_url, headers=headers) - if not 200 <= response.status_code < 300: - raise StorageBootstrapError( - "Supabase Storage bucket verification failed with status " - f"{response.status_code}" - ) - _verify_bucket(_decode_bucket(response), expected) - except httpx.HTTPError as error: - raise StorageBootstrapError( - "Supabase Storage bucket verification request failed" - ) from error - finally: - if owns_client: - active_client.close() # type: ignore[attr-defined] - - return StorageBootstrapResult( - bucket=expected.id, - created=created, - public=expected.public, - environment=settings.environment, - project_ref=settings.project_ref, - ) diff --git a/scripts/bootstrap_v2_supabase_storage.py b/scripts/bootstrap_v2_supabase_storage.py deleted file mode 100644 index 77f133ffe..000000000 --- a/scripts/bootstrap_v2_supabase_storage.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Explicit operator entry point for Stage 8 Supabase Storage bootstrap.""" - -import json - -from policyengine_api.data.v2.settings import load_supabase_storage_settings -from policyengine_api.data.v2.storage_bootstrap import initialize_supabase_storage - - -def main() -> None: - settings = load_supabase_storage_settings() - result = initialize_supabase_storage(settings) - print( - json.dumps( - { - "bucket": result.bucket, - "created": result.created, - "environment": result.environment, - "project_ref": result.project_ref, - "public": result.public, - }, - sort_keys=True, - ) - ) - - -if __name__ == "__main__": - main() diff --git a/scripts/check_stage8_scaffolding_hygiene.py b/scripts/check_stage8_scaffolding_hygiene.py index 0074e4046..6e4d18adb 100644 --- a/scripts/check_stage8_scaffolding_hygiene.py +++ b/scripts/check_stage8_scaffolding_hygiene.py @@ -23,6 +23,12 @@ "bootstrap-payload.json", "scaffold-payload.json", } +PROHIBITED_PATHS = { + "docs/migration/stage-8-supabase-bootstrap.md", + "policyengine_api/data/v2/storage_bootstrap.py", + "scripts/bootstrap_v2_supabase_storage.py", + "tests/unit/v2/test_storage_bootstrap.py", +} ONE_OFF_MARKERS = ("one-off", "one_off", "scratch") @@ -42,6 +48,8 @@ def prohibited_staged_paths(paths: list[str]) -> list[str]: rejected.add(normalized) if path.name.lower() in PROHIBITED_NAMES: rejected.add(normalized) + if normalized in PROHIBITED_PATHS: + rejected.add(normalized) if any(marker in path.name.lower() for marker in ONE_OFF_MARKERS): rejected.add(normalized) return sorted(rejected) diff --git a/tests/unit/v2/test_import_side_effects.py b/tests/unit/v2/test_import_side_effects.py index c9810f89c..3ee987b2c 100644 --- a/tests/unit/v2/test_import_side_effects.py +++ b/tests/unit/v2/test_import_side_effects.py @@ -21,9 +21,6 @@ "V2_MIGRATION_DATABASE_URL", "V2_SUPABASE_PROJECT_REF", "V2_SUPABASE_ENVIRONMENT", - "V2_SUPABASE_STORAGE_URL", - "V2_SUPABASE_STORAGE_ADMIN_KEY", - "V2_SUPABASE_STORAGE_BUCKET", ) diff --git a/tests/unit/v2/test_scaffolding_hygiene.py b/tests/unit/v2/test_scaffolding_hygiene.py index 90585f270..3af9eedee 100644 --- a/tests/unit/v2/test_scaffolding_hygiene.py +++ b/tests/unit/v2/test_scaffolding_hygiene.py @@ -12,26 +12,33 @@ def test_rejects_one_off_supabase_and_secret_shaped_artifacts() -> None: "config/.env", "private/storage-admin.key", "scripts/one-off-supabase.py", + "scripts/bootstrap_v2_supabase_storage.py", + "policyengine_api/data/v2/storage_bootstrap.py", + "tests/unit/v2/test_storage_bootstrap.py", + "docs/migration/stage-8-supabase-bootstrap.md", ] ) == [ ".agent-artifacts/stage8/bootstrap.json", "config/.env", + "docs/migration/stage-8-supabase-bootstrap.md", + "policyengine_api/data/v2/storage_bootstrap.py", "private/storage-admin.key", + "scripts/bootstrap_v2_supabase_storage.py", "scripts/one-off-supabase.py", "supabase/.temp/project-ref", + "tests/unit/v2/test_storage_bootstrap.py", "tmp/stage8-scratch.sql", ] -def test_allows_durable_migrations_bootstrap_tests_and_docs() -> None: +def test_allows_durable_migrations_hygiene_checks_and_docs() -> None: assert ( prohibited_staged_paths( [ "migrations/v2/versions/abc_generated.py", - "scripts/bootstrap_v2_supabase_storage.py", "scripts/check_stage8_scaffolding_hygiene.py", - "tests/unit/v2/test_storage_bootstrap.py", - "docs/migration/stage-8-supabase-bootstrap.md", + "tests/unit/v2/test_scaffolding_hygiene.py", + "docs/migration/stage-8-platform-runbook.md", ".env.example", ] ) diff --git a/tests/unit/v2/test_settings.py b/tests/unit/v2/test_settings.py index 5a444e0c4..ad9a709e7 100644 --- a/tests/unit/v2/test_settings.py +++ b/tests/unit/v2/test_settings.py @@ -7,11 +7,7 @@ V2_RUNTIME_DATABASE_URL, V2_SUPABASE_ENVIRONMENT, V2_SUPABASE_PROJECT_REF, - V2_SUPABASE_STORAGE_ADMIN_KEY, - V2_SUPABASE_STORAGE_BUCKET, - V2_SUPABASE_STORAGE_URL, V2ConfigurationError, - load_supabase_storage_settings, load_v2_migration_database_settings, load_v2_runtime_database_settings, ) @@ -89,43 +85,6 @@ def test_v1_and_debug_settings_never_supply_missing_v2_configuration() -> None: load_v2_migration_database_settings(environment) -def test_storage_settings_require_the_recorded_https_project_origin() -> None: - settings = load_supabase_storage_settings( - { - **TARGET_ENVIRONMENT, - V2_SUPABASE_STORAGE_URL: f"https://{PROJECT_REF}.supabase.co/", - V2_SUPABASE_STORAGE_BUCKET: "policyengine-v2-alpha", - V2_SUPABASE_STORAGE_ADMIN_KEY: "test-storage-admin-key", - } - ) - - assert settings.api_url == f"https://{PROJECT_REF}.supabase.co" - assert settings.bucket == "policyengine-v2-alpha" - assert "test-storage-admin-key" not in repr(settings) - assert settings.admin_key.get_secret_value() == "test-storage-admin-key" - - -@pytest.mark.parametrize( - "api_url", - [ - f"http://{PROJECT_REF}.supabase.co", - "https://another-project.supabase.co", - f"https://{PROJECT_REF}.supabase.co/storage/v1", - f"https://user:password@{PROJECT_REF}.supabase.co", - ], -) -def test_storage_settings_reject_an_inexact_project_origin(api_url: str) -> None: - with pytest.raises(V2ConfigurationError, match=V2_SUPABASE_STORAGE_URL): - load_supabase_storage_settings( - { - **TARGET_ENVIRONMENT, - V2_SUPABASE_STORAGE_URL: api_url, - V2_SUPABASE_STORAGE_BUCKET: "policyengine-v2-alpha", - V2_SUPABASE_STORAGE_ADMIN_KEY: "test-storage-admin-key", - } - ) - - def test_configuration_errors_do_not_echo_secret_values() -> None: secret_url = "postgresql+psycopg://user:do-not-echo@localhost/postgres" diff --git a/tests/unit/v2/test_storage_bootstrap.py b/tests/unit/v2/test_storage_bootstrap.py deleted file mode 100644 index 338d7ba02..000000000 --- a/tests/unit/v2/test_storage_bootstrap.py +++ /dev/null @@ -1,187 +0,0 @@ -"""Isolated contract tests for explicit Supabase Storage initialization.""" - -import json -from pathlib import Path - -import httpx -from pydantic import SecretStr -import pytest - -from policyengine_api.constants import REPO -from policyengine_api.data.v2.settings import SupabaseStorageSettings -from policyengine_api.data.v2.storage_bootstrap import ( - StorageBootstrapError, - initialize_supabase_storage, -) - - -PROJECT_REF = "abcdefghijklmnopqrst" -TARGET_ENVIRONMENT = "test-foundation" -BUCKET = "policyengine-v2-alpha" -ADMIN_KEY = "test-storage-admin-secret" - - -def _settings(**overrides) -> SupabaseStorageSettings: - values = { - "project_ref": PROJECT_REF, - "environment": TARGET_ENVIRONMENT, - "api_url": f"https://{PROJECT_REF}.supabase.co", - "bucket": BUCKET, - "admin_key": SecretStr(ADMIN_KEY), - } - values.update(overrides) - return SupabaseStorageSettings(**values) - - -def _bucket(**overrides) -> dict: - values = { - "id": BUCKET, - "name": BUCKET, - "public": False, - "file_size_limit": None, - "allowed_mime_types": None, - } - values.update(overrides) - return values - - -def _client(handler) -> httpx.Client: - return httpx.Client(transport=httpx.MockTransport(handler)) - - -def test_fresh_bootstrap_creates_then_verifies_private_bucket() -> None: - requests: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - requests.append(request) - if len(requests) == 1: - return httpx.Response( - 400, - json={"statusCode": "404", "code": "NoSuchBucket"}, - ) - if request.method == "POST": - return httpx.Response(200, json={"name": BUCKET}) - return httpx.Response(200, json=_bucket()) - - with _client(handler) as client: - result = initialize_supabase_storage(_settings(), client=client) - - assert result.created is True - assert result.public is False - assert [request.method for request in requests] == ["GET", "POST", "GET"] - payload = json.loads(requests[1].content) - assert payload == _bucket() - assert "authorization" not in requests[1].headers - assert requests[1].headers["apikey"] == ADMIN_KEY - - -def test_second_identical_bootstrap_is_a_read_only_success() -> None: - requests: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - requests.append(request) - return httpx.Response(200, json=_bucket()) - - with _client(handler) as client: - result = initialize_supabase_storage(_settings(), client=client) - - assert result.created is False - assert [request.method for request in requests] == ["GET"] - - -@pytest.mark.parametrize( - ("field", "value"), - [ - ("public", True), - ("name", "wrong-bucket"), - ("file_size_limit", 1024), - ("allowed_mime_types", ["image/png"]), - ], -) -def test_incompatible_bucket_fails_without_update_delete_or_recreate( - field: str, - value, -) -> None: - requests: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - requests.append(request) - return httpx.Response(200, json=_bucket(**{field: value})) - - with _client(handler) as client: - with pytest.raises(StorageBootstrapError, match=field): - initialize_supabase_storage(_settings(), client=client) - - assert [request.method for request in requests] == ["GET"] - - -def test_concurrent_creation_conflict_is_verified_without_overwrite() -> None: - requests: list[httpx.Request] = [] - - def handler(request: httpx.Request) -> httpx.Response: - requests.append(request) - if len(requests) == 1: - return httpx.Response(404) - if request.method == "POST": - return httpx.Response( - 400, - json={"statusCode": "409", "code": "BucketAlreadyExists"}, - ) - return httpx.Response(200, json=_bucket()) - - with _client(handler) as client: - result = initialize_supabase_storage(_settings(), client=client) - - assert result.created is False - assert [request.method for request in requests] == ["GET", "POST", "GET"] - - -def test_target_mismatch_fails_before_any_storage_request() -> None: - def handler(_request: httpx.Request) -> httpx.Response: - raise AssertionError("target mismatch must make no request") - - with _client(handler) as client: - with pytest.raises(StorageBootstrapError, match="configured Supabase target"): - initialize_supabase_storage( - _settings(api_url="https://aaaaaaaaaaaaaaaaaaaa.supabase.co"), - client=client, - ) - - -def test_failures_and_results_never_expose_storage_credentials() -> None: - def handler(_request: httpx.Request) -> httpx.Response: - return httpx.Response(503, text=ADMIN_KEY) - - with _client(handler) as client: - with pytest.raises(StorageBootstrapError) as raised: - initialize_supabase_storage(_settings(), client=client) - - assert ADMIN_KEY not in str(raised.value) - assert ADMIN_KEY not in repr(_settings()) - - -def test_bootstrap_surface_cannot_mutate_application_schema_or_data() -> None: - implementation = (REPO / "policyengine_api/data/v2/storage_bootstrap.py").read_text( - encoding="utf-8" - ) - command = (REPO / "scripts/bootstrap_v2_supabase_storage.py").read_text( - encoding="utf-8" - ) - prohibited = { - "V2_MIGRATION_DATABASE_URL", - "create_all", - "drop_all", - "alembic", - "sqlalchemy", - "/rest/v1/", - "/storage/v1/object", - } - assert all(value not in implementation + command for value in prohibited) - assert "/storage/v1/bucket" in implementation - assert "policyengine_api.api" not in command - - -def test_bootstrap_script_is_durable_tooling_not_one_off_scaffolding() -> None: - path = Path("scripts/bootstrap_v2_supabase_storage.py") - assert (REPO / path).is_file() - assert "supabase/.temp" not in path.as_posix() From 2f8842325f8962547d6ad5aefd72091a66c5c2d6 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 19 Aug 2026 16:51:31 +0300 Subject: [PATCH 17/18] Remove Supabase scaffolding hygiene script --- docs/engineering/skills/testing.md | 6 -- scripts/check_stage8_scaffolding_hygiene.py | 80 --------------------- tests/unit/v2/test_scaffolding_hygiene.py | 46 ------------ 3 files changed, 132 deletions(-) delete mode 100644 scripts/check_stage8_scaffolding_hygiene.py delete mode 100644 tests/unit/v2/test_scaffolding_hygiene.py diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index 44564526a..41e0b626e 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -77,12 +77,6 @@ must not use the persistent Supabase qualification bypass outside explicit disposable-test mode. Continue running the existing isolated v1 MySQL lifecycle whenever either Alembic domain changes. -Repository hygiene for one-time Supabase scaffolding: - -```bash -uv run pytest tests/unit/v2/test_scaffolding_hygiene.py -q -``` - Shared-cache unit behavior and real Redis-compatible integration semantics: ```bash diff --git a/scripts/check_stage8_scaffolding_hygiene.py b/scripts/check_stage8_scaffolding_hygiene.py deleted file mode 100644 index 6e4d18adb..000000000 --- a/scripts/check_stage8_scaffolding_hygiene.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Reject staged one-off Supabase scaffolding and secret-shaped artifacts.""" - -from __future__ import annotations - -from pathlib import PurePosixPath -import subprocess - - -PROHIBITED_PREFIXES = ( - ".agent-artifacts/", - ".artifacts/", - "supabase/", -) -PROHIBITED_SUFFIXES = ( - ".dump", - ".key", - ".pem", - ".sql", - ".sql.gz", -) -PROHIBITED_NAMES = { - ".env", - "bootstrap-payload.json", - "scaffold-payload.json", -} -PROHIBITED_PATHS = { - "docs/migration/stage-8-supabase-bootstrap.md", - "policyengine_api/data/v2/storage_bootstrap.py", - "scripts/bootstrap_v2_supabase_storage.py", - "tests/unit/v2/test_storage_bootstrap.py", -} -ONE_OFF_MARKERS = ("one-off", "one_off", "scratch") - - -def prohibited_staged_paths(paths: list[str]) -> list[str]: - """Return obvious disposable or secret-shaped paths in stable order.""" - - rejected: set[str] = set() - for raw_path in paths: - path = PurePosixPath(raw_path) - normalized = path.as_posix() - if normalized.startswith("./"): - normalized = normalized[2:] - lower = normalized.lower() - if any(normalized.startswith(prefix) for prefix in PROHIBITED_PREFIXES): - rejected.add(normalized) - if lower.endswith(PROHIBITED_SUFFIXES): - rejected.add(normalized) - if path.name.lower() in PROHIBITED_NAMES: - rejected.add(normalized) - if normalized in PROHIBITED_PATHS: - rejected.add(normalized) - if any(marker in path.name.lower() for marker in ONE_OFF_MARKERS): - rejected.add(normalized) - return sorted(rejected) - - -def staged_paths() -> list[str]: - completed = subprocess.run( - ["git", "diff", "--cached", "--name-only", "--diff-filter=ACMR"], - check=True, - capture_output=True, - text=True, - ) - return [line for line in completed.stdout.splitlines() if line] - - -def main() -> None: - rejected = prohibited_staged_paths(staged_paths()) - if rejected: - formatted = "\n".join(f"- {path}" for path in rejected) - raise SystemExit( - "Stage 8 staged-file hygiene rejected one-off or secret-shaped " - f"artifacts:\n{formatted}" - ) - print("Stage 8 staged-file hygiene passed.") - - -if __name__ == "__main__": - main() diff --git a/tests/unit/v2/test_scaffolding_hygiene.py b/tests/unit/v2/test_scaffolding_hygiene.py deleted file mode 100644 index 3af9eedee..000000000 --- a/tests/unit/v2/test_scaffolding_hygiene.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Tests for the staged one-off Supabase artifact guard.""" - -from scripts.check_stage8_scaffolding_hygiene import prohibited_staged_paths - - -def test_rejects_one_off_supabase_and_secret_shaped_artifacts() -> None: - assert prohibited_staged_paths( - [ - "supabase/.temp/project-ref", - ".agent-artifacts/stage8/bootstrap.json", - "tmp/stage8-scratch.sql", - "config/.env", - "private/storage-admin.key", - "scripts/one-off-supabase.py", - "scripts/bootstrap_v2_supabase_storage.py", - "policyengine_api/data/v2/storage_bootstrap.py", - "tests/unit/v2/test_storage_bootstrap.py", - "docs/migration/stage-8-supabase-bootstrap.md", - ] - ) == [ - ".agent-artifacts/stage8/bootstrap.json", - "config/.env", - "docs/migration/stage-8-supabase-bootstrap.md", - "policyengine_api/data/v2/storage_bootstrap.py", - "private/storage-admin.key", - "scripts/bootstrap_v2_supabase_storage.py", - "scripts/one-off-supabase.py", - "supabase/.temp/project-ref", - "tests/unit/v2/test_storage_bootstrap.py", - "tmp/stage8-scratch.sql", - ] - - -def test_allows_durable_migrations_hygiene_checks_and_docs() -> None: - assert ( - prohibited_staged_paths( - [ - "migrations/v2/versions/abc_generated.py", - "scripts/check_stage8_scaffolding_hygiene.py", - "tests/unit/v2/test_scaffolding_hygiene.py", - "docs/migration/stage-8-platform-runbook.md", - ".env.example", - ] - ) - == [] - ) From a0190f28a8120112cd75e01a4ab399c842a75b83 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 19 Aug 2026 17:17:22 +0300 Subject: [PATCH 18/18] Name migration tests by verified behavior --- ...=> v2-platform-foundation-and-shared-runtime-cache.added.md} | 0 docs/engineering/skills/testing.md | 2 +- ...stage8_activation.py => test_report_v1_runtime_selection.py} | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename changelog.d/{stage-8-v2-platform-foundation.added.md => v2-platform-foundation-and-shared-runtime-cache.added.md} (100%) rename tests/unit/v2/{test_stage8_activation.py => test_report_v1_runtime_selection.py} (95%) diff --git a/changelog.d/stage-8-v2-platform-foundation.added.md b/changelog.d/v2-platform-foundation-and-shared-runtime-cache.added.md similarity index 100% rename from changelog.d/stage-8-v2-platform-foundation.added.md rename to changelog.d/v2-platform-foundation-and-shared-runtime-cache.added.md diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index 41e0b626e..4beac5b5e 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -93,7 +93,7 @@ not require network credentials. Startup, deployment, SQLite-removal, and unchanged API migration contracts: ```bash -FLASK_DEBUG=1 uv run pytest tests/unit/v2/test_import_side_effects.py tests/unit/v2/test_stage8_activation.py tests/unit/data/test_orm_sessions.py tests/unit/services/test_direct_orm_local_analysis.py tests/unit/test_app_engine_runtime.py tests/unit/test_cloud_run_deploy_scripts.py tests/unit/test_asgi_factory.py tests/contract/test_v1_route_contracts.py -q +FLASK_DEBUG=1 uv run pytest tests/unit/v2/test_import_side_effects.py tests/unit/v2/test_report_v1_runtime_selection.py tests/unit/data/test_orm_sessions.py tests/unit/services/test_direct_orm_local_analysis.py tests/unit/test_app_engine_runtime.py tests/unit/test_cloud_run_deploy_scripts.py tests/unit/test_asgi_factory.py tests/contract/test_v1_route_contracts.py -q python3 scripts/export_migration_contracts.py python3 scripts/run_quality_guards.py ``` diff --git a/tests/unit/v2/test_stage8_activation.py b/tests/unit/v2/test_report_v1_runtime_selection.py similarity index 95% rename from tests/unit/v2/test_stage8_activation.py rename to tests/unit/v2/test_report_v1_runtime_selection.py index b3b056390..d74a5970a 100644 --- a/tests/unit/v2/test_stage8_activation.py +++ b/tests/unit/v2/test_report_v1_runtime_selection.py @@ -1,4 +1,4 @@ -"""Guards that keep the dormant v2 report schema off production paths.""" +"""Verify that report requests use the existing v1 runtime by default.""" import inspect