diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0fff843 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +# Keep the host's virtualenv out of the build context: `COPY . .` would otherwise merge it into the image's +# own /app/.venv (on Windows that overwrites its pyvenv.cfg with a C:\ interpreter path). +.venv/ +venv/ +# Never bake local env files (live secrets) into an image. +.env +.env.* +!.env.example +.weave_e2e_fixture.json +.git/ +**/__pycache__/ +*.py[cod] +.pytest_cache/ +.ruff_cache/ +.coverage +htmlcov/ +*.log +backups/ diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..c07f05e --- /dev/null +++ b/.env.example @@ -0,0 +1,86 @@ +# Copy to `.env` and adjust (or keep it outside the repo and set KRATER_ENV_FILE; see docs/dev/staging.md +# section 4). See krater/config.py for the full list of settings. + +KRATER_ENV=development +# In docker-compose, the db host is the `db` service; for `bin/dev` locally, use localhost. +KRATER_DATABASE_URL=postgresql+psycopg://krater:krater@db:5432/krater +# Generate a real one for anything beyond local dev: +# python -c "import secrets; print(secrets.token_urlsafe(48))" +# Production refuses short keys and anything containing "change-me". +KRATER_SECRET_KEY=change-me-in-production +KRATER_BASE_URL=http://localhost:8000 + +# Weave (Patchwork Labs identity provider). Weave signs people in and owns their Krater roles. Krater asks +# for the scopes `openid profile email groups roles slack`, and uses a client_credentials token (scope +# `directory`) of the same app for the directory API. See docs/weave-integration.md. +# `stub` replaces Weave with a local fixture of fake users; required outside of production. +KRATER_WEAVE_MODE=stub +KRATER_WEAVE_ISSUER= +KRATER_WEAVE_CLIENT_ID= +KRATER_WEAVE_CLIENT_SECRET= +# Base URL of Weave's directory API. Leave blank to use KRATER_WEAVE_ISSUER. +KRATER_WEAVE_API_BASE_URL= +KRATER_WEAVE_STUB_USERS_FILE= + +# Krater's role keys on its Weave app, and the group slugs Krater falls back to when Weave sends no +# `roles` field at all. The defaults match the Weave setup in docs/weave-integration.md. +# KRATER_WEAVE_ROLE_MEMBER=member +# KRATER_WEAVE_ROLE_REVIEWER=reviewer +# KRATER_WEAVE_ROLE_ADMIN=admin +# KRATER_WEAVE_GROUP_MEMBER=ganymede-members +# KRATER_WEAVE_GROUP_REVIEWER=krater-reviewers +# KRATER_WEAVE_GROUP_ADMIN=krater-admins + +# Which compute layer launches members' machines: `vast` (Krater Compute, docs/compute/) or `off`. Off by default. +KRATER_COMPUTE_MODE=off + + +# Slack integration (review happens in Slack). See docs/SPEC.md "Slack integration" and +# docs/dev/slack-setup.md. `fake` = in-memory Slack for local dev/tests; required outside of production. +KRATER_SLACK_MODE=fake +# Bot token (`xoxb-...`) for Krater's own Slack app -- see docs/dev/slack-setup.md for the required scopes. +KRATER_SLACK_BOT_TOKEN= +# Signing secret from the app's "Basic Information" page, used to verify requests to /slack/interactions. +KRATER_SLACK_SIGNING_SECRET= +# The master feed channel's id (e.g. `C0123456789`), for the one-line post on each new proposal. +KRATER_SLACK_FEED_CHANNEL_ID= +# How often the worker invites new reviewers (anyone granted ganymede:reviewer in Krater), posts missed +# budget warnings, and archives channels of projects that were missed. +KRATER_SLACK_RECONCILE_INTERVAL_MINUTES=10 + +# S3-compatible object storage for gallery screenshots. See docs/dev/storage.md. +# docker-compose runs a temporary SeaweedFS container as `storage`. The app talks to it on the internal endpoint; +# browsers get presigned URLs on the public one. Change the secret for anything beyond local dev. +# `live` uses that container; `fake` is in-memory and hands browsers unusable `fake://` upload URLs. +KRATER_S3_MODE=live +KRATER_S3_ENDPOINT_URL=http://storage:8333 +KRATER_S3_PUBLIC_ENDPOINT_URL=http://localhost:8333 +KRATER_S3_REGION=us-east-1 +KRATER_S3_BUCKET=krater-screenshots +KRATER_S3_ACCESS_KEY_ID=krater +KRATER_S3_SECRET_ACCESS_KEY=krater-dev-secret + +# docker-compose only: the `db` service's own credentials. +POSTGRES_USER=krater +POSTGRES_PASSWORD=krater +POSTGRES_DB=krater +PORT=8000 + +# Production only (docs/dev/production.md). Leave unset for dev and staging. +# Set to 127.0.0.1 so published ports are reachable only from the host, behind the TLS proxy. +# KRATER_BIND_ADDRESS=127.0.0.1 +# Number of reverse proxies in front of the portal (1 with the `proxy` profile), for rate limiting. +# KRATER_TRUSTED_PROXY_COUNT=1 +# SeaweedFS version; defaults to 4.47, the one staging ran. Change it only after a staging run, never to `latest`. +# SEAWEEDFS_VERSION=4.47 +# `--profile proxy`: the public hostnames Caddy gets certificates for. +# KRATER_DOMAIN=krater.example.org +# KRATER_S3_DOMAIN=storage.krater.example.org +# `--profile backup`: where nightly database dumps go, how often, and how long they're kept. +# KRATER_BACKUP_DIR=/srv/krater/backups +# KRATER_BACKUP_INTERVAL_SECONDS=86400 +# KRATER_BACKUP_KEEP_DAYS=14 + +# The public GPU price catalog behind /pricing and the budget estimator (docs/dev/pricing.md). `live` fetches it +# from GitHub; production refuses `fake`. KRATER_SKYPILOT_MODE still works as the old name for one release. +KRATER_PRICING_MODE=fake diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3a59662 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Shell scripts must keep LF endings even in a Windows checkout (core.autocrlf=true): WSL and Linux +# bash reject CRLF, and some dev scripts run in WSL2. +*.sh text eol=lf + +# Self-hosted fonts are binary; never let autocrlf touch them. +*.woff2 binary diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..77c5bf1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,54 @@ +name: CI + +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + + services: + postgres: + image: postgres:16 + env: + POSTGRES_USER: root + POSTGRES_PASSWORD: root + POSTGRES_DB: krater_test + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U root" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + + env: + KRATER_ENV: test + KRATER_DATABASE_URL: postgresql+psycopg://root:root@localhost:5432/krater_test + KRATER_TEST_DATABASE_URL: postgresql+psycopg://root:root@localhost:5432/krater_test + KRATER_SECRET_KEY: ci-test-secret + + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v3 + with: + version: "0.8.17" + enable-cache: true + + - name: Set up Python + run: uv python install + + - name: Install dependencies + run: uv sync --frozen + + - name: Lint (ruff check) + run: uv run ruff check . + + - name: Format check (ruff format) + run: uv run ruff format --check . + + - name: Run tests + run: uv run pytest diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..b53bad4 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,40 @@ +name: Deploy + +# Deploys a commit of this repo to alastor. There's no registry image: the NixOS module +# (patchworklabsorg/infra, modules/krater) builds the image on alastor from the commit written to +# /var/lib/krater/source-rev. This workflow writes that file, builds while the running site keeps +# serving, then restarts krater-web, which migrates and starts; krater-worker follows it. Manual only, +# like Weave's production deploys. See docs/dev/production.md. +# +# Required repository secrets: +# ALASTOR_HOST - alastor's SSH host / IP +# KRATER_DEPLOY_SSH_KEY - private key for the krater-deploy user on alastor (its public key goes in +# the infra repo's hosts/alastor/krater.nix, deployAuthorizedKeys) + +on: + workflow_dispatch: + +concurrency: + group: deploy-krater + cancel-in-progress: false + +jobs: + deploy: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Deploy ${{ github.sha }} + uses: appleboy/ssh-action@v1 + with: + host: ${{ secrets.ALASTOR_HOST }} + username: krater-deploy + key: ${{ secrets.KRATER_DEPLOY_SSH_KEY }} + # The first build of the Python dependencies takes several minutes on alastor. + command_timeout: 50m + script: | + set -e + echo "${{ github.sha }}" > /var/lib/krater/source-rev + sudo systemctl start krater-build + sudo systemctl restart krater-web + systemctl is-active krater-web krater-worker diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a539791 --- /dev/null +++ b/.gitignore @@ -0,0 +1,37 @@ +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +.eggs/ + +# Virtual environments +.venv/ +venv/ + +# uv +# (uv.lock IS committed -- do not ignore it) + +# Environment / secrets +.env +.env.* +!.env.example +# Live-Weave e2e fixture (scripts/dev/weave_e2e_setup.py) -- carries a live OAuth client secret +# and service API key, regenerated on every run. See docs/dev/weave-e2e.md. +.weave_e2e_fixture.json + +# Testing / tooling +.pytest_cache/ +.ruff_cache/ +.coverage +htmlcov/ + +# Editors / OS +.vscode/ +.idea/ +.DS_Store + +# Logs +*.log + +# Database dumps from the db-backup service (docker-compose.yml, --profile backup). +/backups/ diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..ca89aa6 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,90 @@ +# CLAUDE.md + +Guidance for Claude Code (and humans) working in this repo. + +## What this is + +Krater is the Project Ganymede portal (Patchwork Labs): proposal review, compute budget allocation/enforcement, and a +public gallery. **Read `docs/SPEC.md` before changing behavior.** Integration contracts live in +`docs/weave-integration.md` (sign-in and roles) and +`docs/quilt-integration.md` (submission and budget events to Quilt). + +## Stack + +- Python 3.12, managed with **uv** (`pyproject.toml` + `uv.lock`) +- **FastAPI** web app, server-rendered **Jinja2** templates (no SPA; progressive enhancement with HTMX is fine) +- **SQLAlchemy 2.0** (typed `Mapped[...]` models, sync sessions, psycopg 3) + **Alembic** migrations +- **Postgres** (16+) +- **procrastinate** (Postgres-backed) for background jobs and periodic tasks — no Redis +- **Authlib** for OIDC against Weave; Starlette `SessionMiddleware` for the session cookie +- **pytest** against a real Postgres test database; **ruff** for lint + format + +## Commands + +```bash +uv sync # install deps (incl. dev) +uv run alembic upgrade head # migrate (DATABASE_URL must be set) +uv run alembic revision --autogenerate -m "..." # new migration; always review the output +uv run uvicorn krater.web.app:create_app --factory --reload # web +uv run procrastinate --app=krater.worker.app.app worker # worker +uv run pytest # tests (TEST_DATABASE_URL, see tests/conftest.py) +uv run ruff check . && uv run ruff format --check . +docker compose up --build # full stack: portal, worker, db +``` + +## Layout + +``` +krater/ + config.py Settings (pydantic-settings, env vars prefixed KRATER_) + db.py engine, session factory, Base + models/ SQLAlchemy models, one module per aggregate + services/ domain logic (framework-free): approval_policy, projects, budget, audit, users + weave/ WeaveClient protocol: OIDC sign-in, directory API, role mapping, dev stub + quilt/ Quilt patch API client and the outbox sender (the only code that knows Quilt's URLs) + web/ FastAPI app factory, routers, deps (current user, authz), templates/, static/ + worker/ procrastinate app + tasks +alembic/ migrations +tests/ mirrors krater/ layout +``` + +## Conventions + +- **Services own the rules.** Routers parse input, call a service, render. Services take a `Session` plus an explicit + actor and raise domain errors (`krater.services.errors`); they never import FastAPI. +- **Money is integer cents** (`*_cents` columns, `int` in Python). Never floats. +- **Primary keys are UUIDs** (`uuid.uuid4`), safe to show in URLs. +- **Weave owns roles.** Krater's role keys in Weave are `member`, `reviewer` and `admin`, sent in the `roles` claim + and in directory records. Only when the `roles` field is absent does Krater fall back to the group slugs + `ganymede-members`, `krater-reviewers` and `krater-admins`. `krater.weave.roles` is the one place that maps them to + Krater's internal names `ganymede:member`, `ganymede:reviewer`, `ganymede:admin`; nothing else knows the keys or slugs. + Every state-changing action (`fresh_actor`, the Slack click path) re-checks Weave's directory by `sub` through + `krater.services.users.authorize`, and fails closed if Weave is down. `users.roles_cached` is for display only. + Krater has no role tables, no admin users page and no disable switch: those are done in Weave. +- **Weave owns member data.** Krater stores a member's Patchwork ID (`users.weave_sub`), never their name, email or + other personal details: read those from Weave when needed (`krater.services.people`) and record who acted by + Patchwork ID (`docs/weave-integration.md` "Member data"). +- **All Weave access goes through `krater.weave`.** Nothing else imports httpx for Weave or knows Weave's URLs. + Krater asks for `openid profile email groups roles slack`, and calls the directory with a client_credentials token + of its own app (scope `directory`). Its token for Quilt (scope `quilt`) comes from `WeaveClient.quilt_token`. +- **Events for Quilt go through the outbox.** A service that changes a project's status, title, ledger or spend + calls `krater.services.quilt_events.sync_project` in the same transaction. Never post to Quilt from a request. +- **Every admin override writes an `AuditEvent`** (who, what, reason). Budget changes write a `BudgetEntry` (append-only; + never update or delete ledger rows). +- **Stub mode** (`KRATER_WEAVE_MODE=stub`) replaces Weave with a local fixture of fake users for dev and tests. The + fixture carries the same `roles`, `groups` and `slack_id` fields a real Weave sends, and the stub answers the + directory lookups too. Tests change a user mid-test with `StubWeaveClient.set_roles` and friends. The app must + refuse to start in stub mode when `KRATER_ENV=production`. +- Timestamps are timezone-aware UTC (`DateTime(timezone=True)`). +- Tests: every service rule gets a unit test; every route gets at least a happy-path and an authz test. +- Keep comments for the *why*; match the surrounding style. + +## Brand + +Krater uses the Patchwork palette with the same role tokens as Quilt's stylesheet: Ink `#2a2050` text on Quilt +`#f7f5fb`, Grape `#6c3ec1` as the primary accent, Deep Purple `#4b006e` for hover, Lavender `#c7b7e2` in the brand +mark, solid hairline cards, Geist for all text, and a dark mode. The tokens live on `:root` in +`krater/web/static/css/main.css`; components use the role tokens (`--surface`, `--text`, `--accent` ...), never raw +colors. Style through the existing classes (`.page-head`, `.card`, `.stats`, `.button--primary`, `.badge--`, +`.table-scroll` ...). Pages give only their own name in `{% block title %}`; `base.html` adds " · Krater". Every +page must work at 360px wide with no horizontal page scroll. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d90f18f --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +# syntax=docker/dockerfile:1 +# +# Builds for both amd64 and arm64 (e.g. `docker buildx build --platform linux/amd64,linux/arm64 .`): +# both `python:3.12-slim` and the `uv` image below publish multi-arch manifests, and nothing here +# pins an architecture. +FROM python:3.12-slim + +# Copy the `uv` and `uvx` binaries from Astral's official image rather than installing via pip. +COPY --from=ghcr.io/astral-sh/uv:0.8.17 /uv /uvx /usr/local/bin/ + +ENV UV_COMPILE_BYTECODE=1 \ + UV_LINK_MODE=copy \ + UV_PYTHON_DOWNLOADS=never \ + PYTHONUNBUFFERED=1 \ + PATH="/app/.venv/bin:$PATH" + +WORKDIR /app + +# Install dependencies before copying the rest of the source, so `docker build` caches this layer as +# long as pyproject.toml / uv.lock haven't changed. +COPY pyproject.toml uv.lock ./ +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --frozen --no-install-project --no-dev + +COPY . . +RUN --mount=type=cache,target=/root/.cache/uv \ + uv sync --frozen --no-dev + +RUN groupadd --system krater \ + && useradd --system --gid krater --create-home --home-dir /home/krater krater \ + && chown -R krater:krater /app +USER krater + +EXPOSE 8000 + +# The portal is the default; docker-compose.yml overrides `command` for `migrate` and `worker`. +CMD ["uvicorn", "krater.web.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md index 6c955c2..225786c 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,63 @@ # Krater The management utility for Project Ganymede + +Krater is the Project Ganymede portal: proposal review, compute budget allocation and enforcement, and the public +gallery of completed projects. + +## Docs + +- [Spec](docs/SPEC.md): product spec, data model, workflow, deployment, open questions +- [Weave integration](docs/weave-integration.md): sign-in and roles (Weave owns roles; Krater reads them from claims and the directory API) +- [Quilt integration](docs/quilt-integration.md): submission and budget events to Quilt +- [Handoff](docs/HANDOFF.md): start here if you are picking this project up (setup, state, decisions, next steps) +- [Future work](docs/FUTURE.md): what comes after v1, loose ends, and parked ideas + +## Development + +Stack, layout and conventions are documented in [CLAUDE.md](CLAUDE.md). + +First-time setup is one command, given [uv](https://docs.astral.sh/uv/) and a running Docker. It installs Python +3.12 and the dependencies, starts a Postgres container, creates `krater_dev` and `krater_test` and migrates +`krater_dev`. Add `--check` (`-Check` on Windows) to also lint and run the tests. + +```bash +scripts/dev/setup.sh # macOS, Linux, WSL, Git Bash +powershell -ExecutionPolicy Bypass -File scripts\dev\setup.ps1 # Windows +``` + +Or by hand, against your own Postgres: + +```bash +# Once: install Python 3.12 and the project's dependencies. +uv python install 3.12 +uv sync + +# Once: create the local databases (adjust to your Postgres superuser). +createdb krater_dev +createdb krater_test + +# Environment variables (KRATER_DATABASE_URL etc.) are listed in .env.example. For running outside Docker, don't +# save them as .env in the repo root: pydantic-settings loads that file automatically and it leaks into the test +# suite. Use another name (e.g. .env.local) and load it into your shell. (`docker compose` reads a .env too; keep it +# outside the repo with `--env-file` and `KRATER_ENV_FILE`, see docs/dev/staging.md.) + +# Apply migrations, then run the web app and worker (in separate terminals). +uv run alembic upgrade head +uv run uvicorn krater.web.app:create_app --factory --reload +uv run procrastinate --app=krater.worker.app.app worker + +# Lint, format and test. +uv run ruff check . && uv run ruff format --check . +uv run pytest +``` + +`KRATER_TEST_DATABASE_URL` (default `postgresql+psycopg://root:root@localhost:5432/krater_test`) points +tests at a separate database; see `tests/conftest.py`. + +The full stack (Postgres, a one-shot migration, the portal and the worker) also runs under +`docker compose up --build`. + +Weave owns roles. A Weave admin gives people Krater's app roles `member`, `reviewer` and `admin`, and Krater reads +them at sign-in and re-checks them with Weave's directory API before every action. In stub mode (the default for +development) the fixture users in `krater/weave/stub_users.json` carry their roles. See `docs/weave-integration.md`. + diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..78b4d8f --- /dev/null +++ b/alembic.ini @@ -0,0 +1,148 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts. +# this is typically a path given in POSIX (e.g. forward slashes) +# format, relative to the token %(here)s which refers to the location of this +# ini file +script_location = %(here)s/alembic + +# template used to generate migration file names; The default value is %%(rev)s_%%(slug)s +# Uncomment the line below if you want the files to be prepended with date and time +# see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file +# for all available tokens +# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s +# Or organize into date-based subdirectories (requires recursive_version_locations = true) +# file_template = %%(year)d/%%(month).2d/%%(day).2d_%%(hour).2d%%(minute).2d_%%(second).2d_%%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. for multiple paths, the path separator +# is defined by "path_separator" below. +prepend_sys_path = . + + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the tzdata library which can be installed by adding +# `alembic[tz]` to the pip requirements. +# string value is passed to ZoneInfo() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to /versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "path_separator" +# below. +# version_locations = %(here)s/bar:%(here)s/bat:%(here)s/alembic/versions + +# path_separator; This indicates what character is used to split lists of file +# paths, including version_locations and prepend_sys_path within configparser +# files such as alembic.ini. +# The default rendered in new alembic.ini files is "os", which uses os.pathsep +# to provide os-dependent path splitting. +# +# Note that in order to support legacy alembic.ini files, this default does NOT +# take place if path_separator is not present in alembic.ini. If this +# option is omitted entirely, fallback logic is as follows: +# +# 1. Parsing of the version_locations option falls back to using the legacy +# "version_path_separator" key, which if absent then falls back to the legacy +# behavior of splitting on spaces and/or commas. +# 2. Parsing of the prepend_sys_path option falls back to the legacy +# behavior of splitting on spaces, commas, or colons. +# +# Valid values for path_separator are: +# +# path_separator = : +# path_separator = ; +# path_separator = space +# path_separator = newline +# +# Use os.pathsep. Default configuration used for new projects. +path_separator = os + +# set to 'true' to search source files recursively +# in each "version_locations" directory +# new in Alembic version 1.10 +# recursive_version_locations = false + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +# database URL. Deliberately left unset -- env.py sets it from krater.config.get_settings() +# (KRATER_DATABASE_URL), so there's no URL to keep in sync with the app's own settings. +sqlalchemy.url = + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# lint with attempts to fix using "ruff" - use the module runner, against the "ruff" module +# hooks = ruff +# ruff.type = module +# ruff.module = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Alternatively, use the exec runner to execute a binary found on your PATH +# hooks = ruff +# ruff.type = exec +# ruff.executable = ruff +# ruff.options = check --fix REVISION_SCRIPT_FILENAME + +# Logging configuration. This is also consumed by the user-maintained +# env.py script only. +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/README b/alembic/README new file mode 100644 index 0000000..98e4f9c --- /dev/null +++ b/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..87fd17e --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,86 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config, pool + +# Import every model module so they're registered on `Base.metadata` before autogenerate runs. +import krater.models # noqa: F401 +from alembic import context +from krater.config import get_settings +from krater.db import Base + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Always take the database URL from Krater's own settings (KRATER_DATABASE_URL), never from a +# hard-coded value in alembic.ini, so migrations run against whatever the app is configured for. +config.set_main_option("sqlalchemy.url", get_settings().database_url) + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def include_name(name, type_, parent_names) -> bool: + """Keep autogenerate away from procrastinate's schema. + + procrastinate's tables are created by their own migration from its SQL, not from our models, so without this + every autogenerated migration would try to drop them. + """ + if type_ == "table": + return not (name or "").startswith("procrastinate_") + return True + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + include_name=include_name, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata, include_name=include_name) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..1101630 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,28 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + """Upgrade schema.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Downgrade schema.""" + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/153093277378_add_unique_constraint_on_reviews_.py b/alembic/versions/153093277378_add_unique_constraint_on_reviews_.py new file mode 100644 index 0000000..145130b --- /dev/null +++ b/alembic/versions/153093277378_add_unique_constraint_on_reviews_.py @@ -0,0 +1,31 @@ +"""add unique constraint on reviews revision_id reviewer_id + +Revision ID: 153093277378 +Revises: ddcdc402852d +Create Date: 2026-09-26 19:37:31.411192 + +`record_review`'s "already reviewed" check is a SELECT followed by an INSERT, so two concurrent +approvals from the same reviewer (a double-click, or a retried Slack action) could both pass the +check before either has inserted its row. This constraint is what actually prevents the duplicate; +`record_review` turns the resulting `IntegrityError` into a friendly `InvalidState`. +""" + +from collections.abc import Sequence + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "153093277378" +down_revision: str | Sequence[str] | None = "ddcdc402852d" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.create_unique_constraint("uq_reviews_revision_id_reviewer_id", "reviews", ["revision_id", "reviewer_id"]) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_constraint("uq_reviews_revision_id_reviewer_id", "reviews", type_="unique") diff --git a/alembic/versions/29fc5db971e5_add_gpu_prices_and_revision_budget_.py b/alembic/versions/29fc5db971e5_add_gpu_prices_and_revision_budget_.py new file mode 100644 index 0000000..19b892d --- /dev/null +++ b/alembic/versions/29fc5db971e5_add_gpu_prices_and_revision_budget_.py @@ -0,0 +1,56 @@ +"""add gpu prices and revision budget estimate + +Revision ID: 29fc5db971e5 +Revises: 9ed6e4593cff +Create Date: 2026-09-27 03:11:25.600703 + +`gpu_prices` holds the latest aggregated Vast pricing (`krater.services.pricing.refresh_prices` +replaces it wholesale on every refresh -- no history table). `project_revisions.budget_estimate` is the +server-recomputed breakdown from the budget estimator, set only when the submitter used it. See +docs/dev/pricing.md. +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "29fc5db971e5" +down_revision: str | Sequence[str] | None = "9ed6e4593cff" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.create_table( + "gpu_prices", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("accelerator_name", sa.String(length=64), nullable=False), + sa.Column("accelerator_count", sa.Integer(), nullable=False), + sa.Column("vram_gib", sa.Float(), nullable=True), + sa.Column("vcpus_typical", sa.Float(), nullable=True), + sa.Column("memory_gib_typical", sa.Float(), nullable=True), + sa.Column("on_demand_min_cents", sa.Integer(), nullable=False), + sa.Column("on_demand_median_cents", sa.Integer(), nullable=False), + sa.Column("spot_min_cents", sa.Integer(), nullable=True), + sa.Column("offer_count", sa.Integer(), nullable=False), + sa.Column("refreshed_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_gpu_prices")), + sa.UniqueConstraint("accelerator_name", "accelerator_count", name="uq_gpu_prices_accelerator_name"), + ) + op.create_index(op.f("ix_gpu_prices_accelerator_name"), "gpu_prices", ["accelerator_name"], unique=False) + op.add_column( + "project_revisions", + sa.Column("budget_estimate", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + ) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_column("project_revisions", "budget_estimate") + op.drop_index(op.f("ix_gpu_prices_accelerator_name"), table_name="gpu_prices") + op.drop_table("gpu_prices") diff --git a/alembic/versions/4c2a9e7b1d3f_move_roles_into_krater_db.py b/alembic/versions/4c2a9e7b1d3f_move_roles_into_krater_db.py new file mode 100644 index 0000000..d1e764d --- /dev/null +++ b/alembic/versions/4c2a9e7b1d3f_move_roles_into_krater_db.py @@ -0,0 +1,125 @@ +"""move roles into krater's database + +Revision ID: 4c2a9e7b1d3f +Revises: 29fc5db971e5 +Create Date: 2026-09-28 10:00:00.000000 + +By maintainer decision, Krater now runs against Weave's main branch (plain OIDC sign-in, no `groups` +claim, no directory API), so roles live in Krater's own database. Hand-written, then checked against +`alembic revision --autogenerate` (which reports no diff after this runs): + +* `user_roles` and `pending_role_grants` are new (see `krater.models.user_role`). +* `users.groups_cached` (the last-seen Weave `groups` claim) seeds `user_roles` (only `ganymede:*` + values), with one `role_grant` audit event per seeded row, and is then dropped. `downgrade()` + rebuilds it from `user_roles`. +* `users.disabled_at` and `users.email_verified` are new. Existing rows start unverified; the next + sign-in sets it from the id_token. +* `users.slack_user_id` becomes unique, so a Slack click resolves to one user. Any duplicates are + cleared first, keeping the most recently signed-in user's copy. +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "4c2a9e7b1d3f" +down_revision: str | Sequence[str] | None = "29fc5db971e5" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.create_table( + "user_roles", + sa.Column("user_id", sa.UUID(), nullable=False), + sa.Column("role", sa.String(length=64), nullable=False), + sa.Column("granted_by_id", sa.UUID(), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint(["granted_by_id"], ["users.id"], name=op.f("fk_user_roles_granted_by_id_users")), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], name=op.f("fk_user_roles_user_id_users")), + sa.PrimaryKeyConstraint("id", name=op.f("pk_user_roles")), + sa.UniqueConstraint("user_id", "role", name="uq_user_roles_user_id_role"), + ) + op.create_index(op.f("ix_user_roles_user_id"), "user_roles", ["user_id"], unique=False) + op.create_table( + "pending_role_grants", + sa.Column("email", sa.String(length=255), nullable=False), + sa.Column("role", sa.String(length=64), nullable=False), + sa.Column("granted_by_id", sa.UUID(), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["granted_by_id"], ["users.id"], name=op.f("fk_pending_role_grants_granted_by_id_users") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_pending_role_grants")), + sa.UniqueConstraint("email", "role", name="uq_pending_role_grants_email_role"), + ) + op.create_index(op.f("ix_pending_role_grants_email"), "pending_role_grants", ["email"], unique=False) + + op.add_column("users", sa.Column("email_verified", sa.Boolean(), server_default=sa.false(), nullable=False)) + op.add_column("users", sa.Column("disabled_at", sa.DateTime(timezone=True), nullable=True)) + + op.execute( + """ + INSERT INTO user_roles (id, user_id, role, granted_by_id, created_at) + SELECT gen_random_uuid(), u.id, g.role, NULL, now() + FROM users u CROSS JOIN LATERAL unnest(u.groups_cached) AS g(role) + WHERE g.role LIKE 'ganymede:%' + ON CONFLICT (user_id, role) DO NOTHING + """ + ) + op.execute( + """ + INSERT INTO audit_events (id, actor_id, action, project_id, payload, reason, created_at) + SELECT gen_random_uuid(), NULL, 'role_grant', NULL, + jsonb_build_object('user_id', ur.user_id::text, 'role', ur.role), + 'migrated from the Weave groups claim', now() + FROM user_roles ur + """ + ) + op.drop_column("users", "groups_cached") + + op.execute( + """ + UPDATE users SET slack_user_id = NULL + WHERE id IN ( + SELECT id FROM ( + SELECT id, row_number() OVER ( + PARTITION BY slack_user_id ORDER BY last_login_at DESC NULLS LAST, created_at DESC + ) AS rn + FROM users WHERE slack_user_id IS NOT NULL + ) ranked + WHERE rn > 1 + ) + """ + ) + op.create_unique_constraint(op.f("uq_users_slack_user_id"), "users", ["slack_user_id"]) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_constraint(op.f("uq_users_slack_user_id"), "users", type_="unique") + op.add_column( + "users", + sa.Column("groups_cached", postgresql.ARRAY(sa.String()), server_default="{}", nullable=False), + ) + op.execute( + """ + UPDATE users SET groups_cached = COALESCE( + (SELECT array_agg(ur.role ORDER BY ur.role) FROM user_roles ur WHERE ur.user_id = users.id), + '{}' + ) + """ + ) + op.drop_column("users", "disabled_at") + op.drop_column("users", "email_verified") + op.drop_index(op.f("ix_pending_role_grants_email"), table_name="pending_role_grants") + op.drop_table("pending_role_grants") + op.drop_index(op.f("ix_user_roles_user_id"), table_name="user_roles") + op.drop_table("user_roles") diff --git a/alembic/versions/59b55982f8a6_initial_schema.py b/alembic/versions/59b55982f8a6_initial_schema.py new file mode 100644 index 0000000..dae8d56 --- /dev/null +++ b/alembic/versions/59b55982f8a6_initial_schema.py @@ -0,0 +1,287 @@ +"""initial schema + +Revision ID: 59b55982f8a6 +Revises: +Create Date: 2026-09-26 19:04:14.972174 + +Hand-reviewed after `alembic revision --autogenerate`. Two things needed fixing beyond what +autogenerate produced: + +* Each `sa.Enum(..., create_constraint=True)` column already attaches its own CHECK constraint when + the table is built (that's what `create_constraint=True` does); autogenerate additionally rendered + an explicit `sa.CheckConstraint(...)` for the same check, twice, which duplicated the constraint and + made `upgrade()` fail outright (`DuplicateObject`). The redundant explicit constraints are removed + here; the column type still creates the one real constraint. +* `projects.current_revision_id` / `approved_revision_id` and `project_revisions.project_id` form a + genuine circular FK dependency between the two tables. Autogenerate rendered both `use_alter=True` + foreign keys inline inside `create_table('projects', ...)`, which fails because `project_revisions` + doesn't exist yet at that point in the script. They're added below via `op.create_foreign_key(...)` + once both tables exist, and dropped again (via `op.drop_constraint`) before either table is dropped + in `downgrade()`. +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "59b55982f8a6" +down_revision: str | Sequence[str] | None = None +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.create_table( + "approval_policies", + sa.Column( + "stage", + sa.Enum( + "proposal", "completion", name="approval_stage", native_enum=False, create_constraint=True, length=64 + ), + nullable=False, + ), + sa.Column("min_budget_cents", sa.Integer(), nullable=True), + sa.Column("min_approvals", sa.Integer(), server_default="1", nullable=False), + sa.Column("required_group", sa.String(length=255), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_approval_policies")), + ) + op.create_table( + "users", + sa.Column("weave_sub", sa.String(length=64), nullable=False), + sa.Column("display_name", sa.String(length=255), nullable=False), + sa.Column("email", sa.String(length=255), nullable=False), + sa.Column("slack_user_id", sa.String(length=64), nullable=True), + sa.Column("groups_cached", postgresql.ARRAY(sa.String()), server_default="{}", nullable=False), + sa.Column("last_login_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_users")), + sa.UniqueConstraint("weave_sub", name=op.f("uq_users_weave_sub")), + ) + op.create_table( + "projects", + sa.Column("title", sa.String(length=255), nullable=False), + sa.Column("submitter_id", sa.UUID(), nullable=False), + sa.Column( + "status", + sa.Enum( + "draft", + "pending_review", + "changes_requested", + "approved", + "pending_completion_review", + "completion_changes_requested", + "completed", + "withdrawn", + name="project_status", + native_enum=False, + create_constraint=True, + length=64, + ), + server_default="draft", + nullable=False, + ), + # FKs to project_revisions.id are added below, once that table exists (circular dependency). + sa.Column("current_revision_id", sa.UUID(), nullable=True), + sa.Column("approved_revision_id", sa.UUID(), nullable=True), + sa.Column("repo_url", sa.String(length=2048), nullable=True), + sa.Column("slack_channel_id", sa.String(length=64), nullable=True), + sa.Column("skypilot_workspace", sa.String(length=255), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint(["submitter_id"], ["users.id"], name=op.f("fk_projects_submitter_id_users")), + sa.PrimaryKeyConstraint("id", name=op.f("pk_projects")), + ) + op.create_index(op.f("ix_projects_submitter_id"), "projects", ["submitter_id"], unique=False) + op.create_table( + "audit_events", + sa.Column("actor_id", sa.UUID(), nullable=False), + sa.Column("action", sa.String(length=64), nullable=False), + sa.Column("project_id", sa.UUID(), nullable=True), + sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), server_default="{}", nullable=False), + sa.Column("reason", sa.Text(), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint(["actor_id"], ["users.id"], name=op.f("fk_audit_events_actor_id_users")), + sa.ForeignKeyConstraint(["project_id"], ["projects.id"], name=op.f("fk_audit_events_project_id_projects")), + sa.PrimaryKeyConstraint("id", name=op.f("pk_audit_events")), + ) + op.create_index(op.f("ix_audit_events_action"), "audit_events", ["action"], unique=False) + op.create_index(op.f("ix_audit_events_actor_id"), "audit_events", ["actor_id"], unique=False) + op.create_index(op.f("ix_audit_events_project_id"), "audit_events", ["project_id"], unique=False) + op.create_table( + "project_revisions", + sa.Column("project_id", sa.UUID(), nullable=False), + sa.Column("number", sa.Integer(), nullable=False), + sa.Column( + "kind", + sa.Enum( + "proposal", + "amendment", + "completion", + name="revision_kind", + native_enum=False, + create_constraint=True, + length=64, + ), + nullable=False, + ), + sa.Column("write_up", sa.Text(), nullable=False), + sa.Column("budget_requested_cents", sa.Integer(), nullable=False), + sa.Column("demo_url", sa.String(length=2048), nullable=True), + sa.Column("screenshot_keys", postgresql.ARRAY(sa.String()), server_default="{}", nullable=False), + sa.Column("credited_builder_ids", postgresql.ARRAY(sa.UUID()), server_default="{}", nullable=False), + sa.Column("tags", postgresql.ARRAY(sa.String()), server_default="{}", nullable=False), + sa.Column("submitted_at", sa.DateTime(timezone=True), nullable=True), + sa.Column( + "outcome", + sa.Enum( + "pending", + "approved", + "rejected", + "superseded", + name="revision_outcome", + native_enum=False, + create_constraint=True, + length=64, + ), + server_default="pending", + nullable=False, + ), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint(["project_id"], ["projects.id"], name=op.f("fk_project_revisions_project_id_projects")), + sa.PrimaryKeyConstraint("id", name=op.f("pk_project_revisions")), + sa.UniqueConstraint("project_id", "number", name="uq_project_revisions_project_id_number"), + ) + op.create_index(op.f("ix_project_revisions_project_id"), "project_revisions", ["project_id"], unique=False) + + # Both tables now exist: add the circular FKs from `projects` to `project_revisions` via ALTER TABLE. + op.create_foreign_key( + op.f("fk_projects_current_revision_id_project_revisions"), + "projects", + "project_revisions", + ["current_revision_id"], + ["id"], + ) + op.create_foreign_key( + op.f("fk_projects_approved_revision_id_project_revisions"), + "projects", + "project_revisions", + ["approved_revision_id"], + ["id"], + ) + + op.create_table( + "spend_snapshots", + sa.Column("project_id", sa.UUID(), nullable=False), + sa.Column("estimated_spend_cents", sa.Integer(), nullable=False), + sa.Column( + "source", + sa.Enum("skypilot_cost_report", name="spend_source", native_enum=False, create_constraint=True, length=64), + nullable=False, + ), + sa.Column("taken_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.Column("id", sa.UUID(), nullable=False), + sa.ForeignKeyConstraint(["project_id"], ["projects.id"], name=op.f("fk_spend_snapshots_project_id_projects")), + sa.PrimaryKeyConstraint("id", name=op.f("pk_spend_snapshots")), + ) + op.create_index( + "ix_spend_snapshots_project_id_taken_at", "spend_snapshots", ["project_id", "taken_at"], unique=False + ) + op.create_table( + "budget_entries", + sa.Column("project_id", sa.UUID(), nullable=False), + sa.Column( + "kind", + sa.Enum( + "initial_approval", + "amendment", + "admin_adjustment", + "reclaim", + name="budget_entry_kind", + native_enum=False, + create_constraint=True, + length=64, + ), + nullable=False, + ), + sa.Column("amount_cents", sa.Integer(), nullable=False), + sa.Column("actor_id", sa.UUID(), nullable=False), + sa.Column("reason", sa.Text(), nullable=True), + sa.Column("revision_id", sa.UUID(), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint(["actor_id"], ["users.id"], name=op.f("fk_budget_entries_actor_id_users")), + sa.ForeignKeyConstraint(["project_id"], ["projects.id"], name=op.f("fk_budget_entries_project_id_projects")), + sa.ForeignKeyConstraint( + ["revision_id"], ["project_revisions.id"], name=op.f("fk_budget_entries_revision_id_project_revisions") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_budget_entries")), + ) + op.create_index(op.f("ix_budget_entries_actor_id"), "budget_entries", ["actor_id"], unique=False) + op.create_index(op.f("ix_budget_entries_project_id"), "budget_entries", ["project_id"], unique=False) + op.create_index(op.f("ix_budget_entries_revision_id"), "budget_entries", ["revision_id"], unique=False) + op.create_table( + "reviews", + sa.Column("revision_id", sa.UUID(), nullable=False), + sa.Column("reviewer_id", sa.UUID(), nullable=False), + sa.Column( + "decision", + sa.Enum("approve", "reject", name="review_decision", native_enum=False, create_constraint=True, length=64), + nullable=False, + ), + sa.Column("reason", sa.Text(), nullable=True), + sa.Column( + "source", + sa.Enum("slack", "web", name="review_source", native_enum=False, create_constraint=True, length=64), + nullable=False, + ), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint(["reviewer_id"], ["users.id"], name=op.f("fk_reviews_reviewer_id_users")), + sa.ForeignKeyConstraint( + ["revision_id"], ["project_revisions.id"], name=op.f("fk_reviews_revision_id_project_revisions") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_reviews")), + ) + op.create_index(op.f("ix_reviews_reviewer_id"), "reviews", ["reviewer_id"], unique=False) + op.create_index(op.f("ix_reviews_revision_id"), "reviews", ["revision_id"], unique=False) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_index(op.f("ix_reviews_revision_id"), table_name="reviews") + op.drop_index(op.f("ix_reviews_reviewer_id"), table_name="reviews") + op.drop_table("reviews") + op.drop_index(op.f("ix_budget_entries_revision_id"), table_name="budget_entries") + op.drop_index(op.f("ix_budget_entries_project_id"), table_name="budget_entries") + op.drop_index(op.f("ix_budget_entries_actor_id"), table_name="budget_entries") + op.drop_table("budget_entries") + op.drop_index("ix_spend_snapshots_project_id_taken_at", table_name="spend_snapshots") + op.drop_table("spend_snapshots") + + # Drop the circular FKs before either table goes away. + op.drop_constraint(op.f("fk_projects_approved_revision_id_project_revisions"), "projects", type_="foreignkey") + op.drop_constraint(op.f("fk_projects_current_revision_id_project_revisions"), "projects", type_="foreignkey") + + op.drop_index(op.f("ix_project_revisions_project_id"), table_name="project_revisions") + op.drop_table("project_revisions") + op.drop_index(op.f("ix_audit_events_project_id"), table_name="audit_events") + op.drop_index(op.f("ix_audit_events_actor_id"), table_name="audit_events") + op.drop_index(op.f("ix_audit_events_action"), table_name="audit_events") + op.drop_table("audit_events") + op.drop_index(op.f("ix_projects_submitter_id"), table_name="projects") + op.drop_table("projects") + op.drop_table("users") + op.drop_table("approval_policies") diff --git a/alembic/versions/5f3b46edeb95_add_skypilot_allowed_users_to_projects.py b/alembic/versions/5f3b46edeb95_add_skypilot_allowed_users_to_projects.py new file mode 100644 index 0000000..381a046 --- /dev/null +++ b/alembic/versions/5f3b46edeb95_add_skypilot_allowed_users_to_projects.py @@ -0,0 +1,35 @@ +"""add skypilot allowed users to projects + +Revision ID: 5f3b46edeb95 +Revises: b7ba424bfaeb +Create Date: 2026-10-10 12:07:34.645919 + +`projects.skypilot_allowed_users` holds the `allowed_users` list the SkyPilot reconciler last sent to the +project's workspace. The reconciler compares it with the next list to find people it removed because +Weave no longer lists them as active members, and audits each removal once. It starts empty (`NULL`): +those workspaces hold the whole team, and the next reconcile fills it in. +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "5f3b46edeb95" +down_revision: str | Sequence[str] | None = "b7ba424bfaeb" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.add_column( + "projects", + sa.Column("skypilot_allowed_users", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("projects", "skypilot_allowed_users") diff --git a/alembic/versions/67a1656dcb6d_procrastinate_schema.py b/alembic/versions/67a1656dcb6d_procrastinate_schema.py new file mode 100644 index 0000000..e415c21 --- /dev/null +++ b/alembic/versions/67a1656dcb6d_procrastinate_schema.py @@ -0,0 +1,77 @@ +"""procrastinate schema + +Revision ID: 67a1656dcb6d +Revises: 59b55982f8a6 +Create Date: 2026-09-26 19:06:29.266229 + +procrastinate ships its schema as raw SQL (`procrastinate.sql.schema.sql`) and normally applies it +itself (`procrastinate schema --apply`, or a job store's `apply_schema()`). We use Alembic as the only +migration path instead, so this migration pulls that same SQL from the installed `procrastinate` +package via `procrastinate.schema.SchemaManager.get_schema()` -- the approach procrastinate's own docs +point to for projects that manage migrations with a different tool -- and executes it here. Bump this +migration (or add a new one) if `procrastinate` is upgraded to a version with schema changes. + +`downgrade()` can't reuse a procrastinate helper the same way (it doesn't ship a "drop everything" +function), so it drops procrastinate's tables, functions and types explicitly. Functions are dropped +by matching `procrastinate_%` in `pg_proc` rather than by listing exact signatures, since those change +between procrastinate versions. +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from procrastinate.schema import SchemaManager + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "67a1656dcb6d" +down_revision: str | Sequence[str] | None = "59b55982f8a6" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +_DROP_FUNCTIONS_SQL = """ +DO $$ +DECLARE + proc RECORD; +BEGIN + FOR proc IN + SELECT p.oid::regprocedure AS signature + FROM pg_proc p + JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = 'public' AND p.proname LIKE 'procrastinate\\_%' ESCAPE '\\' + LOOP + EXECUTE 'DROP FUNCTION IF EXISTS ' || proc.signature || ' CASCADE'; + END LOOP; +END; +$$; +""" + +_TABLES = ( + "procrastinate_events", + "procrastinate_periodic_defers", + "procrastinate_jobs", + "procrastinate_workers", +) + +_TYPES = ( + "procrastinate_job_status", + "procrastinate_job_event_type", + "procrastinate_job_to_defer_v1", +) + + +def upgrade() -> None: + """Upgrade schema.""" + # `op.execute` wraps a plain string in `sa.text(...)` itself; checked that this SQL has no bare + # `:name`-style substrings that `text()` would otherwise mistake for bind parameters. + op.execute(SchemaManager.get_schema()) + + +def downgrade() -> None: + """Downgrade schema.""" + for table in _TABLES: + op.execute(sa.text(f"DROP TABLE IF EXISTS {table} CASCADE")) + op.execute(sa.text(_DROP_FUNCTIONS_SQL)) + for type_name in _TYPES: + op.execute(sa.text(f"DROP TYPE IF EXISTS {type_name} CASCADE")) diff --git a/alembic/versions/83af68b415ca_add_quilt_outbox.py b/alembic/versions/83af68b415ca_add_quilt_outbox.py new file mode 100644 index 0000000..1f4e4f2 --- /dev/null +++ b/alembic/versions/83af68b415ca_add_quilt_outbox.py @@ -0,0 +1,70 @@ +"""add quilt outbox + +Revision ID: 83af68b415ca +Revises: e3f1a7c2b9d4 +Create Date: 2026-10-10 16:06:56.374626 + +`quilt_outbox` holds the events Krater sends to Quilt's patch API. The services write a row in the same +transaction as the change it reports, and the worker sends the rows. See docs/quilt-integration.md. + +The `state` column's `sa.Enum(..., create_constraint=True)` attaches its own CHECK constraint, so there is +no separate `sa.CheckConstraint` here (see the initial schema migration). +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "83af68b415ca" +down_revision: str | Sequence[str] | None = "e3f1a7c2b9d4" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.create_table( + "quilt_outbox", + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("seq", sa.BigInteger(), sa.Identity(always=True), nullable=False), + sa.Column("type", sa.String(length=64), nullable=False), + sa.Column("external_id", sa.String(length=200), nullable=False), + sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column("occurred_at", sa.DateTime(timezone=True), nullable=False), + sa.Column( + "state", + sa.Enum( + "pending", + "sent", + "failed", + "skipped", + name="quilt_outbox_state", + native_enum=False, + create_constraint=True, + length=64, + ), + server_default="pending", + nullable=False, + ), + sa.Column("attempts", sa.Integer(), server_default="0", nullable=False), + sa.Column("next_attempt_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("sent_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("last_status", sa.Integer(), nullable=True), + sa.Column("last_error", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("pk_quilt_outbox")), + sa.UniqueConstraint("seq", name=op.f("uq_quilt_outbox_seq")), + ) + op.create_index("ix_quilt_outbox_external_id_seq", "quilt_outbox", ["external_id", "seq"], unique=False) + op.create_index("ix_quilt_outbox_state_seq", "quilt_outbox", ["state", "seq"], unique=False) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_index("ix_quilt_outbox_state_seq", table_name="quilt_outbox") + op.drop_index("ix_quilt_outbox_external_id_seq", table_name="quilt_outbox") + op.drop_table("quilt_outbox") diff --git a/alembic/versions/9ed6e4593cff_add_slack_integration_fields.py b/alembic/versions/9ed6e4593cff_add_slack_integration_fields.py new file mode 100644 index 0000000..58acfd2 --- /dev/null +++ b/alembic/versions/9ed6e4593cff_add_slack_integration_fields.py @@ -0,0 +1,57 @@ +"""add slack integration fields + +Revision ID: 9ed6e4593cff +Revises: cdd8685b4aa7 +Create Date: 2026-09-26 22:41:17.333902 + +Adds what `krater.services.slack_notify` needs: a project's channel-archived flag (kept separate from +`slack_channel_id`, which stays set after archiving so the project page can still link to it), the +review message a revision's decision later updates in place, and `slack_notifications` -- tracking which +`budget_warning`/`budget_teardown` audit events the periodic Slack reconcile has already posted, so it +never posts one twice. +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "9ed6e4593cff" +down_revision: str | Sequence[str] | None = "cdd8685b4aa7" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.add_column( + "projects", sa.Column("slack_channel_archived", sa.Boolean(), server_default=sa.false(), nullable=False) + ) + op.add_column("project_revisions", sa.Column("slack_message_channel_id", sa.String(length=64), nullable=True)) + op.add_column("project_revisions", sa.Column("slack_message_ts", sa.String(length=32), nullable=True)) + + op.create_table( + "slack_notifications", + sa.Column("audit_event_id", sa.UUID(), nullable=False), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["audit_event_id"], ["audit_events.id"], name=op.f("fk_slack_notifications_audit_event_id_audit_events") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_slack_notifications")), + ) + op.create_index( + op.f("ix_slack_notifications_audit_event_id"), "slack_notifications", ["audit_event_id"], unique=True + ) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_index(op.f("ix_slack_notifications_audit_event_id"), table_name="slack_notifications") + op.drop_table("slack_notifications") + + op.drop_column("project_revisions", "slack_message_ts") + op.drop_column("project_revisions", "slack_message_channel_id") + op.drop_column("projects", "slack_channel_archived") diff --git a/alembic/versions/a7c1e5d9f2b4_stop_storing_member_data.py b/alembic/versions/a7c1e5d9f2b4_stop_storing_member_data.py new file mode 100644 index 0000000..9f447f6 --- /dev/null +++ b/alembic/versions/a7c1e5d9f2b4_stop_storing_member_data.py @@ -0,0 +1,45 @@ +"""stop storing member data from Weave (patchworklabsorg/krater#7) + +Drops `users.display_name`, `users.email` and `users.email_verified`: Krater reads names and emails from +Weave's directory when it needs them. Audit events gain `actor_sub`, filled in for existing rows, and lose the +emails earlier SkyPilot access-removal events carried. The emails in the unused `projects.skypilot_allowed_users` +are cleared. + +Revision ID: a7c1e5d9f2b4 +Revises: 83af68b415ca +Create Date: 2026-10-12 03:00:00.000000 + +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "a7c1e5d9f2b4" +down_revision: str | Sequence[str] | None = "83af68b415ca" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + op.add_column("audit_events", sa.Column("actor_sub", sa.String(length=64), nullable=True)) + op.execute("UPDATE audit_events SET actor_sub = users.weave_sub FROM users WHERE audit_events.actor_id = users.id") + op.execute("UPDATE audit_events SET payload = payload - 'email' WHERE payload ? 'email'") + # Krater Compute drops this column; on a database that has gone that way first there's nothing to clear. + if "skypilot_allowed_users" in {c["name"] for c in sa.inspect(op.get_bind()).get_columns("projects")}: + op.execute("UPDATE projects SET skypilot_allowed_users = NULL") + op.drop_column("users", "email_verified") + op.drop_column("users", "email") + op.drop_column("users", "display_name") + + +def downgrade() -> None: + # The names and emails are gone; a downgrade gets placeholders until each member signs in again. + op.add_column("users", sa.Column("display_name", sa.String(length=255), nullable=False, server_default="")) + op.add_column("users", sa.Column("email", sa.String(length=255), nullable=False, server_default="")) + op.add_column("users", sa.Column("email_verified", sa.Boolean(), nullable=False, server_default=sa.false())) + op.execute("UPDATE users SET display_name = weave_sub") + op.drop_column("audit_events", "actor_sub") diff --git a/alembic/versions/b7ba424bfaeb_weave_owns_roles.py b/alembic/versions/b7ba424bfaeb_weave_owns_roles.py new file mode 100644 index 0000000..023fc00 --- /dev/null +++ b/alembic/versions/b7ba424bfaeb_weave_owns_roles.py @@ -0,0 +1,93 @@ +"""weave owns roles + +Revision ID: b7ba424bfaeb +Revises: 4c2a9e7b1d3f +Create Date: 2026-10-07 15:33:28.797985 + +By maintainer decision, Weave owns Krater's roles again (the `roles` claim and the directory API), so +the Krater-side role tables and disable switch from 4c2a9e7b1d3f go away: + +* `users.roles_cached` is new: the Krater role names Weave last reported, for display only. It starts + from the user's rows in `user_roles`; the next sign-in or directory lookup overwrites it. +* `user_roles` and `pending_role_grants` are dropped. +* `users.disabled_at` is dropped. Disabling someone is now done in Weave. + +`users.email_verified` and the unique `users.slack_user_id` stay. Past `role_grant` and similar audit +events stay as history. `downgrade()` recreates the tables, seeds `user_roles` from `roles_cached`, +and brings back `disabled_at` (empty). +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "b7ba424bfaeb" +down_revision: str | Sequence[str] | None = "4c2a9e7b1d3f" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.add_column( + "users", + sa.Column("roles_cached", postgresql.ARRAY(sa.String()), server_default="{}", nullable=False), + ) + op.execute( + """ + UPDATE users SET roles_cached = COALESCE( + (SELECT array_agg(ur.role ORDER BY ur.role) FROM user_roles ur WHERE ur.user_id = users.id), + '{}' + ) + """ + ) + op.drop_index(op.f("ix_pending_role_grants_email"), table_name="pending_role_grants") + op.drop_table("pending_role_grants") + op.drop_index(op.f("ix_user_roles_user_id"), table_name="user_roles") + op.drop_table("user_roles") + op.drop_column("users", "disabled_at") + + +def downgrade() -> None: + """Downgrade schema.""" + op.add_column("users", sa.Column("disabled_at", sa.DateTime(timezone=True), nullable=True)) + op.create_table( + "user_roles", + sa.Column("user_id", sa.UUID(), nullable=False), + sa.Column("role", sa.String(length=64), nullable=False), + sa.Column("granted_by_id", sa.UUID(), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint(["granted_by_id"], ["users.id"], name=op.f("fk_user_roles_granted_by_id_users")), + sa.ForeignKeyConstraint(["user_id"], ["users.id"], name=op.f("fk_user_roles_user_id_users")), + sa.PrimaryKeyConstraint("id", name=op.f("pk_user_roles")), + sa.UniqueConstraint("user_id", "role", name="uq_user_roles_user_id_role"), + ) + op.create_index(op.f("ix_user_roles_user_id"), "user_roles", ["user_id"], unique=False) + op.create_table( + "pending_role_grants", + sa.Column("email", sa.String(length=255), nullable=False), + sa.Column("role", sa.String(length=64), nullable=False), + sa.Column("granted_by_id", sa.UUID(), nullable=True), + sa.Column("id", sa.UUID(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False), + sa.ForeignKeyConstraint( + ["granted_by_id"], ["users.id"], name=op.f("fk_pending_role_grants_granted_by_id_users") + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_pending_role_grants")), + sa.UniqueConstraint("email", "role", name="uq_pending_role_grants_email_role"), + ) + op.create_index(op.f("ix_pending_role_grants_email"), "pending_role_grants", ["email"], unique=False) + op.execute( + """ + INSERT INTO user_roles (id, user_id, role, granted_by_id, created_at) + SELECT gen_random_uuid(), u.id, r.role, NULL, now() + FROM users u CROSS JOIN LATERAL unnest(u.roles_cached) AS r(role) + ON CONFLICT (user_id, role) DO NOTHING + """ + ) + op.drop_column("users", "roles_cached") diff --git a/alembic/versions/cdd8685b4aa7_make_audit_events_actor_id_nullable.py b/alembic/versions/cdd8685b4aa7_make_audit_events_actor_id_nullable.py new file mode 100644 index 0000000..1330d04 --- /dev/null +++ b/alembic/versions/cdd8685b4aa7_make_audit_events_actor_id_nullable.py @@ -0,0 +1,32 @@ +"""make audit_events.actor_id nullable + +Revision ID: cdd8685b4aa7 +Revises: 153093277378 +Create Date: 2026-09-26 21:14:38.215192 + +The SkyPilot reconciler (`krater.services.skypilot_sync`) writes `budget_warning`, `budget_teardown` +and workspace-teardown audit events as a system actor with no human `User` behind it -- there's no one +to attribute a periodic reconcile run to. Every human-triggered action still always supplies an actor. +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "cdd8685b4aa7" +down_revision: str | Sequence[str] | None = "153093277378" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.alter_column("audit_events", "actor_id", existing_type=sa.UUID(), nullable=True) + + +def downgrade() -> None: + """Downgrade schema.""" + op.alter_column("audit_events", "actor_id", existing_type=sa.UUID(), nullable=False) diff --git a/alembic/versions/ddcdc402852d_add_reviewer_groups_snapshot_to_reviews.py b/alembic/versions/ddcdc402852d_add_reviewer_groups_snapshot_to_reviews.py new file mode 100644 index 0000000..cb75569 --- /dev/null +++ b/alembic/versions/ddcdc402852d_add_reviewer_groups_snapshot_to_reviews.py @@ -0,0 +1,36 @@ +"""add reviewer_groups snapshot to reviews + +Revision ID: ddcdc402852d +Revises: 67a1656dcb6d +Create Date: 2026-09-26 19:19:22.841148 + +Snapshots the reviewer's Weave groups onto the `Review` row at the moment they review, so +`ApprovalPolicy.required_group` can be checked against the groups the reviewer actually held at +review time rather than their current (possibly since-changed) groups. +""" + +from collections.abc import Sequence + +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "ddcdc402852d" +down_revision: str | Sequence[str] | None = "67a1656dcb6d" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.add_column( + "reviews", + sa.Column("reviewer_groups", postgresql.ARRAY(sa.String()), server_default="{}", nullable=False), + ) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_column("reviews", "reviewer_groups") diff --git a/alembic/versions/e3f1a7c2b9d4_add_per_project_hourly_cost_cap.py b/alembic/versions/e3f1a7c2b9d4_add_per_project_hourly_cost_cap.py new file mode 100644 index 0000000..5ce8cc1 --- /dev/null +++ b/alembic/versions/e3f1a7c2b9d4_add_per_project_hourly_cost_cap.py @@ -0,0 +1,32 @@ +"""add per-project hourly cost cap + +Revision ID: e3f1a7c2b9d4 +Revises: 5f3b46edeb95 +Create Date: 2026-10-10 18:40:00.000000 + +`projects.max_hourly_cost_cents`: an admin-set hourly price cap for this project's SkyPilot launches. +Nullable, and null for every existing project, which keeps using the global default +(`KRATER_SKYPILOT_MAX_HOURLY_COST_CENTS`). +""" + +from collections.abc import Sequence + +import sqlalchemy as sa + +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "e3f1a7c2b9d4" +down_revision: str | Sequence[str] | None = "5f3b46edeb95" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.add_column("projects", sa.Column("max_hourly_cost_cents", sa.Integer(), nullable=True)) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_column("projects", "max_hourly_cost_cents") diff --git a/deploy/Caddyfile b/deploy/Caddyfile new file mode 100644 index 0000000..72f9e8b --- /dev/null +++ b/deploy/Caddyfile @@ -0,0 +1,23 @@ +# Caddy in front of Krater: TLS (certificates from Let's Encrypt, renewed automatically) and one public +# hostname per service. Used by docker-compose.yml's `proxy` service (`--profile proxy`); see +# docs/dev/production.md. Skip it when the host already runs its own reverse proxy, and point that +# proxy at the same two upstreams instead. + +{ + # The *.localhost defaults (docker-compose.yml) get certificates from Caddy's own local CA; + # there's no system trust store worth writing it into inside the container. + skip_install_trust +} + +# The portal: UI, sign-in, Slack's interactivity endpoint, and the runner API rented machines call +# (/runner/v1/, docs/compute/runner.md). +{$KRATER_DOMAIN} { + encode gzip + reverse_proxy portal:8000 +} + +# Gallery screenshots. Browsers upload straight to the bucket and load thumbnails from it with +# presigned URLs, so KRATER_S3_PUBLIC_ENDPOINT_URL must be https:// (docs/dev/storage.md). +{$KRATER_S3_DOMAIN} { + reverse_proxy storage:8333 +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4dc6d59 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,150 @@ +services: + db: + image: postgres:16 + restart: unless-stopped + environment: + POSTGRES_USER: ${POSTGRES_USER:-krater} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-krater} + POSTGRES_DB: ${POSTGRES_DB:-krater} + volumes: + - krater_db_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-krater} -d ${POSTGRES_DB:-krater}"] + interval: 5s + timeout: 5s + retries: 10 + + # One-shot deploy setup: database migrations, then the screenshot bucket and its CORS rule (skipped + # when KRATER_S3_MODE=fake). Both steps are safe to re-run; see docs/dev/storage.md. + migrate: + build: . + env_file: ${KRATER_ENV_FILE:-.env} + command: ["/bin/sh", "-c", "alembic upgrade head && python -m krater.storage.ensure_bucket"] + depends_on: + db: + condition: service_healthy + storage: + condition: service_started + + portal: + build: . + restart: unless-stopped + env_file: ${KRATER_ENV_FILE:-.env} + # KRATER_BIND_ADDRESS=127.0.0.1 in production keeps the published ports off the network, so the + # only way in is through the TLS proxy (docs/dev/production.md). Applies to every port below. + ports: + - "${KRATER_BIND_ADDRESS:-0.0.0.0}:${PORT:-8000}:8000" + depends_on: + migrate: + condition: service_completed_successfully + + worker: + build: . + restart: unless-stopped + env_file: ${KRATER_ENV_FILE:-.env} + command: ["procrastinate", "--app=krater.worker.app.app", "worker"] + # The Vast key, for Krater Compute (KRATER_COMPUTE_MODE=vast): mounted into the worker only, never the portal. + # Point KRATER_VAST_KEY_FILE at the key on the host and set KRATER_VAST_API_KEY_FILE=/run/secrets/vast_api_key. + volumes: + - ${KRATER_VAST_KEY_FILE:-/dev/null}:/run/secrets/vast_api_key:ro + depends_on: + migrate: + condition: service_completed_successfully + + # Temporary self-hosted S3-compatible storage (gallery screenshots) until a real provider is chosen; see + # docs/SPEC.md "Open questions". SeaweedFS rather than MinIO: MinIO stopped publishing community images in 2025. + storage: + # 4.47 is what staging ran (2026-10-10). Bump deliberately, after a staging run on the new version. + image: chrislusf/seaweedfs:${SEAWEEDFS_VERSION:-4.47} + restart: unless-stopped + environment: + S3_ACCESS_KEY: ${KRATER_S3_ACCESS_KEY_ID:-krater} + S3_SECRET_KEY: ${KRATER_S3_SECRET_ACCESS_KEY:-krater-dev-secret} + # The S3 gateway reads its credentials from a JSON file; write it from the env so no secret lives in the repo. + entrypoint: ["/bin/sh", "-c"] + command: + - >- + printf '{"identities":[{"name":"krater","credentials":[{"accessKey":"%s","secretKey":"%s"}],"actions":["Admin","Read","List","Tagging","Write"]}]}' + "$$S3_ACCESS_KEY" "$$S3_SECRET_KEY" > /tmp/s3.json && + exec weed server -dir=/data -master.volumeSizeLimitMB=1024 -s3 -s3.port=8333 -s3.config=/tmp/s3.json + ports: + - "${KRATER_BIND_ADDRESS:-0.0.0.0}:${S3_PORT:-8333}:8333" + volumes: + - krater_storage_data:/data + + # -- Production extras (docs/dev/production.md) -------------------------------------------------- + # Opt-in, so dev and staging stay as they are: `--profile backup` and `--profile proxy`. + + # Nightly `pg_dump` of Krater's database: the budget ledger and audit log live only there. Dumps land + # in KRATER_BACKUP_DIR on the host, which is what to copy off the machine; files older than + # KRATER_BACKUP_KEEP_DAYS are deleted. Restoring one is in docs/dev/production.md. + db-backup: + image: postgres:16 + profiles: ["backup"] + restart: unless-stopped + environment: + PGHOST: db + PGUSER: ${POSTGRES_USER:-krater} + PGPASSWORD: ${POSTGRES_PASSWORD:-krater} + PGDATABASE: ${POSTGRES_DB:-krater} + BACKUP_INTERVAL_SECONDS: ${KRATER_BACKUP_INTERVAL_SECONDS:-86400} + BACKUP_KEEP_DAYS: ${KRATER_BACKUP_KEEP_DAYS:-14} + entrypoint: ["/bin/sh", "-c"] + # A one-item list so the script reaches the shell intact. Each dump is written under a temporary + # name and renamed when complete, so a half-written file is never mistaken for a good backup. + command: + - | + set -eu + while true; do + name="krater-$$(date -u +%Y%m%dT%H%M%SZ).dump" + if pg_dump --format=custom --file="/backups/$$name.partial"; then + mv "/backups/$$name.partial" "/backups/$$name" + echo "backup written: $$name" + else + rm -f "/backups/$$name.partial" + echo "backup FAILED at $$(date -u +%FT%TZ)" >&2 + fi + find /backups -name 'krater-*.dump' -mtime +"$$BACKUP_KEEP_DAYS" -delete + sleep "$$BACKUP_INTERVAL_SECONDS" + done + volumes: + - ${KRATER_BACKUP_DIR:-./backups}:/backups + depends_on: + db: + condition: service_healthy + + # TLS in front of the portal and the screenshot bucket, with certificates + # from Let's Encrypt. Routes are in deploy/Caddyfile. Leave this profile off when the host already + # runs a reverse proxy, and point that one at the same services instead. + proxy: + image: caddy:${CADDY_VERSION:-2.10.2} + profiles: ["proxy"] + restart: unless-stopped + environment: + KRATER_DOMAIN: ${KRATER_DOMAIN:-} + # *.localhost names keep a site unserved (Caddy's local CA, never Let's Encrypt): screenshots kept + # with a hosted provider. + KRATER_S3_DOMAIN: ${KRATER_S3_DOMAIN:-storage.localhost} + # Checked here rather than with Compose's `${KRATER_DOMAIN:?}`, which would fail every + # `docker compose` command, with or without this profile. + entrypoint: ["/bin/sh", "-c"] + command: + - >- + : "$${KRATER_DOMAIN:?set KRATER_DOMAIN to the portal's public hostname}" && + exec caddy run --config /etc/caddy/Caddyfile --adapter caddyfile + ports: + - "80:80" + - "443:443" + - "443:443/udp" + volumes: + - ./deploy/Caddyfile:/etc/caddy/Caddyfile:ro + - krater_caddy_data:/data + - krater_caddy_config:/config + depends_on: + - portal + +volumes: + krater_db_data: + krater_storage_data: + krater_caddy_data: + krater_caddy_config: diff --git a/docs/FUTURE.md b/docs/FUTURE.md new file mode 100644 index 0000000..85a4fac --- /dev/null +++ b/docs/FUTURE.md @@ -0,0 +1,95 @@ +# Future work + +Things we've decided to do later, with enough context to pick each one up cold. v1 scope is in [SPEC.md](SPEC.md); +this list covers what comes after, plus loose ends from building v1. + +## Before v1 goes live + +- **Staging run on the maintainer's machine** ([dev/staging.md](dev/staging.md)). The Compose stack itself has now + been run with stub credentials (HANDOFF section 4), so what's left needs real accounts: + - a real Vast launch; + - the 80% warning and 100% teardown with real spend; + - how far `cost_report` drifts from the Vast bill. That sets the safety margin, if any, to take off the ceiling. +- **Production deployment** ([dev/production.md](dev/production.md)). Compose now has opt-in `proxy` (Caddy, TLS) + and `backup` (nightly `pg_dump`) profiles, restart policies and a bind address. For alastor, a NixOS module in + `patchworklabsorg/infra` (`modules/krater`) builds the image on the host. Still open: merging that module, its + secrets and DNS, off-machine backup copies, a screenshot volume backup, and alerting when the reconcile job fails. + None of it has run on a real host yet. +- **Set up Krater's roles in Weave** on each real deployment: a Weave superadmin creates `member`, `reviewer` and + `admin` on the Krater app page, an admin adds the `directory` scope to the app, and admins give people the roles + (see [weave-integration.md](weave-integration.md)). +- **Create Krater's Slack app** in the Patchwork workspace ([dev/slack-setup.md](dev/slack-setup.md)), and a tunnel + for local testing. +- **Pick a long-term screenshot storage provider.** Today it's a temporary self-hosted SeaweedFS container. The options + are R2/B2/S3, or keeping SeaweedFS with backups (a reminder was set for early October 2026). +- **Decide what happens exactly at the ceiling** (patchworklabsorg/krater#2). Today warnings go out at 80% and teardown happens at 100% with no + grace period, which can kill a long training run just past the limit. Consider a small admin-configurable grace margin. + +## Weave follow-ups + +- **[weave#118](https://github.com/patchworklabsorg/weave/issues/118): require joining Slack as part of membership.** + Krater's gate uses Weave's `slack_member` when Weave sends it, and asks Slack otherwise. *Requiring* Slack as part + of signing up is still a Weave product decision. +- **Weave admin loose ends** (found while fixing Weave's admin users privilege bug, see HANDOFF section 3): there's no + `admin/users/new` view in Weave, so creating a user from Weave's admin panel errors. +- **Reviewer tiers need Weave role keys.** `ganymede:reviewer:` has no Weave role key yet. Add one to the + Krater app in Weave and to `krater.weave.roles` before using tiered approval policies. + +## Product features parked in the spec + +- **In-progress project visibility:** let members see active, not-yet-completed projects, to avoid duplicate work. +- **Multi-approval and reviewer tiers by budget size.** Already supported by `ApprovalPolicy` rows; this is a policy + decision plus a small admin UI to edit the rows (today they're read-only in `/admin`). +- **Mirroring Slack comments into Krater,** if linking to the channel turns out not to be enough. + +> SkyPilot was dropped on 2026-10-11; Krater Compute, Krater's own Vast layer, replaces it in its own release. The +> notes below are from when Krater ran on SkyPilot; the donated-GPU idea would now need Krater Compute to rent from +> more than Vast. + +## SkyPilot + +- **Untested with 0.13.0's CLI:** `sky exec`, `sky jobs launch` and `sky serve up` going through the launch gate for + real. They have no `--dryrun`, so they need a real Vast key. Resource labels on Vast are also untested. See + `docs/dev/skypilot-spike.md` (removed with SkyPilot; see git history). +- **Image allowlist in the launch gate,** to stop crypto mining or other abuse. It becomes more important if donated + compute happens (below). +- **Shared rate limiting.** The limiter is per-process in memory, so limits multiply with the number of web workers. Move + it to Postgres if abuse shows up. + +## Donated idle compute (idea, parked) + +Let members donate idle GPU time on their own machines to Ganymede through SkyPilot. + +**Feasible with SkyPilot's existing pieces.** SkyPilot treats a Kubernetes cluster as another infrastructure. Its SSH +Node Pools feature (`sky ssh up`) turns existing machines into one using **k3s** (confirmed in SkyPilot's source). +Project workspaces could allow a "donated" Kubernetes pool alongside Vast. Donated nodes cost $0 to the optimizer by +default, so SkyPilot would prefer them and fall back to Vast. + +**Proposed shape** +- **One Krater-run k3s cluster; donors join as worker nodes** over a private mesh such as Tailscale or Headscale. Home + machines sit behind NAT, and k3s needs the server and agents to reach each other. +- **Pause and resume rather than boot and kill.** The k3s agent stays installed. A small donor-side agent (a systemd + service) watches for idleness: no input for N minutes, GPU unused, optionally on AC power, within donor-set hours. + When idle, it **uncordons** the node. When the owner returns, it **cordons and drains** it. This is more reliable than + joining and leaving the cluster repeatedly. +- **Treat donated nodes like spot instances.** Drains evict jobs, so only SkyPilot **managed jobs** (which auto-recover, + including onto Vast) should run there, and they must checkpoint, for example to the S3 storage. The launch gate can + enforce "donated pool ⇒ managed jobs only". +- **Credit donors.** Track donated GPU-hours per donor through node labels; that's a good line in gallery credits. + +**Open problems to decide first** +1. **Trust runs both ways.** Donors run other members' code (containers aren't a strong boundary on a GPU box), and + donors with root can see a job's data and models. Only viable among vetted members. It helps that completed + projects are meant to be open source anyway. +2. **Windows donors are hard.** k3s is Linux. GPU access in WSL2 exists, but k3s plus the NVIDIA device plugin inside + WSL2 is fiddly. A pilot should be Linux-only. +3. **Heterogeneous hardware and home bandwidth.** Mixed GPUs and drivers, and slow downloads of images and datasets. + Suited to small, single-node jobs; no multi-node training. +4. **Budget accounting.** Donated compute is $0 to SkyPilot, so Krater's dollar budgets wouldn't count it. Either + donated compute is free (budgets cover Vast only), or it's charged at an internal rate. +5. **Abuse and donor cost.** Crypto mining, and electricity. Needs an image allowlist in the launch gate and donor-set + caps and hours. + +**Suggested first step:** a pilot with 2–3 trusted Linux GPU machines on Tailscale, the cordon/uncordon agent, and +managed jobs with checkpointing. Before that, do a quick local check that SkyPilot's Kubernetes support handles +cordon/drain and managed-job recovery the way this needs. diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md new file mode 100644 index 0000000..b000fc7 --- /dev/null +++ b/docs/HANDOFF.md @@ -0,0 +1,226 @@ +# Krater handoff + +> **Historical.** Written for v1, when Krater ran on SkyPilot. On 2026-10-11 SkyPilot was removed; this release runs +> with compute off, and Krater Compute (Krater's own Vast layer) adds machines in its own release. The SkyPilot +> sections below describe code that no longer exists. + +Everything a fresh Claude Code session (or a human) needs to pick up Krater. Written 2026-09-27 at the end of the +cloud session that built v1. + +**If you're a Claude session: read this file, then `CLAUDE.md`, then `docs/SPEC.md`, before changing anything.** + +--- + +## 1. What exists + +**Krater** is the Project Ganymede portal (Patchwork Labs). It covers proposal review, dollar compute budgets +enforced on SkyPilot/Vast.ai, Slack-based review, a public gallery and a GPU pricing page. + +| Repo | Where | Branch | State | +| --- | --- | --- | --- | +| **Krater** | https://github.com/patchworklabsorg/krater | `claude/exciting-sagan-7oh2zh` | **Draft PR [#1](https://github.com/patchworklabsorg/krater/pull/1)**, CI green, 603 tests | +| **Weave** (identity provider) | https://github.com/patchworklabsorg/weave | needs unmerged work | Weave owns Krater's roles: needs the stack #156 to #161, #165 and #166 (section 3) | + +`main` in Krater is only the initial commit. All the work is on the PR branch. + +## 2. Clone and run Krater locally + +### One command + +With [uv](https://docs.astral.sh/uv/) installed and Docker running, `scripts/dev/setup.ps1` (Windows) or +`scripts/dev/setup.sh` (macOS, Linux, WSL, Git Bash) runs steps 2 and 3 below and migrates `krater_dev`. It's safe to +re-run. Add `-Check` / `--check` to also run step 4. + +```powershell +powershell -ExecutionPolicy Bypass -File scripts\dev\setup.ps1 -Check +``` + +The manual steps follow. + +### Windows (PowerShell) with Docker Desktop + +```powershell +# 1. Clone the PR branch +git clone -b claude/exciting-sagan-7oh2zh https://github.com/patchworklabsorg/krater.git +cd Krater + +# 2. Install uv (Python tool manager) if needed, then Python 3.12 and the dependencies +powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex" +uv python install 3.12 +uv sync + +# 3. A Postgres for dev and tests (matches the tests' default URL: root:root@localhost:5432) +docker run -d --name krater-pg -e POSTGRES_USER=root -e POSTGRES_PASSWORD=root -p 5432:5432 postgres:16 +docker exec krater-pg psql -U root -d postgres -c "CREATE DATABASE krater_dev" -c "CREATE DATABASE krater_test" + +# 4. Tests and lint +uv run pytest +uv run ruff check . ; uv run ruff format --check . + +# 5. Run the app in stub mode (fake users, fake SkyPilot/Slack/S3), then open http://localhost:8000 +$env:KRATER_DATABASE_URL = "postgresql+psycopg://root:root@localhost:5432/krater_dev" +uv run alembic upgrade head +uv run uvicorn krater.web.app:create_app --factory --reload +``` + +- **Don't create a `.env` in the repo root while running tests.** pydantic-settings auto-loads it, and live settings + leak into the test suite. Use a separately named file and load it into your shell. +- Background worker (optional locally): `uv run procrastinate --app=krater.worker.app.app worker`. +- **Full stack in Docker:** copy `.env.example` to an env file outside the repo, then run + `docker compose --env-file up --build` with `KRATER_ENV_FILE` set to the same path (a repo-root `.env` + works too, but leaks into pytest). Add `--profile skypilot` for the SkyPilot server and sign-in proxy. See + `docs/dev/staging.md` section 4. + +### macOS / Linux / WSL +The same steps with bash syntax (install uv with `curl -LsSf https://astral.sh/uv/install.sh | sh`). + +## 3. Weave: Weave owns Krater's roles + +**Decision (maintainer, 2026-10-07): Weave owns roles.** This replaces the 2026-09-28 design where roles, a disable +switch and Slack links lived in Krater's database. Details are in `docs/weave-integration.md` and `docs/SPEC.md` +"Roles & authentication". In short: + +- Krater asks for the scopes `openid profile email groups roles slack`. It reads `sub`, `name`, `email`, + `email_verified`, `roles`, `groups`, `slack_id` and `slack_member`. +- Krater's role keys on its Weave app are `member`, `reviewer` and `admin`. The `roles` claim is the source of truth. + Only when the `roles` field is absent does Krater fall back to the group slugs `ganymede-members`, + `krater-reviewers` and `krater-admins`. Inside Krater the roles keep their `ganymede:*` names. +- Every action re-checks Weave's directory API by `sub` (`GET /api/v1/directory/users/{sub}`), with a + client_credentials token of Krater's own app (scope `directory`). Reviewer invites use + `GET /api/v1/directory/users?role=reviewer`. If Weave is down, actions fail closed. +- There is no `/admin/users` page, no `KRATER_BOOTSTRAP_ADMINS` and no Krater-side disable switch. To shut someone + out, remove their roles or app access in Weave and revoke their tokens. + +**The Weave work is merged and on Weave `main`** (2026-10-07): the stack patchworklabsorg/weave#156 to #161, plus +app roles (patchworklabsorg/weave#165) and the directory API (patchworklabsorg/weave#166). It is deployed to Weave +staging. Before Krater can sign anyone in, a Weave superadmin creates the roles `member`, `reviewer` and `admin` on +the Krater app page, and an admin adds `directory` to the Krater app's scopes. + +**Weave's sign-in and security fixes are merged** in patchworklabsorg/weave#119 (2026-09-28): the two CSP fixes +that broke OAuth sign-in for returning users in Chrome and Safari (patches `0002`/`0003`), Slack event signature +checks, the admin takeover fix, and the OAuth cutoff for locked users. The old `fix/security-hardening` branch has +nothing that `main` does not already have. + +**First admin on a real deployment:** give yourself the `member` and `admin` roles on the Krater app in Weave, then +sign in. + +## 4. What's done and verified + +- **v1 features:** + - Weave OIDC sign-in, with roles owned by Weave (the `roles` claim, group-slug fallback) and re-checked with + Weave's directory API on every action; + - the proposal → review → approval workflow, amendments, and completion review; + - the configurable approval policy and the append-only budget ledger; + - SkyPilot: the launch gate and the reconcile job (workspaces, spend, 80% warning, 100% teardown). Workspace + access follows Weave: someone who loses `member` leaves every project workspace on the next reconcile; + - Slack review channels; + - screenshot uploads and the gallery; + - the `/pricing` page and the budget estimator; + - production hardening. +- **Verified here:** 603 tests pass, CI is green, and pip-audit is clean. A security review found 7 issues and + all are fixed. Live tests ran against a real Weave, a real SkyPilot 0.13.0 server (`scripts/dev/skypilot_contract.sh`) + and a real SeaweedFS. Details are in the PR description and `docs/dev/*.md`. +- **Docker Compose stack: verified** on the maintainer's Windows machine (Docker Desktop; stub Weave, fake Slack, + empty Vast key): migrations, portal, worker jobs, stub sign-in, SeaweedFS screenshot upload and gallery, and + `--profile skypilot` (service-token bootstrap, live reconcile creating workspaces, and a `sky launch --dryrun` + through both launch-gate hops). Eight bugs were fixed on the way, among them: the SkyPilot container never + started and instead printed every secret to its log; SkyPilot and oauth2-proxy were handed all of Krater's + secrets; basic auth could never work and, once on, blocked member sign-in; the server-side launch-gate call + pointed at the SkyPilot container itself; restarting the container wiped every project workspace; and the + image baked in the host's `.venv`. Details in `docs/dev/staging.md`. +- **Not verified yet** (`docs/dev/staging.md` covers all of it): + - a real Weave sign-in through the Dockerized portal (check Weave's discovery `issuer` matches + `KRATER_WEAVE_ISSUER`, e.g. `host.docker.internal` vs `localhost`); + - a full oauth2-proxy / `sky api login` round trip, and `host.docker.internal` from WSL2; + - a real Vast launch, and billing drift; + - spot machines and their recovery; + - real Slack (including the invite changes in section 6). +- **SkyPilot contract CI.** `.github/workflows/skypilot-contract.yml` runs `scripts/dev/skypilot_contract.sh` (a real + SkyPilot API server, a real Krater process, and a `sky launch --dryrun` walk through the launch gate) nightly, on + pushes touching the SkyPilot integration or its pin, and manually (Actions, "SkyPilot contract", "Run workflow"). + The SkyPilot version is pinned only in `scripts/dev/skypilot-requirements.txt`: to upgrade, bump it and the + `berkeleyskypilot/skypilot` tag in `docker-compose.yml` together (CI fails if they differ). Not run on GitHub yet; + verified in a Linux container that mimics it. Running it with internet found and fixed two reconciler bugs: + finished projects' workspaces were never deleted (SkyPilot's "No live services." `ClusterNotUpError` wasn't + recognized), and one finished project whose workspace had vanished blocked every project's provisioning. The + reconciler now isolates failures per project and records an already-vanished workspace as torn down. + +## 5. Decisions and standing instructions (don't re-litigate) + +- **PR #1 stays a draft until the maintainer explicitly approves marking it ready.** +- Don't push to branches other than `claude/exciting-sagan-7oh2zh` without permission. +- **Stack:** Python 3.12 / FastAPI / SQLAlchemy 2 / Alembic / Postgres / procrastinate (no Redis). See `CLAUDE.md` + for conventions: services own the rules, money is integer cents, Weave owns roles. +- **Weave owns roles** (maintainer decision, 2026-10-07; this **replaces** the 2026-09-28 rule "roles live in + Krater"). Krater reads the `roles` claim (group slugs only when `roles` is absent) and re-checks Weave's directory + by `sub` before every action. No role tables, no `/admin/users`, no `KRATER_BOOTSTRAP_ADMINS`, no Krater-side + disable switch. +- **SkyPilot is pinned to 0.13.0.** Krater talks to it over plain REST, with no `skypilot` package dependency + (it's 453 MB and conflicts with Krater's dependencies). One private, Vast-only workspace per project; members + sign in to SkyPilot with Weave via oauth2-proxy. +- **The launch gate is publicly reachable.** Members' own machines call it, and its URL token is visible to them. + So it must stay free of side effects and only enforce on the server-side call. +- **Storage** is a temporary SeaweedFS container. MinIO was rejected (its community edition was archived in 2025). + The long-term provider is undecided. +- **Prices** come from SkyPilot's public Vast catalog CSV, which is the same data SkyPilot's own price listing reads. +- **Staging** runs on the maintainer's own Windows machine: Docker Desktop, with the `sky` CLI in WSL2. +- **Donated idle compute** is parked. The design notes are in `docs/FUTURE.md`. + +## 6. What to do next + +**The maintainer:** +1. Merge the Weave work Krater needs (section 3), then do the staging run (`docs/dev/staging.md`) with your Krater + roles set in Weave, and bring any failures back to a session to fix on the PR. +2. Create Krater's Slack app (`docs/dev/slack-setup.md`). +3. Pick a long-term screenshot storage provider. +4. Decide what happens at 100% of a budget (patchworklabsorg/krater#2). +5. Approve PR #1 out of draft when ready. + +**A Claude session, in suggested order:** +1. ~~**Local dev setup script.**~~ Done: `scripts/dev/setup.ps1` and `scripts/dev/setup.sh` (section 2). +2. ~~**Weave follow-ups**~~ Done: Weave owns roles again (section 3). `krater/weave/roles.py` maps Weave's role keys + and group slugs; `krater.services.users.authorize` re-checks the directory before every action. The Slack + membership gate (`krater/services/slack_membership.py`) prefers Weave's `slack_member` and asks Slack otherwise. + Channel invites (`krater/services/slack_notify.py`) take reviewers from Weave's directory and still invite with + Slack's `force` flag: without it, Slack invited nobody whenever any one invitee failed, including people already + in the channel. (Found from Slack's documented behavior; not yet seen against real Slack.) +3. ~~**A CI job for the SkyPilot contract test**~~ Done (section 4). +4. ~~**Live Weave role tests.**~~ Done: `scripts/dev/weave_e2e_provision.rb` creates Krater's app roles in Weave + and gives them to the fixture users. All 8 tests in `tests/live/test_weave_live.py` pass against Weave `main` + (`docs/dev/weave-e2e.md`). +5. ~~**Production deployment pieces.**~~ Done in Compose: `proxy` and `backup` profiles, restart policies, + `KRATER_BIND_ADDRESS`, and the runbook `docs/dev/production.md`. For alastor there's a NixOS module instead, on + the unpushed `krater-module` branch of `patchworklabsorg/infra`; it builds the image on the host, so there's no + registry (runbook section 0.1, and `.github/workflows/deploy.yml`). None of it has run on a real host yet. +6. The remaining items in `docs/FUTURE.md`. + +## 7. Gotchas learned the hard way + +- **Test real servers, not just fakes.** Real-server testing caught 3 SkyPilot wire-format bugs and 2 Weave browser + bugs that mocks never would have. Re-run `scripts/dev/skypilot_contract.sh` after touching `krater/skypilot/`. It + needs a venv with `skypilot[vast]==0.13.0`; see `docs/dev/skypilot-contract.md`. +- **Keep autogenerate away from procrastinate's tables.** `alembic/env.py` filters out `procrastinate_*` tables; + without that, autogenerated migrations try to drop them. +- **Postgres `now()` is frozen per transaction**, so timestamps that must order rows within one transaction are + stamped in Python (see `SpendSnapshot`). +- **Rate limiting is off when `KRATER_ENV=test`.** Its own tests switch it on explicitly. +- **SkyPilot's serve-status call fails in sandboxes without direct internet** (its network probe). That's expected + there; on a normal machine it isn't. +- **Test the Weave sign-in flow in a real browser.** Weave's CSP `form-action` applies to the whole redirect chain, + so curl-based tests miss breakage. +- **Earlier patch files are now obsolete.** `secfix-wip.patch` and `weave-krater-integration-fixes.patch` are + superseded: the security fixes are merged on the PR branch, and the Weave fixes are merged in patchworklabsorg/weave#119. + Patch `0001` and the `0004` diff are replaced by the Weave stack #156 to #161, #165 and #166. +- **CI is Linux-only, so Windows breakage slips through.** `strftime("%-d")` is glibc-only and raises + `ValueError: Invalid format string` on Windows (it broke 13 tests there); use `.day` instead. Shell scripts must + stay LF (`.gitattributes` enforces it) or WSL bash rejects them. + +## 8. Prompt to start a fresh local Claude Code session + +Open a terminal in the cloned `Krater` folder, start Claude Code, and paste: + +> You're continuing work on Krater (this repo, branch `claude/exciting-sagan-7oh2zh`, draft PR +> https://github.com/patchworklabsorg/krater/pull/1). Read `docs/HANDOFF.md` first, then `CLAUDE.md` and `docs/SPEC.md`. +> Follow the standing instructions in HANDOFF section 5; in particular, never mark the PR ready without my explicit +> approval. First, get the test suite running locally (HANDOFF section 2) and tell me the result. Then propose what +> to work on from HANDOFF section 6 before starting. diff --git a/docs/SPEC.md b/docs/SPEC.md new file mode 100644 index 0000000..01bcc2f --- /dev/null +++ b/docs/SPEC.md @@ -0,0 +1,294 @@ +# Krater — Project Ganymede Approval & Compute Allocation Portal + +Revised Sep 26, 2026. Supersedes the Sep 24 draft (drafted with @Adam). Roles live in Weave (maintainer decision, +Oct 7, 2026; see "Roles & authentication"). +The previous draft was checked against the Weave codebase. This revision fixes the places where it assumed +capabilities that don't exist. The integration details are in [weave-integration.md](weave-integration.md) and +[quilt-integration.md](quilt-integration.md). + +**Compute (Oct 11, 2026):** SkyPilot was dropped. This release runs with compute off: it approves and tracks budgets +but launches no machines. Krater Compute, Krater's own Vast.ai layer, follows in its own release and adds launching, +spend tracking and the budget hard stop. + +## What changed from the Sep 24 draft + +| Area | Sep 24 draft | This revision | Why | +| --- | --- | --- | --- | +| Budget enforcement | Handed to "SkyPilot's internal budgeting" | **Krater enforces it** itself (with Krater Compute) | SkyPilot has no dollar budgets, only per-instance `max_hourly_cost` and estimated `cost-report`; it was dropped | +| Reviewer role | "A claim Krater reads from Weave" | **Weave owns roles** as app-defined roles on Krater's Weave app, sent in a `roles` claim and re-checked through Weave's directory API | Weave's original claims had no roles; the Weave work adds app roles and a directory API | +| Slack membership | "Weave users are already Slack members" | **Only full Slack members can submit**, checked with Weave's `slack_member`, else with Slack itself | Weave signup is open, new users are single-channel guests, and some never join Slack | +| Reviews | Attached to the project | Attached to a **project revision** | Otherwise approvals of an old version count toward a resubmitted one | +| Amendments | Approved → back through review | Approved project **stays approved** while an amendment revision is reviewed | Otherwise compute is cut off during review | +| Budget | `budget_approved` column + `BudgetReclaim` table | **Append-only budget ledger** | One place for the ceiling and for the audit trail | +| Comments | Mirrored from Slack into Krater's DB | **Not mirrored in v1**; Krater links to the channel | Mirroring means handling Slack edits, deletes and threads, for little gain | +| Deployment | "Two containers" | Portal, worker, Postgres, object storage | That's what it actually takes | + +## Overview & scope + +**Krater** is the web portal for Project Ganymede (Patchwork Labs). It does three things: + +1. Reviews and approves member-submitted project proposals. +2. Allocates a dollar-denominated compute budget to approved projects. +3. Hosts a public gallery of completed Ganymede projects. + +**Not in this release:** running compute. Approved projects show "Launching machines from Krater isn't open yet." +Krater Compute, Krater's own Vast.ai layer, adds launching machines against a project's budget, spend tracking and +the hard stop at the ceiling. + +**Core capabilities** + +- Proposal submission, with drafts and resubmission after rejection +- A configurable approval policy (reviewer sign-off), with review happening in a private Slack channel per project +- Dollar-based compute budgets (enforced once Krater Compute launches machines) +- A two-stage lifecycle: proposal review, then a separate completion review before a project goes public +- A public gallery of completed, open-source projects +- Admin override at every stage, fully audited + +## Roles & authentication + +Three roles: + +- **Submitter:** any Ganymede member who is also a **full Slack member** (see below). Can draft, submit, edit and + resubmit after rejection, amend an approved project, and submit their project for completion review. +- **Reviewer:** can approve or reject proposals and completion requests. Can't review a project they submitted or are a + credited builder on. +- **Admin:** a *Ganymede* admin. Can override the normal flow at any point: approve or reject at either stage, and adjust + or reclaim budget, and set a project's hourly cap. Not on a project they submitted or are a credited builder on, + though: there they can't decide it, add budget or raise its cap (cutting budget or the cap, reclaiming and withdrawing + are still fine). This is **not** Weave's existing `admin` flag, which means Weave operations admin. + +A "logged in but can't submit" tier and a public/anonymous tier are still deferred. The gallery itself is public. + +**Authentication** is OIDC against Weave (Authorization Code + PKCE; Weave enforces PKCE). Krater asks for the scopes +`openid profile email groups roles slack`. All Weave access goes through one adapter module (`WeaveClient`) so Weave +changes don't spread through the app. The contract is in [weave-integration.md](weave-integration.md). + +**Weave owns roles** (maintainer decision, Oct 7, 2026). Krater never stores a role as its own source of truth: + +- Krater is an OAuth app in Weave with three app-defined role keys: `member`, `reviewer` and `admin`. A Weave admin + gives people these roles. Weave sends the keys a user holds in the `roles` claim and in each directory record. +- The `roles` field is the source of truth. Only when Weave sends no `roles` field at all (absent, not empty) does Krater + fall back to group slugs linked to the Krater app: `ganymede-members`, `krater-reviewers` and `krater-admins`. The keys + and slugs are settings (`KRATER_WEAVE_ROLE_*`, `KRATER_WEAVE_GROUP_*`). +- Inside Krater the roles keep the names `ganymede:member`, `ganymede:reviewer` and `ganymede:admin`, so review snapshots + and approval policies read the same. `krater.weave` translates at the boundary. Later tiers + (`ganymede:reviewer:`) will need a matching Weave role key. +- Sign-in refuses anyone without `member` ("ask a Ganymede admin to give you the member role in Weave"). +- Every state-changing action re-checks Weave: web actions through `fresh_actor`, Slack Approve/Reject clicks through + the same `users.authorize`. Krater asks Weave's directory API for the user by `sub`, with an access token of its own + OAuth app (client_credentials). A 404, an inactive account or a missing `member` role refuses the action. If Weave + can't be reached, the action fails rather than guessing. These lookups are never cached; only page views may reuse + a directory answer, for up to 60 seconds. +- Krater keeps `users.roles_cached` (what Weave said last) for display and navigation only. +- To shut someone out of Krater, an admin removes their role or the app access in Weave, and revokes their tokens there. + Krater has no disable switch of its own. + +**Slack membership gate.** Signing in through Weave doesn't guarantee Slack membership. Weave signup is open, new users +join Slack as single-channel guests until they accept the code of conduct, and Slack won't add guests to another +channel. + +- On draft → submit, Krater uses Weave's `slack_member` from the fresh directory record when Weave gives one. +- When Weave doesn't say, Krater asks Slack about the user's `users.slack_user_id` (Weave's `slack_id`) with + `users.info`: the account must exist and not be deleted, `is_restricted` or `is_ultra_restricted`. No Slack id + from Weave means the check fails. Krater keeps no emails, so it never looks Slack accounts up by email. +- If Slack can't be reached, the submission fails with an error rather than guessing either way. +- If the check fails, the user can still save drafts but sees "Join the Patchwork Labs Slack and accept the code of + conduct to submit", with a link to Weave. +- Channel invites use the team's stored Slack ids (from Weave). Reviewers come from Weave's directory (`?role=reviewer`), with Weave's `slack_id`. Krater doesn't pre-filter + guests: invites use Slack's `force` flag, and anyone Slack refuses (a guest, a deactivated or stale id) is skipped and + logged, so one person's Slack account never stops the channel, the rest of the team's invites or the review message. + +## Data model + +Postgres. All money is stored as **integer cents** (`*_cents`). Names are suggestions; refine during implementation. + +**User** (Krater's handle on a Weave account; no names, emails or role data of its own, see +[weave-integration.md](weave-integration.md#member-data)) +- `id`, `weave_sub` (Weave's `p_id`, e.g. `PWL5A1B2C3D4`; unique), `slack_user_id` (nullable, unique; Weave's + `slack_id`), `roles_cached` (the Krater role names Weave reported last; display only), `last_login_at` + +**Project** +- `id`, `title`, `submitter_id`, `status`, `current_revision_id`, `approved_revision_id`, `repo_url`, + `slack_channel_id`, `max_hourly_cost_cents` (nullable; the admin's per-project hourly cap), `created_at`, + `updated_at` +- `status` values: + - `draft` + - `pending_review` + - `changes_requested` + - `approved` + - `pending_completion_review` + - `completion_changes_requested` + - `completed` + - `withdrawn` + +**ProjectRevision** (an immutable snapshot of what was reviewed) +- `id`, `project_id`, `number`, `kind` (`proposal` | `amendment` | `completion`), `write_up`, + `budget_requested_cents`, completion fields (`demo_url`, `screenshot_keys[]`, `credited_builder_ids[]`, `tags[]`), + `submitted_at`, `outcome` (`pending` | `approved` | `rejected` | `superseded`) +- Drafts are edited in place. Submitting freezes them into a revision, and resubmitting creates the next one. + +**Review** +- `id`, `revision_id`, `reviewer_id`, `decision` (`approve` | `reject`), `reason` (required on reject), `source` + (`slack` | `web`), `created_at` + +**BudgetEntry** (append-only ledger; the project's ceiling is the sum of its entries) +- `id`, `project_id`, `kind` (`initial_approval` | `amendment` | `admin_adjustment` | `reclaim`), `amount_cents` + (signed), `actor_id`, `reason`, `revision_id` (nullable), `created_at` + +**SpendSnapshot** (written by the compute layer; none until Krater Compute) +- `id`, `project_id`, `estimated_spend_cents`, `source`, `taken_at` + +**ApprovalPolicy** (configuration, not code) +- `id`, `stage` (`proposal` | `completion`), `min_budget_cents` (nullable; for tiering by budget size), + `min_approvals` (currently 1), `required_group` (nullable; e.g. `ganymede:reviewer:senior`) + +**AuditEvent** +- `id`, `actor_id`, `action` (e.g. `admin_approve`, `admin_reject`, `budget_adjust`, `policy_change`, + `launch_blocked`, `teardown`), `project_id`, `payload` (jsonb), `reason`, `created_at` + +**GalleryEntry:** a view over `completed` projects plus their approved completion revision. It isn't a separate table. + +## Proposal & review workflow + +1. **Draft.** The submitter fills in a title, requested budget and write-up (repo link optional). They can save and come + back later. +2. **Submit.** Krater runs the Slack membership check, freezes the draft into revision N, sets `pending_review`, creates + the project's private Slack channel if it doesn't exist yet, invites the submitter and all current reviewers + (every active user Weave's directory lists with the `reviewer` role), posts the review message with Approve/Reject + buttons, and posts a line in the master feed channel. +3. **Review.** Reviewers discuss freely in the channel. Decisions come in through the buttons, or through the web UI as a + fallback. For every decision, Krater: + - finds the clicker's Krater user by stored Slack id (from Weave's `slack_id`), then checks their roles with Weave's + directory by `sub` right now (not from anything cached); + - rejects self-review (submitter or credited builder); + - records the Review against the current revision; + - asks `ApprovalPolicyService` whether the policy is now satisfied. +4. **Approve.** When the policy is satisfied, the revision is approved, a `BudgetEntry(initial_approval)` is written, and the + project moves to `approved`. +5. **Reject.** A reason is required. The project moves to `changes_requested`, and the submitter edits and resubmits + (step 2, same project, new revision). +6. **Amend.** An `approved` project can have its scope or budget amended. This creates an `amendment` revision reviewed + under the same policy. **The project stays `approved` and keeps its current ceiling while the amendment is + reviewed.** If the amendment is approved, the budget difference is written to the ledger. If it's rejected, nothing + changes. +7. **Withdraw.** The submitter or an admin can withdraw a project at any point. The channel is archived, unspent budget + is reclaimed, and the workspace is torn down. + +Only approvals on the **current** revision count. A new revision marks earlier pending ones as `superseded`. + +`ApprovalPolicyService` is its own module from day one. Given a revision, it finds the matching `ApprovalPolicy` rows +(by stage and budget size) and decides whether the approvals so far meet them. Multi-approval and reviewer tiers are then +just new rows. + +## Slack integration + +Slack is where review happens, not just where notifications go. Krater gets **its own Slack app**. Weave's app already +uses the one interactivity URL a Slack app can have, for code-of-conduct acceptance. + +**Per-project private channel:** created on first submission and reused for amendments and the completion review. +Members are the submitter, credited builders (once added), and all reviewers. When someone gets the `reviewer` role in +Weave, a periodic job invites them to open project channels. The channel is +archived when the project ends up `completed` or `withdrawn`. + +**Master feed channel:** one top-level post per new submission, for visibility across Ganymede. It carries no decisions. + +**Source of truth:** Krater's database holds decisions, budgets and state. Slack holds discussion. Comments are **not** +mirrored into Krater in v1; the project page links to the channel instead. + +**App requirements** +- Bot scopes: `groups:write`, `groups:write.invites`, `chat:write`, `users:read` (the membership gate, when Weave has + no `slack_member` for someone). +- An HTTPS interactivity endpoint on the portal, reachable by Slack. +- Slack signature verification (`X-Slack-Signature` with the signing secret, and a check on how old the timestamp is). + Weave's `SlackSignatureVerification` concern is a working reference. +- Slack must get an acknowledgement within 3s. Do the real work in the worker, then update the message. +- Invites and posts must be safe to retry: `already_in_channel` counts as success, and failures are retried from the + worker. Invites use `force`: without it Slack invites nobody if any one invitee fails, including people who are + already in the channel. + +**Not needed:** a separate completion announcement. The gallery is enough (revisit later). + +## Completion flow & public gallery + +The submitter fills in the completion fields (final write-up, screenshots, demo link, repo link, credited builders, +tags) and submits them as a `completion` revision. The project moves to `pending_completion_review`. Review happens in +the same channel, under the `completion` stage policy. + +- **Approved:** the project becomes `completed`. It's published to the gallery, spend is frozen at its final value, + unspent budget is reclaimed to the ledger, and (with compute) its machines are destroyed. +- **Rejected:** the project moves to `completion_changes_requested`. The submitter revises and resubmits. + +**Public gallery** (no auth). Each entry shows the write-up, screenshots, demo link, repo link, credited builders, +compute spent (latest reconciled estimate, labelled as an estimate), and tags for browsing. + +**Screenshot storage:** S3-compatible object storage, as a placeholder until a provider is chosen (see Open questions). +Uploads use presigned URLs, and only objects from approved completion revisions are served publicly. + +## Budget handling + +- The **ceiling** is the sum of the project's `BudgetEntry` rows. Approvals, amendments, admin adjustments and reclaims + all add entries, so the ledger doubles as the audit trail. +- **Spend** comes from the compute layer, shown as "≈ $X estimated". With compute off it stays at zero. +- **Enforcement** arrives with Krater Compute: launches only for `approved` projects with budget left, every machine + capped at the project's hourly cap, a warning in the project channel at 80% of the ceiling, and its machines + stopped at 100%. +- **No automatic expiry.** Stalled projects are reclaimed manually by an admin (`BudgetEntry(reclaim)`). +- **Quilt.** Krater tells Quilt (Patchwork Labs finance) about each submission, each ledger entry and each change of + spend, through Quilt's patch API. The events go through an outbox table in the same transaction as the change. + See [quilt-integration.md](quilt-integration.md). + +## Admin overrides + +At any point an admin can: +- approve or reject at either review stage, bypassing `ApprovalPolicy`; +- adjust an approved project's budget up or down, or reclaim unspent funds; +- set a project's own hourly cap for machines, or put it back to the default; +- withdraw a project. + +The exception is the admin's own project (they submitted it, or are a credited builder on its current revision): they +can't decide it, add to its budget or raise its hourly cap, so an admin can never fund their own work without someone +else signing off. Another +admin, or the normal review, has to do it. Cutting or reclaiming its budget and withdrawing it are still allowed. + +Every override writes an `AuditEvent` with who, what, when and a required reason. Budget changes also write a +`BudgetEntry`. Overrides are posted in the project's channel so reviewers can see them. + +## Deployment architecture + +Everything runs in Docker and is defined in one Compose file, so it can run on any host, including the existing arm64 +host (alastor). + +| Service | What it is | +| --- | --- | +| `portal` | FastAPI web app: UI, OIDC, Slack endpoints, public gallery | +| `worker` | Same image, runs background jobs: Slack work, reviewer-channel sync, price refresh, events to Quilt | +| `db` | Postgres | +| `storage` | Temporary self-hosted S3-compatible storage (SeaweedFS) for gallery screenshots, until a provider is chosen | + +Networking: Slack must reach `portal` over public HTTPS. See [dev/production.md](dev/production.md). + +**Stack:** Python + FastAPI + Postgres. + +## Open questions + +1. **Screenshot storage provider.** Temporarily a self-hosted SeaweedFS container (`storage` in docker-compose). + MinIO was ruled out because its community edition stopped publishing images in 2025. Still to pick a long-term + provider (e.g. R2/B2/S3, or keep SeaweedFS with backups). A reminder is set to circle back. +2. **How members run compute:** decided. Not SkyPilot (dropped Oct 11, 2026): Krater Compute, Krater's own Vast.ai + layer, in its own release. +3. **What happens at the ceiling.** Proposed: warn at 80%, block new launches and tear down at 100%, with no grace + period. Consider a small admin-configurable grace so a running training job isn't killed at 100.1%. +4. **Weave changes:** resolved. Krater needs the Weave stack patchworklabsorg/weave#156 to #161, plus app roles + (patchworklabsorg/weave#165) and the directory API (patchworklabsorg/weave#166), tracked in + patchworklabsorg/weave#163. All of it was merged to Weave `main` on 2026-10-07. Each deployment still needs Krater's + roles and the `directory` scope set up on its Weave app. See [weave-integration.md](weave-integration.md). + +## Parked / future work + +Designed for, not built now: + +- **In-progress project visibility:** letting members see active projects, to avoid duplicate work. +- **Multi-approval policy:** new `ApprovalPolicy` rows with `min_approvals > 1`. +- **Reviewer tiers by budget size:** `ApprovalPolicy.min_budget_cents` plus `required_group`. +- **Comment mirroring from Slack,** if the channel link turns out not to be enough. +- **Reconciling against real Vast billing** (Krater Compute does this). diff --git a/docs/dev/pricing.md b/docs/dev/pricing.md new file mode 100644 index 0000000..010cf1a --- /dev/null +++ b/docs/dev/pricing.md @@ -0,0 +1,147 @@ +# GPU pricing: source, refresh, and how the estimator uses it + +> Krater no longer runs SkyPilot (removed on 2026-10-11), but `/pricing` and the budget estimator +> still read SkyPilot's public Vast catalog CSV, now through `krater/pricing/catalog.py`. The "why" below is kept as +> written; its references to the SkyPilot integration and spike docs are history (see git). + +`/pricing` (public) and the proposal-form budget estimator both need "what does a GPU cost on Vast" +figures. This documents where those figures come from, why, how they refresh, what happens when the +source is unreachable, and how to run a refresh by hand. + +## The question: REST accelerator listing, or the catalog CSV directly? + +`docs/dev/skypilot-spike.md` pinned down SkyPilot 0.13.0's wire formats for everything Krater already +calls (workspaces, `cost_report`, launch gating) but never looked at pricing/catalog listing, so this +is fresh spike work, done by reading the installed `skypilot==0.13.0` wheel +(`skyvenv/lib/python3.12/site-packages/sky`) and confirming the network calls it implies. + +**Option A** — SkyPilot's REST `list_accelerators` (what `sky show-gpus`/the SDK's `list_accelerators` +call hits): `POST /list_accelerators` on a running API server (`sky/server/server.py:1394`), async like +every other admin-plane endpoint (`x-skypilot-request-id` + poll `GET /api/get`), body +`{"clouds": ["vast"], ...}` (`payloads.ListAcceleratorsBody`). + +**Option B** — SkyPilot's public catalog CSV for Vast, fetched directly with no server involved. + +**What the code actually does, traced end to end:** + +- `sky/server/server.py`'s `/list_accelerators` handler calls `catalog.list_accelerators(clouds=...)` + (`sky/catalog/__init__.py:57`), which for `clouds=["vast"]` dispatches to + `sky/catalog/vast_catalog.py`'s `list_accelerators(...)` (`common.list_accelerators_impl('Vast', _df, + ...)`). +- `vast_catalog.py`'s module-level `_df = common.read_catalog('vast/vms.csv')` (line 19) is the *only* + place Vast pricing data comes from. `common.read_catalog` (`sky/catalog/common.py`) downloads + `{HOSTED_CATALOG_DIR_URL}/{CATALOG_SCHEMA_VERSION}/{filename}` on first read and caches it locally. + Reading `sky/skylet/constants.py`: `HOSTED_CATALOG_DIR_URL = + 'https://raw.githubusercontent.com/skypilot-org/skypilot-catalog/master/catalogs'` and + `CATALOG_SCHEMA_VERSION = 'v8'` — i.e. the exact file is + `https://raw.githubusercontent.com/skypilot-org/skypilot-catalog/master/catalogs/v8/vast/vms.csv` + (an S3-mirror fallback exists at `HOSTED_CATALOG_DIR_URL_S3_MIRROR`, same path shape, unused here). +- **So Option A *is* Option B**, wrapped in a running API server, an admin bearer token, and + async-request-id polling: the REST endpoint's answer for Vast is computed by reading this exact CSV, + nothing else. There's no separate, more "live" price source behind the REST call. +- Confirmed reachable from this environment: `curl` against that exact URL returned `HTTP 200`, 65 + data rows, 11.6 KB, 17 distinct canonical accelerator names (A100, H100, H200, L40S, RTX3060/3090/ + 4060/4070/4090/5070/5090, RTX5880-Ada, RTX6000-Ada, RTXA5000, RTXA6000, RTXPRO6000WS, V100), columns + `InstanceType,AcceleratorName,AcceleratorCount,vCPUs,MemoryGiB,GpuInfo,Price,SpotPrice,Region`. A + sample row: `2x-RTX_5090-32-65536,RTX5090,2,32,64.0,"{'Gpus': [...], + 'TotalGpuMemoryInMiB': 65214}",0.93,0.00,"Taiwan, TW, AS"`. +- **Does the `default` workspace's disabled clouds affect this?** No — checked by reading the call + chain: `catalog.list_accelerators`/`vast_catalog.list_accelerators` never look at + `enabled_clouds`/workspace config at all; they're a pure read of the cached CSV, filtered only by the + function's own `gpus_only`/`name_filter`/`quantity_filter` arguments. The workspace's cloud + allow/deny-list (`docs/skypilot-integration.md`'s "no positive allowlist, deny every other cloud" + design) is enforced by a completely different code path — `sky check` / the optimizer's resource + search, which decides what a given workspace may *launch*, not what the catalog *lists*. This also + matches `sky/client/cli/command.py`'s `_show_gpus_impl`: `enabled_clouds` there only toggles whether + Kubernetes/SSH/Slurm-specific sections are shown; the plain per-cloud GPU table comes from + `sdk.list_accelerators(...)` regardless of what's enabled. **No workspace is needed either** — the + call takes no workspace argument at all. + +**Decision: Option B — fetch the CSV directly**, no SkyPilot API server involved for pricing at all. + +Reasoning, weighed the same way the spike weighed SDK-vs-REST for everything else: + +1. **Identical data.** Per the trace above, Option A's answer for Vast is this CSV; going through the + server adds nothing. +2. **Fewer moving parts.** Option A needs a running API server, an admin service-account token, and + async request-id polling just to read a static file. Option B is one `httpx.get`. Krater's pricing + page and periodic refresh have no other reason to need a live SkyPilot server up — unlike budget + enforcement (`docs/skypilot-integration.md`), pricing isn't project-specific and doesn't need + SkyPilot's admin plane at all. +3. **No auth/availability coupling.** `/pricing` is a public page (no login). Sourcing it from a plain + public GitHub URL means it never depends on `KRATER_SKYPILOT_SERVICE_TOKEN`/the API server being up; + sourcing it from Option A would make a public page's data depend on an internal admin credential and + an extra service's uptime for no benefit. +4. **Consistent with the spike's SDK-vs-REST call** (`docs/dev/skypilot-spike.md` section 6): prefer the + smallest, most direct path to the same bytes SkyPilot itself reads, over routing through more of + SkyPilot's own machinery than the data requires. + +The trade-off, noted for completeness: Option A would also read whatever *locally modified* catalog a +real SkyPilot deployment might have overridden (`is_catalog_modified`/`get_modified_catalog_file_mounts` +in `common.py`). Ganymede's SkyPilot deployment doesn't override catalogs (nothing in +`docker-compose.yml`/the server config touches `~/.sky/catalogs`), so this doesn't apply here — but if +that ever changes, Krater's prices would silently diverge from what the optimizer actually uses. Worth +a one-line check the next time `docs/skypilot-integration.md` is revisited. + +## What Krater stores + +`krater.pricing.catalog` (`LiveCatalog`, or `FakeCatalog` with `KRATER_PRICING_MODE=fake`) `.list_gpu_prices()` +returns one `GpuOffer` per CSV row (accelerator name/count, vCPUs, host memory, device (VRAM) memory +parsed out of the `GpuInfo` column, on-demand price, spot price, region). `krater.services.pricing` +aggregates these **per (accelerator name, accelerator count)** — the granularity the estimator and the +`/pricing` table both want — into a `GpuPrice` row: + +- `vram_gib`, `vcpus_typical`, `memory_gib_typical`: the **median** across that group's offers (typical + machine shape; a plain `min`/`max` would be misleading across regions with very different host specs + for the "same" GPU count). +- `on_demand_min_cents` / `on_demand_median_cents`: min and median of `Price` across the group. +- `spot_min_cents`: min of `SpotPrice` across the group, **only over rows with a positive spot price** + (the CSV has plenty of `SpotPrice=0.00` rows — not a real $0/hr offer, just "no spot price quoted" — + so a plain `min()` including zeros would make the page and the estimator claim absurd $0.00 spot + rates); `null` if no row in the group has one. +- `offer_count`: how many catalog rows (regions) fed the group, shown on `/pricing` as a rough liquidity + signal. +- `refreshed_at`: when this refresh ran (one timestamp for the whole batch). + +A row with `Price <= 0` is dropped before aggregating (not a usable on-demand quote). + +## Refresh: schedule, atomicity, and failing soft + +- **Periodic task**: `krater.worker.app.pricing_refresh`, daily (`0 7 * * *` — chosen off-peak, an + arbitrary early-UTC hour; the whole catalog is 65 rows, so run time doesn't matter, but every launch + of `/pricing` should show a stable "as of" time rather than the DB churning throughout the day). + Wraps `krater.services.pricing.refresh_prices`. +- **CLI**: `uv run python -m krater.pricing.refresh_once` — same call, for a manual refresh or a first + seed against a fresh database. +- **Atomic replace**: `refresh_prices` fetches and aggregates *before* touching the database, then does + `DELETE FROM gpu_prices` + re-`INSERT` inside the caller's transaction. Since the fetch/parse/ + aggregate step happens first and raises straight through on any failure, a failure never reaches the + delete — existing rows are left exactly as they were. The caller (the periodic task, the CLI) commits + only on success. +- **Fail soft**: if the GitHub fetch fails (network error, non-200, unparseable CSV), `refresh_prices` + raises `CatalogError` and the periodic task logs and returns + without committing anything — last-refreshed prices (and their `refreshed_at`) stay exactly as they + were, and `/pricing` keeps showing them with their true (now older) "as of" date. There's no separate + DB write for "stale" — staleness is just `refreshed_at` getting old, which the page always shows. + +## Manual refresh + +```bash +KRATER_DATABASE_URL=postgresql+psycopg://root:root@localhost:5432/krater_dev \ + uv run python -m krater.pricing.refresh_once +``` + +Prints the number of `(accelerator, count)` groups written. Uses `KRATER_PRICING_MODE` (old name +`KRATER_SKYPILOT_MODE`): `fake` (dev default) refreshes from `FakeCatalog`'s canned catalog, `live` fetches the real CSV +from `KRATER_PRICING_CATALOG_URL` (defaults to the GitHub URL above), with nothing but network access to GitHub. + +## Estimator: never trust a client-sent rate + +The budget estimator on the proposal/amendment draft form posts the *inputs* (GPU key, count, hours, +pricing basis, safety margin) to the server; `krater.services.pricing.estimate_cost` looks up the +**current** `GpuPrice` row and computes the rate/subtotal/margin/total itself. Nothing about the +resulting dollar figure is ever accepted from the client — only the choice of which GPU/count/hours/ +basis/margin to price. The computed breakdown (including which `refreshed_at` snapshot priced it) is +what gets stored on `ProjectRevision.budget_estimate` when the submitter used the estimator, and it's +recomputed again, from scratch, at draft-save time — a tampered rate in a resubmitted form changes +nothing. diff --git a/docs/dev/production.md b/docs/dev/production.md new file mode 100644 index 0000000..45b763e --- /dev/null +++ b/docs/dev/production.md @@ -0,0 +1,164 @@ +# Production runbook + +How to run Krater for real members: public HTTPS, restarts, backups and the settings that differ from staging. Do the +staging run first ([staging.md](staging.md)); this page assumes everything there already worked. + +## 0. Where it runs + +- **On alastor: the NixOS module.** Weave on alastor isn't run with Compose: its containers are systemd units defined + in [patchworklabsorg/infra](https://github.com/patchworklabsorg/infra), behind the host's Traefik, with agenix + secrets. Krater has the same: `modules/krater` and `hosts/alastor/krater.nix` in that repo (section 0.1). +- **On any other Docker host: this Compose file**, with the opt-in `proxy` (Caddy, automatic TLS) and `backup` + profiles. Sections 2 to 5 describe that path. + +Section 1 (DNS, Weave, Slack) applies to both. The image builds for arm64, which alastor is, as well as amd64 +(checked under emulation; not yet run on arm64 hardware). + +### 0.1 On alastor (NixOS) + +There's no registry image. `krater-build` fetches one pushed commit of this repo, builds it on alastor with this +repo's `Dockerfile` and tags it `krater:`. A deploy is the manual `Deploy` workflow +(`.github/workflows/deploy.yml`), which runs as the `krater-deploy` user: + +```bash +echo > /var/lib/krater/source-rev +sudo systemctl start krater-build # the running site keeps serving meanwhile +sudo systemctl restart krater-web # migrates, then starts; krater-worker follows +``` + +Settings live in `hosts/alastor/krater.nix` (hostnames, Weave client id, the fallback commit). The proposed hostnames +are `krater` and `krater-storage` under `patchworklabs.org`, added by a PR to +[patchworklabsorg/dns](https://github.com/patchworklabsorg/dns). They're proxied through Cloudflare like Weave's, +whose edge certificate covers only one subdomain level. Behind Cloudflare and Traefik, `KRATER_TRUSTED_PROXY_COUNT` +probably needs to be 2 (see the host file). Secrets are one agenix +file, `secrets/krater-env.age`, with the keys listed at the top of `modules/krater/default.nix`. The module splits them +so each container gets only its own. Getting a module change +onto alastor takes a merge in `patchworklabsorg/infra`, then `nix flake update patchwork-infra` and a deploy from +`jaspermayone/infra`. Krater releases don't need that, only the `Deploy` workflow. + +Backups are a `krater-backup` timer (03:30 UTC) writing to `/var/lib/krater/backups`. Restore as in section 4, with +`docker exec -i krater-db` in place of `docker compose ... exec -T db`, user and database `krater`. + +**No compute in this release.** Krater runs proposals, review, budgets, Slack, the gallery and Quilt events, and +rents no machines (`KRATER_COMPUTE_MODE=off`, its only mode until Krater Compute). With nothing launched, the 80% and +100% budget alerts never fire. `KRATER_PRICING_MODE=live` (or its old name, `KRATER_SKYPILOT_MODE`) fetches the public +GPU catalog the budget estimator reads; it needs no server or credentials. Krater Compute's release adds the settings +for renting machines. + +## 1. Before the first deploy + +- **DNS:** point these A/AAAA records at the host. The names are examples. + + | Name | Serves | Setting | + | --- | --- | --- | + | `krater.example.org` | the portal and Slack's endpoint | `KRATER_DOMAIN` | + | `storage.krater.example.org` | screenshot uploads and thumbnails | `KRATER_S3_DOMAIN` | + + Leave `KRATER_S3_DOMAIN` unset if storage lives with a hosted provider (R2, B2 or S3). Its `*.localhost` default keeps + that site unserved. +- **Ports 80 and 443** open to the internet. Let's Encrypt needs port 80 to issue certificates. +- **Weave** (production Weave, not staging), following [weave-integration.md](../weave-integration.md): + - Krater's app has redirect URI `https:///auth/callback` and scopes + `openid profile email groups roles slack directory`. + - A Weave superadmin creates the roles `member`, `reviewer` and `admin` on it, and gives you `member` and `admin`. +- **Slack:** the app's interactivity URL is `https:///slack/interactions` ([slack-setup.md](slack-setup.md)). + +## 2. The env file + +Start from `.env.example`. Keep the file outside the repo, readable only by the deploy user, and pass it with +`--env-file `, with `KRATER_ENV_FILE` set to the same path. Krater refuses to start in production with stub or +fake modes, a short or placeholder `KRATER_SECRET_KEY`, a non-https `KRATER_BASE_URL`, or missing +Weave, Slack or S3 secrets (`krater/config.py`). Settings that differ from staging: + +```bash +KRATER_ENV=production +KRATER_BASE_URL=https://krater.example.org +KRATER_TRUSTED_PROXY_COUNT=1 # Caddy; rate limits key on the real client IP +KRATER_BIND_ADDRESS=127.0.0.1 # published ports reachable only from the host itself + +KRATER_DOMAIN=krater.example.org +KRATER_S3_DOMAIN=storage.krater.example.org + +# Every secret new for production, never reused from staging: +# python -c "import secrets; print(secrets.token_urlsafe(48))" +KRATER_SECRET_KEY=... +POSTGRES_PASSWORD=... # and the same password in KRATER_DATABASE_URL +KRATER_S3_SECRET_ACCESS_KEY=... + +KRATER_WEAVE_MODE=live +KRATER_PRICING_MODE=live +KRATER_SLACK_MODE=live +KRATER_S3_MODE=live +KRATER_S3_PUBLIC_ENDPOINT_URL=https://storage.krater.example.org + +SEAWEEDFS_VERSION=4.47 # the version staging ran; never `latest` +KRATER_BACKUP_DIR=/srv/krater/backups # outside the checkout +``` + +Set `POSTGRES_PASSWORD` **before the first start**. Postgres only reads it when it creates an empty data volume. + +## 3. Bring it up + +```bash +docker compose --env-file "$KRATER_ENV_FILE" --profile proxy --profile backup up -d --build +``` + +Every long-running service has `restart: unless-stopped`, so the stack comes back after a reboot as long as Docker +itself starts at boot. `migrate` runs once per `up` and is safe to re-run. + +### Check it + +- `https:///` loads with a valid certificate, and you can sign in with Weave in a real browser. +- Upload a screenshot to a completed project. If it fails, check `KRATER_S3_PUBLIC_ENDPOINT_URL` ([storage.md](storage.md)). +- `ls "$KRATER_BACKUP_DIR"` shows a `krater-*.dump` from the first run of `db-backup`. + +## 4. Backups + +`db-backup` runs `pg_dump --format=custom` at start and then every `KRATER_BACKUP_INTERVAL_SECONDS` (default one day). +It writes `krater-.dump` into `KRATER_BACKUP_DIR` and deletes dumps older than `KRATER_BACKUP_KEEP_DAYS` +(default 14). A failed dump logs `backup FAILED` and leaves no file behind. + +The dumps sit on the same disk as the database, so **copy `KRATER_BACKUP_DIR` off the machine** (rclone, restic or the +host's own backup job). The database holds the budget ledger and the audit log, and nothing else has them. + +Not covered by `db-backup`: + +- **Screenshots** (the `krater_storage_data` volume). They're user uploads that can't be recreated. Back up the volume, + or move to a hosted provider (open question 1 in SPEC). + +### Restore + +Load the env file into your shell first (`set -a; . "$KRATER_ENV_FILE"; set +a`), so `$POSTGRES_USER` and +`$POSTGRES_DB` are set. + +```bash +docker compose --env-file "$KRATER_ENV_FILE" stop portal worker +docker compose --env-file "$KRATER_ENV_FILE" exec -T db \ + pg_restore --clean --if-exists --no-owner -U "$POSTGRES_USER" -d "$POSTGRES_DB" < /srv/krater/backups/krater-