diff --git a/Cargo.lock b/Cargo.lock index 0de6dcf0..3cb7de96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10468,6 +10468,7 @@ name = "tinycloud-node" version = "1.25.17" dependencies = [ "aes-gcm", + "age", "anyhow", "async-trait", "aws-config", @@ -10492,6 +10493,7 @@ dependencies = [ "lazy_static", "libc", "libp2p-identity", + "lru 0.12.5", "multer", "native-tls", "opentelemetry", @@ -10530,6 +10532,7 @@ dependencies = [ "thiserror 2.0.18", "tikv-jemallocator", "time", + "tinycloud-at-rest-escrow", "tinycloud-auth", "tinycloud-core", "tinycloud-sdk-wasm", @@ -10546,6 +10549,7 @@ dependencies = [ "uuid", "x25519-dalek", "x509-parser 0.16.0", + "zeroize", ] [[package]] diff --git a/docs/at-rest-escrow.md b/docs/at-rest-escrow.md new file mode 100644 index 00000000..30ac7bb1 --- /dev/null +++ b/docs/at-rest-escrow.md @@ -0,0 +1,219 @@ +# At-rest keys and escrow (TC-887) + +The node provisions at-rest keys only when `at_rest` is configured (below). The default config does nothing: no key reads, no rows. + +The at-rest root key (ARK) is 32 random bytes made by the node and sealed under a wrap key derived from a root. The root is one of: +- **`root_source = "escrow"` (default):** the node's own root key, with the ARK split 2-of-3 to escrow holders and the escrow gate below; +- **`root_source = "env"`:** a 32-byte key the operator supplies in `TINYCLOUD_AT_REST_ROOT_KEY`, with no escrow (see [Root from the environment](#root-from-the-environment)). + +Everything below the ARK (per-space data keys, object names, the key copies in the block store) is the same for both, so moving from one root to the other re-seals the ARK rows and rewrites no data. + +- `tinycloud-at-rest-escrow` (a node workspace member) holds the escrow format: + - the 2-of-3 split of the ARK; + - shares sealed to age recipients; + - the bundle and its fingerprint; + - the canary; + - the verification record; + - the recovery wire format. +- The `tinycloud-at-rest` tool that holders and operators run for the escrow ceremony (a Cargo workspace of its own, `tinycloud-at-rest-cli`) lands with the hardware-key escrow work; the commands below describe it. + +## Formats + +`lp(x)` is `u32 BE length ‖ x`. Digests are SHA-256, written in lowercase hex. + +- **Share.** + - The ARK is split with Shamir over GF(256) (`vsss-rs` 5.4.0): threshold 2 of 3, and share `i` has x-coordinate `i`. + - The sealed plaintext is `"TCS1" ‖ ark_id(16) ‖ generation u32 BE ‖ i ‖ share(33)`, 58 bytes. +- **Recipients.** Each share is age-encrypted (`age` 0.12.1, ASCII armor) to one holder. The recipient type can be: + - `age1yubikey1…` (`piv-p256` stanza). This is what released age-plugin-yubikey (0.5.x) prints and decrypts. The stanza is produced exactly as the plugin's own `wrap_file_key`, so the TEE needs no plugin. + - `age1tag1…` (`p256tag` stanza). This is native age 1.3 tagged P-256. The released age-plugin-yubikey 0.5.1 **cannot** decrypt it, so every command warns when a holder uses one. + - `age1…` X25519. These are software keys, refused unless software recipients are explicitly allowed (`--allow-software` on the CLI; tests and self-hosted nodes only). Every command prints a warning while that is in effect. + - A recipient format proves which kind of key decrypts, **not** where the private key lives. That a P-256 key was generated on a YubiKey is established in the escrow ceremony. +- **Commitment.** `SHA-256(lp("tinycloud/at-rest/escrow-share-commit/v1") ‖ ark_id ‖ generation ‖ i ‖ lp(share))`. +- **Canary.** + - `canary = HKDF-SHA256(ikm = ARK, salt = ark_id, info = lp("tinycloud/at-rest/escrow-canary/v1") ‖ generation)`. + - `canary_fp = SHA-256(lp("tinycloud/at-rest/escrow-canary-fp/v1") ‖ ark_id ‖ generation ‖ lp(canary))`. +- **Bundle.** JSON with `version`, `ark_id`, `generation`, `threshold`, `canary_fp`, and `shares[{index, holder, recipient, commitment, ciphertext}]`. Unknown fields are rejected. +- **EFP (escrow fingerprint).** `SHA-256(lp("tinycloud/at-rest/escrow-bundle/v1") ‖ version ‖ ark_id ‖ generation ‖ threshold ‖ canary_fp ‖ n ‖ for each share: i ‖ lp(holder) ‖ lp(recipient) ‖ commitment ‖ SHA-256(ciphertext))`. +- **Verification record.** + - It is JSON naming the EFP, all three indices and all three pairs, with an HMAC-SHA256 under `HKDF(ARK, salt = ark_id, info = lp("tinycloud/at-rest/escrow-record/v1"))`. + - Only the ARK holder (the TEE) can make or check the MAC. `record-efp` checks only the public fields and says so. +- **Recovery binding.** + - The binding string is `xyz.tinycloud.at-rest/recovery-key/v1\0:::`, where `\0` is two characters. + - A TDX quote's REPORTDATA must be `SHA-256(binding) ‖ 32 zero bytes`. + +## Commands + +Every command that reads a bundle: +- refuses software holders unless `--allow-software` is given; +- accepts `--expect-efp E` (case-insensitive) and refuses a bundle with any other EFP. + +Bundles, records and the node's recovery response carry a `version`, which is checked before any other field. A newer format is reported as "version N; this tool reads version 1". + +Exit codes: +- `0`: OK. +- `1`: REFUSED. A check failed: don't release a share and don't pin a value. +- `2`: usage error. +- `3`: ERROR. A file, stdin or the PCCS couldn't be read. + +| Command | Reads | Prints | +|---|---|---| +| `recipients --holder NAME=RECIPIENT` ×3 | — | `[at_rest.escrow]` TOML and the compose env line | +| `self-test --recipient R [--out DIR]` | — | a throwaway escrow whose share 1 is sealed to `R` by the TEE's own code (ceremony step 0) | +| `nonce` | — | 32 random bytes, base64url | +| `inspect --bundle B` | bundle | EFP, ark_id, generation, canary_fp, each holder's recipient kind | +| `extract-share --bundle B --holder H` | bundle | holder H's armored age file | +| `check-share --bundle B` | decrypted share on **stdin** | which share it is; refuses on a commitment mismatch (e.g. a share from an earlier split) | +| `verify-compose --app-compose A --trust T` | the deployment's `app-compose.json` | OK if it hashes to `compose_hash` and its compose YAML passes the strict check below; also lists the other app-compose settings to review | +| `verify-node --bundle B --recovery R --nonce N --trust T [--collateral C \| --pccs-url U]` | node's recovery response | OK (with node state and shares held) or REFUSED | +| `share-wrap` (same flags) | decrypted share on **stdin** | after `verify-node` passes: the share sealed to the node's recovery key | +| `record-efp --bundle B --record V` | record | `TINYCLOUD_AT_REST__ESCROW__VERIFIED_FP=`, marked NOT AUTHENTICATED: the record's MAC is checked only in the TEE | + +A decrypted share only ever arrives on stdin (`age -d -i share.age | tinycloud-at-rest …`). It never goes on argv or to a file. + +### What `verify-node` and `share-wrap` check before releasing a share + +All of these must hold, or the command prints `REFUSED: …` and exits 1: + +1. The node answered as an attested TEE: `mode` is `dstack` and a quote is present. +2. The node's `efp`, `ark_id` and `generation` equal the holder's own bundle. +3. The node's binding is exactly the one recomputed from the recovery recipient, that EFP, and the holder's nonce (at least 22 characters). A response made for any other nonce is stale. +4. The quote passes Intel DCAP verification with `dcap-qvl` 0.7.0: signature, PCK chain, CRLs, TCB info and QE identity. Collateral comes from the PCCS (Phala's by default; Intel PCS also works) with a 60 s timeout, or from `--collateral`, and is checked at the current time. +5. The TCB status is one fixed class: + - `UpToDate` passes. + - `SWHardeningNeeded`, `ConfigurationNeeded` and `ConfigurationAndSWHardeningNeeded` pass only if the policy lists them and pins every advisory ID the TCB level carries. + - `OutOfDate*`, `TDRelaunchAdvised*`, `Revoked` and unknown statuses never pass. + - On a pass, the status, the advisory IDs and the collateral dates are printed. +6. The TD debug bit is clear. +7. MRTD and RTMR0-2 equal the trust policy. +8. The dstack event log replays to the quoted RTMR3: + - each `boot-mr-done` and `system-ready` marker appears once, in that order; + - exactly one `app-id` and one `compose-hash` event appear, each before `boot-mr-done`, and each equals the trust policy; + - v2 events must carry their canonical preimage. +9. The quoted REPORTDATA is `SHA-256(binding) ‖ 0³²`. + +Not pinned: `key-provider`, `instance-id`, `os-image-hash`, XFAM, MRCONFIGID, MROWNER and MROWNERCONFIG. RTMR0-2 and the compose hash cover what the per-boot, in-memory recovery key depends on. + +`compose_hash` pins container images only if `app-compose.json` references every image by digest. `verify-compose` reads it with a real parser (`tinycloud_at_rest_escrow::compose`, shared with the node) and accepts only a small subset: +- `runner` is `docker-compose`; +- the compose text is newlines and printable ASCII only, because YAML parsers disagree on Unicode line breaks such as U+2028; +- exactly one document; +- no anchors, aliases, merge keys, tags or duplicate keys; +- a `services` mapping, and no `include`; +- every service has a string `image` of the form `[:tag]@sha256:<64 lowercase hex>`, with no `build` or `extends`; +- `TINYCLOUD_AT_REST__ESCROW__RECIPIENTS` and `TINYCLOUD_AT_REST__ESCROW__VERIFIED_FP`, if set, are literal values: no `$` interpolation and no pass-through from the runtime environment. + +Anything else is refused. `verify-compose` prints the escrow values the compose sets and every other app-compose setting in full (scripts included). All untrusted text is escaped, so control characters and terminal escapes show up as `\u{…}`. Run it when building the trust policy. + +A PCCS can't forge collateral, which is Intel-signed. A malicious PCCS can, however, serve older collateral that is still valid (TCB info is valid for about 30 days), which could hide a TCB recovery for that long. Use Intel PCS when that matters. + +Trust policy (`--trust`), from a source independent of the node being checked: + +```json +{ + "app_id": "<40 hex>", + "compose_hash": "<64 hex: SHA-256 of app-compose.json>", + "mrtd": "<96 hex>", + "rtmr0": "<96 hex>", + "rtmr1": "<96 hex>", + "rtmr2": "<96 hex>", + "allowed_tcb_status": ["UpToDate"], + "allowed_advisories": [] +} +``` + +## Node configuration and endpoints + +```toml +[at_rest] +mode = "off" # off | encrypt | strict +root_source = "escrow" # escrow | env +escrow_none = false # self-hosted Static-root nodes only (refused in a TEE) + +[at_rest.escrow] +recipients = [ { holder = "sam", recipient = "age1yubikey1…" }, … ] # exactly 3 +verified_fp = "" # after the ceremony (record-efp) +allow_software = false # X25519 holders; tests and self-host only, refused in a TEE +allow_resplit_before_verify = false # before the ceremony only: corrected holders get a fresh root key +``` + +The same settings as compose env: `TINYCLOUD_AT_REST__MODE`, `TINYCLOUD_AT_REST__ROOT_SOURCE`, `TINYCLOUD_AT_REST__ESCROW_NONE`, `TINYCLOUD_AT_REST__ESCROW__RECIPIENTS=[{holder="…",recipient="…"},…]`, `TINYCLOUD_AT_REST__ESCROW__VERIFIED_FP`, `TINYCLOUD_AT_REST__ESCROW__ALLOW_SOFTWARE`, `TINYCLOUD_AT_REST__ESCROW__ALLOW_RESPLIT_BEFORE_VERIFY`. The root keys are not config fields: they are read from `TINYCLOUD_AT_REST_ROOT_KEY` and `TINYCLOUD_AT_REST_PREVIOUS_ROOT_KEY` (single underscores) only. + +- **Recipients set, `mode = "off"`:** + - at boot the node creates the ARK (sealed under its root) and the escrow bundle; + - it keeps a copy of the sealed key, the bundle, the ceremony record and each space's wrapped data key in the block store (`_tc/at-rest/` in S3, `{datadir}/at-rest/_tc/at-rest/` on disk). Data-key objects are named by a keyed hash, not the space; + - in this mode a copy that can't be written is logged and the boot goes on (nothing depends on it yet). With `encrypt` or `strict`, every boot-time copy must be written: a block store that refuses the write stops the boot; + - nothing is encrypted. +- **`encrypt` and `strict` are refused at boot until both KV block storage and SQL artifacts seal their writes.** Those are the external, authoritative payloads (KV blocks in the block store, SQL and DuckDB artifacts in the database); until both seal, `/info` never shows `encrypting: true`. With both in place, `encrypt` or `strict` refuses to start unless the gate passes: with escrow, the recipients equal the bundle's, `verified_fp` equals its EFP, and the stored ceremony record verifies under the key inside the TEE. A Static-root node with no recipients may use `escrow_none = true` instead, which acknowledges that no escrow exists. With `root_source = "env"`, the gate needs only unlocked keys. +- **What is sealed, and what isn't.** External authoritative payloads are sealed. Local working files (SQL and DuckDB working and cache files, upload staging, exports) are plaintext on the node's disk and rely on the CVM's encrypted storage. Operators running without a TEE must protect the local paths, swap and file-level backups themselves (for example, the `data.tar.gz` backup that `deploy/personal-node/node.sh` writes is not encrypted). A node in `encrypt` or `strict` without a TEE logs a warning saying so at boot. +- **In a TEE, the escrow recipients and `verified_fp` must be literal values in the measured compose.** With any at-rest config, the node reads its `app-compose.json` from the guest agent (`/Info`, `tcb_info.app_compose`), checks it against `compose_hash`, and refuses to boot unless `TINYCLOUD_AT_REST__ESCROW__RECIPIENTS` and `TINYCLOUD_AT_REST__ESCROW__VERIFIED_FP` are set there as literals (no `${…}`, no pass-through) equal to the effective config. Values from the encrypted environment, a file or anywhere else are refused. If the app-compose can't be read or parsed, the node refuses to boot. The `/Info` shape must be confirmed on a live CVM before the escrow ceremony. +- **The escrow bundle is made or changed only at boot, never by a request.** Every endpoint below only reads it. + - **An ARK never gets a second bundle for other holders.** The node refuses to boot (`escrow_holders_changed`) if the configured recipients differ from any existing bundle for its ARK: the active bundle, any stored escrow row, a ceremony record in the block store whose MAC verifies under the ARK, or a bundle object in the block store naming the ARK. This holds after a restore too, when the database lost its escrow rows. The block-store bundle object is unauthenticated, so it can only refuse, never be used. Changing a holder after the ceremony needs an ARK rotation (TC-1345). + - **Fixing a wrong holder before the ceremony:** set `allow_resplit_before_verify = true` together with the corrected recipients. The node then **replaces the root key**: it makes a fresh ARK, splits it for the corrected holders, and retires the old ARK and its bundle. Shares of the old bundle become worthless: any two of them rebuild only the retired key, which never protected anything and is never loaded again (a W-authenticated retirement marker in the block store drops it even from a database restored to before the fix). The node does this only if no fingerprint is pinned, no ceremony record exists, and no data key was ever made, checking both the database and the block store. If any of those fails, it still refuses to boot. Remove the flag after the fix. + - With `verified_fp` set, the node uses only the pinned bundle, restoring it from the block store if the database lost it, and refuses to boot if that bundle is not for the configured recipients and the newest key. +- **If the node's root can no longer unseal the key** (new app id or KMS), it boots **locked**: + - `/info` shows `at_rest.state = "locked"` and encryption is unavailable; + - two holders recover it through the endpoints below, which needs `verified_fp` to be pinned; + - the node then re-seals the key under its new root, with no restart and no data rewrite. + +| Endpoint | Answer | +|---|---| +| `GET /at-rest/status` | `mode`, `state`, `encrypting`, `root_source`, `root_fingerprint` (env root only), `efp`, `generation`, `verified` | +| `GET /at-rest/escrow` | the active escrow bundle (or, on a locked node, the pinned one) | +| `GET /at-rest/escrow/record` | the ceremony record, only if its MAC verifies under the live key | +| `GET /at-rest/recovery?nonce=N` | `RecoveryInfo`: this boot's recovery recipient, bound with `N` and the EFP into a TDX quote (`mode = "classic"` with no quote outside a TEE) | +| `POST /at-rest/recovery/shares` | body: one share age-sealed to the recovery recipient (≤ 4 KiB). Answers `held` (with `shares_held`), `verified` (all three shares rebuilt the key), or `unlocked` (locked node recovered), with the gate result | + +The share endpoint: +- is rate limited per IP. Until TC-1357, the client IP comes from a proxy header trusted from any peer, so per-client limits are best effort; the per-boot budget below doesn't depend on them; +- checks every share against the bundle's commitments before it takes one of three slots; +- counts only shares that authenticate (decrypt and match a commitment) against the node's per-boot share budget, so junk can't starve genuine holders. Rejected shares are limited per client (5, then one a minute); +- never logs share bytes or ciphertexts. + +Shares live in memory for the current boot only. + +A keyring failure answers `503` when a retry may help (`store_unavailable`, `mirror_unavailable`, `locked`, `conflict: …`, `dek_unavailable: …`) and `500` when an operator must act (`inconsistent: …`, `escrow_pinned`, `escrow_holders_changed`, `escrow: …`). The reason never contains key material or backend detail; the server log has the rest. + +**Recovery is per node.** Each boot has its own recovery key, so a share wrapped for one node is rejected by another. With several nodes on one database: +- recover one node, by its own address (not through a load balancer); +- then restart the others; a locked node doesn't pick up another node's recovery until it restarts; +- every node on one database must use the same root. A node under another root boots locked, and recovering it re-seals the rows under its root, which locks the others out at their next restart. + +Production runs one node per database. Multi-node recovery is a known limitation. + +### Key continuity + +- **Database restored to an earlier point.** At boot the node takes back a key generation or chain from the block-store copy only once it authenticates: the newest under this node's root, each older one by its chain. During recovery, the recovered key authenticates them. Anything else in the copy (forged, junk, a gap, an oversized object) is skipped with a warning and never stored, so objects planted in the bucket can neither lock the node nor stop it booting while the database is healthy. A block store that refuses the node's own writes is different: with `encrypt` or `strict` it stops the boot (see above). + - Retirement markers (`_tc/at-rest/retired-.json`, written when a holder fix replaces the root key) follow the same rule. A forged or junk marker is skipped. One that can't be read is skipped too, unless it names a key the node has stored or mirrored and the mode is `encrypt` or `strict`: skipping it then could bring a retired key back, so the boot stops until the object reads. With `mode = "off"` a marker never stops the boot. +- **Data keys after a restore.** A space's data key missing from the database is restored from its copy only if its wrap authenticates for that space. A copy that exists but doesn't authenticate is never ignored and never overwritten: it may be the genuine key under a generation the node hasn't loaded (for example after a failed read at boot). That space answers `503 dek_unavailable` until the generation loads or an operator removes the object; no new key is made over it. + - **Runbook, `dek_unavailable` that doesn't clear:** first restart once the block store reads cleanly; a genuine key under an unloaded generation then authenticates and comes back. If the error stays, the server log names the space's data-key object as not authenticating under any loaded generation. Check no newer key generation exists anywhere (database backups, other nodes); only then delete that object under `_tc/at-rest/` and the space gets a new key on its next write. Deleting a genuine key makes everything sealed under it unreadable. +- **No key is ever made over existing material.** The node creates a new root key only when the database has no key, escrow or data-key rows and the block store has nothing under `_tc/at-rest/`. A store or mirror error stops the boot instead, and so does an empty database whose block-store copy holds nothing usable. +- **Escrow and ceremony record after a restore.** A bundle or record the database lost is restored from the block store at boot, so a restore doesn't force a new ceremony: the pinned bundle (the pin authenticates it), or, unpinned, a bundle whose mirrored ceremony record verifies under the key. A record is restored only if its MAC verifies under the key. + - **Runbook, `escrow_holders_changed` you didn't expect:** the server log names the evidence. If it is a block-store object (`_tc/at-rest/escrow-…`) and the configured recipients are right, check it isn't a genuine escrow: compare it with the bundle the holders hold (`tinycloud-at-rest inspect`) and the pinned `efp`. Only then remove that object and restart. A planted object can only stop a boot whose database lost its escrow rows; it never changes who can rebuild the key. +- **Generations.** Each older generation is sealed under the next one's key (its chain), so escrowing the newest generation recovers all of them. At boot under the same root, a missing or broken chain is rebuilt and logged. During recovery under a new root it can't be rebuilt, and the shares are refused with the reason. The newest generation never keeps a chain: a stale one is cleared at boot. +- The key tables' migration `down` refuses while they hold rows. + +### Root from the environment + +With `root_source = "env"` the ARK is sealed under a key from `TINYCLOUD_AT_REST_ROOT_KEY` instead of the node's root, and there is no escrow. Anyone who holds that key and can read the stored key rows (database or block store) can open the ARK and every data key, so the key must be generated offline, stored only in the deploy's secret store and one backup, and never printed. + +- **Format:** the standard base64 (with padding, 44 characters) of exactly 32 random bytes, for example `head -c32 /dev/urandom | base64`. A trailing newline is ignored. Anything else is refused at boot, and the error never contains the value. +- **`mode = "off"` uses the key.** With `root_source = "env"`, even in mode `off` the node provisions with the key: it creates the ARK, seals it under the key, and stores the sealed rows in the database and their copy in the block store. Only data writes stay plaintext. So staging a key in mode `off` commits to it: from then on the node starts only with that key (or with it as the previous root). +- **In the process.** At startup, before the config loader runs, the node copies both root variables into memory that is wiped when freed, overwrites their bytes in the environment and unsets them, so `/proc//environ`, the config and child processes don't see them. With either variable set it also turns core dumps off for the process (Linux: not dumpable, core size limit 0) and refuses to start if it can't. The decoded key lives only in wiped buffers. Code that runs inside the process (or reads its memory with root on the host) can still get the key. +- **Fingerprint:** the node logs `root_fingerprint` once at boot and shows it in `/at-rest/status`. It is `hex(SHA-256("tinycloud/at-rest/root-fingerprint/v1" ‖ key))[..16]`, so a copy of the key can be matched without showing it: + ```sh + (printf 'tinycloud/at-rest/root-fingerprint/v1'; printf '%s' "$KEY" | base64 -d) | sha256sum | cut -c1-16 + ``` + The fingerprint identifies a key; it doesn't authenticate one (anyone can compute it from the key, and 64 bits is a label, not a check). Equal fingerprints also show that two deployments share a root. +- **The node refuses to start** when: + - `root_source = "env"` and `TINYCLOUD_AT_REST_ROOT_KEY` is missing or malformed; + - `TINYCLOUD_AT_REST_ROOT_KEY` is set but `root_source` is `escrow`; + - `root_source = "env"` is combined with any `at_rest.escrow` setting or `escrow_none`; + - the key doesn't open the stored keys (there is no escrow to recover with). The error names the key's fingerprint; + - a dstack TEE is indicated (`tee.mode = "Dstack"`, `TINYCLOUD_TEE_MODE=dstack`, `TINYCLOUD_KEYS_TYPE=Dstack`, `DSTACK_SIMULATOR_ENDPOINT` or a dstack socket) but the guest agent's Info isn't available: the TEE checks below can't run, so the node doesn't boot as if there were no TEE; + - in a TEE: the measured compose doesn't set `TINYCLOUD_AT_REST__ROOT_SOURCE` to the literal `env`; any setting of a root key, in any service, is anything but one plain reference to the encrypted environment (`${NAME}`, or `${NAME:?message}` / `${NAME?message}`): a literal, a default or alternate value (`${NAME:-…}`, `${NAME-…}`, `${NAME:+…}`), a pass-through, or a compose this check can't read; the measured app-compose allows `DSTACK_AUTHORIZED_KEYS`; or the guest agent doesn't report a production dstack OS image (the hashes pinned in `PRODUCTION_OS_IMAGE_HASHES` in `tinycloud-node-server/src/at_rest.rs`). Moving to another OS image adds its hash there in a node release first. +- **`--validate-config`** runs the same checks as startup's first step (`at_rest::check_config`): everything above that needs no key store, database or guest agent. It opens no socket, so with a TEE indicated it refuses at-rest: the TEE checks run only at boot inside the TEE. +- **Changing the root.** Deploy once with the new root in place and the old one in `TINYCLOUD_AT_REST_PREVIOUS_ROOT_KEY`. When the stored rows don't open under the new root but do under the previous one, the node re-seals every ARK generation under the new root (database and block-store copy) and logs both fingerprints. Data keys and data are unchanged. Remove the previous key on the next deploy; the node refuses to start with a previous key equal to the current one. + - **A new env key:** `TINYCLOUD_AT_REST_ROOT_KEY=`, `TINYCLOUD_AT_REST_PREVIOUS_ROOT_KEY=`. + - **Moving to escrow:** `root_source = "escrow"` with the escrow settings, and the env key as `TINYCLOUD_AT_REST_PREVIOUS_ROOT_KEY`. The ARK is re-sealed under the node's root and escrowed. + - Re-sealing changes which root opens the current rows. A copy of the rows made before the change (a database backup, an older block-store object version) still opens under the old root, and with it every data key that existed then. Retiring an old root fully also needs an ARK rotation and new data keys for later writes (TC-1345), and then the old copies to age out. diff --git a/docs/at-rest/INTERFACES.md b/docs/at-rest/INTERFACES.md index 198de792..41784634 100644 --- a/docs/at-rest/INTERFACES.md +++ b/docs/at-rest/INTERFACES.md @@ -1,9 +1,9 @@ # Encryption at rest: interfaces (TC-887) -> **Status (TC-887):** on `main` today: the envelope, key types, `DataKeyProvider` and the -> harness. Not on `main` yet: key continuity (`docs/at-rest-escrow.md`, `at_rest::keyring`, -> node wiring) and sealed KV storage (§7). They land with PRs #325–#327 and #329/#333/#336 -> (#336: the S3 store seals KV blocks). +> **Status (TC-887):** the envelope, key types, `DataKeyProvider`, the keyring +> (`at_rest::keyring`), the node wiring (`docs/at-rest-escrow.md`) and the storage view +> (`at_rest::{AtRestMode, AtRestView, AtRestSource}`) are in. Sealed KV storage (§7) and sealed +> SQL artifacts follow; until both seal, the node refuses `encrypt` and `strict`. Normative interface and format for `tinycloud_core::at_rest` and the node's sealed KV block storage. Plan: TC-887 (encryption at rest inside the TEE). Key continuity and escrow: @@ -65,10 +65,19 @@ pub trait DataKeyProvider: Send + Sync { /// by deliberate crypto-shred). async fn key(&self, space: &str, dek_id: DekId) -> Result; + /// Before work that may read `space`'s sealed blobs under the invoke write + /// lock: put back any of the space's keys the database lost (for example + /// from an authenticated mirror). Never creates a key. + async fn prepare_read(&self, space: &str) -> Result<(), DataKeyError>; + /// NK for sealed S3 object names. Same value for the life of the deployment. fn object_name_key(&self) -> Result; } +/// Runs `f` as work under the invoke write lock: providers make no key-store +/// writes there; a key that would need creating or restoring is `NotPrepared`. +pub async fn write_locked(f: F) -> F::Output; + #[derive(Debug, thiserror::Error)] pub enum DataKeyError { /// Node booted locked (W cannot unseal ARK). Callers map to HTTP 503. @@ -81,6 +90,9 @@ pub enum DataKeyError { /// PG/S3/unwrap failure (incl. a wrapped DEK failing AEAD). Opaque, no key bytes. #[error("data key backend: {0}")] Backend(#[source] Box), + /// A key needed creating or restoring inside `write_locked`. Callers map to 503. + #[error("the space's data key was not prepared before the write lock")] + NotPrepared, } ``` diff --git a/test/m1-realdata-e2e/Cargo.lock b/test/m1-realdata-e2e/Cargo.lock index de7732f4..ee19b9f7 100644 --- a/test/m1-realdata-e2e/Cargo.lock +++ b/test/m1-realdata-e2e/Cargo.lock @@ -8637,11 +8637,13 @@ dependencies = [ "if-addrs", "lazy_static", "libc", + "lru", "multer", "native-tls", "opentelemetry", "opentelemetry-otlp", "opentelemetry_sdk", + "parking_lot", "percent-encoding", "pin-project", "prometheus", @@ -8667,6 +8669,7 @@ dependencies = [ "thiserror 2.0.18", "tikv-jemallocator", "time", + "tinycloud-at-rest-escrow", "tinycloud-auth", "tinycloud-core", "tokio", @@ -8680,6 +8683,7 @@ dependencies = [ "tracing-opentelemetry", "tracing-subscriber", "uuid", + "zeroize", ] [[package]] diff --git a/test/w5-policy-runtime-node-e2e/Cargo.lock b/test/w5-policy-runtime-node-e2e/Cargo.lock index f8df2565..a54256f7 100644 --- a/test/w5-policy-runtime-node-e2e/Cargo.lock +++ b/test/w5-policy-runtime-node-e2e/Cargo.lock @@ -9022,11 +9022,13 @@ dependencies = [ "if-addrs", "lazy_static", "libc", + "lru", "multer", "native-tls", "opentelemetry", "opentelemetry-otlp", "opentelemetry_sdk", + "parking_lot", "percent-encoding", "pin-project", "prometheus", @@ -9052,6 +9054,7 @@ dependencies = [ "thiserror 2.0.18", "tikv-jemallocator", "time", + "tinycloud-at-rest-escrow", "tinycloud-auth", "tinycloud-core", "tokio", @@ -9065,6 +9068,7 @@ dependencies = [ "tracing-opentelemetry", "tracing-subscriber", "uuid", + "zeroize", ] [[package]] diff --git a/tinycloud-at-rest-escrow/src/compose.rs b/tinycloud-at-rest-escrow/src/compose.rs index 7e0c783c..dfc653d9 100644 --- a/tinycloud-at-rest-escrow/src/compose.rs +++ b/tinycloud-at-rest-escrow/src/compose.rs @@ -360,20 +360,21 @@ impl AppCompose { .collect() } - /// The literal value of `key` in the services' `environment`, if set. - /// - /// Refused: a value that docker compose would interpolate (`$`), a key - /// passed through from the runtime environment (`- KEY` or `KEY:` - /// with no value), and different values in different services. - pub fn literal_env(&self, key: &str) -> Result> { - let mut found: Option = None; + /// Every setting of `key` in the services' `environment`, one per service + /// that sets it (`KEY: value`, or `- KEY=value` in list form). A value + /// that is empty or a YAML null, and a list entry without `=`, pass the + /// variable through from the runtime environment: [`EnvSetting::FromRuntime`]. + /// Errors (never quoting a value): a non-string setting, a key set twice + /// in one service, an environment that is not a mapping or list. + pub fn env_settings(&self, key: &str) -> Result> { + let mut settings = Vec::new(); for (service, fields) in services(&self.top)? { let service = escape(service); let value = match get(fields, "environment") { None => None, Some(Node::Map(env)) => match get(env, key) { None => None, - Some(Node::Scalar(v)) => Some(v.clone()), + Some(Node::Scalar(v)) => Some(Some(v.clone())), Some(_) => { return err(format!("service {service:?} sets {key} to a non-string")) } @@ -387,16 +388,12 @@ impl AppCompose { )); }; let (k, v) = match item.split_once('=') { - Some((k, v)) => (k, Some(v)), + Some((k, v)) => (k, Some(v.to_owned())), None => (item.as_str(), None), }; if k == key { ensure!(value.is_none(), "service {service:?} sets {key} twice"); - value = Some(v.map(str::to_owned).ok_or_else(|| { - EscrowError::Compose(format!( - "service {service:?} passes {key} through from the runtime environment; it must be a literal value" - )) - })?); + value = Some(v); } } value @@ -410,10 +407,32 @@ impl AppCompose { let Some(value) = value else { continue }; // Empty or a YAML 1.2 null (any spelling docker compose reads as // null) passes the variable through from the runtime. - ensure!( - !matches!(value.as_str(), "" | "~" | "null" | "Null" | "NULL"), - "service {service:?} passes {key} through from the runtime environment; it must be a literal value" - ); + settings.push(match value { + Some(v) if !matches!(v.as_str(), "" | "~" | "null" | "Null" | "NULL") => { + EnvSetting::Value { service, value: v } + } + _ => EnvSetting::FromRuntime { service }, + }); + } + Ok(settings) + } + + /// The literal value of `key` in the services' `environment`, if set. + /// + /// Refused: a value that docker compose would interpolate (`$`), a key + /// passed through from the runtime environment (`- KEY` or `KEY:` + /// with no value), and different values in different services. + pub fn literal_env(&self, key: &str) -> Result> { + let mut found: Option = None; + for setting in self.env_settings(key)? { + let (service, value) = match setting { + EnvSetting::FromRuntime { service } => { + return err(format!( + "service {service:?} passes {key} through from the runtime environment; it must be a literal value" + )) + } + EnvSetting::Value { service, value } => (service, value), + }; ensure!( !value.contains('$'), "service {service:?} sets {key} by interpolation; it must be a literal value" @@ -427,6 +446,24 @@ impl AppCompose { } } +/// One service's setting of an environment variable ([`AppCompose::env_settings`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum EnvSetting { + /// `KEY: value` or `- KEY=value`, exactly as written (before docker + /// compose interpolation). `service` is escaped for display. + Value { service: String, value: String }, + /// The variable is passed through from the runtime environment. + FromRuntime { service: String }, +} + +impl EnvSetting { + pub fn service(&self) -> &str { + match self { + Self::Value { service, .. } | Self::FromRuntime { service } => service, + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/tinycloud-core/src/at_rest/mod.rs b/tinycloud-core/src/at_rest/mod.rs index 7b652a5d..4487642b 100644 --- a/tinycloud-core/src/at_rest/mod.rs +++ b/tinycloud-core/src/at_rest/mod.rs @@ -1,5 +1,5 @@ -//! Encryption at rest (TC-887). Nothing in the node calls this module yet; it -//! is switched off until the key-continuity gate exists. +//! Encryption at rest (TC-887): the TCE1 envelope, the keyring, and the +//! view storage reads of the node's at-rest state. Off unless configured. mod envelope; mod hierarchy; @@ -13,6 +13,7 @@ mod stream; mod testing; #[cfg(test)] mod tests; +mod view; pub use envelope::{ ciphertext_len, format_version, open, plaintext_len, seal, BlobIdentity, EnvelopeError, Header, @@ -27,3 +28,4 @@ pub use testing::FixedKeyProvider; pub use keyring::{AtRestKeyring, KeyringConfig, KeyringState}; pub use store::{AtRestStore, KeyMirror}; +pub use view::{AtRestMode, AtRestSource, AtRestView}; diff --git a/tinycloud-core/src/at_rest/view.rs b/tinycloud-core/src/at_rest/view.rs new file mode 100644 index 00000000..a3e199c0 --- /dev/null +++ b/tinycloud-core/src/at_rest/view.rs @@ -0,0 +1,68 @@ +//! What storage sees of the node's at-rest state, per operation (TC-887). +//! +//! The node owns the keyring and the mode; block stores and the SQL artifact +//! repository read a fresh [`AtRestView`] from an [`AtRestSource`] for each +//! operation, because keys arrive after migrations and may unlock later. +//! +//! Rules every store follows (`docs/at-rest/INTERFACES.md`): +//! - mode `off` writes plaintext exactly as before; +//! - any other mode seals iff [`AtRestView::encrypting`], else the write fails +//! (503), never a plaintext fallback; +//! - sealed data is read whenever keys exist; legacy plaintext is read unless +//! the mode is `strict`. + +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; + +use super::{DataKeyProvider, ObjectNameKey}; + +/// `at_rest.mode`. +#[derive(Serialize, Deserialize, Debug, Default, Clone, Copy, Hash, PartialEq, Eq)] +#[serde(rename_all = "lowercase")] +pub enum AtRestMode { + /// New writes are plaintext; sealed data is still read when keys exist. + #[default] + Off, + /// New writes are sealed; legacy plaintext is still read. + Encrypt, + /// New writes are sealed; legacy plaintext is refused. + Strict, +} + +/// A snapshot of the node's at-rest state for one storage operation. +#[derive(Clone)] +pub struct AtRestView { + pub mode: AtRestMode, + /// The gate passed and the keys are unlocked: new writes are sealed. + pub encrypting: bool, + /// Set once the keyring is installed (locked or not). + pub provider: Option>, +} + +impl AtRestView { + /// The state of a node without at-rest wiring: `off`, no keys. + pub fn off() -> Self { + Self { + mode: AtRestMode::Off, + encrypting: false, + provider: None, + } + } + + /// The object-name key, if keys are set up and unlocked. + pub fn object_name_key(&self) -> Option { + self.provider.as_ref()?.object_name_key().ok() + } + + /// Whether legacy plaintext may be read (every mode but `strict`). + pub fn legacy_readable(&self) -> bool { + self.mode != AtRestMode::Strict + } +} + +/// Where storage reads its at-rest state from: the node's runtime, or a +/// test double. +pub trait AtRestSource: Send + Sync { + fn view(&self) -> AtRestView; +} diff --git a/tinycloud-node-server/Cargo.toml b/tinycloud-node-server/Cargo.toml index 59c1a04a..f0084161 100644 --- a/tinycloud-node-server/Cargo.toml +++ b/tinycloud-node-server/Cargo.toml @@ -59,6 +59,9 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_with = { version = "3.0", features = ["base64", "hex"] } sha2 = "0.10" +parking_lot = "0.12" +lru = "0.12" +zeroize = "1" serde_ipld_dagcbor.workspace = true toml = "0.8" thiserror = "2.0" @@ -107,6 +110,10 @@ features = ["sqlite", "postgres", "mysql", "tokio"] [dependencies.tinycloud-auth] path = "../tinycloud-auth/" +# TC-887: escrow bundle and recovery wire types for the at-rest routes. +[dependencies.tinycloud-at-rest-escrow] +path = "../tinycloud-at-rest-escrow/" + # TC-409: only for test builds, enable tinycloud-core's opt-in # verification-count test hook so node route tests can assert exactly-once # signature verification. Cargo only activates dev-dependency features when @@ -119,6 +126,8 @@ default-features = false features = ["sqlite", "postgres", "mysql", "tokio", "verification-count-test-hook", "test-util"] [dev-dependencies] +# TC-887: escrow holders decrypting their shares in tests/at_rest_ceremony.rs. +age = { version = "=0.12.1", default-features = false, features = ["armor"] } # Parse production Rust items in the DID resolver source guard (tests only). syn = { version = "2", features = ["full", "visit"] } quote = "1" diff --git a/tinycloud-node-server/src/at_rest.rs b/tinycloud-node-server/src/at_rest.rs new file mode 100644 index 00000000..264871b5 --- /dev/null +++ b/tinycloud-node-server/src/at_rest.rs @@ -0,0 +1,1412 @@ +//! Node wiring for encryption at rest (TC-887 S1): config → keyring, the +//! boot-time escrow gate, and the runtime state the routes use. +//! +//! Two phases, because block storage opens before migrations and the +//! keyring needs its tables: [`AtRestRuntime::new`] before storage, then +//! [`setup`] after migrations installs the keyring. +//! +//! In this build no storage seals anything yet: nothing calls +//! [`AtRestRuntime::declare_seal_path`], so `setup` refuses every mode +//! but `off` and `encrypting()` is always false. +//! +//! With the default config (`mode = "off"`, `root_source = "escrow"`, no +//! recipients, no `escrow_none`) nothing is set up: no key reads, no rows, no +//! mirror. +//! +//! The root that seals the at-rest root key (ARK) is the node's own root +//! (`root_source = "escrow"`, with the escrow gate) or a 32-byte key from +//! [`ROOT_KEY_ENV`] (`root_source = "env"`, no escrow). Either way the ARK +//! and everything below it are the same, so moving between them re-seals +//! the ARK rows ([`PREVIOUS_ROOT_KEY_ENV`]) and rewrites no data. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, OnceLock}; + +use anyhow::{anyhow, bail, ensure, Context, Result}; +use sha2::Digest as _; +use tinycloud_at_rest_escrow::compose::{AppCompose, EnvSetting, RECIPIENTS_ENV, VERIFIED_FP_ENV}; +use tinycloud_at_rest_escrow::{Efp, HolderRecipient, RecipientPolicy}; +use tinycloud_core::{ + at_rest::{ + keyring::{GatePass, GateRefusal}, + AtRestKeyring, AtRestSource, AtRestStore, AtRestView, DataKeyProvider, KeyMirror, + KeyringConfig, KeyringState, + }, + storage::either::Either, + RecoveringConnection, +}; +use zeroize::Zeroizing; + +use crate::config::{AtRestConfig, AtRestEscrowConfig, AtRestMode, AtRestRootSource, Storage}; +use crate::dstack::MeasuredInfo; +use crate::storage::at_rest_mirror::{FsKeyMirror, S3KeyMirror}; +use crate::tee::TeeContext; + +/// The at-rest root when `root_source = "env"`: 32 bytes, standard base64. +pub const ROOT_KEY_ENV: &str = "TINYCLOUD_AT_REST_ROOT_KEY"; +/// The root the stored ARK rows were sealed under before a root change. The +/// node re-seals them under the current root at boot; remove it afterwards. +pub const PREVIOUS_ROOT_KEY_ENV: &str = "TINYCLOUD_AT_REST_PREVIOUS_ROOT_KEY"; +/// In a TEE, `root_source = "env"` must be this literal in the measured compose. +pub const ROOT_SOURCE_ENV: &str = "TINYCLOUD_AT_REST__ROOT_SOURCE"; +/// The variable that gives a dstack CVM SSH authorized keys. +const SSH_KEYS_ENV: &str = "DSTACK_AUTHORIZED_KEYS"; +/// dstack production (not dev) OS images a TEE node may use +/// `root_source = "env"` on: `dstack-0.5.12`, as pinned by the deploy +/// (`PHALA_OS_IMAGE_HASH` in `.github/workflows/docker.yml`). Moving to +/// another OS image adds its hash here in a node release first. +pub const PRODUCTION_OS_IMAGE_HASHES: &[&str] = + &["8db40cb23008d6213668b5816702d100b37b859a8a5bfd18d938ae3a1b3715bd"]; + +/// Whether this build's storage seals every write when at-rest is set up: +/// both KV block storage and SQL artifacts ([`SealPath`]). `--validate-config` +/// decides the seal-path refusal from this; startup checks that the seal +/// paths storage actually declared agree with it. +pub const STORAGE_SEALS_EVERY_WRITE: bool = false; + +/// The root variables as read from the process environment, held only in +/// zeroizing buffers. Debug is redacted. +#[derive(Default)] +pub struct RootEnv { + current: Option>, + previous: Option>, +} + +impl std::fmt::Debug for RootEnv { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RootEnv") + .field("current", &self.current.is_some()) + .field("previous", &self.previous.is_some()) + .finish() + } +} + +/// [`RootEnv::take`] in `main`, before the config loader and any other +/// thread start; the node takes it from here at boot. +static STASHED_ROOT_ENV: std::sync::Mutex> = std::sync::Mutex::new(None); + +impl RootEnv { + /// Explicit values (tests, and callers that already hold them). + pub fn new(current: Option>, previous: Option>) -> Self { + Self { current, previous } + } + + /// Takes [`ROOT_KEY_ENV`] and [`PREVIOUS_ROOT_KEY_ENV`] out of the process + /// environment: copies each value into a zeroizing buffer, overwrites the + /// environment's own bytes, and unsets the variable, so neither the + /// config loader nor `/proc//environ` sees it again. With either set, + /// it also turns core dumps off for the process (Linux: not dumpable, + /// core size limit 0), and refuses if that fails. + /// + /// Call it before other threads run where possible (`main` does): it + /// writes the environment. + pub fn take() -> Result { + let env = Self { + current: take_var(ROOT_KEY_ENV), + previous: take_var(PREVIOUS_ROOT_KEY_ENV), + }; + if env.current.is_some() || env.previous.is_some() { + disable_core_dumps()?; + } + Ok(env) + } + + /// Keeps `self` for [`Self::for_boot`]. + pub fn stash(self) { + *STASHED_ROOT_ENV.lock().unwrap_or_else(|p| p.into_inner()) = Some(self); + } + + /// The stashed root variables if `main` took them, else [`Self::take`]. + pub fn for_boot() -> Result { + match STASHED_ROOT_ENV + .lock() + .unwrap_or_else(|p| p.into_inner()) + .take() + { + Some(env) => Ok(env), + None => Self::take(), + } + } +} + +fn take_var(name: &str) -> Option> { + use std::os::unix::ffi::OsStringExt as _; + let raw = Zeroizing::new(std::env::var_os(name)?.into_vec()); + // The environment block holds its own copy: overwrite it, then unset. + if let Ok(c_name) = std::ffi::CString::new(name) { + // SAFETY: getenv returns NUL-terminated memory owned by the + // environment; we only overwrite the value's own bytes before + // unsetting it. `main` calls this before other threads start. + unsafe { + let value = libc::getenv(c_name.as_ptr()); + if !value.is_null() { + std::ptr::write_bytes(value, 0, libc::strlen(value)); + } + } + } + std::env::remove_var(name); + // A value that isn't UTF-8 isn't a key; keep a stand-in that fails to + // parse rather than a lossy copy of the bytes. + let mut value = Zeroizing::new(String::new()); + match std::str::from_utf8(&raw) { + Ok(text) => value.push_str(text), + Err(_) => value.push('\u{fffd}'), + } + Some(value) +} + +fn disable_core_dumps() -> Result<()> { + #[cfg(target_os = "linux")] + // SAFETY: plain syscalls on this process with valid arguments. + unsafe { + ensure!( + libc::prctl(libc::PR_SET_DUMPABLE, 0, 0, 0, 0) == 0, + "could not turn off core dumps (prctl PR_SET_DUMPABLE)" + ); + let none = libc::rlimit { + rlim_cur: 0, + rlim_max: 0, + }; + ensure!( + libc::setrlimit(libc::RLIMIT_CORE, &none) == 0, + "could not turn off core dumps (RLIMIT_CORE)" + ); + } + Ok(()) +} + +/// An operator-supplied at-rest root: exactly 32 bytes. Debug is redacted; +/// only [`Self::fingerprint`] is ever logged. +pub struct RootKey(Zeroizing<[u8; 32]>); + +impl RootKey { + /// Standard base64 with padding (44 characters, canonical) of 32 bytes. + /// A trailing newline is ignored; anything else is refused, as is a key + /// of one repeated byte (a placeholder, not a key). Decoding and the + /// canonical check use fixed zeroizing buffers, no heap copies. + pub fn parse(value: &str) -> Result { + let value = value + .strip_suffix("\r\n") + .or_else(|| value.strip_suffix('\n')) + .unwrap_or(value); + ensure!( + value.len() == 44, + "must be the standard base64 of exactly 32 bytes (44 characters)" + ); + // 44 characters decode to at most 33 bytes. + let mut decoded = Zeroizing::new([0u8; 33]); + let len = base64::decode_config_slice(value, base64::STANDARD, &mut decoded[..]) + .map_err(|_| anyhow!("is not standard base64"))?; + ensure!(len == 32, "does not decode to exactly 32 bytes"); + let mut key = Zeroizing::new([0u8; 32]); + key.copy_from_slice(&decoded[..32]); + let mut encoded = Zeroizing::new([0u8; 44]); + base64::encode_config_slice(key.as_slice(), base64::STANDARD, &mut encoded[..]); + ensure!( + encoded.as_slice() == value.as_bytes(), + "is not canonical standard base64" + ); + ensure!( + key.iter().any(|b| *b != key[0]), + "is one repeated byte, not a key" + ); + Ok(Self(key)) + } + + pub fn bytes(&self) -> &[u8] { + self.0.as_slice() + } + + /// `hex(SHA-256("tinycloud/at-rest/root-fingerprint/v1" ‖ key))[..16]`: + /// names the key in logs and `/at-rest/status`, and can be computed + /// offline (`(printf '