From 69110312a779bc1c5b919dfe8fdc0708a0d9de8b Mon Sep 17 00:00:00 2001 From: "tinycloud-bot[bot]" <348376441+tinycloud-bot[bot]@users.noreply.github.com> Date: Sun, 8 Nov 2026 01:11:18 +0000 Subject: [PATCH 1/2] test(TC-887): the at-rest harness runs plaintext and encrypt legs harness.sh gains TC887_AT_REST_ROOT_SOURCE: with env it generates a per-run root key, passes it as TINYCLOUD_AT_REST_ROOT_KEY and points the checks at it (TC887_NODE_ROOT_ENCODING). With mode encrypt or strict the checks now expect KV and SQL both sealed. CI runs the harness as a matrix: plaintext must stay RED, encrypt (root from the environment) must be GREEN. --- .github/workflows/rust.yml | 32 ++++++++---- scripts/at-rest-harness/README.md | 22 ++++---- scripts/at-rest-harness/harness.sh | 51 +++++++++++++++---- .../tests/at_rest_harness.rs | 22 +++++--- 4 files changed, 92 insertions(+), 35 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 1d07235d..1749ffdf 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -373,17 +373,21 @@ jobs: # containers). MinIO/mc are pinned community releases whose sha256 is pinned # in harness.sh; PostgreSQL is the runner image's own native install. The # green checks prove KV puts land in MinIO and SQL writes land in - # database_artifact (checkpoint and WAL delta). The RED checks assert - # encryption at rest; until TC-887 S3/S4 land, the step passes only if both - # fail with their "TC-887 RED:" findings (`--expect red`), and fails loudly - # if they error or unexpectedly pass. When S3 lands, start the harness with - # the at-rest mode set and expect `kv=green,sql=red`; when S4 lands, `green`. + # database_artifact (checkpoint and WAL delta). The TC-887 checks run in two + # legs: plaintext (the default node) must find plaintext (`--expect red`, + # failing loudly if a check errors or unexpectedly passes), and encrypt (mode + # encrypt, root from the environment) must find KV blocks and SQL artifacts + # sealed (`--expect green`). at-rest-harness: - name: at-rest harness (native PostgreSQL + MinIO) + name: at-rest harness (native PostgreSQL + MinIO, ${{ matrix.at_rest }}) runs-on: ubuntu-latest timeout-minutes: 45 permissions: contents: read + strategy: + fail-fast: false + matrix: + at_rest: [plaintext, encrypt] steps: - name: Checkout TinyCloud repository uses: actions/checkout@v4 @@ -399,7 +403,7 @@ jobs: with: shared-key: rust-1.97.1-at-rest-harness workspaces: . -> target - save-if: ${{ github.ref == 'refs/heads/main' }} + save-if: ${{ github.ref == 'refs/heads/main' && matrix.at_rest == 'plaintext' }} - name: Harness paths run: | @@ -413,26 +417,36 @@ jobs: run: bash scripts/at-rest-harness/harness.sh bins - name: Start the harness (native PostgreSQL, MinIO, tinycloud-node) + env: + TC887_AT_REST_MODE: ${{ matrix.at_rest == 'encrypt' && 'encrypt' || '' }} + TC887_AT_REST_ROOT_SOURCE: ${{ matrix.at_rest == 'encrypt' && 'env' || '' }} run: bash scripts/at-rest-harness/harness.sh up - name: Harness checks (writes reach MinIO and PostgreSQL) run: bash scripts/at-rest-harness/harness.sh test green "$TC887_STATE" - - name: TC-887 RED checks (must be RED until S3/S4) - run: bash scripts/at-rest-harness/harness.sh test red --expect red "$TC887_STATE" + - name: TC-887 checks (RED when plaintext, GREEN when encrypting) + run: >- + bash scripts/at-rest-harness/harness.sh test red + --expect ${{ matrix.at_rest == 'encrypt' && 'green' || 'red' }} "$TC887_STATE" + # These write their own objects into the harness bucket, so they run + # after the checks above, and in one leg only (they don't use the node). - name: S3 block store (against the harness MinIO) + if: matrix.at_rest == 'plaintext' run: | (set -a; . "$TC887_RUN_DIR/harness.env"; set +a cargo test --locked -p tinycloud-node --test s3_block_store -- --ignored) - name: S3 block store with encryption at rest (against the harness MinIO) + if: matrix.at_rest == 'plaintext' timeout-minutes: 10 run: | (set -a; . "$TC887_RUN_DIR/harness.env"; set +a cargo test --locked -p tinycloud-node --test at_rest_s3_store -- --ignored) - name: SQL object store (against the harness MinIO) + if: matrix.at_rest == 'plaintext' timeout-minutes: 10 run: | (set -a; . "$TC887_RUN_DIR/harness.env"; set +a diff --git a/scripts/at-rest-harness/README.md b/scripts/at-rest-harness/README.md index 364b3c36..dd93caee 100644 --- a/scripts/at-rest-harness/README.md +++ b/scripts/at-rest-harness/README.md @@ -69,10 +69,10 @@ exits 0 only if every check matches: | 4 | a check errored: compile failure, harness failure, a precondition (`harness no-op`/`harness gap`) or a node error, i.e. not a RED finding (per check; wins over 3) | Defaults: `test green` expects `kv=green,sql=green`. `test red` expects -`kv=red,sql=red`, or `kv=green,sql=red` if the run was started with -`TC887_AT_REST_MODE=encrypt` or `strict`, because S3 seals KV before S4 seals -SQL. When S4 lands, the SQL default flips to green (`default_expect` in -`harness.sh`). CI runs `test red --expect red` until S3 lands. +`kv=red,sql=red`, or `kv=green,sql=green` if the run was started with +`TC887_AT_REST_MODE=encrypt` or `strict` (KV blocks and SQL artifacts both +seal). CI runs both: a plaintext leg with `--expect red` and an encrypt leg +(root from the environment) with `--expect green`. ## At-rest mode @@ -83,11 +83,15 @@ the top of `harness.sh`: |---|---| | `TC887_AT_REST_MODE` (`off`/`encrypt`/`strict`) | `TINYCLOUD_AT_REST__MODE` | | `TC887_AT_REST_ESCROW_NONE` (`true`/`false`: the Static self-host acknowledgement that no escrow exists, plan §5) | `TINYCLOUD_AT_REST__ESCROW_NONE` | - -If either is unset, the harness sets nothing and the node runs its default -(today: no encryption at rest). Other values are refused before anything -starts. The simplest encrypting node is the harness's Static root plus -`TC887_AT_REST_MODE=encrypt TC887_AT_REST_ESCROW_NONE=true`. +| `TC887_AT_REST_ROOT_SOURCE` (`escrow`/`env`) | `TINYCLOUD_AT_REST__ROOT_SOURCE`; with `env`, also `TINYCLOUD_AT_REST_ROOT_KEY` from a per-run key in `secrets/at-rest-root-key` | + +If one is unset, the harness sets nothing for it and the node runs its default +(no encryption at rest). Other values are refused before anything starts. The +checks re-derive the key hierarchy from the root the node seals under: the +Static node root, or with `env` the per-run root key (`TC887_NODE_ROOT_FILE` +and `TC887_NODE_ROOT_ENCODING` in `harness.env`). Simplest encrypting nodes: +`TC887_AT_REST_MODE=encrypt` with `TC887_AT_REST_ESCROW_NONE=true`, or with +`TC887_AT_REST_ROOT_SOURCE=env`. ## Checks diff --git a/scripts/at-rest-harness/harness.sh b/scripts/at-rest-harness/harness.sh index a74f7d0a..371b8d92 100755 --- a/scripts/at-rest-harness/harness.sh +++ b/scripts/at-rest-harness/harness.sh @@ -39,14 +39,19 @@ MC_URL=https://github.com/minio/mc/releases/download/$MC_RELEASE/mc.linux-amd64. # TC887_AT_REST_MODE -> TINYCLOUD_AT_REST__MODE (off|encrypt|strict) # TC887_AT_REST_ESCROW_NONE -> TINYCLOUD_AT_REST__ESCROW_NONE (true|false: the Static # self-host acknowledgement that no escrow exists, plan §5) -# Unset: the harness sets nothing and the node runs its default (today: no -# encryption at rest). Simplest encrypting node: Static root (always, here) + -# TC887_AT_REST_MODE=encrypt + TC887_AT_REST_ESCROW_NONE=true. With mode -# encrypt/strict, `test red` defaults to expecting kv=green,sql=red: S3 seals -# KV before S4 seals SQL. When S4 lands, flip the SQL default to green -# (default_expect below). +# TC887_AT_REST_ROOT_SOURCE -> TINYCLOUD_AT_REST__ROOT_SOURCE (escrow|env). With env, +# `up` also generates a per-run root key (standard base64 +# of 32 bytes) into secrets/at-rest-root-key and passes it +# as TINYCLOUD_AT_REST_ROOT_KEY; the checks derive from it. +# Unset: the harness sets nothing and the node runs its default (no encryption +# at rest). Simplest encrypting nodes: TC887_AT_REST_MODE=encrypt with either +# TC887_AT_REST_ESCROW_NONE=true (Static node root) or +# TC887_AT_REST_ROOT_SOURCE=env. With mode encrypt/strict, `test red` defaults +# to expecting both checks green (default_expect below). AT_REST_MODE_VAR=TINYCLOUD_AT_REST__MODE AT_REST_ESCROW_NONE_VAR=TINYCLOUD_AT_REST__ESCROW_NONE +AT_REST_ROOT_SOURCE_VAR=TINYCLOUD_AT_REST__ROOT_SOURCE +AT_REST_ROOT_KEY_VAR=TINYCLOUD_AT_REST_ROOT_KEY # --------------------------------------------------------------------------- BUCKET=tinycloud-blocks @@ -251,10 +256,13 @@ cmd_up() { command -v python3 >/dev/null || die "python3 is required" command -v curl >/dev/null || die "curl is required" local at_rest_mode=${TC887_AT_REST_MODE:-} at_rest_escrow_none=${TC887_AT_REST_ESCROW_NONE:-} + local at_rest_root_source=${TC887_AT_REST_ROOT_SOURCE:-} [[ -z "$at_rest_mode" ]] || valid_mode "$at_rest_mode" \ || die "TC887_AT_REST_MODE must be off, encrypt or strict" [[ -z "$at_rest_escrow_none" ]] || valid_bool "$at_rest_escrow_none" \ || die "TC887_AT_REST_ESCROW_NONE must be true or false" + [[ -z "$at_rest_root_source" ]] || [[ "$at_rest_root_source" =~ ^(escrow|env)$ ]] \ + || die "TC887_AT_REST_ROOT_SOURCE must be escrow or env" cmd_bins local pg_bin @@ -295,13 +303,22 @@ cmd_up() { # Recorded by every write_state call, including a failed up's. export STATE_PG_BIN=$pg_bin STATE_NODE_BIN=$node_bin STATE_PG_PORT=$pg_port \ STATE_MINIO_PORT=$minio_port STATE_CONSOLE_PORT=$console_port STATE_NODE_PORT=$node_port \ - STATE_AT_REST_MODE=$at_rest_mode STATE_AT_REST_ESCROW_NONE=$at_rest_escrow_none + STATE_AT_REST_MODE=$at_rest_mode STATE_AT_REST_ESCROW_NONE=$at_rest_escrow_none \ + STATE_AT_REST_ROOT_SOURCE=$at_rest_root_source # Per-run secrets: generated here, kept in 0600 files, never on argv. random_hex | secret_file "$run/secrets/pg-password" printf 'tc887%s\n' "$(python3 -c 'import secrets; print(secrets.token_hex(8))')" | secret_file "$run/secrets/minio-user" random_hex | secret_file "$run/secrets/minio-password" python3 -c 'import secrets; print(secrets.token_urlsafe(32))' | secret_file "$run/secrets/node-static-secret" + # The root the at-rest keys are sealed under, and how the checks decode it. + local root_file=$run/secrets/node-static-secret root_encoding=base64url at_rest_root_key= + if [[ "$at_rest_root_source" == env ]]; then + python3 -c 'import base64, secrets; print(base64.b64encode(secrets.token_bytes(32)).decode())' \ + | secret_file "$run/secrets/at-rest-root-key" + root_file=$run/secrets/at-rest-root-key root_encoding=base64 + at_rest_root_key=$(<"$root_file") + fi local pg_password minio_user minio_password node_secret pg_password=$(<"$run/secrets/pg-password") minio_user=$(<"$run/secrets/minio-user") @@ -322,7 +339,8 @@ TC887_S3_BUCKET=$BUCKET TC887_S3_REGION=$REGION TC887_S3_ACCESS_KEY_ID=$minio_user TC887_S3_SECRET_ACCESS_KEY=$minio_password -TC887_NODE_ROOT_FILE=$run/secrets/node-static-secret +TC887_NODE_ROOT_FILE=$root_file +TC887_NODE_ROOT_ENCODING=$root_encoding TC887_SPACES_LOG=$run/spaces.log TC887_PUTS_LOG=$run/puts.log TC887_BOOT_SNAPSHOT=$run/boot-snapshot @@ -361,6 +379,14 @@ EOF printf '%s=%s\n' "$AT_REST_ESCROW_NONE_VAR" "$at_rest_escrow_none" >>"$run/node.env" log "at-rest: $AT_REST_ESCROW_NONE_VAR=$at_rest_escrow_none" fi + if [[ -n "$at_rest_root_source" ]]; then + printf '%s=%s\n' "$AT_REST_ROOT_SOURCE_VAR" "$at_rest_root_source" >>"$run/node.env" + log "at-rest: $AT_REST_ROOT_SOURCE_VAR=$at_rest_root_source" + fi + if [[ -n "$at_rest_root_key" ]]; then + printf '%s=%s\n' "$AT_REST_ROOT_KEY_VAR" "$at_rest_root_key" >>"$run/node.env" + log "at-rest: $AT_REST_ROOT_KEY_VAR set from secrets/at-rest-root-key" + fi printf '[global]\nlog_level = "normal"\n' >"$run/tinycloud.toml" write_state "$run" starting @@ -467,7 +493,8 @@ write_state() { NODE_PID=$UP_NODE_PID NODE_START=$UP_NODE_START \ BIN_DIR=$BIN_DIR MINIO_RELEASE=$MINIO_RELEASE MC_RELEASE=$MC_RELEASE \ BUCKET=$BUCKET REGION=$REGION PG_USER=$PG_USER PG_DB=$PG_DB \ - AT_REST_MODE_VAR=$AT_REST_MODE_VAR AT_REST_ESCROW_NONE_VAR=$AT_REST_ESCROW_NONE_VAR python3 - <<'PY' + AT_REST_MODE_VAR=$AT_REST_MODE_VAR AT_REST_ESCROW_NONE_VAR=$AT_REST_ESCROW_NONE_VAR \ + AT_REST_ROOT_SOURCE_VAR=$AT_REST_ROOT_SOURCE_VAR python3 - <<'PY' import json, os, datetime e = os.environ run = e["RUN"] @@ -488,8 +515,10 @@ state = { "at_rest": { "mode": text("STATE_AT_REST_MODE"), "escrow_none": {"true": True, "false": False}.get(e.get("STATE_AT_REST_ESCROW_NONE", "")), + "root_source": text("STATE_AT_REST_ROOT_SOURCE"), "mode_var": e["AT_REST_MODE_VAR"], "escrow_none_var": e["AT_REST_ESCROW_NONE_VAR"], + "root_source_var": e["AT_REST_ROOT_SOURCE_VAR"], }, "node": { "pid": num("NODE_PID"), @@ -630,8 +659,8 @@ default_expect() { local kind=$1 check=$2 state=$3 [[ "$kind" == green ]] && { echo green; return; } case $(state_get "$state" at_rest.mode) in - # S3 seals KV before S4 seals SQL; flip sql to green when S4 lands. - encrypt|strict) [[ "$check" == kv ]] && echo green || echo red ;; + # Both KV blocks and SQL artifacts seal once a mode encrypts. + encrypt|strict) echo green ;; *) echo red ;; esac } diff --git a/tinycloud-node-server/tests/at_rest_harness.rs b/tinycloud-node-server/tests/at_rest_harness.rs index 37b4afed..24a3468c 100644 --- a/tinycloud-node-server/tests/at_rest_harness.rs +++ b/tinycloud-node-server/tests/at_rest_harness.rs @@ -93,8 +93,10 @@ struct Harness { s3: aws_sdk_s3::Client, bucket: String, pg: DatabaseConnection, - /// The node's Static root (`TINYCLOUD_KEYS__SECRET`, base64url): the - /// test re-derives the at-rest key hierarchy from it. Never printed. + /// The root that seals the at-rest keys: the node's Static root + /// (`TINYCLOUD_KEYS__SECRET`, base64url) or, with `root_source = "env"`, + /// `TINYCLOUD_AT_REST_ROOT_KEY` (standard base64). The test re-derives + /// the at-rest key hierarchy from it. Never printed. root: Vec, /// Every space any check signed in to on this harness, one per line. spaces_log: std::path::PathBuf, @@ -122,7 +124,9 @@ impl Harness { state["at_rest"]["mode"].as_str(), Some("encrypt" | "strict") ), - escrow_none: state["at_rest"]["escrow_none"] == true, + // An env root has no escrow either: no bundle or record may exist. + escrow_none: state["at_rest"]["escrow_none"] == true + || state["at_rest"]["root_source"] == "env", }; let env_file = state["env_file"] .as_str() @@ -170,9 +174,15 @@ impl Harness { let root_file = var("TC887_NODE_ROOT_FILE")?; let root = std::fs::read_to_string(&root_file) .with_context(|| format!("read the node root file {root_file}"))?; - // The decode error could quote secret bytes; don't pass it on. - let root = base64::decode_config(root.trim(), base64::URL_SAFE_NO_PAD) - .map_err(|_| anyhow!("{root_file} does not hold unpadded base64url"))?; + // `harness.sh up` names the encoding of the root it wrote. The decode + // error could quote secret bytes; don't pass it on. + let config = match env.get("TC887_NODE_ROOT_ENCODING").map(String::as_str) { + None | Some("base64url") => base64::URL_SAFE_NO_PAD, + Some("base64") => base64::STANDARD, + Some(other) => bail!("unknown TC887_NODE_ROOT_ENCODING {other:?}"), + }; + let root = base64::decode_config(root.trim(), config) + .map_err(|_| anyhow!("{root_file} does not hold the root in its encoding"))?; ensure!( root.len() >= 32, "{root_file}: the node root is shorter than 32 bytes" From fcf1531cc061ccbb1aea2fb1aee6127aab8628d7 Mon Sep 17 00:00:00 2001 From: "tinycloud-bot[bot]" <348376441+tinycloud-bot[bot]@users.noreply.github.com> Date: Sun, 8 Nov 2026 03:29:36 +0000 Subject: [PATCH 2/2] test(TC-887): the harness SQL check reads every sealed artifact cold The SQL check's payload heuristics can't see random bytes appended to a sealed part or a dropped WAL delta: both still look like ciphertext. The check now also opens every `database_artifact` row of the harness spaces with keys it re-derives itself from its root and the authenticated `_tc/` DEK mirrors (`key_mirror::recover_keys`), through `open_row` (manifest tag, each part's AEAD and exact length), and requires the parts to decode to a SQLite database that passes `PRAGMA integrity_check` and holds exactly the probe rows written. `selftest_sql_check_flags_appended_and_omitted_sealed_parts` proves an appended checkpoint or delta and a dropped delta each turn the check RED. The RED notices no longer say "plaintext", since a RED SQL check can now mean bytes that fail authentication. --- scripts/at-rest-harness/README.md | 23 +- scripts/at-rest-harness/harness.sh | 6 +- .../tests/at_rest_harness.rs | 382 ++++++++++++++++-- 3 files changed, 365 insertions(+), 46 deletions(-) diff --git a/scripts/at-rest-harness/README.md b/scripts/at-rest-harness/README.md index dd93caee..1b62bcac 100644 --- a/scripts/at-rest-harness/README.md +++ b/scripts/at-rest-harness/README.md @@ -112,7 +112,7 @@ then insert (a WAL delta). | `green_kv_put_reaches_minio` | every put round-trips through the node and adds a non-empty object to the bucket | pass | | `green_sql_write_reaches_postgres` | the writes round-trip, and `database_artifact` has a `payload` and a non-NULL `delta_payload` | pass | | `red_kv_blocks_are_sealed_in_minio` | object keys contain no identity of any space the harness created (space id or wallet address: raw, lowercase, hex, base64 and base64url at any alignment, of the text and of the bytes) and no blake3 fingerprint of the value (b64url padded/unpadded, base64 padded/unpadded, hex; raw digest and multihash); raw bytes hold no marker; each put's objects together are at least as large as the value; the 64 KiB object doesn't compress (zlib ratio > 0.95); every object of a put in one space shares no 32-byte run with every object of the same put in the other (equal object counts required); every `_tc/` key-mirror object passes the checks below | **fail** | -| `red_sql_artifacts_are_sealed_in_postgres` | `payload` doesn't start with `SQLite format 3\0` and `delta_payload` doesn't start with the WAL magic; neither holds a marker, compresses (ratio > 0.95) or is smaller than the 64 KiB value it holds; the two spaces' payloads and deltas share no 32-byte run | **fail** | +| `red_sql_artifacts_are_sealed_in_postgres` | `payload` doesn't start with `SQLite format 3\0` and `delta_payload` doesn't start with the WAL magic; neither holds a marker, compresses (ratio > 0.95) or is smaller than the 64 KiB value it holds; the two spaces' payloads and deltas share no 32-byte run; and an authenticated cold read of every row succeeds (below) | **fail** | The RED checks re-run the green preconditions first, so a RED result is never a harness no-op. The 32-byte limit: two sealed blobs of the same plaintext @@ -127,6 +127,22 @@ across writes once a per-write salt (SQLite's WAL salts) is mixed in. flagged, a salted deflate-9 blob is flagged by the size floor, and sealed-like random blobs pass. +**SQL cold read.** The payload checks above are heuristics: random bytes +appended to a sealed part, or a dropped WAL delta, still look like +ciphertext. So the SQL check also opens every `database_artifact` row of the +harness spaces the way a node hydrating it would, with keys the harness +re-derived itself (root → ARK → KEK → each space's DEK from its +authenticated `_tc/` mirror, as the KV check does): the row must be sealed +(`payload_encoding`/`delta_encoding` 1 and a `manifest` tag), +`database_artifacts::open_row` must authenticate the manifest tag and every +part (AEAD and exact stored length), and the checkpoint plus WAL must be a +SQLite database that passes `PRAGMA integrity_check` and holds exactly the +four `probe` rows the harness wrote. Anything else is a finding (RED). +`selftest_sql_check_flags_appended_and_omitted_sealed_parts` runs in plain +`cargo test`: a sealed fixture row passes, and random bytes appended to its +checkpoint or delta, or its delta dropped (with the unkeyed hash and size +reset), each turn the SQL check RED. + ### At-rest key mirror (`_tc/…` objects) With at-rest keys on, the node writes control objects next to its blocks. The @@ -234,3 +250,8 @@ bypass), `selftest_key_mirror_refuses_escrow_contents_and_derived_keys`, installed here, so the node runs with a per-run Static root; S1 adds a Dstack-root variant. - DuckDB artifacts (`service=duckdb`, an opt-in feature): S4 follow-up. +- The node's local working files (SQL and DuckDB working and cache files, + upload staging, exports) are not scanned: they are plaintext by design and + rely on the CVM's encrypted storage (`docs/at-rest/INTERFACES.md` §8). +- A row replaced by an older, self-consistent copy of itself passes the SQL + cold read, as it passes the node's own read (INTERFACES §8). diff --git a/scripts/at-rest-harness/harness.sh b/scripts/at-rest-harness/harness.sh index 371b8d92..a5bbf328 100755 --- a/scripts/at-rest-harness/harness.sh +++ b/scripts/at-rest-harness/harness.sh @@ -733,7 +733,7 @@ cmd_test() { rc=4 elif [[ "$CHECK_OUTCOME" == "${want[$check]}" ]]; then if [[ "$CHECK_OUTCOME" == red ]]; then - annotate notice "$label RED as expected: $label data is plaintext at rest" + annotate notice "$label RED as expected: $label data is not sealed and authenticated at rest" else annotate notice "$label $kind check GREEN as expected" fi @@ -741,13 +741,13 @@ cmd_test() { annotate error "$label $kind check UNEXPECTEDLY PASSED under expect $check=red: the at-rest property holds for $label; expect $check=green and update CI" ((rc == 4)) || rc=3 else - annotate error "$label $kind check still RED ($label data is plaintext at rest) but expected $check=green" + annotate error "$label $kind check still RED ($label data is not sealed and authenticated at rest) but expected $check=green" ((rc == 4)) || rc=3 fi done if ((rc == 0)); then if [[ "$kind" == red && "${want[kv]}" == red && "${want[sql]}" == red ]]; then - annotate notice "RED as expected: both TC-887 RED checks report plaintext at rest (KV and SQL)" + annotate notice "RED as expected: both TC-887 RED checks report data not sealed at rest (KV and SQL)" else annotate notice "$kind checks as expected (kv=${want[kv]}, sql=${want[sql]})" fi diff --git a/tinycloud-node-server/tests/at_rest_harness.rs b/tinycloud-node-server/tests/at_rest_harness.rs index 24a3468c..b5926b0c 100644 --- a/tinycloud-node-server/tests/at_rest_harness.rs +++ b/tinycloud-node-server/tests/at_rest_harness.rs @@ -29,8 +29,13 @@ //! exact typed schema, carry no raw key, identity, fingerprint or marker, and //! hold key material that authenticates under the harness's Static root (see //! [`key_mirror`]). +//! +//! The SQL RED check also reads every artifact row cold with keys it +//! re-derives from its root and the mirrors: the manifest tag and every part +//! must authenticate, and the parts must decode to a SQLite database holding +//! exactly the rows written (see [`sql_cold_read_findings`]). -use std::collections::{BTreeSet, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::io::Write; use std::time::Duration; @@ -2126,6 +2131,45 @@ mod key_mirror { policy: Policy, ) -> Vec { let mut findings = Vec::new(); + let (arks, docs, keys) = read_snapshot(label, objects, root, spaces, policy, &mut findings); + for (what, doc) in &docs { + findings.extend(secret_findings(what, doc, &keys.secrets)); + } + for block in blocks { + findings.extend(block_secret_findings(label, block, &keys.secrets)); + findings.extend(salt_findings(label, block, &keys.secrets)); + } + if policy.encrypt { + require_mirrors(label, blocks, puts, spaces, &arks, &keys, &mut findings); + } + findings + } + + /// The DEKs of `spaces` whose mirrors in `objects` authenticate under the + /// harness's `root`, re-derived as [`check_snapshot`] does (its findings + /// are that check's to report). + pub fn recover_keys( + label: &str, + objects: &[ObjectRef], + root: &[u8], + spaces: &[String], + policy: Policy, + ) -> Recovered { + read_snapshot(label, objects, root, spaces, policy, &mut Vec::new()).2 + } + + /// Parses and authenticates the `_tc/` objects of one snapshot: the + /// ARK documents, every document with its label, and what + /// authentication recovered. + #[allow(clippy::type_complexity)] + fn read_snapshot( + label: &str, + objects: &[ObjectRef], + root: &[u8], + spaces: &[String], + policy: Policy, + findings: &mut Vec, + ) -> (Vec, Vec<(String, Value)>, Recovered) { let mut arks: Vec = Vec::new(); let mut deks: Vec = Vec::new(); let mut bundles: Vec = Vec::new(); @@ -2166,30 +2210,24 @@ mod key_mirror { }; let (encoded, halves): (&[&str], &[&str]) = match kind { Kind::Ark { generation, ark_id } => { - strict::(&what, "MirroredArk", object.body, &mut findings); + strict::(&what, "MirroredArk", object.body, findings); arks.extend(ark_doc( &what, object.key, map, (generation, ark_id), - &mut findings, + findings, )); (&["ark_id", "sealed", "chain"], &[]) } Kind::Dek { tag, dek_id } => { - strict::(&what, "MirroredDek", object.body, &mut findings); - deks.extend(dek_doc( - &what, - object.key, - map, - (tag, dek_id), - &mut findings, - )); + strict::(&what, "MirroredDek", object.body, findings); + deks.extend(dek_doc(&what, object.key, map, (tag, dek_id), findings)); (&["wrapped"], &[]) } Kind::Escrow { efp } => { - strict::(&what, "EscrowBundle", object.body, &mut findings); - bundles.extend(bundle_doc(&what, object.key, map, efp, &mut findings)); + strict::(&what, "EscrowBundle", object.body, findings); + bundles.extend(bundle_doc(&what, object.key, map, efp, findings)); ( &[ "ark_id", @@ -2202,8 +2240,8 @@ mod key_mirror { ) } Kind::Record { efp } => { - strict::(&what, "VerificationRecord", object.body, &mut findings); - records.extend(record_doc(&what, object.key, map, efp, &mut findings)); + strict::(&what, "VerificationRecord", object.body, findings); + records.extend(record_doc(&what, object.key, map, efp, findings)); (&["efp", "ark_id", "canary_fp", "mac"], &[]) } }; @@ -2211,26 +2249,9 @@ mod key_mirror { docs.push((what, doc)); } let keys = authenticate( - label, - root, - spaces, - &arks, - &deks, - &bundles, - &records, - &mut findings, + label, root, spaces, &arks, &deks, &bundles, &records, findings, ); - for (what, doc) in &docs { - findings.extend(secret_findings(what, doc, &keys.secrets)); - } - for block in blocks { - findings.extend(block_secret_findings(label, block, &keys.secrets)); - findings.extend(salt_findings(label, block, &keys.secrets)); - } - if policy.encrypt { - require_mirrors(label, blocks, puts, spaces, &arks, &keys, &mut findings); - } - findings + (arks, docs, keys) } /// An authenticated DEK of one space. @@ -2780,6 +2801,8 @@ struct SqlRun { markers: [String; 2], /// Bytes of the smaller 64 KiB value; every payload and delta holds one. big_len: usize, + /// The `probe` rows the writes leave, in `id` order. + expected: Vec, spaces: Vec, } @@ -2835,9 +2858,12 @@ async fn sql_run(harness: &Harness) -> Result { sql_reached_postgres(&space)?; spaces.push(space); } + let [big0, big1] = big; + let [marker0, marker1] = markers.clone(); Ok(SqlRun { markers, big_len, + expected: vec![marker0, big0, marker1, big1], spaces, }) } @@ -3048,13 +3074,17 @@ async fn red_kv_blocks_are_sealed_in_minio() -> Result<()> { Ok(()) } -#[tokio::test] -#[ignore = "TC-887 RED: plaintext at rest until S3/S4"] -async fn red_sql_artifacts_are_sealed_in_postgres() -> Result<()> { - let harness = Harness::connect().await?; - let run = sql_run(&harness).await?; +/// The findings for each stored `database_artifact` row: the plaintext +/// heuristics (file magic, markers, compressibility, size floor). +async fn sql_row_findings( + rows: &[database_artifact::Model], + markers: &[String], + big_len: usize, + keys: &dyn tinycloud_core::at_rest::DataKeyProvider, + expected: &[String], +) -> Result> { let mut findings = Vec::new(); - for row in run.spaces.iter().flat_map(|space| &space.rows) { + for row in rows { let id = row_id(row); if row.payload.starts_with(SQLITE_MAGIC) { findings.push(format!( @@ -3080,7 +3110,7 @@ async fn red_sql_artifacts_are_sealed_in_postgres() -> Result<()> { ("delta_payload", row.delta_payload.as_ref()), ] { let Some(bytes) = bytes else { continue }; - for marker in &run.markers { + for marker in markers { if let Some(offset) = find(bytes, marker.as_bytes()) { findings.push(format!( "{id}.{column} contains the plaintext marker {marker:?} at byte {offset}" @@ -3091,11 +3121,145 @@ async fn red_sql_artifacts_are_sealed_in_postgres() -> Result<()> { check_size_floor( &format!("{id}.{column}"), bytes.len(), - run.big_len, + big_len, &mut findings, ); } + findings.extend(sql_cold_read_findings(row, keys, expected).await?); } + Ok(findings) +} + +/// An authenticated cold read of one row, as a node hydrating it would, +/// but with the keys the harness re-derived: every part must be sealed +/// (TCE1 encodings and a manifest tag), the row must open with +/// `database_artifacts::open_row` (manifest tag, each part's AEAD and exact +/// length), and the checkpoint plus WAL must be a SQLite database that +/// passes `PRAGMA integrity_check` and whose `probe` rows are exactly +/// `expected` (each harness space holds only the harness database), so a +/// dropped delta can't pass. +async fn sql_cold_read_findings( + row: &database_artifact::Model, + keys: &dyn tinycloud_core::at_rest::DataKeyProvider, + expected: &[String], +) -> Result> { + let id = row_id(row); + if row.payload_encoding != Some(1) + || (row.delta_payload.is_some() && row.delta_encoding != Some(1)) + || row.manifest.is_none() + { + return Ok(vec![format!( + "{id} is not sealed at rest (payload_encoding {:?}, delta_encoding {:?}, manifest tag {})", + row.payload_encoding, + row.delta_encoding, + if row.manifest.is_some() { "present" } else { "absent" } + )]); + } + let (checkpoint, wal) = + match tinycloud_core::database_artifacts::open_row(row.clone(), Some(keys)).await { + Ok(parts) => parts, + Err(e) => { + return Ok(vec![format!( + "{id}: a cold read with the harness-derived keys does not authenticate: {e}" + )]) + } + }; + let dir = tempfile::tempdir()?; + let path = dir.path().join("cold.db"); + std::fs::write(&path, &checkpoint)?; + if let Some(wal) = &wal { + std::fs::write(dir.path().join("cold.db-wal"), wal)?; + } + let opened = rusqlite::Connection::open(&path).and_then(|db| { + let check: String = db.query_row("PRAGMA integrity_check", [], |r| r.get(0))?; + let mut select = db.prepare("SELECT secret FROM probe ORDER BY id")?; + let rows = select + .query_map([], |r| r.get::<_, String>(0))? + .collect::>>()?; + Ok((check, rows)) + }); + Ok(match opened { + Err(e) => vec![format!( + "{id}: authenticates, but its checkpoint and WAL are not a readable SQLite database \ + with the harness's probe table: {e}" + )], + Ok((check, _)) if check != "ok" => vec![format!( + "{id}: authenticates, but PRAGMA integrity_check says {check:?}" + )], + Ok((_, rows)) if rows != expected => vec![format!( + "{id}: authenticates, but holds {} probe row(s), not the {} the harness wrote", + rows.len(), + expected.len() + )], + Ok(_) => Vec::new(), + }) +} + +/// The DEKs the harness re-derived from its root and the mirrors, as the +/// key provider a cold read opens sealed SQL artifacts with. Read-only: +/// it never creates a key. +struct HarnessKeys(BTreeMap>); + +#[async_trait::async_trait] +impl tinycloud_core::at_rest::DataKeyProvider for HarnessKeys { + async fn current_key( + &self, + _space: &str, + ) -> Result { + Err(tinycloud_core::at_rest::DataKeyError::Locked) + } + + async fn key( + &self, + space: &str, + dek_id: tinycloud_core::at_rest::DekId, + ) -> Result { + self.0 + .get(space) + .and_then(|deks| deks.iter().find(|d| d.dek_id == dek_id.get())) + .map(|d| tinycloud_core::at_rest::DataKey::new(dek_id, d.key)) + .ok_or(tinycloud_core::at_rest::DataKeyError::UnknownKey { dek_id }) + } + + async fn prepare_read( + &self, + _space: &str, + ) -> Result<(), tinycloud_core::at_rest::DataKeyError> { + Ok(()) + } + + fn object_name_key( + &self, + ) -> Result { + Err(tinycloud_core::at_rest::DataKeyError::Locked) + } +} + +#[tokio::test] +#[ignore = "TC-887 RED: plaintext at rest until S3/S4"] +async fn red_sql_artifacts_are_sealed_in_postgres() -> Result<()> { + let harness = Harness::connect().await?; + let run = sql_run(&harness).await?; + let rows: Vec = run + .spaces + .iter() + .flat_map(|space| space.rows.iter().cloned()) + .collect(); + // The keys come from the mirrors in the bucket, authenticated under the + // harness's own root, never from the node. + let control = harness.control_objects().await?; + let keys = HarnessKeys( + key_mirror::recover_keys( + "sql", + &object_refs(&control), + &harness.root, + &harness.known_spaces()?, + harness.policy, + ) + .deks, + ); + let mut findings = + sql_row_findings(&rows, &run.markers, run.big_len, &keys, &run.expected).await?; for a in &run.spaces[0].rows { let b = run.spaces[1] .rows @@ -5113,3 +5277,137 @@ fn selftest_base32_secret_before_other_characters() -> Result<()> { ); Ok(()) } + +/// A real SQLite database saved as the node saves one, sealed: the artifact +/// repository in `encrypt` stores a checkpoint holding the first write +/// (`markers[0]`, `big[0]`), then a WAL delta holding the second. Returns the +/// stored row and the keys it was sealed under. +async fn sealed_sql_row( + markers: &[String; 2], + big: &[String; 2], +) -> Result<( + database_artifact::Model, + std::sync::Arc, +)> { + use tinycloud_core::{ + at_rest::{AtRestMode, DataKeyProvider, FixedKeyProvider, TestAtRestSource}, + database_artifacts::{ + ArtifactExpectation, DatabaseArtifactRepository, SeaOrmDatabaseArtifactRepository, + }, + RecoveringConnection, + }; + let dir = tempfile::tempdir()?; + let path = dir.path().join("probe.db"); + let db = rusqlite::Connection::open(&path)?; + db.query_row("PRAGMA journal_mode = WAL", [], |_| Ok(()))?; + db.query_row("PRAGMA wal_autocheckpoint = 0", [], |_| Ok(()))?; + db.execute( + "CREATE TABLE probe (id INTEGER PRIMARY KEY, secret TEXT NOT NULL)", + [], + )?; + let insert = "INSERT INTO probe (secret) VALUES (?1)"; + db.execute(insert, [&markers[0]])?; + db.execute(insert, [&big[0]])?; + db.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |_| Ok(()))?; + let checkpoint = std::fs::read(&path)?; + db.execute(insert, [&markers[1]])?; + db.execute(insert, [&big[1]])?; + let wal = std::fs::read(dir.path().join("probe.db-wal"))?; + drop(db); + + let url = format!("sqlite://{}?mode=rwc", dir.path().join("caps.db").display()); + let conn = RecoveringConnection::new(Database::connect(url).await?); + tinycloud_core::db::migrate(&conn).await?; + let keys: std::sync::Arc = + std::sync::Arc::new(FixedKeyProvider::new([7; 32])); + let repo = SeaOrmDatabaseArtifactRepository::new(conn.clone()).with_at_rest( + TestAtRestSource::new(AtRestMode::Encrypt, true, Some(keys.clone())), + ); + let space = "tinycloud:key:harness-selftest:default"; + repo.save( + "sql", + space, + SQL_DATABASE, + checkpoint, + ArtifactExpectation::Absent, + ) + .await?; + repo.save_delta("sql", space, SQL_DATABASE, wal, ArtifactExpectation::Any) + .await?; + let row = database_artifact::Entity::find() + .one(&conn) + .await? + .context("the sealed artifact row")?; + ensure!( + row.payload_encoding == Some(1) && row.delta_encoding == Some(1), + "harness self-test: the fixture row is not sealed" + ); + Ok((row, keys)) +} + +/// The SQL check must catch what a payload heuristic can't: a sealed row +/// whose parts were extended with random bytes, or whose WAL delta was +/// dropped (delta columns cleared, the unkeyed hash and size reset), still +/// looks like ciphertext. Each must give the SQL check a finding (RED); the +/// untouched row gives none. +#[tokio::test] +async fn selftest_sql_check_flags_appended_and_omitted_sealed_parts() -> Result<()> { + let markers = [marker("SQL-WRITE1"), marker("SQL-WRITE2")]; + let big = [ + repeated(&markers[0], BIG_VALUE_LEN), + repeated(&markers[1], BIG_VALUE_LEN), + ]; + let big_len = big[0].len().min(big[1].len()); + let (row, keys) = sealed_sql_row(&markers, &big).await?; + let expected = [ + markers[0].clone(), + big[0].clone(), + markers[1].clone(), + big[1].clone(), + ]; + let findings = sql_row_findings( + std::slice::from_ref(&row), + &markers, + big_len, + keys.as_ref(), + &expected, + ) + .await?; + ensure!( + findings.is_empty(), + "the untouched sealed row has findings: {findings:?}" + ); + type Edit = fn(&mut database_artifact::Model); + let cases: [(&str, Edit); 3] = [ + ("random bytes appended to the checkpoint", |row| { + row.payload.extend(random_bytes(4096)) + }), + ("random bytes appended to the delta", |row| { + row.delta_payload + .as_mut() + .expect("the fixture has a delta") + .extend(random_bytes(4096)) + }), + ("the delta dropped", |row| { + row.delta_payload = None; + row.delta_encoding = None; + row.delta_content_hash = None; + row.delta_size_bytes = 0; + row.content_hash = row.checkpoint_content_hash.clone(); + row.size_bytes = row.checkpoint_size_bytes; + row.storage_mode = "database-blob".into(); + }), + ]; + for (what, edit) in cases { + let mut tampered = row.clone(); + edit(&mut tampered); + let findings = + sql_row_findings(&[tampered], &markers, big_len, keys.as_ref(), &expected).await?; + ensure!( + !findings.is_empty(), + "{what}: the SQL check found nothing, so it would report GREEN" + ); + println!("{what}: {findings:?}"); + } + Ok(()) +}