test(TC-887): the at-rest harness runs plaintext and encrypt legs

.github/workflows/rust.yml

@@ -373,17 +373,21 @@ jobs:
   # containers). MinIO/mc are pinned community releases whose sha256 is pinned
   # in harness.sh; PostgreSQL is the runner image's own native install. The
   # green checks prove KV puts land in MinIO and SQL writes land in
-  # database_artifact (checkpoint and WAL delta). The RED checks assert
-  # encryption at rest; until TC-887 S3/S4 land, the step passes only if both
-  # fail with their "TC-887 RED:" findings (`--expect red`), and fails loudly
-  # if they error or unexpectedly pass. When S3 lands, start the harness with
-  # the at-rest mode set and expect `kv=green,sql=red`; when S4 lands, `green`.
+  # database_artifact (checkpoint and WAL delta). The TC-887 checks run in two
+  # legs: plaintext (the default node) must find plaintext (`--expect red`,
+  # failing loudly if a check errors or unexpectedly passes), and encrypt (mode
+  # encrypt, root from the environment) must find KV blocks and SQL artifacts
+  # sealed (`--expect green`).
   at-rest-harness:
-    name: at-rest harness (native PostgreSQL + MinIO)
+    name: at-rest harness (native PostgreSQL + MinIO, ${{ matrix.at_rest }})
     runs-on: ubuntu-latest
     timeout-minutes: 45
     permissions:
       contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        at_rest: [plaintext, encrypt]
     steps:
     - name: Checkout TinyCloud repository
       uses: actions/checkout@v4
@@ -399,7 +403,7 @@ jobs:
       with:
         shared-key: rust-1.97.1-at-rest-harness
         workspaces: . -> target
-        save-if: ${{ github.ref == 'refs/heads/main' }}
+        save-if: ${{ github.ref == 'refs/heads/main' && matrix.at_rest == 'plaintext' }}
 
     - name: Harness paths
       run: |
@@ -413,26 +417,36 @@ jobs:
       run: bash scripts/at-rest-harness/harness.sh bins
 
     - name: Start the harness (native PostgreSQL, MinIO, tinycloud-node)
+      env:
+        TC887_AT_REST_MODE: ${{ matrix.at_rest == 'encrypt' && 'encrypt' || '' }}
+        TC887_AT_REST_ROOT_SOURCE: ${{ matrix.at_rest == 'encrypt' && 'env' || '' }}
       run: bash scripts/at-rest-harness/harness.sh up
 
     - name: Harness checks (writes reach MinIO and PostgreSQL)
       run: bash scripts/at-rest-harness/harness.sh test green "$TC887_STATE"
 
-    - name: TC-887 RED checks (must be RED until S3/S4)
-      run: bash scripts/at-rest-harness/harness.sh test red --expect red "$TC887_STATE"
+    - name: TC-887 checks (RED when plaintext, GREEN when encrypting)
+      run: >-
+        bash scripts/at-rest-harness/harness.sh test red
+        --expect ${{ matrix.at_rest == 'encrypt' && 'green' || 'red' }} "$TC887_STATE"
 
+    # These write their own objects into the harness bucket, so they run
+    # after the checks above, and in one leg only (they don't use the node).
     - name: S3 block store (against the harness MinIO)
+      if: matrix.at_rest == 'plaintext'
       run: |
         (set -a; . "$TC887_RUN_DIR/harness.env"; set +a
          cargo test --locked -p tinycloud-node --test s3_block_store -- --ignored)
 
     - name: S3 block store with encryption at rest (against the harness MinIO)
+      if: matrix.at_rest == 'plaintext'
       timeout-minutes: 10
       run: |
         (set -a; . "$TC887_RUN_DIR/harness.env"; set +a
          cargo test --locked -p tinycloud-node --test at_rest_s3_store -- --ignored)
 
     - name: SQL object store (against the harness MinIO)
+      if: matrix.at_rest == 'plaintext'
       timeout-minutes: 10
       run: |
         (set -a; . "$TC887_RUN_DIR/harness.env"; set +a

scripts/at-rest-harness/README.md

@@ -69,10 +69,10 @@ exits 0 only if every check matches:
 | 4 | a check errored: compile failure, harness failure, a precondition (`harness no-op`/`harness gap`) or a node error, i.e. not a RED finding (per check; wins over 3) |
 
 Defaults: `test green` expects `kv=green,sql=green`. `test red` expects
-`kv=red,sql=red`, or `kv=green,sql=red` if the run was started with
-`TC887_AT_REST_MODE=encrypt` or `strict`, because S3 seals KV before S4 seals
-SQL. When S4 lands, the SQL default flips to green (`default_expect` in
-`harness.sh`). CI runs `test red --expect red` until S3 lands.
+`kv=red,sql=red`, or `kv=green,sql=green` if the run was started with
+`TC887_AT_REST_MODE=encrypt` or `strict` (KV blocks and SQL artifacts both
+seal). CI runs both: a plaintext leg with `--expect red` and an encrypt leg
+(root from the environment) with `--expect green`.
 
 ## At-rest mode
 
@@ -83,11 +83,15 @@ the top of `harness.sh`:
 |---|---|
 | `TC887_AT_REST_MODE` (`off`/`encrypt`/`strict`) | `TINYCLOUD_AT_REST__MODE` |
 | `TC887_AT_REST_ESCROW_NONE` (`true`/`false`: the Static self-host acknowledgement that no escrow exists, plan §5) | `TINYCLOUD_AT_REST__ESCROW_NONE` |
+| `TC887_AT_REST_ROOT_SOURCE` (`escrow`/`env`) | `TINYCLOUD_AT_REST__ROOT_SOURCE`; with `env`, also `TINYCLOUD_AT_REST_ROOT_KEY` from a per-run key in `secrets/at-rest-root-key` |
 
-If either is unset, the harness sets nothing and the node runs its default
-(today: no encryption at rest). Other values are refused before anything
-starts. The simplest encrypting node is the harness's Static root plus
-`TC887_AT_REST_MODE=encrypt TC887_AT_REST_ESCROW_NONE=true`.
+If one is unset, the harness sets nothing for it and the node runs its default
+(no encryption at rest). Other values are refused before anything starts. The
+checks re-derive the key hierarchy from the root the node seals under: the
+Static node root, or with `env` the per-run root key (`TC887_NODE_ROOT_FILE`
+and `TC887_NODE_ROOT_ENCODING` in `harness.env`). Simplest encrypting nodes:
+`TC887_AT_REST_MODE=encrypt` with `TC887_AT_REST_ESCROW_NONE=true`, or with
+`TC887_AT_REST_ROOT_SOURCE=env`.
 
 ## Checks
 
@@ -108,7 +112,7 @@ then insert (a WAL delta).
 | `green_kv_put_reaches_minio` | every put round-trips through the node and adds a non-empty object to the bucket | pass |
 | `green_sql_write_reaches_postgres` | the writes round-trip, and `database_artifact` has a `payload` and a non-NULL `delta_payload` | pass |
 | `red_kv_blocks_are_sealed_in_minio` | object keys contain no identity of any space the harness created (space id or wallet address: raw, lowercase, hex, base64 and base64url at any alignment, of the text and of the bytes) and no blake3 fingerprint of the value (b64url padded/unpadded, base64 padded/unpadded, hex; raw digest and multihash); raw bytes hold no marker; each put's objects together are at least as large as the value; the 64 KiB object doesn't compress (zlib ratio > 0.95); every object of a put in one space shares no 32-byte run with every object of the same put in the other (equal object counts required); every `_tc/` key-mirror object passes the checks below | **fail** |
-| `red_sql_artifacts_are_sealed_in_postgres` | `payload` doesn't start with `SQLite format 3\0` and `delta_payload` doesn't start with the WAL magic; neither holds a marker, compresses (ratio > 0.95) or is smaller than the 64 KiB value it holds; the two spaces' payloads and deltas share no 32-byte run | **fail** |
+| `red_sql_artifacts_are_sealed_in_postgres` | `payload` doesn't start with `SQLite format 3\0` and `delta_payload` doesn't start with the WAL magic; neither holds a marker, compresses (ratio > 0.95) or is smaller than the 64 KiB value it holds; the two spaces' payloads and deltas share no 32-byte run; and an authenticated cold read of every row succeeds (below) | **fail** |
 
 The RED checks re-run the green preconditions first, so a RED result is never
 a harness no-op. The 32-byte limit: two sealed blobs of the same plaintext
@@ -123,6 +127,22 @@ across writes once a per-write salt (SQLite's WAL salts) is mixed in.
 flagged, a salted deflate-9 blob is flagged by the size floor, and sealed-like
 random blobs pass.
 
+**SQL cold read.** The payload checks above are heuristics: random bytes
+appended to a sealed part, or a dropped WAL delta, still look like
+ciphertext. So the SQL check also opens every `database_artifact` row of the
+harness spaces the way a node hydrating it would, with keys the harness
+re-derived itself (root → ARK → KEK → each space's DEK from its
+authenticated `_tc/` mirror, as the KV check does): the row must be sealed
+(`payload_encoding`/`delta_encoding` 1 and a `manifest` tag),
+`database_artifacts::open_row` must authenticate the manifest tag and every
+part (AEAD and exact stored length), and the checkpoint plus WAL must be a
+SQLite database that passes `PRAGMA integrity_check` and holds exactly the
+four `probe` rows the harness wrote. Anything else is a finding (RED).
+`selftest_sql_check_flags_appended_and_omitted_sealed_parts` runs in plain
+`cargo test`: a sealed fixture row passes, and random bytes appended to its
+checkpoint or delta, or its delta dropped (with the unkeyed hash and size
+reset), each turn the SQL check RED.
+
 ### At-rest key mirror (`_tc/…` objects)
 
 With at-rest keys on, the node writes control objects next to its blocks. The
@@ -230,3 +250,8 @@ bypass), `selftest_key_mirror_refuses_escrow_contents_and_derived_keys`,
   installed here, so the node runs with a per-run Static root; S1 adds a
   Dstack-root variant.
 - DuckDB artifacts (`service=duckdb`, an opt-in feature): S4 follow-up.
+- The node's local working files (SQL and DuckDB working and cache files,
+  upload staging, exports) are not scanned: they are plaintext by design and
+  rely on the CVM's encrypted storage (`docs/at-rest/INTERFACES.md` §8).
+- A row replaced by an older, self-consistent copy of itself passes the SQL
+  cold read, as it passes the node's own read (INTERFACES §8).

scripts/at-rest-harness/harness.sh

@@ -39,14 +39,19 @@ MC_URL=https://github.com/minio/mc/releases/download/$MC_RELEASE/mc.linux-amd64.
 #   TC887_AT_REST_MODE        -> TINYCLOUD_AT_REST__MODE         (off|encrypt|strict)
 #   TC887_AT_REST_ESCROW_NONE -> TINYCLOUD_AT_REST__ESCROW_NONE  (true|false: the Static
 #                                self-host acknowledgement that no escrow exists, plan §5)
-# Unset: the harness sets nothing and the node runs its default (today: no
-# encryption at rest). Simplest encrypting node: Static root (always, here) +
-# TC887_AT_REST_MODE=encrypt + TC887_AT_REST_ESCROW_NONE=true. With mode
-# encrypt/strict, `test red` defaults to expecting kv=green,sql=red: S3 seals
-# KV before S4 seals SQL. When S4 lands, flip the SQL default to green
-# (default_expect below).
+#   TC887_AT_REST_ROOT_SOURCE -> TINYCLOUD_AT_REST__ROOT_SOURCE  (escrow|env). With env,
+#                                `up` also generates a per-run root key (standard base64
+#                                of 32 bytes) into secrets/at-rest-root-key and passes it
+#                                as TINYCLOUD_AT_REST_ROOT_KEY; the checks derive from it.
+# Unset: the harness sets nothing and the node runs its default (no encryption
+# at rest). Simplest encrypting nodes: TC887_AT_REST_MODE=encrypt with either
+# TC887_AT_REST_ESCROW_NONE=true (Static node root) or
+# TC887_AT_REST_ROOT_SOURCE=env. With mode encrypt/strict, `test red` defaults
+# to expecting both checks green (default_expect below).
 AT_REST_MODE_VAR=TINYCLOUD_AT_REST__MODE
 AT_REST_ESCROW_NONE_VAR=TINYCLOUD_AT_REST__ESCROW_NONE
+AT_REST_ROOT_SOURCE_VAR=TINYCLOUD_AT_REST__ROOT_SOURCE
+AT_REST_ROOT_KEY_VAR=TINYCLOUD_AT_REST_ROOT_KEY
 # ---------------------------------------------------------------------------
 
 BUCKET=tinycloud-blocks
@@ -251,10 +256,13 @@ cmd_up() {
   command -v python3 >/dev/null || die "python3 is required"
   command -v curl >/dev/null || die "curl is required"
   local at_rest_mode=${TC887_AT_REST_MODE:-} at_rest_escrow_none=${TC887_AT_REST_ESCROW_NONE:-}
+  local at_rest_root_source=${TC887_AT_REST_ROOT_SOURCE:-}
   [[ -z "$at_rest_mode" ]] || valid_mode "$at_rest_mode" \
     || die "TC887_AT_REST_MODE must be off, encrypt or strict"
   [[ -z "$at_rest_escrow_none" ]] || valid_bool "$at_rest_escrow_none" \
     || die "TC887_AT_REST_ESCROW_NONE must be true or false"
+  [[ -z "$at_rest_root_source" ]] || [[ "$at_rest_root_source" =~ ^(escrow|env)$ ]] \
+    || die "TC887_AT_REST_ROOT_SOURCE must be escrow or env"
   cmd_bins
 
   local pg_bin
@@ -295,13 +303,22 @@ cmd_up() {
   # Recorded by every write_state call, including a failed up's.
   export STATE_PG_BIN=$pg_bin STATE_NODE_BIN=$node_bin STATE_PG_PORT=$pg_port \
     STATE_MINIO_PORT=$minio_port STATE_CONSOLE_PORT=$console_port STATE_NODE_PORT=$node_port \
-    STATE_AT_REST_MODE=$at_rest_mode STATE_AT_REST_ESCROW_NONE=$at_rest_escrow_none
+    STATE_AT_REST_MODE=$at_rest_mode STATE_AT_REST_ESCROW_NONE=$at_rest_escrow_none \
+    STATE_AT_REST_ROOT_SOURCE=$at_rest_root_source
 
   # Per-run secrets: generated here, kept in 0600 files, never on argv.
   random_hex | secret_file "$run/secrets/pg-password"
   printf 'tc887%s\n' "$(python3 -c 'import secrets; print(secrets.token_hex(8))')" | secret_file "$run/secrets/minio-user"
   random_hex | secret_file "$run/secrets/minio-password"
   python3 -c 'import secrets; print(secrets.token_urlsafe(32))' | secret_file "$run/secrets/node-static-secret"
+  # The root the at-rest keys are sealed under, and how the checks decode it.
+  local root_file=$run/secrets/node-static-secret root_encoding=base64url at_rest_root_key=
+  if [[ "$at_rest_root_source" == env ]]; then
+    python3 -c 'import base64, secrets; print(base64.b64encode(secrets.token_bytes(32)).decode())' \
+      | secret_file "$run/secrets/at-rest-root-key"
+    root_file=$run/secrets/at-rest-root-key root_encoding=base64
+    at_rest_root_key=$(<"$root_file")
+  fi
   local pg_password minio_user minio_password node_secret
   pg_password=$(<"$run/secrets/pg-password")
   minio_user=$(<"$run/secrets/minio-user")
@@ -322,7 +339,8 @@ TC887_S3_BUCKET=$BUCKET
 TC887_S3_REGION=$REGION
 TC887_S3_ACCESS_KEY_ID=$minio_user
 TC887_S3_SECRET_ACCESS_KEY=$minio_password
-TC887_NODE_ROOT_FILE=$run/secrets/node-static-secret
+TC887_NODE_ROOT_FILE=$root_file
+TC887_NODE_ROOT_ENCODING=$root_encoding
 TC887_SPACES_LOG=$run/spaces.log
 TC887_PUTS_LOG=$run/puts.log
 TC887_BOOT_SNAPSHOT=$run/boot-snapshot
@@ -361,6 +379,14 @@ EOF
     printf '%s=%s\n' "$AT_REST_ESCROW_NONE_VAR" "$at_rest_escrow_none" >>"$run/node.env"
     log "at-rest: $AT_REST_ESCROW_NONE_VAR=$at_rest_escrow_none"
   fi
+  if [[ -n "$at_rest_root_source" ]]; then
+    printf '%s=%s\n' "$AT_REST_ROOT_SOURCE_VAR" "$at_rest_root_source" >>"$run/node.env"
+    log "at-rest: $AT_REST_ROOT_SOURCE_VAR=$at_rest_root_source"
+  fi
+  if [[ -n "$at_rest_root_key" ]]; then
+    printf '%s=%s\n' "$AT_REST_ROOT_KEY_VAR" "$at_rest_root_key" >>"$run/node.env"
+    log "at-rest: $AT_REST_ROOT_KEY_VAR set from secrets/at-rest-root-key"
+  fi
   printf '[global]\nlog_level = "normal"\n' >"$run/tinycloud.toml"
   write_state "$run" starting
 
@@ -467,7 +493,8 @@ write_state() {
     NODE_PID=$UP_NODE_PID NODE_START=$UP_NODE_START \
     BIN_DIR=$BIN_DIR MINIO_RELEASE=$MINIO_RELEASE MC_RELEASE=$MC_RELEASE \
     BUCKET=$BUCKET REGION=$REGION PG_USER=$PG_USER PG_DB=$PG_DB \
-    AT_REST_MODE_VAR=$AT_REST_MODE_VAR AT_REST_ESCROW_NONE_VAR=$AT_REST_ESCROW_NONE_VAR python3 - <<'PY'
+    AT_REST_MODE_VAR=$AT_REST_MODE_VAR AT_REST_ESCROW_NONE_VAR=$AT_REST_ESCROW_NONE_VAR \
+    AT_REST_ROOT_SOURCE_VAR=$AT_REST_ROOT_SOURCE_VAR python3 - <<'PY'
 import json, os, datetime
 e = os.environ
 run = e["RUN"]
@@ -488,8 +515,10 @@ state = {
     "at_rest": {
         "mode": text("STATE_AT_REST_MODE"),
         "escrow_none": {"true": True, "false": False}.get(e.get("STATE_AT_REST_ESCROW_NONE", "")),
+        "root_source": text("STATE_AT_REST_ROOT_SOURCE"),
         "mode_var": e["AT_REST_MODE_VAR"],
         "escrow_none_var": e["AT_REST_ESCROW_NONE_VAR"],
+        "root_source_var": e["AT_REST_ROOT_SOURCE_VAR"],
     },
     "node": {
         "pid": num("NODE_PID"),
@@ -630,8 +659,8 @@ default_expect() {
   local kind=$1 check=$2 state=$3
   [[ "$kind" == green ]] && { echo green; return; }
   case $(state_get "$state" at_rest.mode) in
-    # S3 seals KV before S4 seals SQL; flip sql to green when S4 lands.
-    encrypt|strict) [[ "$check" == kv ]] && echo green || echo red ;;
+    # Both KV blocks and SQL artifacts seal once a mode encrypts.
+    encrypt|strict) echo green ;;
     *) echo red ;;
   esac
 }
@@ -704,21 +733,21 @@ cmd_test() {
       rc=4
     elif [[ "$CHECK_OUTCOME" == "${want[$check]}" ]]; then
       if [[ "$CHECK_OUTCOME" == red ]]; then
-        annotate notice "$label RED as expected: $label data is plaintext at rest"
+        annotate notice "$label RED as expected: $label data is not sealed and authenticated at rest"
       else
         annotate notice "$label $kind check GREEN as expected"
       fi
     elif [[ "$CHECK_OUTCOME" == green ]]; then
       annotate error "$label $kind check UNEXPECTEDLY PASSED under expect $check=red: the at-rest property holds for $label; expect $check=green and update CI"
       ((rc == 4)) || rc=3
     else
-      annotate error "$label $kind check still RED ($label data is plaintext at rest) but expected $check=green"
+      annotate error "$label $kind check still RED ($label data is not sealed and authenticated at rest) but expected $check=green"
       ((rc == 4)) || rc=3
     fi
   done
   if ((rc == 0)); then
     if [[ "$kind" == red && "${want[kv]}" == red && "${want[sql]}" == red ]]; then
-      annotate notice "RED as expected: both TC-887 RED checks report plaintext at rest (KV and SQL)"
+      annotate notice "RED as expected: both TC-887 RED checks report data not sealed at rest (KV and SQL)"
     else
       annotate notice "$kind checks as expected (kv=${want[kv]}, sql=${want[sql]})"
     fi