feat(TC-887): node wiring for at-rest keys, with a root from the environment

docs/at-rest/INTERFACES.md

@@ -1,9 +1,9 @@
 # Encryption at rest: interfaces (TC-887)
 
-> **Status (TC-887):** on `main` today: the envelope, key types, `DataKeyProvider` and the
-> harness. Not on `main` yet: key continuity (`docs/at-rest-escrow.md`, `at_rest::keyring`,
-> node wiring) and sealed KV storage (§7). They land with PRs #325–#327 and #329/#333/#336
-> (#336: the S3 store seals KV blocks).
+> **Status (TC-887):** the envelope, key types, `DataKeyProvider`, the keyring
+> (`at_rest::keyring`), the node wiring (`docs/at-rest-escrow.md`) and the storage view
+> (`at_rest::{AtRestMode, AtRestView, AtRestSource}`) are in. Sealed KV storage (§7) and sealed
+> SQL artifacts follow; until both seal, the node refuses `encrypt` and `strict`.
 
 Normative interface and format for `tinycloud_core::at_rest` and the node's sealed KV block
 storage. Plan: TC-887 (encryption at rest inside the TEE). Key continuity and escrow:
@@ -65,10 +65,19 @@ pub trait DataKeyProvider: Send + Sync {
     /// by deliberate crypto-shred).
     async fn key(&self, space: &str, dek_id: DekId) -> Result<DataKey, DataKeyError>;
 
+    /// Before work that may read `space`'s sealed blobs under the invoke write
+    /// lock: put back any of the space's keys the database lost (for example
+    /// from an authenticated mirror). Never creates a key.
+    async fn prepare_read(&self, space: &str) -> Result<(), DataKeyError>;
+
     /// NK for sealed S3 object names. Same value for the life of the deployment.
     fn object_name_key(&self) -> Result<ObjectNameKey, DataKeyError>;
 }
 
+/// Runs `f` as work under the invoke write lock: providers make no key-store
+/// writes there; a key that would need creating or restoring is `NotPrepared`.
+pub async fn write_locked<F: Future>(f: F) -> F::Output;
+
 #[derive(Debug, thiserror::Error)]
 pub enum DataKeyError {
     /// Node booted locked (W cannot unseal ARK). Callers map to HTTP 503.
@@ -81,6 +90,9 @@ pub enum DataKeyError {
     /// PG/S3/unwrap failure (incl. a wrapped DEK failing AEAD). Opaque, no key bytes.
     #[error("data key backend: {0}")]
     Backend(#[source] Box<dyn std::error::Error + Send + Sync>),
+    /// A key needed creating or restoring inside `write_locked`. Callers map to 503.
+    #[error("the space's data key was not prepared before the write lock")]
+    NotPrepared,
 }
 ```
 

tinycloud-at-rest-escrow/src/compose.rs

@@ -360,20 +360,21 @@ impl AppCompose {
             .collect()
     }
 
-    /// The literal value of `key` in the services' `environment`, if set.
-    ///
-    /// Refused: a value that docker compose would interpolate (`$`), a key
-    /// passed through from the runtime environment (`- KEY` or `KEY:`
-    /// with no value), and different values in different services.
-    pub fn literal_env(&self, key: &str) -> Result<Option<String>> {
-        let mut found: Option<String> = None;
+    /// Every setting of `key` in the services' `environment`, one per service
+    /// that sets it (`KEY: value`, or `- KEY=value` in list form). A value
+    /// that is empty or a YAML null, and a list entry without `=`, pass the
+    /// variable through from the runtime environment: [`EnvSetting::FromRuntime`].
+    /// Errors (never quoting a value): a non-string setting, a key set twice
+    /// in one service, an environment that is not a mapping or list.
+    pub fn env_settings(&self, key: &str) -> Result<Vec<EnvSetting>> {
+        let mut settings = Vec::new();
         for (service, fields) in services(&self.top)? {
             let service = escape(service);
             let value = match get(fields, "environment") {
                 None => None,
                 Some(Node::Map(env)) => match get(env, key) {
                     None => None,
-                    Some(Node::Scalar(v)) => Some(v.clone()),
+                    Some(Node::Scalar(v)) => Some(Some(v.clone())),
                     Some(_) => {
                         return err(format!("service {service:?} sets {key} to a non-string"))
                     }
@@ -387,16 +388,12 @@ impl AppCompose {
                             ));
                         };
                         let (k, v) = match item.split_once('=') {
-                            Some((k, v)) => (k, Some(v)),
+                            Some((k, v)) => (k, Some(v.to_owned())),
                             None => (item.as_str(), None),
                         };
                         if k == key {
                             ensure!(value.is_none(), "service {service:?} sets {key} twice");
-                            value = Some(v.map(str::to_owned).ok_or_else(|| {
-                                EscrowError::Compose(format!(
-                                    "service {service:?} passes {key} through from the runtime environment; it must be a literal value"
-                                ))
-                            })?);
+                            value = Some(v);
                         }
                     }
                     value
@@ -410,10 +407,32 @@ impl AppCompose {
             let Some(value) = value else { continue };
             // Empty or a YAML 1.2 null (any spelling docker compose reads as
             // null) passes the variable through from the runtime.
-            ensure!(
-                !matches!(value.as_str(), "" | "~" | "null" | "Null" | "NULL"),
-                "service {service:?} passes {key} through from the runtime environment; it must be a literal value"
-            );
+            settings.push(match value {
+                Some(v) if !matches!(v.as_str(), "" | "~" | "null" | "Null" | "NULL") => {
+                    EnvSetting::Value { service, value: v }
+                }
+                _ => EnvSetting::FromRuntime { service },
+            });
+        }
+        Ok(settings)
+    }
+
+    /// The literal value of `key` in the services' `environment`, if set.
+    ///
+    /// Refused: a value that docker compose would interpolate (`$`), a key
+    /// passed through from the runtime environment (`- KEY` or `KEY:`
+    /// with no value), and different values in different services.
+    pub fn literal_env(&self, key: &str) -> Result<Option<String>> {
+        let mut found: Option<String> = None;
+        for setting in self.env_settings(key)? {
+            let (service, value) = match setting {
+                EnvSetting::FromRuntime { service } => {
+                    return err(format!(
+                        "service {service:?} passes {key} through from the runtime environment; it must be a literal value"
+                    ))
+                }
+                EnvSetting::Value { service, value } => (service, value),
+            };
             ensure!(
                 !value.contains('$'),
                 "service {service:?} sets {key} by interpolation; it must be a literal value"
@@ -427,6 +446,24 @@ impl AppCompose {
     }
 }
 
+/// One service's setting of an environment variable ([`AppCompose::env_settings`]).
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum EnvSetting {
+    /// `KEY: value` or `- KEY=value`, exactly as written (before docker
+    /// compose interpolation). `service` is escaped for display.
+    Value { service: String, value: String },
+    /// The variable is passed through from the runtime environment.
+    FromRuntime { service: String },
+}
+
+impl EnvSetting {
+    pub fn service(&self) -> &str {
+        match self {
+            Self::Value { service, .. } | Self::FromRuntime { service } => service,
+        }
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

tinycloud-core/src/at_rest/mod.rs

@@ -1,5 +1,5 @@
-//! Encryption at rest (TC-887). Nothing in the node calls this module yet; it
-//! is switched off until the key-continuity gate exists.
+//! Encryption at rest (TC-887): the TCE1 envelope, the keyring, and the
+//! view storage reads of the node's at-rest state. Off unless configured.
 
 mod envelope;
 mod hierarchy;
@@ -13,6 +13,7 @@ mod stream;
 mod testing;
 #[cfg(test)]
 mod tests;
+mod view;
 
 pub use envelope::{
     ciphertext_len, format_version, open, plaintext_len, seal, BlobIdentity, EnvelopeError, Header,
@@ -27,3 +28,4 @@ pub use testing::FixedKeyProvider;
 
 pub use keyring::{AtRestKeyring, KeyringConfig, KeyringState};
 pub use store::{AtRestStore, KeyMirror};
+pub use view::{AtRestMode, AtRestSource, AtRestView};

tinycloud-core/src/at_rest/view.rs

@@ -0,0 +1,68 @@
+//! What storage sees of the node's at-rest state, per operation (TC-887).
+//!
+//! The node owns the keyring and the mode; block stores and the SQL artifact
+//! repository read a fresh [`AtRestView`] from an [`AtRestSource`] for each
+//! operation, because keys arrive after migrations and may unlock later.
+//!
+//! Rules every store follows (`docs/at-rest/INTERFACES.md`):
+//! - mode `off` writes plaintext exactly as before;
+//! - any other mode seals iff [`AtRestView::encrypting`], else the write fails
+//!   (503), never a plaintext fallback;
+//! - sealed data is read whenever keys exist; legacy plaintext is read unless
+//!   the mode is `strict`.
+
+use std::sync::Arc;
+
+use serde::{Deserialize, Serialize};
+
+use super::{DataKeyProvider, ObjectNameKey};
+
+/// `at_rest.mode`.
+#[derive(Serialize, Deserialize, Debug, Default, Clone, Copy, Hash, PartialEq, Eq)]
+#[serde(rename_all = "lowercase")]
+pub enum AtRestMode {
+    /// New writes are plaintext; sealed data is still read when keys exist.
+    #[default]
+    Off,
+    /// New writes are sealed; legacy plaintext is still read.
+    Encrypt,
+    /// New writes are sealed; legacy plaintext is refused.
+    Strict,
+}
+
+/// A snapshot of the node's at-rest state for one storage operation.
+#[derive(Clone)]
+pub struct AtRestView {
+    pub mode: AtRestMode,
+    /// The gate passed and the keys are unlocked: new writes are sealed.
+    pub encrypting: bool,
+    /// Set once the keyring is installed (locked or not).
+    pub provider: Option<Arc<dyn DataKeyProvider>>,
+}
+
+impl AtRestView {
+    /// The state of a node without at-rest wiring: `off`, no keys.
+    pub fn off() -> Self {
+        Self {
+            mode: AtRestMode::Off,
+            encrypting: false,
+            provider: None,
+        }
+    }
+
+    /// The object-name key, if keys are set up and unlocked.
+    pub fn object_name_key(&self) -> Option<ObjectNameKey> {
+        self.provider.as_ref()?.object_name_key().ok()
+    }
+
+    /// Whether legacy plaintext may be read (every mode but `strict`).
+    pub fn legacy_readable(&self) -> bool {
+        self.mode != AtRestMode::Strict
+    }
+}
+
+/// Where storage reads its at-rest state from: the node's runtime, or a
+/// test double.
+pub trait AtRestSource: Send + Sync {
+    fn view(&self) -> AtRestView;
+}

tinycloud-node-server/Cargo.toml

@@ -59,6 +59,9 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_with = { version = "3.0", features = ["base64", "hex"] }
 sha2 = "0.10"
+parking_lot = "0.12"
+lru = "0.12"
+zeroize = "1"
 serde_ipld_dagcbor.workspace = true
 toml = "0.8"
 thiserror = "2.0"
@@ -107,6 +110,10 @@ features = ["sqlite", "postgres", "mysql", "tokio"]
 [dependencies.tinycloud-auth]
 path = "../tinycloud-auth/"
 
+# TC-887: escrow bundle and recovery wire types for the at-rest routes.
+[dependencies.tinycloud-at-rest-escrow]
+path = "../tinycloud-at-rest-escrow/"
+
 # TC-409: only for test builds, enable tinycloud-core's opt-in
 # verification-count test hook so node route tests can assert exactly-once
 # signature verification. Cargo only activates dev-dependency features when
@@ -119,6 +126,8 @@ default-features = false
 features = ["sqlite", "postgres", "mysql", "tokio", "verification-count-test-hook", "test-util"]
 
 [dev-dependencies]
+# TC-887: escrow holders decrypting their shares in tests/at_rest_ceremony.rs.
+age = { version = "=0.12.1", default-features = false, features = ["armor"] }
 # Parse production Rust items in the DID resolver source guard (tests only).
 syn = { version = "2", features = ["full", "visit"] }
 quote = "1"