diff --git a/adapter/encryption_admin.go b/adapter/encryption_admin.go index c35ccc894..bfbe5f99d 100644 --- a/adapter/encryption_admin.go +++ b/adapter/encryption_admin.go @@ -30,6 +30,7 @@ import ( type EncryptionAdminServer struct { sidecarPath string fullNodeID uint64 + writerRegistry encryption.WriterRegistryStore buildSHA string latestAppliedIndex func() uint64 // proposer is the raw raft proposer used for cleartext-only @@ -45,6 +46,10 @@ type EncryptionAdminServer struct { // the raft envelope. postCutoverProposer raftengine.Proposer leaderView raftengine.LeaderView + // recoveryLeaderView is the authority for sidecar/registry recovery. + // In multi-group deployments it points at the default group even when + // this server is registered on another group's listener. + recoveryLeaderView raftengine.LeaderView // capabilityFanout, when wired, runs the §4 Voters ∪ Learners // fan-out before the §7.1 Phase 1 cutover entry is proposed. // A nil value short-circuits EnableStorageEnvelope with @@ -182,6 +187,19 @@ func WithEncryptionAdminFullNodeID(id uint64) EncryptionAdminServerOption { } } +// WithEncryptionAdminWriterRegistry wires the §4.1 writer-registry +// store that read-only sidecar recovery RPCs use to project +// writer_registry_for_caller. A nil argument is a no-op so tests and +// encryption-disabled nodes keep the pre-Stage-7 empty-map posture. +func WithEncryptionAdminWriterRegistry(reg encryption.WriterRegistryStore) EncryptionAdminServerOption { + return func(s *EncryptionAdminServer) { + if reg == nil { + return + } + s.writerRegistry = reg + } +} + // WithEncryptionAdminBuildSHA overrides the auto-detected // runtime/debug build SHA. Tests use this to pin a deterministic // value; production wiring leaves it empty. @@ -258,6 +276,17 @@ func WithEncryptionAdminLeaderView(v raftengine.LeaderView) EncryptionAdminServe } } +// WithEncryptionAdminRecoveryLeaderView registers the default-group +// leadership oracle used by ResyncSidecar. A nil value preserves the +// single-group fallback to leaderView. +func WithEncryptionAdminRecoveryLeaderView(v raftengine.LeaderView) EncryptionAdminServerOption { + return func(s *EncryptionAdminServer) { + if v != nil { + s.recoveryLeaderView = v + } + } +} + // WithEncryptionAdminCutoverBarrier wires the §7.1 quiescence // barrier controller used by EnableRaftEnvelope. A nil argument is // a no-op (the server stays in the cutover-disabled posture); @@ -374,10 +403,10 @@ func (s *EncryptionAdminServer) GetCapability(_ context.Context, _ *pb.Empty) (* // pointers; the wrapped material is leakage-safe because it is // KEK-wrapped, which is the same property the on-disk sidecar has. // -// The writer_registry_for_caller map is empty until Stage 7 wires -// the registry. Callers in the §7.1 cutover path tolerate an empty -// map because the §5.6 step 1a batch is sourced from the -// GetCapability fan-out, not from this RPC. +// When the writer registry is wired, writer_registry_for_caller +// carries this node's recorded last_seen_local_epoch per sidecar DEK. +// Unwired tests and encryption-disabled nodes keep the historical +// empty non-nil map. func (s *EncryptionAdminServer) GetSidecarState(_ context.Context, _ *pb.Empty) (*pb.SidecarStateReport, error) { if s.sidecarPath == "" { return nil, grpcStatusError(codes.FailedPrecondition, "encryption: sidecar path is not configured on this node") @@ -386,6 +415,10 @@ func (s *EncryptionAdminServer) GetSidecarState(_ context.Context, _ *pb.Empty) if err != nil { return nil, statusFromSidecarErr(err) } + writerRegistry, err := s.writerRegistryForCaller(sc, s.fullNodeID, codes.Internal) + if err != nil { + return nil, err + } resp := &pb.SidecarStateReport{ ActiveStorageId: sc.Active.Storage, ActiveRaftId: sc.Active.Raft, @@ -393,7 +426,7 @@ func (s *EncryptionAdminServer) GetSidecarState(_ context.Context, _ *pb.Empty) RaftEnvelopeCutoverIndex: sc.RaftEnvelopeCutoverIndex, LatestAppliedIndex: s.appliedIndex(sc.RaftAppliedIndex), WrappedDeksById: wrappedDEKMap(sc), - WriterRegistryForCaller: map[uint32]uint32{}, + WriterRegistryForCaller: writerRegistry, } return resp, nil } @@ -410,15 +443,7 @@ func (s *EncryptionAdminServer) GetSidecarState(_ context.Context, _ *pb.Empty) // state to a recovering follower and silently overwrite recent // rotations. func (s *EncryptionAdminServer) ResyncSidecar(ctx context.Context, req *pb.ResyncSidecarRequest) (*pb.ResyncSidecarResponse, error) { - // req.CallerFullNodeId is intentionally unused for the - // recovery payload itself in PR-B; Stage 7 will use it to - // scope the writer-registry projection to that specific - // caller per §5.5. Recording it here keeps the field on the - // hot path so a future leader-side audit log can correlate - // resyncs to the requesting member without a wire-format - // change. - _ = req - if err := s.requireLeader(ctx); err != nil { + if err := s.requireRecoveryLeader(ctx); err != nil { return nil, err } if s.sidecarPath == "" { @@ -428,20 +453,62 @@ func (s *EncryptionAdminServer) ResyncSidecar(ctx context.Context, req *pb.Resyn if err != nil { return nil, statusFromSidecarErr(err) } + var callerFullNodeID uint64 + if req != nil { + callerFullNodeID = req.GetCallerFullNodeId() + } + writerRegistry, err := s.writerRegistryForCaller(sc, callerFullNodeID, codes.InvalidArgument) + if err != nil { + return nil, err + } return &pb.ResyncSidecarResponse{ WrappedDeksById: wrappedDEKMap(sc), ActiveStorageId: sc.Active.Storage, ActiveRaftId: sc.Active.Raft, LeaderLatestAppliedIndex: s.appliedIndex(sc.RaftAppliedIndex), - // §5.5 follower-repair: leader's recorded - // last_seen_local_epoch per (dek_id, caller). Stage 7 - // fills this from the writer registry. PR-A returns an - // empty non-nil map because a node recovering before - // the registry exists has nothing to re-derive. - WriterRegistryForCaller: map[uint32]uint32{}, + WriterRegistryForCaller: writerRegistry, }, nil } +func (s *EncryptionAdminServer) writerRegistryForCaller(sc *encryption.Sidecar, fullNodeID uint64, missingIDCode codes.Code) (map[uint32]uint32, error) { + out := map[uint32]uint32{} + if s.writerRegistry == nil { + return out, nil + } + if fullNodeID == 0 { + return nil, grpcStatusError(missingIDCode, + "encryption: full_node_id is required to project writer_registry_for_caller") + } + nodeID16 := encryption.NodeID16(fullNodeID) + for idStr := range sc.Keys { + dekID, err := parseSidecarKeyID(idStr) + if err != nil { + return nil, grpcStatusErrorf(codes.Internal, + "encryption: sidecar key id %q could not be projected into writer registry: %v", idStr, err) + } + raw, ok, err := s.writerRegistry.GetRegistryRow(encryption.RegistryKey(dekID, nodeID16)) + if err != nil { + return nil, grpcStatusErrorf(codes.Internal, + "encryption: read writer registry row for dek_id=%d full_node_id=%#x: %v", dekID, fullNodeID, err) + } + if !ok { + continue + } + row, err := encryption.DecodeRegistryValue(raw) + if err != nil { + return nil, grpcStatusErrorf(codes.Internal, + "encryption: decode writer registry row for dek_id=%d full_node_id=%#x: %v", dekID, fullNodeID, err) + } + if row.FullNodeID != fullNodeID { + return nil, grpcStatusErrorf(codes.Internal, + "encryption: writer registry node_id collision for dek_id=%d caller_full_node_id=%#x registry_full_node_id=%#x", + dekID, fullNodeID, row.FullNodeID) + } + out[dekID] = uint32(row.LastSeenLocalEpoch) + } + return out, nil +} + func (s *EncryptionAdminServer) appliedIndex(sidecarValue uint64) uint64 { if s.latestAppliedIndex == nil { return sidecarValue @@ -1860,11 +1927,23 @@ func proposeErrorToStatus(err error, opcode byte) error { // a follower's recovery flow could pull an outdated DEK // set from a stranded leader and miss recent rotations. func (s *EncryptionAdminServer) requireLeader(ctx context.Context) error { - if s.leaderView == nil { + return requireEncryptionLeader(ctx, s.leaderView) +} + +func (s *EncryptionAdminServer) requireRecoveryLeader(ctx context.Context) error { + view := s.recoveryLeaderView + if view == nil { + view = s.leaderView + } + return requireEncryptionLeader(ctx, view) +} + +func requireEncryptionLeader(ctx context.Context, view raftengine.LeaderView) error { + if view == nil { return nil } - if s.leaderView.State() != raftengine.StateLeader { - leader := s.leaderView.Leader() + if view.State() != raftengine.StateLeader { + leader := view.Leader() if leader.ID == "" && leader.Address == "" { return grpcStatusError(codes.FailedPrecondition, "encryption: not leader (no known leader)") } @@ -1872,7 +1951,7 @@ func (s *EncryptionAdminServer) requireLeader(ctx context.Context) error { "encryption: not leader (current leader id=%q address=%q)", leader.ID, leader.Address) } - if err := s.leaderView.VerifyLeader(ctx); err != nil { + if err := view.VerifyLeader(ctx); err != nil { return verifyLeaderErrorToStatus(err) } return nil diff --git a/adapter/encryption_admin_test.go b/adapter/encryption_admin_test.go index 659fb06eb..21718ff21 100644 --- a/adapter/encryption_admin_test.go +++ b/adapter/encryption_admin_test.go @@ -159,6 +159,59 @@ func TestEncryptionAdmin_GetSidecarState_ShipsWrappedDEKs(t *testing.T) { assertSidecarWrappedDEKs(t, got) } +func TestEncryptionAdmin_GetSidecarState_ProjectsWriterRegistryForLocalNode(t *testing.T) { + t.Parallel() + const localFullNodeID uint64 = 0xCAFE_BABE_0000_1234 + path := writeSidecarFixture(t, &encryption.Sidecar{ + RaftAppliedIndex: 42, + Active: encryption.ActiveKeys{Storage: 1, Raft: 2}, + Keys: map[string]encryption.SidecarKey{ + "1": {Purpose: "storage", Wrapped: []byte("wrapped-1")}, + "2": {Purpose: "raft", Wrapped: []byte("wrapped-2")}, + "3": {Purpose: "storage", Wrapped: []byte("wrapped-old-storage")}, + }, + }) + reg := newTestWriterRegistry() + reg.seed(t, 1, localFullNodeID, 4, 9) + reg.seed(t, 2, localFullNodeID, 5, 10) + // No row for DEK 3: the projection omits absent rows instead of + // inventing a zero epoch that could be mistaken for a real record. + srv := NewEncryptionAdminServer( + WithEncryptionAdminSidecarPath(path), + WithEncryptionAdminFullNodeID(localFullNodeID), + WithEncryptionAdminWriterRegistry(reg), + ) + + got, err := srv.GetSidecarState(context.Background(), &pb.Empty{}) + if err != nil { + t.Fatalf("GetSidecarState: %v", err) + } + want := map[uint32]uint32{1: 9, 2: 10} + if fmt.Sprint(got.WriterRegistryForCaller) != fmt.Sprint(want) { + t.Fatalf("WriterRegistryForCaller=%v, want %v", got.WriterRegistryForCaller, want) + } +} + +func TestEncryptionAdmin_GetSidecarState_RejectsMissingLocalNodeIDAsInternal(t *testing.T) { + t.Parallel() + path := writeSidecarFixture(t, &encryption.Sidecar{ + Active: encryption.ActiveKeys{Storage: 1, Raft: 2}, + Keys: map[string]encryption.SidecarKey{ + "1": {Purpose: "storage", Wrapped: []byte("wrapped-1")}, + "2": {Purpose: "raft", Wrapped: []byte("wrapped-2")}, + }, + }) + srv := NewEncryptionAdminServer( + WithEncryptionAdminSidecarPath(path), + WithEncryptionAdminWriterRegistry(newTestWriterRegistry()), + ) + + _, err := srv.GetSidecarState(context.Background(), &pb.Empty{}) + if status.Code(err) != codes.Internal { + t.Fatalf("GetSidecarState status=%v, want Internal for missing local full node id (err=%v)", status.Code(err), err) + } +} + func assertSidecarHeader(t *testing.T, got *pb.SidecarStateReport) { t.Helper() if got.ActiveStorageId != 1 || got.ActiveRaftId != 2 { @@ -184,10 +237,10 @@ func assertSidecarWrappedDEKs(t *testing.T, got *pb.SidecarStateReport) { t.Errorf("wrapped[2]=%q, want %q", got.WrappedDeksById[2], "wrapped-2") } if got.WriterRegistryForCaller == nil { - t.Errorf("WriterRegistryForCaller=nil, want empty non-nil map (PR-A contract)") + t.Errorf("WriterRegistryForCaller=nil, want empty non-nil map when registry is unwired") } if len(got.WriterRegistryForCaller) != 0 { - t.Errorf("WriterRegistryForCaller=%v, want empty in PR-A", got.WriterRegistryForCaller) + t.Errorf("WriterRegistryForCaller=%v, want empty when registry is unwired", got.WriterRegistryForCaller) } } @@ -245,14 +298,88 @@ func TestEncryptionAdmin_ResyncSidecar_ShipsWrappedDEKs(t *testing.T) { if string(got.WrappedDeksById[3]) != "ws" || string(got.WrappedDeksById[4]) != "wr" { t.Errorf("wrapped=%v, want id3=ws id4=wr", got.WrappedDeksById) } - // Mirror the GetSidecarState contract: non-nil empty map until - // Stage 7 wires the writer registry. Locks in the §5.5 promise - // so a future change to the field cannot silently degrade to nil. if got.WriterRegistryForCaller == nil { - t.Errorf("WriterRegistryForCaller=nil, want empty non-nil map (PR-A contract)") + t.Errorf("WriterRegistryForCaller=nil, want empty non-nil map when registry is unwired") } if len(got.WriterRegistryForCaller) != 0 { - t.Errorf("WriterRegistryForCaller=%v, want empty in PR-A", got.WriterRegistryForCaller) + t.Errorf("WriterRegistryForCaller=%v, want empty when registry is unwired", got.WriterRegistryForCaller) + } +} + +func TestEncryptionAdmin_ResyncSidecar_ProjectsWriterRegistryForCaller(t *testing.T) { + t.Parallel() + const callerFullNodeID uint64 = 0x1111_2222_3333_4444 + path := writeSidecarFixture(t, &encryption.Sidecar{ + RaftAppliedIndex: 17, + Active: encryption.ActiveKeys{Storage: 3, Raft: 4}, + Keys: map[string]encryption.SidecarKey{ + "3": {Purpose: "storage", Wrapped: []byte("ws")}, + "4": {Purpose: "raft", Wrapped: []byte("wr")}, + "5": {Purpose: "storage", Wrapped: []byte("old")}, + }, + }) + reg := newTestWriterRegistry() + reg.seed(t, 3, callerFullNodeID, 1, 6) + reg.seed(t, 4, callerFullNodeID, 2, 8) + srv := NewEncryptionAdminServer( + WithEncryptionAdminSidecarPath(path), + WithEncryptionAdminLeaderView(stubLeaderView{state: raftengine.StateLeader}), + WithEncryptionAdminWriterRegistry(reg), + ) + + got, err := srv.ResyncSidecar(context.Background(), &pb.ResyncSidecarRequest{CallerFullNodeId: callerFullNodeID}) + if err != nil { + t.Fatalf("ResyncSidecar: %v", err) + } + want := map[uint32]uint32{3: 6, 4: 8} + if fmt.Sprint(got.WriterRegistryForCaller) != fmt.Sprint(want) { + t.Fatalf("WriterRegistryForCaller=%v, want %v", got.WriterRegistryForCaller, want) + } +} + +func TestEncryptionAdmin_ResyncSidecar_RejectsMissingCallerNodeIDAsInvalidArgument(t *testing.T) { + t.Parallel() + path := writeSidecarFixture(t, &encryption.Sidecar{ + Active: encryption.ActiveKeys{Storage: 3, Raft: 4}, + Keys: map[string]encryption.SidecarKey{ + "3": {Purpose: "storage", Wrapped: []byte("ws")}, + "4": {Purpose: "raft", Wrapped: []byte("wr")}, + }, + }) + srv := NewEncryptionAdminServer( + WithEncryptionAdminSidecarPath(path), + WithEncryptionAdminLeaderView(stubLeaderView{state: raftengine.StateLeader}), + WithEncryptionAdminWriterRegistry(newTestWriterRegistry()), + ) + + _, err := srv.ResyncSidecar(context.Background(), &pb.ResyncSidecarRequest{}) + if status.Code(err) != codes.InvalidArgument { + t.Fatalf("ResyncSidecar status=%v, want InvalidArgument for missing caller full node id (err=%v)", status.Code(err), err) + } +} + +func TestEncryptionAdmin_ResyncSidecar_RejectsWriterRegistryNodeCollision(t *testing.T) { + t.Parallel() + const callerFullNodeID uint64 = 0x10001 + const collidingFullNodeID uint64 = 0x20001 + path := writeSidecarFixture(t, &encryption.Sidecar{ + Active: encryption.ActiveKeys{Storage: 7, Raft: 8}, + Keys: map[string]encryption.SidecarKey{ + "7": {Purpose: "storage", Wrapped: []byte("ws")}, + "8": {Purpose: "raft", Wrapped: []byte("wr")}, + }, + }) + reg := newTestWriterRegistry() + reg.seed(t, 7, collidingFullNodeID, 1, 2) + srv := NewEncryptionAdminServer( + WithEncryptionAdminSidecarPath(path), + WithEncryptionAdminLeaderView(stubLeaderView{state: raftengine.StateLeader}), + WithEncryptionAdminWriterRegistry(reg), + ) + + _, err := srv.ResyncSidecar(context.Background(), &pb.ResyncSidecarRequest{CallerFullNodeId: callerFullNodeID}) + if status.Code(err) != codes.Internal { + t.Fatalf("ResyncSidecar status=%v, want Internal for writer-registry node collision (err=%v)", status.Code(err), err) } } @@ -873,6 +1000,24 @@ func TestEncryptionAdmin_ResyncSidecar_RejectsStaleLeader(t *testing.T) { } } +func TestEncryptionAdmin_ResyncSidecar_UsesRecoveryLeaderView(t *testing.T) { + t.Parallel() + srv := NewEncryptionAdminServer( + WithEncryptionAdminLeaderView(stubLeaderView{state: raftengine.StateLeader}), + WithEncryptionAdminRecoveryLeaderView(stubLeaderView{ + state: raftengine.StateFollower, + leader: raftengine.LeaderInfo{ID: "default-leader", Address: "n2:50051"}, + }), + ) + _, err := srv.ResyncSidecar(context.Background(), &pb.ResyncSidecarRequest{}) + if status.Code(err) != codes.FailedPrecondition { + t.Fatalf("ResyncSidecar status=%v, want FailedPrecondition from default-group recovery view", status.Code(err)) + } + if !strings.Contains(err.Error(), "default-leader") { + t.Fatalf("ResyncSidecar error %q does not identify the default-group leader", err) + } +} + // TestEncryptionAdmin_RotateDEK_VerifyLeader_PreservesContextCodes // pins the context-code mapping: when VerifyLeader returns // context.Canceled / context.DeadlineExceeded (the caller's ctx @@ -1115,6 +1260,48 @@ func writeSidecarFixture(t *testing.T, sc *encryption.Sidecar) string { return path } +type testWriterRegistry struct { + rows map[string][]byte + getErr error +} + +func newTestWriterRegistry() *testWriterRegistry { + return &testWriterRegistry{rows: map[string][]byte{}} +} + +func (r *testWriterRegistry) GetRegistryRow(key []byte) ([]byte, bool, error) { + if r.getErr != nil { + return nil, false, r.getErr + } + raw, ok := r.rows[string(key)] + if !ok { + return nil, false, nil + } + out := append([]byte(nil), raw...) + return out, true, nil +} + +func (r *testWriterRegistry) SetRegistryRow(key, value []byte) error { + if r.rows == nil { + r.rows = map[string][]byte{} + } + r.rows[string(key)] = append([]byte(nil), value...) + return nil +} + +func (r *testWriterRegistry) seed(t *testing.T, dekID uint32, fullNodeID uint64, firstSeen, lastSeen uint16) { + t.Helper() + key := encryption.RegistryKey(dekID, encryption.NodeID16(fullNodeID)) + val := encryption.EncodeRegistryValue(encryption.RegistryValue{ + FullNodeID: fullNodeID, + FirstSeenLocalEpoch: firstSeen, + LastSeenLocalEpoch: lastSeen, + }) + if err := r.SetRegistryRow(key, val); err != nil { + t.Fatalf("seed registry row: %v", err) + } +} + // fixedCapabilityFanout returns a closure that yields the supplied // result regardless of context — lets tests drive the §4 fan-out // branches deterministically without spinning real clients. A diff --git a/adapter/redis_lua_context.go b/adapter/redis_lua_context.go index 3b10a46ac..260c6f861 100644 --- a/adapter/redis_lua_context.go +++ b/adapter/redis_lua_context.go @@ -48,7 +48,8 @@ type luaScriptContext struct { // buggy script from probing unbounded unique non-existent keys and // blowing up memory. Once full, further misses fall back to the // server-side probe (still correct, just not cached). - negativeType map[string]bool + negativeType map[string]bool + negativeTypeLimit int // keyTypeProbeCount counts how many times the server-side keyTypeAt // helper was invoked during this Eval. Only read by tests via @@ -261,21 +262,22 @@ func newLuaScriptContext(ctx context.Context, server *RedisServer) (*luaScriptCo } startTS := server.readTS() return &luaScriptContext{ - server: server, - startTS: startTS, - readPin: server.pinReadTS(startTS), - ctx: ctx, - touched: map[string]struct{}{}, - deleted: map[string]bool{}, - everDeleted: map[string]bool{}, - negativeType: map[string]bool{}, - strings: map[string]*luaStringState{}, - lists: map[string]*luaListState{}, - hashes: map[string]*luaHashState{}, - sets: map[string]*luaSetState{}, - zsets: map[string]*luaZSetState{}, - streams: map[string]*luaStreamState{}, - ttls: map[string]*luaTTLState{}, + server: server, + startTS: startTS, + readPin: server.pinReadTS(startTS), + ctx: ctx, + touched: map[string]struct{}{}, + deleted: map[string]bool{}, + everDeleted: map[string]bool{}, + negativeType: map[string]bool{}, + negativeTypeLimit: maxNegativeTypeCacheEntries, + strings: map[string]*luaStringState{}, + lists: map[string]*luaListState{}, + hashes: map[string]*luaHashState{}, + sets: map[string]*luaSetState{}, + zsets: map[string]*luaZSetState{}, + streams: map[string]*luaStreamState{}, + ttls: map[string]*luaTTLState{}, }, nil } @@ -479,7 +481,7 @@ func (c *luaScriptContext) keyType(key []byte) (redisValueType, error) { if err != nil { return redisTypeNone, err } - if typ == redisTypeNone && len(c.negativeType) < maxNegativeTypeCacheEntries { + if typ == redisTypeNone && len(c.negativeType) < c.effectiveNegativeTypeLimit() { // Pin the absence result for the rest of this Eval so repeated // BullMQ-style polling of a missing key (e.g. a "delayed" zset) // does not re-run the ~8-seek rawKeyTypeAt probe on every @@ -493,6 +495,13 @@ func (c *luaScriptContext) keyType(key []byte) (redisValueType, error) { return typ, nil } +func (c *luaScriptContext) effectiveNegativeTypeLimit() int { + if c.negativeTypeLimit > 0 { + return c.negativeTypeLimit + } + return maxNegativeTypeCacheEntries +} + func (c *luaScriptContext) ensureKeyNotExpired(key []byte) error { ttl, err := c.loadTTL(key) if err != nil { diff --git a/adapter/redis_lua_negative_type_cache_test.go b/adapter/redis_lua_negative_type_cache_test.go index 6253f7e30..6a3a29db3 100644 --- a/adapter/redis_lua_negative_type_cache_test.go +++ b/adapter/redis_lua_negative_type_cache_test.go @@ -156,17 +156,19 @@ func TestLuaNegativeTypeCache_BoundedSize(t *testing.T) { sc, err := newLuaScriptContext(ctx, nodes[0].redisServer) require.NoError(t, err) defer sc.Close() + const cacheLimit = 4 + sc.negativeTypeLimit = cacheLimit // Probe cap+overflow unique missing keys. Each probe is a miss // (redisTypeNone); only the first `cap` should be memoized. - const overflow = 50 - total := maxNegativeTypeCacheEntries + overflow + const overflow = 2 + total := cacheLimit + overflow for i := 0; i < total; i++ { typ, kerr := sc.keyType([]byte(fmt.Sprintf("lua:neg:cap:%d", i))) require.NoError(t, kerr) require.Equal(t, redisTypeNone, typ) } - require.Equal(t, maxNegativeTypeCacheEntries, len(sc.negativeType), + require.Equal(t, cacheLimit, len(sc.negativeType), "negativeType map must be capped at maxNegativeTypeCacheEntries") // Each unique key above required exactly one probe on first access. @@ -182,11 +184,11 @@ func TestLuaNegativeTypeCache_BoundedSize(t *testing.T) { require.Equal(t, total, sc.keyTypeProbeCount, "a key inserted before the cap must remain cached") - overflowKey := []byte(fmt.Sprintf("lua:neg:cap:%d", maxNegativeTypeCacheEntries+1)) + overflowKey := []byte(fmt.Sprintf("lua:neg:cap:%d", cacheLimit+1)) _, kerr = sc.keyType(overflowKey) require.NoError(t, kerr) require.Equal(t, total+1, sc.keyTypeProbeCount, "a key probed after the cap was reached must fall back to the server probe") - require.Equal(t, maxNegativeTypeCacheEntries, len(sc.negativeType), + require.Equal(t, cacheLimit, len(sc.negativeType), "fallback probe must NOT grow the bounded cache") } diff --git a/docs/design/2026_04_29_partial_data_at_rest_encryption.md b/docs/design/2026_04_29_partial_data_at_rest_encryption.md index 0d77a8784..91af40095 100644 --- a/docs/design/2026_04_29_partial_data_at_rest_encryption.md +++ b/docs/design/2026_04_29_partial_data_at_rest_encryption.md @@ -1,6 +1,6 @@ # Data-at-rest encryption for elastickv -Status: Partial — Stages 0–6 and Stage 8 shipped (5E deferred); Stage 7 partially shipped; Stage 9 open +Status: Partial — Stages 0–8 and 9A–9B shipped (5E deferred); remaining Stage 9 work open Author: bootjp Date: 2026-04-29 @@ -30,9 +30,11 @@ Date: 2026-04-29 | 6D | §6.6 `enable-storage-envelope` admin RPC + §7.1 Phase-1 storage cutover (§6.2 toggle ON) + Voters ∪ Learners capability gate + production storage-envelope write-path wiring | shipped | `2026_05_18_implemented_6d_enable_storage_envelope.md` + `2026_05_25_implemented_6d6c2_production_storage_envelope_wiring.md` | | 6E | §6.6 `enable-raft-envelope` admin RPC + §7.1 Phase-2 raft cutover + `raft_envelope_cutover_index` sidecar record + `internal/raftengine/etcd/engine.go` `applyNormalEntry` unwrap hook activation + `ErrRaftUnwrapFailed` HaltApply path + `kv/coordinator.go` / `kv/sharded_coordinator.go` wrap-on-propose switch (Phase-2 leader-side §6.3 proposal-payload wrap) + §7.1 steps 1–6 proposal quiescence barrier (block new user proposal intake, drain in-flight queue, source-tag exemption for the cutover entry itself) + production runtime wiring for startup/apply-time wrap install and post-cutover admin proposals + 6C-4 fail-closed guards. | shipped | 2026_05_31_implemented_6e_enable_raft_envelope.md | | 6F | §6.5 `--encryption-rotate-on-startup` request flag + leader-elected rotation proposal on the default encryption Raft group. The leader rotates every active DEK purpose once before listeners open; followers keep an in-memory pending request and fire only if they acquire leadership in the same process uptime. | shipped | this PR | -| 7 | Writer registry + deterministic nonce (§4.1). Implemented slices: process-start registration, storage-layer registration gate, runtime re-registration for cutover and rotation, and conf-change-time pre-registration. Still open: §5.5 registry projection in `GetSidecarState` / `ResyncSidecar` (`WriterRegistryForCaller`). | partial | `2026_05_26_implemented_7a_process_start_registration.md` + `2026_05_26_implemented_7a2_storage_layer_registration_enforcement.md` + `2026_05_28_implemented_7b_runtime_reregistration.md` + `2026_05_28_implemented_7b_prime_runtime_reregistration_rotation.md` + `2026_05_29_implemented_7c_confchange_time_registration.md` | +| 7 | Writer registry + deterministic nonce (§4.1). Covers process-start registration, storage-layer registration gate, runtime re-registration for cutover and rotation, conf-change-time pre-registration, and §5.5 registry projection in `GetSidecarState` / `ResyncSidecar` (`WriterRegistryForCaller`). | shipped | `2026_05_26_implemented_7a_process_start_registration.md` + `2026_05_26_implemented_7a2_storage_layer_registration_enforcement.md` + `2026_05_28_implemented_7b_runtime_reregistration.md` + `2026_05_28_implemented_7b_prime_runtime_reregistration_rotation.md` + `2026_05_29_implemented_7c_confchange_time_registration.md` + `2026_07_11_implemented_7d_sidecar_registry_projection.md` | | 8 | Snapshot header v2 (§4.4); WAL coverage closure (§4.3 / §4.6) | shipped | [`2026_05_29_implemented_8a_snapshot_header_v2.md`](2026_05_29_implemented_8a_snapshot_header_v2.md) + [`2026_06_01_implemented_8b_wal_coverage_closure.md`](2026_06_01_implemented_8b_wal_coverage_closure.md) | -| 9 | KMS-backed wrappers, compression, rotation/retire/rewrite, Jepsen (§5.2, §5.4, §6.4, §8) | open | — | +| 9A | Compress-then-encrypt, authenticated compression flag, encrypted-store Pebble compression policy, storage benchmark (§6.4, §8.3) | shipped | `2026_07_18_implemented_9a_encryption_compression.md` | +| 9B | AWS KMS, GCP KMS, Vault Transit, and test/CI env KEK providers; mutually-exclusive source loader and loaded-provider mutator gate (§5.1, §6.1, §6.5) | shipped | `2026_07_18_implemented_9b_kek_providers.md` | +| 9C+ | Rotation budget/rewrap/retire/rewrite, metrics, remaining benchmarks and encrypted Jepsen (§5.2, §5.4, §6.5, §8, §9.2) | open | — | Stages 0–4 ship the entire byte-tag pipeline (storage envelope, raft envelope, FSM dispatch, halt-on-error) but leave it **production @@ -451,14 +453,16 @@ the way out. Envelope format (single byte stream, stored as the Pebble value): ```text -+--------+------+---------+----------+----------------+--------+ -| 0x01 | flag | key_id | nonce | ciphertext | tag | -| 1 byte | 1 B | 4 bytes | 12 bytes | N bytes | 16 B | -+--------+------+---------+----------+----------------+--------+ ++---------+------+---------+----------+----------------+--------+ +| version | flag | key_id | nonce | ciphertext | tag | +| 1 byte | 1 B | 4 bytes | 12 bytes | N bytes | 16 B | ++---------+------+---------+----------+----------------+--------+ ``` -- `0x01` — envelope-version byte. Future authenticated formats reserve - `0x02..0x0F` (see §11.3). The byte itself is **not** used to +- `version` — `0x01` is the original uncompressed format and requires + `flag=0`. `0x02` is the compression-capable storage format. Compressed + writes use `0x02`, causing V1-only readers to reject rather than return + Snappy-framed plaintext during rolling rollback. The byte itself is **not** used to discriminate cleartext from ciphertext on the read path — that decision lives in MVCC metadata, not in the value bytes (see §7.1). - `flag` — bit 0: `1` if `ciphertext` is the encryption of a @@ -1016,9 +1020,11 @@ Two-tier hierarchy: - **KEK (Key Encryption Key).** Held outside the cluster, never on the cluster's disks. Sources, in order of preference: - 1. AWS KMS: `--kekUri=aws-kms://arn:aws:kms:...`. The KMS key never + 1. AWS KMS: `--kekUri=aws-kms://arn:aws:kms:...:key/...` with an + immutable key ARN (mutable alias ARNs are rejected). The KMS key never leaves AWS; we call `Encrypt` / `Decrypt` to wrap/unwrap DEKs. - 2. GCP KMS: `--kekUri=gcp-kms://projects/.../keys/...`. Same shape. + 2. GCP KMS: `--kekUri=gcp-kms://projects//locations//keyRings//cryptoKeys/`. + Same shape. 3. HashiCorp Vault Transit: `--kekUri=vault-transit://...`. 4. Static file: `--kekFile=/etc/elastickv/kek.bin` — 32 bytes raw. Recommended only when the file lives on a tmpfs or sealed @@ -1028,7 +1034,10 @@ Two-tier hierarchy: inspection); supported only for tests and CI. No default. If `--encryption-enabled` is set without a KEK source, - the process refuses to start. + the process refuses to start. Before encryption mutators are exposed, every + configured provider must also complete a random 32-byte DEK wrap/unwrap + preflight; this catches credentials, reachability, permission, key-binding, + and malformed-response failures before a Raft entry can commit. - **DEK (Data Encryption Key).** 32-byte AES key. Two DEKs are issued in v1: `dek_storage` (used by §4.1) and `dek_raft` @@ -1849,6 +1858,12 @@ so we never pay CPU twice. Because the flag is part of the AES-GCM AAD (§4.1), an attacker cannot flip it post-hoc to crash the decompressor. +Compressed values use envelope version `0x02`; uncompressed values remain +version `0x01`. This format split is the reader-capability boundary: a +pre-compression V1 reader fails closed on V2 instead of serving compressed +bytes. Visibility-only operations authenticate the envelope and value header +without allocating the decompressed value. + This is also the reason we cannot rely on Pebble's built-in block compression alone: by the time bytes reach Pebble's block writer they are already ciphertext, so Pebble's compression is wasted CPU. We @@ -2674,8 +2689,9 @@ eventual code change. top of value-level encryption, accepting the complexity tax. Not needed for the v1 threat model. -3. **Envelope-format versioning.** v1 ships envelope version `0x01`. - We will reserve `0x02..0x0F` for future authenticated formats +3. **Envelope-format versioning.** Uncompressed storage and Raft + envelopes use `0x01`; compression-capable storage envelopes use + `0x02`. We reserve `0x03..0x0F` for future authenticated formats (e.g., AES-256-GCM-SIV if nonce-misuse resistance matters, ChaCha20-Poly1305 for non-AES-NI hosts). The decrypt path dispatches on the version byte; anything unknown errors loudly. diff --git a/docs/design/2026_05_29_implemented_7c_confchange_time_registration.md b/docs/design/2026_05_29_implemented_7c_confchange_time_registration.md index 6e13370f9..84113a53e 100644 --- a/docs/design/2026_05_29_implemented_7c_confchange_time_registration.md +++ b/docs/design/2026_05_29_implemented_7c_confchange_time_registration.md @@ -463,8 +463,8 @@ that bypass the standard workflow) must rely on the `ErrNodeIDCollision` startup membership pre-check before issuing the ConfChange. -This closes the 7c ConfChange-time registration slice. The parent -Stage 7 remains partial until the §5.5 `WriterRegistryForCaller` -projection is wired into `GetSidecarState` / `ResyncSidecar`. +This closes the 7c ConfChange-time registration slice. The remaining +Stage 7 §5.5 `WriterRegistryForCaller` projection is tracked by +`2026_07_11_implemented_7d_sidecar_registry_projection.md`. Stage 8 (snapshot header v2) and Stage 9 (KMS + compress + rotation/retire/rewrite + Jepsen) follow. diff --git a/docs/design/2026_07_11_implemented_7d_sidecar_registry_projection.md b/docs/design/2026_07_11_implemented_7d_sidecar_registry_projection.md new file mode 100644 index 000000000..b84f51a9a --- /dev/null +++ b/docs/design/2026_07_11_implemented_7d_sidecar_registry_projection.md @@ -0,0 +1,52 @@ +# Implemented 7d: sidecar registry projection + +Status: Implemented +Author: bootjp +Date: 2026-07-11 + +## Scope + +This closes the remaining Stage 7 §5.5 recovery surface for +`WriterRegistryForCaller`. + +The prior Stage 7 slices already registered writers at process start, +storage-layer admission, runtime cutover/rotation, and membership +change time. The open gap was the compaction-fallback recovery RPC: +a node whose sidecar fell behind a compacted Raft log needs the +leader's per-DEK `last_seen_local_epoch` record for that specific +caller so it can choose a strictly monotonic replacement +`local_epoch`. + +## Implementation + +`EncryptionAdminServer` now accepts the production +`encryption.WriterRegistryStore` through +`WithEncryptionAdminWriterRegistry`. + +`GetSidecarState` projects registry rows for the local full node ID. +`ResyncSidecar` projects rows for `ResyncSidecarRequest.caller_full_node_id`. +Both responses keep returning a non-nil empty map when the registry is +not wired, preserving the encryption-disabled and test posture. + +For each DEK present in the sidecar, the server reads +`RegistryKey(dek_id, NodeID16(full_node_id))` and returns the decoded +`LastSeenLocalEpoch`. Missing rows are omitted instead of zero-filled, +because zero could be mistaken for a real registry observation. A +decoded row whose stored `FullNodeID` does not match the requested +caller is rejected as an internal node-ID collision. + +Production wiring stores the writer registry on each +`raftGroupRuntime` as the group is built, then passes it into the +per-shard `EncryptionAdmin` registration in `startRaftServers`. + +## Validation + +- `go test ./adapter -run 'TestEncryptionAdmin_(GetSidecarState|ResyncSidecar)' -count=1 -timeout=240s` +- `go test ./adapter -run TestEncryptionAdmin -count=1 -timeout=300s` +- `go test ./store ./internal/encryption . -count=1 -timeout=180s` +- `golangci-lint run ./adapter ./store ./internal/encryption . --timeout=5m` + +The broader `go test ./adapter ./store ./internal/encryption . -count=1 +-timeout=300s` run timed out in the full adapter package. The targeted +encryption admin tests and the directly affected non-adapter packages +passed. diff --git a/docs/design/2026_07_18_implemented_9a_encryption_compression.md b/docs/design/2026_07_18_implemented_9a_encryption_compression.md new file mode 100644 index 000000000..4e9082ff3 --- /dev/null +++ b/docs/design/2026_07_18_implemented_9a_encryption_compression.md @@ -0,0 +1,45 @@ +# Stage 9A data-at-rest compression + +Status: Implemented +Author: bootjp +Date: 2026-07-18 + +## Scope + +This milestone implements the compress-then-encrypt storage path from +`2026_04_29_partial_data_at_rest_encryption.md` §6.4. + +- Snappy runs on plaintext before AES-256-GCM. +- The compressed representation is selected only when it is strictly smaller. +- `FlagCompressed` is authenticated as part of the envelope AAD. +- Reads decompress only after GCM authentication succeeds. +- V1 envelopes require `flag=0`; compressed values use envelope V2. +- Unknown version/flag combinations fail closed. +- The cleartext-rebadge guard verifies both valid compression-flag variants. +- Pebble block compression is disabled for encryption-wired stores, including + database reopen and snapshot-restore temporary databases. + +Raft proposal envelopes remain uncompressed. Their apply path is latency +sensitive and the storage layer already performs the only value-compression +pass after proposal decryption. + +## Compatibility + +Existing V1 envelopes have `flag=0` and remain readable. Compressed writes use +V2 with `FlagCompressed`, so a V1-only reader rejects the version instead of +returning Snappy-framed bytes as plaintext during a rolling rollback. Legacy +cleartext operation and stores with no encryption wiring retain Pebble's +default block-compression policy. Visibility-only scans authenticate V2 rows +without decompressing values they will discard. + +## Verification + +- Storage round trips for compressed, uncompressed, empty, and already + compressed values. +- Authenticated compression-flag tamper rejection. +- V1-reader fail-closed rejection of compressed V2 envelopes. +- Fail-closed handling for an authenticated malformed Snappy payload. +- Prefix-delete visibility checks that authenticate without Snappy expansion. +- Property testing over arbitrary byte slices for compression framing. +- Pebble compression-policy tests for encrypted and legacy stores. +- Paired cleartext/encrypted 1 KiB storage benchmark for `benchstat` review. diff --git a/docs/design/2026_07_18_implemented_9b_kek_providers.md b/docs/design/2026_07_18_implemented_9b_kek_providers.md new file mode 100644 index 000000000..d178ea543 --- /dev/null +++ b/docs/design/2026_07_18_implemented_9b_kek_providers.md @@ -0,0 +1,76 @@ +# Stage 9B: production KEK providers + +Status: Implemented +Author: bootjp +Date: 2026-07-18 + +## Scope + +Stage 9B completes the §5.1 KEK source matrix: + +- `--kekUri=aws-kms://` uses an immutable AWS KMS `key/` ARN with + `Encrypt` / `Decrypt`, derives + the client region from the ARN, and binds every request to the fixed + `elastickv-purpose=dek-wrap-v1` encryption context. +- `--kekUri=gcp-kms://` uses Google Cloud KMS with fixed + AAD. Request and response CRC32C values are supplied and verified before a + wrapped DEK or plaintext DEK is accepted. +- `--kekUri=vault-transit:///` uses Vault Transit through standard + `VAULT_*` client configuration, including nested mount paths with the final + path segment interpreted as the key name. The existing key is read before + encrypt to prevent implicit key creation, and encrypt/decrypt use fixed AAD. + Binary DEKs are base64 encoded for the API; only + `vault:v:` ciphertexts are accepted. +- `ELASTICKV_KEK_BASE64` supplies a 32-byte test/CI-only static KEK. The + variable is unset immediately after the decode attempt, including malformed + input. It is also unset when source ambiguity is rejected. +- `--kekFile` keeps its existing owner-only regular-file contract. + +File, URI, and environment sources are mutually exclusive. Ambiguous +configuration fails closed instead of selecting a source by precedence. + +## Runtime wiring + +The startup loader returns the actual loaded `kek.Wrapper`. Startup guards now +derive `KEKConfigured` from that non-nil wrapper rather than from `--kekFile` +text. The same loaded-state boolean is threaded to the EncryptionAdmin mutator +gate, so URI and environment providers can bootstrap/rotate while a failed or +absent provider cannot expose mutating RPCs. + +Before that gate can open, startup performs a real random 32-byte DEK +wrap/unwrap round trip and compares the result in constant time. This proves +provider reachability, credentials, encrypt/decrypt permission, key binding, +and response shape on a fresh data directory where no sidecar DEK exists to +exercise the provider. A failed preflight refuses process start before Raft or +the mutating RPC surface is available. + +Remote wrappers use a 30-second operation deadline and reject nil, empty, +integrity-invalid, or non-32-byte provider responses before sidecar state can be +updated. Provider clients are safe for concurrent use and are hidden behind +narrow interfaces for deterministic tests. + +## Compatibility and verification + +No envelope, sidecar, Raft opcode, or snapshot format changes. Existing wrapped +DEKs remain provider-specific opaque bytes, and the `kek.Wrapper` contract is +unchanged. + +Verification includes: + +- fake-client unit tests for AWS encryption context, GCP AAD/CRC32C, and Vault + key existence, nested mount, AAD, and binary request/response encoding; +- malformed provider response, strict Vault ciphertext, immutable AWS key ARN, + wrong DEK length, invalid URI, source conflict, environment-unset, and + startup wrap/unwrap preflight tests; +- a production-wiring integration test that loads the environment KEK through + the source selector, bootstraps both DEKs, enables the storage envelope, + writes encrypted data, snapshots it, restores it into an encrypted Pebble + store, and reads the original plaintext; +- startup/mutator caller audit confirming the gate consumes loaded-wrapper + state rather than the legacy file flag. + +## Remaining work + +Stage 5E discovery batching and Stage 9C+ rotation-budget, rewrap, rewrite, +retirement, metrics, remaining benchmark, and encrypted Jepsen requirements are +not part of this milestone. diff --git a/go.mod b/go.mod index b9c9df389..431a2970a 100644 --- a/go.mod +++ b/go.mod @@ -5,11 +5,13 @@ go 1.26 toolchain go1.26.5 require ( + cloud.google.com/go/kms v1.31.0 github.com/Jille/grpc-multi-resolver v1.3.0 github.com/aws/aws-sdk-go-v2 v1.42.1 github.com/aws/aws-sdk-go-v2/config v1.32.29 github.com/aws/aws-sdk-go-v2/credentials v1.19.28 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.60.0 + github.com/aws/aws-sdk-go-v2/service/kms v1.54.1 github.com/aws/smithy-go v1.27.3 github.com/cockroachdb/errors v1.14.0 github.com/cockroachdb/pebble/v2 v2.1.6 @@ -17,7 +19,9 @@ require ( github.com/emirpasic/gods v1.18.1 github.com/getsentry/sentry-go v0.47.0 github.com/goccy/go-json v0.10.6 + github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e github.com/hanwen/go-fuse/v2 v2.10.1 + github.com/hashicorp/vault/api v1.23.0 github.com/klauspost/compress v1.19.0 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.23.2 @@ -40,6 +44,12 @@ require ( ) require ( + cloud.google.com/go v0.123.0 // indirect + cloud.google.com/go/auth v0.20.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.7.0 // indirect + cloud.google.com/go/longrunning v0.9.0 // indirect github.com/DataDog/zstd v1.5.7 // indirect github.com/RaduBerinde/axisds v0.1.0 // indirect github.com/RaduBerinde/btreemap v0.0.0-20250419174037-3d62b7205d54 // indirect @@ -55,6 +65,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.0 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.44.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b // indirect github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect @@ -64,24 +75,40 @@ require ( github.com/coreos/go-systemd/v22 v22.7.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.3.1 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e // indirect github.com/google/go-cmp v0.7.0 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.17 // indirect + github.com/googleapis/gax-go/v2 v2.21.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-retryablehttp v0.7.8 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect + github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.7 // indirect + github.com/hashicorp/hcl v1.0.1-vault-7 // indirect github.com/jonboulle/clockwork v0.5.0 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.16.1 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/ryanuber/go-glob v1.0.0 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/tidwall/btree v1.1.0 // indirect github.com/tidwall/match v1.1.1 // indirect @@ -92,6 +119,7 @@ require ( go.etcd.io/etcd/pkg/v3 v3.7.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 // indirect go.opentelemetry.io/otel v1.43.0 // indirect go.opentelemetry.io/otel/metric v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect @@ -101,7 +129,11 @@ require ( golang.org/x/crypto v0.52.0 // indirect golang.org/x/exp v0.0.0-20240325151524-a685a6edb6d8 // indirect golang.org/x/net v0.55.0 // indirect + golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/text v0.37.0 // indirect + golang.org/x/time v0.15.0 // indirect + google.golang.org/api v0.274.0 // indirect + google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 1203c7d30..d3710ade0 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,17 @@ +cloud.google.com/go v0.123.0 h1:2NAUJwPR47q+E35uaJeYoNhuNEM9kM8SjgRgdeOJUSE= +cloud.google.com/go v0.123.0/go.mod h1:xBoMV08QcqUGuPW65Qfm1o9Y4zKZBpGS+7bImXLTAZU= +cloud.google.com/go/auth v0.20.0 h1:kXTssoVb4azsVDoUiF8KvxAqrsQcQtB53DcSgta74CA= +cloud.google.com/go/auth v0.20.0/go.mod h1:942/yi/itH1SsmpyrbnTMDgGfdy2BUqIKyd0cyYLc5Q= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.7.0 h1:JD3zh0C6LHl16aCn5Akff0+GELdp1+4hmh6ndoFLl8U= +cloud.google.com/go/iam v1.7.0/go.mod h1:tetWZW1PD/m6vcuY2Zj/aU0eCHNPuxedbnbRTyKXvdY= +cloud.google.com/go/kms v1.31.0 h1:LS8N92OxFDgOLg5NCo3OmbvjtQAIVT5gUHVLKIDHaFE= +cloud.google.com/go/kms v1.31.0/go.mod h1:YIyXZym11R5uovJJt4oN5eUL3oPmirF3yKeIh6QAf4U= +cloud.google.com/go/longrunning v0.9.0 h1:0EzbDEGsAvOZNbqXopgniY0w0a1phvu5IdUFq8grmqY= +cloud.google.com/go/longrunning v0.9.0/go.mod h1:pkTz846W7bF4o2SzdWJ40Hu0Re+UoNT6Q5t+igIcb8E= github.com/DataDog/zstd v1.5.7 h1:ybO8RBeh29qrxIhCA9E8gKY6xfONU9T6G6aP9DTKfLE= github.com/DataDog/zstd v1.5.7/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Jille/grpc-multi-resolver v1.3.0 h1:cbVm1TtWP7YxdiCCZ8gU4/78pYO2OXpzZSFAAUMdFLs= @@ -30,6 +44,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.12.7 h1:uqsK github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.12.7/go.mod h1:Js/P8Zbwe1mRejnD+OpFLyQiJ8ioQlo3GMAg7Dfxk7w= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 h1:/Z5jmNrKsSD7EmDjzAPsm/3L9IuOkzaynklJZ1qX7S4= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30/go.mod h1:lEzEZnOosE7zi8Z6royW1cFJTD9fpab4Ul1SBrllewk= +github.com/aws/aws-sdk-go-v2/service/kms v1.54.1 h1:aeJAJyvWS3gQ679pJbz8ZdOh3MViD1zvEdoZMVEawbg= +github.com/aws/aws-sdk-go-v2/service/kms v1.54.1/go.mod h1:0RXNc6Yf3AvSMldGD6Lcch96Ojlw2TtGnHsqfD/L4u8= github.com/aws/aws-sdk-go-v2/service/signin v1.4.0 h1:sLzmJGCMv+C8KqiJgEqDLB6vxaJGmobRh4rr//ZpA3w= github.com/aws/aws-sdk-go-v2/service/signin v1.4.0/go.mod h1:mxC0nT/C8wMMS97DemZPzvUZxvIt+2Iq+eS3JdFZGgg= github.com/aws/aws-sdk-go-v2/service/sso v1.32.0 h1:qjMmry/cBDee1E/2gyvel0uRYCi3mwRZ2hf6N+GAodo= @@ -46,8 +62,12 @@ github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 h1:aBangftG7EVZoUb69Os8IaYg++6uMOdKK83QtkkvJik= +github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2/go.mod h1:qwXFYgsP6T7XnJtbKlf1HP8AjxZZyzxMmc+Lq5GjlU4= github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b h1:SHlYZ/bMx7frnmeqCu+xm0TCxXLzX3jQIVuFbnFGtFU= github.com/cockroachdb/crlib v0.0.0-20241112164430-1264a2edc35b/go.mod h1:Gq51ZeKaFCXk6QwuGM0w1dnaOqc/F5zKT2zA9D6Xeac= github.com/cockroachdb/datadriven v1.0.3-0.20250407164829-2945557346d5 h1:UycK/E0TkisVrQbSoxvU827FwgBBcZ95nRRmpj/12QI= @@ -71,23 +91,37 @@ github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03V github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= +github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= +github.com/envoyproxy/go-control-plane/envoy v1.37.0 h1:u3riX6BoYRfF4Dr7dwSOroNfdSbEPe9Yyl09/B6wBrQ= +github.com/envoyproxy/go-control-plane/envoy v1.37.0/go.mod h1:DReE9MMrmecPy+YvQOAOHNYMALuowAnbjjEMkkWOi6A= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/getsentry/sentry-go v0.47.0 h1:AnSMSyrYA5qZCIN/2xpgAAwv63sVULV+vBq37ajouc8= github.com/getsentry/sentry-go v0.47.0/go.mod h1:h+b4VHpKnK7aUXB5wc+KDnPgp9ZtfliRD4eV85FbiSA= github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9 h1:r5GgOLGbza2wVHRzK7aAj6lWZjfbAwiu/RDCVOKjRyM= github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U= +github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU= github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -100,12 +134,41 @@ github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e h1:4bw4WeyTYPp0sma github.com/golang/snappy v0.0.5-0.20231225225746-43d5d4cd4e0e/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.17 h1:73NfMHdiqo9JFU9+7a5ExpVa10/R29pXfZIaW559nrg= +github.com/googleapis/enterprise-certificate-proxy v0.3.17/go.mod h1:rSEsBUemEBZEexP2y6jPp16LUmUbjmSbcPMQizR0o4k= +github.com/googleapis/gax-go/v2 v2.21.0 h1:h45NjjzEO3faG9Lg/cFrBh2PgegVVgzqKzuZl/wMbiI= +github.com/googleapis/gax-go/v2 v2.21.0/go.mod h1:But/NJU6TnZsrLai/xBAQLLz+Hc7fHZJt/hsCz3Fih4= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= github.com/hanwen/go-fuse/v2 v2.10.1 h1:QAqZuc9+aBtTou+OPruU/hkYQYCkgPtQd2QaepHkTTs= github.com/hanwen/go-fuse/v2 v2.10.1/go.mod h1:aU7NkGYZUmuJrZapoI3mEcNve7PZTySUOLBuch/vR6U= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= +github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= +github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 h1:U+kC2dOhMFQctRfhK0gRctKAPTloZdMU5ZJxaesJ/VM= +github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0/go.mod h1:Ll013mhdmsVDuoIXVfBtvgGJsXDYkTw1kooNcoCXuE0= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= +github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw= +github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw= +github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I= +github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= +github.com/hashicorp/vault/api v1.23.0 h1:gXgluBsSECfRWTSW9niY2jwg2e9mMJc4WoHNv4g3h6A= +github.com/hashicorp/vault/api v1.23.0/go.mod h1:zransKiB9ftp+kgY8ydjnvCU7Wk8i9L0DYWpXeMj9ko= github.com/jonboulle/clockwork v0.5.0 h1:Hyh9A8u51kptdkR+cqRpT1EebBwTn1oK9YfGYbdFz6I= github.com/jonboulle/clockwork v0.5.0/go.mod h1:3mZlmanh0g2NDKO5TWZVJAfofYk64M7XN3SzBPjZF60= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -120,8 +183,16 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882 h1:0lgqHvJWHLGW5TuObJrfyEi6+ASTKDBWikGvPqy9Yiw= github.com/minio/minlz v1.0.1-0.20250507153514-87eb42fe8882/go.mod h1:qT0aEB35q79LLornSzeDH75LBf3aH1MV+jB5w9Wasec= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9KouLrg= github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= @@ -131,6 +202,8 @@ github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTw github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= @@ -146,6 +219,8 @@ github.com/redis/go-redis/v9 v9.21.0/go.mod h1:v/M13XI1PVCDcm01VtPFOADfZtHf8YW3b github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= @@ -186,6 +261,8 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 h1:0Qx7VGBacMm9ZENQ7TnNObTYI4ShC+lHI16seduaxZo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0/go.mod h1:Sje3i3MjSPKTSPvVWCaL8ugBzJwik3u4smCjUeuupqg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0 h1:OyrsyzuttWTSur2qN/Lm0m2a8yqyIjUVBZcxFPuXq2o= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.67.0/go.mod h1:C2NGBr+kAB4bk3xtMXfZ94gqFDtg/GkI7e9zqGh5Beg= go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= @@ -223,6 +300,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -237,6 +316,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -247,6 +328,10 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/api v0.274.0 h1:aYhycS5QQCwxHLwfEHRRLf9yNsfvp1JadKKWBE54RFA= +google.golang.org/api v0.274.0/go.mod h1:JbAt7mF+XVmWu6xNP8/+CTiGH30ofmCmk9nM8d8fHew= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7 h1:XzmzkmB14QhVhgnawEVsOn6OFsnpyxNPRY9QV01dNB0= +google.golang.org/genproto v0.0.0-20260319201613-d00831a3d3e7/go.mod h1:L43LFes82YgSonw6iTXTxXUX1OlULt4AQtkik4ULL/I= google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478 h1:yQugLulqltosq0B/f8l4w9VryjV+N/5gcW0jQ3N8Qec= google.golang.org/genproto/googleapis/api v0.0.0-20260414002931-afd174a4e478/go.mod h1:C6ADNqOxbgdUUeRTU+LCHDPB9ttAMCTff6auwCVa4uc= google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw= diff --git a/internal/encryption/envelope.go b/internal/encryption/envelope.go index 0d221d3c3..d41499570 100644 --- a/internal/encryption/envelope.go +++ b/internal/encryption/envelope.go @@ -8,13 +8,14 @@ import ( // Public constants for the §4.1 wire format. const ( - // EnvelopeVersionV1 is the current envelope format version. §11.3 - // reserves 0x02..0x0F for future authenticated formats. The current - // build only understands 0x01; ANY other version byte (including the - // 0x02..0x0F reserved range) causes DecodeEnvelope to return - // ErrEnvelopeVersion. Future decoders that know how to handle the - // reserved range will widen this check. + // EnvelopeVersionV1 is the original uncompressed storage and Raft + // format. It rejects FlagCompressed so V1 readers never return a + // Snappy frame as user plaintext. EnvelopeVersionV1 byte = 0x01 + // EnvelopeVersionV2 identifies storage envelopes whose reader + // understands the authenticated compression flag. V1-only readers + // reject this version before decrypting, making rollback fail closed. + EnvelopeVersionV2 byte = 0x02 // FlagCompressed (bit 0) is set when ciphertext encrypts a Snappy- // compressed plaintext (§6.4). The flag participates in the AAD so a @@ -82,13 +83,12 @@ type Envelope struct { // (uninitialised Version, truncated Body) fails here with a clear // stack trace, rather than surfacing later as a confusing // DecodeEnvelope or Cipher.Decrypt failure on the read side. Returns: -// - ErrEnvelopeVersion if Version is not EnvelopeVersionV1. +// - ErrEnvelopeVersion if Version is unsupported. // - ErrEnvelopeShort if Body is shorter than TagSize (every valid // body must contain at least the GCM tag). func (e *Envelope) Encode() ([]byte, error) { - if e.Version != EnvelopeVersionV1 { - return nil, errors.Wrapf(ErrEnvelopeVersion, - "encode: got 0x%02x, want 0x%02x", e.Version, EnvelopeVersionV1) + if err := validateEnvelopeVersionFlag(e.Version, e.Flag); err != nil { + return nil, errors.Wrap(err, "encode") } if len(e.Body) < TagSize { return nil, errors.Wrapf(ErrEnvelopeShort, @@ -112,9 +112,8 @@ func DecodeEnvelope(src []byte) (*Envelope, error) { return nil, errors.Wrapf(ErrEnvelopeShort, "got %d bytes, want >= %d", len(src), HeaderSize+TagSize) } - if src[versionOffset] != EnvelopeVersionV1 { - return nil, errors.Wrapf(ErrEnvelopeVersion, - "got 0x%02x, want 0x%02x", src[versionOffset], EnvelopeVersionV1) + if err := validateEnvelopeVersionFlag(src[versionOffset], src[flagOffset]); err != nil { + return nil, err } e := &Envelope{ Version: src[versionOffset], @@ -128,6 +127,26 @@ func DecodeEnvelope(src []byte) (*Envelope, error) { return e, nil } +func validateEnvelopeVersionFlag(version, flag byte) error { + switch version { + case EnvelopeVersionV1: + if flag != 0 { + return errors.Wrapf(ErrEnvelopeFlag, + "version 0x%02x requires flag 0x00, got 0x%02x", version, flag) + } + case EnvelopeVersionV2: + if flag&^FlagCompressed != 0 { + return errors.Wrapf(ErrEnvelopeFlag, + "version 0x%02x got flag 0x%02x, allowed mask 0x%02x", version, flag, FlagCompressed) + } + default: + return errors.Wrapf(ErrEnvelopeVersion, + "got 0x%02x, supported versions are 0x%02x and 0x%02x", + version, EnvelopeVersionV1, EnvelopeVersionV2) + } + return nil +} + // HeaderAADBytes returns the first 6 bytes of the envelope header // (version, flag, key_id) in their on-disk order. These bytes participate // in the §4.1 storage-layer AAD (storage AAD = HeaderAADBytes ‖ pebble_key) diff --git a/internal/encryption/envelope_test.go b/internal/encryption/envelope_test.go index 74b89552f..49a550b6a 100644 --- a/internal/encryption/envelope_test.go +++ b/internal/encryption/envelope_test.go @@ -2,6 +2,7 @@ package encryption_test import ( "bytes" + "fmt" "testing" "github.com/bootjp/elastickv/internal/encryption" @@ -25,7 +26,7 @@ func TestEnvelope_RoundTrip(t *testing.T) { { name: "compressed flag set", env: encryption.Envelope{ - Version: encryption.EnvelopeVersionV1, + Version: encryption.EnvelopeVersionV2, Flag: encryption.FlagCompressed, KeyID: 0xCAFE_F00D, Body: bytes.Repeat([]byte{0x55}, 64+encryption.TagSize), @@ -34,8 +35,8 @@ func TestEnvelope_RoundTrip(t *testing.T) { { name: "max key_id", env: encryption.Envelope{ - Version: encryption.EnvelopeVersionV1, - Flag: 0xff, + Version: encryption.EnvelopeVersionV2, + Flag: encryption.FlagCompressed, KeyID: 0xFFFF_FFFF, Body: bytes.Repeat([]byte{0x77}, 1024+encryption.TagSize), }, @@ -119,7 +120,7 @@ func TestEnvelope_Decode_TooShort(t *testing.T) { } func TestEnvelope_Decode_RejectsUnknownVersion(t *testing.T) { - for _, version := range []byte{0x00, 0x02, 0x10, 0xFE, 0xFF} { + for _, version := range []byte{0x00, 0x03, 0x10, 0xFE, 0xFF} { buf := make([]byte, encryption.HeaderSize+encryption.TagSize) buf[0] = version _, err := encryption.DecodeEnvelope(buf) @@ -129,6 +130,49 @@ func TestEnvelope_Decode_RejectsUnknownVersion(t *testing.T) { } } +func TestEnvelope_V1RejectsCompressedFlag(t *testing.T) { + t.Parallel() + env := encryption.Envelope{ + Version: encryption.EnvelopeVersionV1, + Flag: encryption.FlagCompressed, + KeyID: 1, + Body: bytes.Repeat([]byte{0x55}, encryption.TagSize), + } + if _, err := env.Encode(); !errors.Is(err, encryption.ErrEnvelopeFlag) { + t.Fatalf("V1 compressed encode: expected ErrEnvelopeFlag, got %v", err) + } + buf := make([]byte, encryption.HeaderSize+encryption.TagSize) + buf[0] = encryption.EnvelopeVersionV1 + buf[1] = encryption.FlagCompressed + if _, err := encryption.DecodeEnvelope(buf); !errors.Is(err, encryption.ErrEnvelopeFlag) { + t.Fatalf("V1 compressed decode: expected ErrEnvelopeFlag, got %v", err) + } +} + +func TestEnvelope_RejectsUnknownFlagBits(t *testing.T) { + t.Parallel() + for _, flag := range []byte{0x02, 0x80, 0xff} { + t.Run(fmt.Sprintf("flag_%02x", flag), func(t *testing.T) { + env := encryption.Envelope{ + Version: encryption.EnvelopeVersionV1, + Flag: flag, + KeyID: 1, + Body: bytes.Repeat([]byte{0x55}, encryption.TagSize), + } + if _, err := env.Encode(); !errors.Is(err, encryption.ErrEnvelopeFlag) { + t.Fatalf("Encode flag=%#x: expected ErrEnvelopeFlag, got %v", flag, err) + } + + buf := make([]byte, encryption.HeaderSize+encryption.TagSize) + buf[0] = encryption.EnvelopeVersionV1 + buf[1] = flag + if _, err := encryption.DecodeEnvelope(buf); !errors.Is(err, encryption.ErrEnvelopeFlag) { + t.Fatalf("DecodeEnvelope flag=%#x: expected ErrEnvelopeFlag, got %v", flag, err) + } + }) + } +} + func TestEnvelope_Decode_DoesNotAliasInput(t *testing.T) { src := make([]byte, encryption.HeaderSize+encryption.TagSize+8) src[0] = encryption.EnvelopeVersionV1 @@ -147,8 +191,8 @@ func TestEnvelope_Decode_DoesNotAliasInput(t *testing.T) { } func TestHeaderAADBytes_Layout(t *testing.T) { - got := encryption.HeaderAADBytes(encryption.EnvelopeVersionV1, encryption.FlagCompressed, 0x12345678) - want := []byte{0x01, 0x01, 0x12, 0x34, 0x56, 0x78} + got := encryption.HeaderAADBytes(encryption.EnvelopeVersionV2, encryption.FlagCompressed, 0x12345678) + want := []byte{0x02, 0x01, 0x12, 0x34, 0x56, 0x78} if !bytes.Equal(got, want) { t.Fatalf("HeaderAADBytes layout mismatch:\n got %x\n want %x", got, want) } @@ -158,8 +202,8 @@ func TestAppendHeaderAADBytes_AppendsAndAllocFree(t *testing.T) { // Append onto an existing buffer. The result should be the original // bytes followed by the 6-byte header. prefix := []byte{0xCA, 0xFE} - got := encryption.AppendHeaderAADBytes(prefix, encryption.EnvelopeVersionV1, encryption.FlagCompressed, 0x12345678) - want := []byte{0xCA, 0xFE, 0x01, 0x01, 0x12, 0x34, 0x56, 0x78} + got := encryption.AppendHeaderAADBytes(prefix, encryption.EnvelopeVersionV2, encryption.FlagCompressed, 0x12345678) + want := []byte{0xCA, 0xFE, 0x02, 0x01, 0x12, 0x34, 0x56, 0x78} if !bytes.Equal(got, want) { t.Fatalf("AppendHeaderAADBytes mismatch:\n got %x\n want %x", got, want) } @@ -177,7 +221,7 @@ func TestAppendHeaderAADBytes_AppendsAndAllocFree(t *testing.T) { func TestEnvelope_Encode_RejectsBadVersion(t *testing.T) { env := encryption.Envelope{ - Version: 0x02, // not EnvelopeVersionV1 + Version: 0x03, Body: bytes.Repeat([]byte{0xAA}, encryption.TagSize), } _, err := env.Encode() diff --git a/internal/encryption/errors.go b/internal/encryption/errors.go index 80bfeb8de..ee7feaa73 100644 --- a/internal/encryption/errors.go +++ b/internal/encryption/errors.go @@ -35,6 +35,12 @@ var ( // current build does not know how to parse. Reserved values per §11.3. ErrEnvelopeVersion = errors.New("encryption: unknown envelope version") + // ErrEnvelopeFlag indicates an envelope set a flag bit that its version + // does not define. V1 permits no flags; V2 permits FlagCompressed. Accepting + // other combinations would make extensions ambiguous and could route + // authenticated plaintext through the wrong post-decrypt transform. + ErrEnvelopeFlag = errors.New("encryption: unknown envelope flag bits") + // ErrNilKeystore indicates NewCipher was called with a nil Keystore. // Surfaced at construction time so a wiring mistake is caught // before the first Encrypt/Decrypt would otherwise nil-deref panic. @@ -153,7 +159,7 @@ var ( // and the operator's clear intent (enable encryption) cannot be // satisfied. Fail fast at startup rather than discover the // mismatch later via a halted apply loop. - ErrKEKRequiredWithFlag = errors.New("encryption: --encryption-enabled is set but no KEK source (--kekFile) was provided; refusing to start (set --kekFile or unset --encryption-enabled)") + ErrKEKRequiredWithFlag = errors.New("encryption: --encryption-enabled is set but no KEK source was provided; refusing to start (set --kekFile, --kekUri, or ELASTICKV_KEK_BASE64, or unset --encryption-enabled)") // ErrKEKMismatch is the §9.1 startup-refusal guard raised when // the data dir contains a sidecar whose wrapped DEKs do NOT @@ -165,7 +171,7 @@ var ( // DEK. Recovery requires the operator to either point // --kekFile at the correct KEK file or restore the data dir // from a backup that matches the supplied KEK. - ErrKEKMismatch = errors.New("encryption: configured KEK cannot unwrap one or more wrapped DEKs in the sidecar; refusing to start (verify --kekFile matches the KEK that bootstrapped this data dir)") + ErrKEKMismatch = errors.New("encryption: configured KEK cannot unwrap one or more wrapped DEKs in the sidecar; refusing to start (verify the configured KEK source matches the KEK that bootstrapped this data dir)") // ErrLocalEpochExhausted is the §9.1 startup-refusal guard // raised when any active DEK in the sidecar has reached the diff --git a/internal/encryption/kek/aws_kms.go b/internal/encryption/kek/aws_kms.go new file mode 100644 index 000000000..3f7beed32 --- /dev/null +++ b/internal/encryption/kek/aws_kms.go @@ -0,0 +1,106 @@ +package kek + +import ( + "context" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/arn" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + awskms "github.com/aws/aws-sdk-go-v2/service/kms" + "github.com/cockroachdb/errors" +) + +const awsEncryptionContextKey = "elastickv-purpose" +const awsEncryptionContextValue = "dek-wrap-v1" + +type awsKMSClient interface { + Encrypt(context.Context, *awskms.EncryptInput, ...func(*awskms.Options)) (*awskms.EncryptOutput, error) + Decrypt(context.Context, *awskms.DecryptInput, ...func(*awskms.Options)) (*awskms.DecryptOutput, error) +} + +// AWSKMSWrapper wraps DEKs using an AWS KMS symmetric ENCRYPT_DECRYPT key. +type AWSKMSWrapper struct { + client awsKMSClient + keyARN string + timeout time.Duration +} + +// NewAWSKMSWrapper loads the standard AWS credential chain and derives the KMS +// endpoint region from keyARN. +func NewAWSKMSWrapper(ctx context.Context, keyARN string) (*AWSKMSWrapper, error) { + parsed, err := arn.Parse(keyARN) + if err != nil || parsed.Service != "kms" || parsed.Region == "" || parsed.AccountID == "" || + !strings.HasPrefix(parsed.Resource, "key/") || strings.TrimPrefix(parsed.Resource, "key/") == "" { + return nil, errors.Wrapf(ErrInvalidKEKURI, "invalid AWS KMS key ARN %q", keyARN) + } + cfg, err := awsconfig.LoadDefaultConfig(ctx, awsconfig.WithRegion(parsed.Region)) + if err != nil { + return nil, errors.Wrap(err, "kek: load AWS configuration") + } + return newAWSKMSWrapper(awskms.NewFromConfig(cfg), keyARN), nil +} + +func newAWSKMSWrapper(client awsKMSClient, keyARN string) *AWSKMSWrapper { + return &AWSKMSWrapper{client: client, keyARN: keyARN, timeout: providerRequestTimeout} +} + +// Wrap calls AWS KMS Encrypt with a fixed encryption context that is required +// again by Unwrap. +func (w *AWSKMSWrapper) Wrap(dek []byte) ([]byte, error) { + if w == nil || w.client == nil { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: AWS KMS client is nil") + } + if err := validateDEK(dek); err != nil { + return nil, err + } + ctx, cancel := requestContext(w.timeout) + defer cancel() + out, err := w.client.Encrypt(ctx, &awskms.EncryptInput{ + KeyId: aws.String(w.keyARN), + Plaintext: dek, + EncryptionContext: map[string]string{ + awsEncryptionContextKey: awsEncryptionContextValue, + }, + }) + if err != nil { + return nil, errors.Wrap(err, "kek: AWS KMS Encrypt") + } + if out == nil || len(out.CiphertextBlob) == 0 { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: AWS KMS returned empty ciphertext") + } + return append([]byte(nil), out.CiphertextBlob...), nil +} + +// Unwrap calls AWS KMS Decrypt and rejects any non-32-byte plaintext response. +func (w *AWSKMSWrapper) Unwrap(wrapped []byte) ([]byte, error) { + if w == nil || w.client == nil { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: AWS KMS client is nil") + } + if len(wrapped) == 0 { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: AWS KMS ciphertext is empty") + } + ctx, cancel := requestContext(w.timeout) + defer cancel() + out, err := w.client.Decrypt(ctx, &awskms.DecryptInput{ + CiphertextBlob: wrapped, + KeyId: aws.String(w.keyARN), + EncryptionContext: map[string]string{ + awsEncryptionContextKey: awsEncryptionContextValue, + }, + }) + if err != nil { + return nil, errors.Wrap(err, "kek: AWS KMS Decrypt") + } + if out == nil { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: AWS KMS returned nil plaintext") + } + if err := validateDEK(out.Plaintext); err != nil { + return nil, errors.Wrap(err, "kek: AWS KMS plaintext") + } + return append([]byte(nil), out.Plaintext...), nil +} + +// Name returns the provider and configured key ARN. +func (w *AWSKMSWrapper) Name() string { return "aws-kms:" + w.keyARN } diff --git a/internal/encryption/kek/aws_kms_test.go b/internal/encryption/kek/aws_kms_test.go new file mode 100644 index 000000000..f8bb012f7 --- /dev/null +++ b/internal/encryption/kek/aws_kms_test.go @@ -0,0 +1,96 @@ +package kek + +import ( + "bytes" + "context" + "testing" + + awskms "github.com/aws/aws-sdk-go-v2/service/kms" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +type fakeAWSKMSClient struct { + encryptInput *awskms.EncryptInput + decryptInput *awskms.DecryptInput + encryptOutput *awskms.EncryptOutput + decryptOutput *awskms.DecryptOutput + err error +} + +func (f *fakeAWSKMSClient) Encrypt(_ context.Context, input *awskms.EncryptInput, _ ...func(*awskms.Options)) (*awskms.EncryptOutput, error) { + f.encryptInput = input + return f.encryptOutput, f.err +} + +func (f *fakeAWSKMSClient) Decrypt(_ context.Context, input *awskms.DecryptInput, _ ...func(*awskms.Options)) (*awskms.DecryptOutput, error) { + f.decryptInput = input + return f.decryptOutput, f.err +} + +func TestAWSKMSWrapperRequestBinding(t *testing.T) { + const keyARN = "arn:aws:kms:us-east-1:123456789012:key/1234abcd" + dek := bytes.Repeat([]byte{0x42}, fileKEKSize) + client := &fakeAWSKMSClient{ + encryptOutput: &awskms.EncryptOutput{CiphertextBlob: []byte("wrapped")}, + decryptOutput: &awskms.DecryptOutput{Plaintext: append([]byte(nil), dek...)}, + } + wrapper := newAWSKMSWrapper(client, keyARN) + + wrapped, err := wrapper.Wrap(dek) + require.NoError(t, err) + require.Equal(t, []byte("wrapped"), wrapped) + require.Equal(t, keyARN, *client.encryptInput.KeyId) + require.Equal(t, dek, client.encryptInput.Plaintext) + require.Equal(t, awsEncryptionContextValue, client.encryptInput.EncryptionContext[awsEncryptionContextKey]) + + plain, err := wrapper.Unwrap(wrapped) + require.NoError(t, err) + require.Equal(t, dek, plain) + require.Equal(t, keyARN, *client.decryptInput.KeyId) + require.Equal(t, awsEncryptionContextValue, client.decryptInput.EncryptionContext[awsEncryptionContextKey]) + require.Equal(t, "aws-kms:"+keyARN, wrapper.Name()) +} + +func TestAWSKMSWrapperRejectsProviderFailures(t *testing.T) { + const keyARN = "arn:aws:kms:us-east-1:123456789012:key/1234abcd" + dek := bytes.Repeat([]byte{0x42}, fileKEKSize) + for _, tc := range []struct { + name string + client *fakeAWSKMSClient + unwrap bool + want error + }{ + {name: "encrypt error", client: &fakeAWSKMSClient{err: errors.New("denied")}, want: errors.New("sentinel")}, + {name: "empty ciphertext", client: &fakeAWSKMSClient{encryptOutput: &awskms.EncryptOutput{}}, want: ErrInvalidProviderResponse}, + {name: "short plaintext", client: &fakeAWSKMSClient{decryptOutput: &awskms.DecryptOutput{Plaintext: []byte("short")}}, unwrap: true, want: ErrInvalidDEKLength}, + } { + t.Run(tc.name, func(t *testing.T) { + wrapper := newAWSKMSWrapper(tc.client, keyARN) + var err error + if tc.unwrap { + _, err = wrapper.Unwrap([]byte("wrapped")) + } else { + _, err = wrapper.Wrap(dek) + } + require.Error(t, err) + if errors.Is(tc.want, ErrInvalidProviderResponse) || errors.Is(tc.want, ErrInvalidDEKLength) { + require.ErrorIs(t, err, tc.want) + } + }) + } +} + +func TestNewAWSKMSWrapperRejectsInvalidARNBeforeConfigLoad(t *testing.T) { + for _, keyARN := range []string{ + "not-an-arn", + "arn:aws:kms:us-east-1:123456789012:key/", + "arn:aws:kms:us-east-1:123456789012:alias/current", + "arn:aws:kms:us-east-1:123456789012:alias/", + } { + t.Run(keyARN, func(t *testing.T) { + _, err := NewAWSKMSWrapper(context.Background(), keyARN) + require.ErrorIs(t, err, ErrInvalidKEKURI) + }) + } +} diff --git a/internal/encryption/kek/env.go b/internal/encryption/kek/env.go new file mode 100644 index 000000000..59fa5c48a --- /dev/null +++ b/internal/encryption/kek/env.go @@ -0,0 +1,91 @@ +package kek + +import ( + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/base64" + "os" + + "github.com/cockroachdb/errors" +) + +// EnvVar is the test/CI-only environment variable accepted as a static KEK +// source. Production deployments should use a remote KMS provider. +const EnvVar = "ELASTICKV_KEK_BASE64" + +var ErrNilEnvWrapper = errors.New("kek: EnvWrapper is nil or uninitialised; construct with NewEnvWrapper") + +// EnvWrapper wraps DEKs with a process-local AES-256-GCM key loaded from +// EnvVar. NewEnvWrapper removes EnvVar immediately after decoding it so the +// raw KEK is not retained in the process environment. +type EnvWrapper struct { + aead cipher.AEAD +} + +// NewEnvWrapper reads a standard-base64 encoded 32-byte KEK. EnvVar is unset +// after the decode attempt on both success and failure paths. +func NewEnvWrapper() (*EnvWrapper, error) { + encoded, ok := os.LookupEnv(EnvVar) + if !ok { + return nil, errors.Errorf("kek: %s is not set", EnvVar) + } + raw, decodeErr := base64.StdEncoding.DecodeString(encoded) + unsetErr := os.Unsetenv(EnvVar) + if decodeErr != nil { + return nil, errors.Wrapf(decodeErr, "kek: decode %s", EnvVar) + } + defer clear(raw) + if unsetErr != nil { + return nil, errors.Wrapf(unsetErr, "kek: unset %s", EnvVar) + } + if err := validateDEK(raw); err != nil { + return nil, errors.Wrapf(err, "kek: %s", EnvVar) + } + block, err := aes.NewCipher(raw) + if err != nil { + return nil, errors.Wrap(err, "kek: env aes.NewCipher") + } + aead, err := cipher.NewGCM(block) + if err != nil { + return nil, errors.Wrap(err, "kek: env cipher.NewGCM") + } + return &EnvWrapper{aead: aead}, nil +} + +// Wrap returns nonce || AES-GCM(KEK, DEK), matching FileWrapper's local +// provider format. +func (w *EnvWrapper) Wrap(dek []byte) ([]byte, error) { + if w == nil || w.aead == nil { + return nil, errors.WithStack(ErrNilEnvWrapper) + } + if err := validateDEK(dek); err != nil { + return nil, err + } + nonce := make([]byte, fileNonceSize) + if _, err := rand.Read(nonce); err != nil { + return nil, errors.Wrap(err, "kek: env random nonce") + } + out := make([]byte, 0, fileNonceSize+fileKEKSize+fileTagSize) + out = append(out, nonce...) + return w.aead.Seal(out, nonce, dek, nil), nil +} + +// Unwrap authenticates and decrypts an EnvWrapper payload. +func (w *EnvWrapper) Unwrap(wrapped []byte) ([]byte, error) { + if w == nil || w.aead == nil { + return nil, errors.WithStack(ErrNilEnvWrapper) + } + if len(wrapped) != fileNonceSize+fileKEKSize+fileTagSize { + return nil, errors.Errorf("kek: env wrapped DEK is %d bytes, want %d", + len(wrapped), fileNonceSize+fileKEKSize+fileTagSize) + } + plain, err := w.aead.Open(nil, wrapped[:fileNonceSize], wrapped[fileNonceSize:], nil) + if err != nil { + return nil, errors.Wrap(err, "kek: env AES-GCM Open") + } + return plain, nil +} + +// Name identifies the environment-backed provider without exposing key bytes. +func (*EnvWrapper) Name() string { return "env" } diff --git a/internal/encryption/kek/env_test.go b/internal/encryption/kek/env_test.go new file mode 100644 index 000000000..a63cdb8ba --- /dev/null +++ b/internal/encryption/kek/env_test.go @@ -0,0 +1,52 @@ +package kek + +import ( + "bytes" + "encoding/base64" + "os" + "testing" + + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +func TestEnvWrapperRoundTripAndUnset(t *testing.T) { + key := bytes.Repeat([]byte{0x41}, fileKEKSize) + t.Setenv(EnvVar, base64.StdEncoding.EncodeToString(key)) + wrapper, err := NewEnvWrapper() + require.NoError(t, err) + _, stillSet := os.LookupEnv(EnvVar) + require.False(t, stillSet) + require.Equal(t, "env", wrapper.Name()) + + dek := bytes.Repeat([]byte{0xA5}, fileKEKSize) + wrapped, err := wrapper.Wrap(dek) + require.NoError(t, err) + require.NotEqual(t, dek, wrapped) + plain, err := wrapper.Unwrap(wrapped) + require.NoError(t, err) + require.Equal(t, dek, plain) + + wrapped[len(wrapped)-1] ^= 0x01 + _, err = wrapper.Unwrap(wrapped) + require.Error(t, err) +} + +func TestEnvWrapperInvalidInputStillUnsets(t *testing.T) { + t.Setenv(EnvVar, "not-base64") + _, err := NewEnvWrapper() + require.Error(t, err) + _, stillSet := os.LookupEnv(EnvVar) + require.False(t, stillSet) +} + +func TestEnvWrapperRejectsInvalidDEKLength(t *testing.T) { + t.Setenv(EnvVar, base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{1}, fileKEKSize))) + wrapper, err := NewEnvWrapper() + require.NoError(t, err) + _, err = wrapper.Wrap([]byte("short")) + require.ErrorIs(t, err, ErrInvalidDEKLength) + var nilWrapper *EnvWrapper + _, err = nilWrapper.Wrap(bytes.Repeat([]byte{1}, fileKEKSize)) + require.True(t, errors.Is(err, ErrNilEnvWrapper)) +} diff --git a/internal/encryption/kek/gcp_kms.go b/internal/encryption/kek/gcp_kms.go new file mode 100644 index 000000000..418df7cdc --- /dev/null +++ b/internal/encryption/kek/gcp_kms.go @@ -0,0 +1,137 @@ +package kek + +import ( + "context" + "hash/crc32" + "strings" + "time" + + gcpkms "cloud.google.com/go/kms/apiv1" + "cloud.google.com/go/kms/apiv1/kmspb" + "github.com/cockroachdb/errors" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +var ( + gcpKMSAAD = []byte("elastickv-dek-wrap-v1") + crc32cCastagnoli = crc32.MakeTable(crc32.Castagnoli) +) + +type gcpKMSClient interface { + Encrypt(context.Context, *kmspb.EncryptRequest) (*kmspb.EncryptResponse, error) + Decrypt(context.Context, *kmspb.DecryptRequest) (*kmspb.DecryptResponse, error) +} + +type gcpSDKClient struct { + client *gcpkms.KeyManagementClient +} + +func (c gcpSDKClient) Encrypt(ctx context.Context, req *kmspb.EncryptRequest) (*kmspb.EncryptResponse, error) { + out, err := c.client.Encrypt(ctx, req) + return out, errors.Wrap(err, "GCP KMS SDK Encrypt") +} + +func (c gcpSDKClient) Decrypt(ctx context.Context, req *kmspb.DecryptRequest) (*kmspb.DecryptResponse, error) { + out, err := c.client.Decrypt(ctx, req) + return out, errors.Wrap(err, "GCP KMS SDK Decrypt") +} + +// GCPKMSWrapper wraps DEKs using a Google Cloud KMS symmetric CryptoKey. +type GCPKMSWrapper struct { + client gcpKMSClient + keyName string + timeout time.Duration +} + +// NewGCPKMSWrapper uses Application Default Credentials to construct a Cloud +// KMS client for keyName. +func NewGCPKMSWrapper(ctx context.Context, keyName string) (*GCPKMSWrapper, error) { + if !validGCPKeyName(keyName) { + return nil, errors.Wrapf(ErrInvalidKEKURI, "invalid GCP KMS CryptoKey name %q", keyName) + } + client, err := gcpkms.NewKeyManagementClient(ctx) + if err != nil { + return nil, errors.Wrap(err, "kek: create GCP KMS client") + } + return newGCPKMSWrapper(gcpSDKClient{client: client}, keyName), nil +} + +func newGCPKMSWrapper(client gcpKMSClient, keyName string) *GCPKMSWrapper { + return &GCPKMSWrapper{client: client, keyName: keyName, timeout: providerRequestTimeout} +} + +func validGCPKeyName(name string) bool { + parts := strings.Split(name, "/") + return len(parts) == 8 && parts[0] == "projects" && parts[1] != "" && + parts[2] == "locations" && parts[3] != "" && parts[4] == "keyRings" && + parts[5] != "" && parts[6] == "cryptoKeys" && parts[7] != "" +} + +func crc32c(data []byte) *wrapperspb.Int64Value { + return wrapperspb.Int64(int64(crc32.Checksum(data, crc32cCastagnoli))) +} + +func checksumMatches(data []byte, checksum *wrapperspb.Int64Value) bool { + return checksum != nil && crc32c(data).Value == checksum.Value +} + +// Wrap calls Cloud KMS Encrypt with fixed AAD and verifies request/response +// CRC32C integrity metadata. +func (w *GCPKMSWrapper) Wrap(dek []byte) ([]byte, error) { + if w == nil || w.client == nil { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: GCP KMS client is nil") + } + if err := validateDEK(dek); err != nil { + return nil, err + } + ctx, cancel := requestContext(w.timeout) + defer cancel() + out, err := w.client.Encrypt(ctx, &kmspb.EncryptRequest{ + Name: w.keyName, + Plaintext: dek, + AdditionalAuthenticatedData: gcpKMSAAD, + PlaintextCrc32C: crc32c(dek), + AdditionalAuthenticatedDataCrc32C: crc32c(gcpKMSAAD), + }) + if err != nil { + return nil, errors.Wrap(err, "kek: GCP KMS Encrypt") + } + if out == nil || len(out.Ciphertext) == 0 || !out.VerifiedPlaintextCrc32C || + !out.VerifiedAdditionalAuthenticatedDataCrc32C || !checksumMatches(out.Ciphertext, out.CiphertextCrc32C) { + return nil, errors.WithStack(ErrInvalidProviderResponse) + } + return append([]byte(nil), out.Ciphertext...), nil +} + +// Unwrap calls Cloud KMS Decrypt and verifies the plaintext CRC32C before +// accepting the 32-byte DEK. +func (w *GCPKMSWrapper) Unwrap(wrapped []byte) ([]byte, error) { + if w == nil || w.client == nil { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: GCP KMS client is nil") + } + if len(wrapped) == 0 { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: GCP KMS ciphertext is empty") + } + ctx, cancel := requestContext(w.timeout) + defer cancel() + out, err := w.client.Decrypt(ctx, &kmspb.DecryptRequest{ + Name: w.keyName, + Ciphertext: wrapped, + AdditionalAuthenticatedData: gcpKMSAAD, + CiphertextCrc32C: crc32c(wrapped), + AdditionalAuthenticatedDataCrc32C: crc32c(gcpKMSAAD), + }) + if err != nil { + return nil, errors.Wrap(err, "kek: GCP KMS Decrypt") + } + if out == nil || !checksumMatches(out.Plaintext, out.PlaintextCrc32C) { + return nil, errors.WithStack(ErrInvalidProviderResponse) + } + if err := validateDEK(out.Plaintext); err != nil { + return nil, errors.Wrap(err, "kek: GCP KMS plaintext") + } + return append([]byte(nil), out.Plaintext...), nil +} + +// Name returns the provider and configured CryptoKey resource name. +func (w *GCPKMSWrapper) Name() string { return "gcp-kms:" + w.keyName } diff --git a/internal/encryption/kek/gcp_kms_test.go b/internal/encryption/kek/gcp_kms_test.go new file mode 100644 index 000000000..a0c394aaa --- /dev/null +++ b/internal/encryption/kek/gcp_kms_test.go @@ -0,0 +1,80 @@ +package kek + +import ( + "bytes" + "context" + "testing" + + "cloud.google.com/go/kms/apiv1/kmspb" + "github.com/stretchr/testify/require" +) + +type fakeGCPKMSClient struct { + encryptInput *kmspb.EncryptRequest + decryptInput *kmspb.DecryptRequest + encryptOutput *kmspb.EncryptResponse + decryptOutput *kmspb.DecryptResponse + err error +} + +func (f *fakeGCPKMSClient) Encrypt(_ context.Context, input *kmspb.EncryptRequest) (*kmspb.EncryptResponse, error) { + f.encryptInput = input + return f.encryptOutput, f.err +} + +func (f *fakeGCPKMSClient) Decrypt(_ context.Context, input *kmspb.DecryptRequest) (*kmspb.DecryptResponse, error) { + f.decryptInput = input + return f.decryptOutput, f.err +} + +func TestGCPKMSWrapperRequestAndResponseIntegrity(t *testing.T) { + const keyName = "projects/p/locations/global/keyRings/r/cryptoKeys/k" + dek := bytes.Repeat([]byte{0x52}, fileKEKSize) + ciphertext := []byte("gcp-wrapped") + client := &fakeGCPKMSClient{ + encryptOutput: &kmspb.EncryptResponse{ + Ciphertext: ciphertext, + CiphertextCrc32C: crc32c(ciphertext), + VerifiedPlaintextCrc32C: true, + VerifiedAdditionalAuthenticatedDataCrc32C: true, + }, + decryptOutput: &kmspb.DecryptResponse{Plaintext: dek, PlaintextCrc32C: crc32c(dek)}, + } + wrapper := newGCPKMSWrapper(client, keyName) + + wrapped, err := wrapper.Wrap(dek) + require.NoError(t, err) + require.Equal(t, ciphertext, wrapped) + require.Equal(t, keyName, client.encryptInput.Name) + require.Equal(t, gcpKMSAAD, client.encryptInput.AdditionalAuthenticatedData) + require.True(t, checksumMatches(dek, client.encryptInput.PlaintextCrc32C)) + + plain, err := wrapper.Unwrap(wrapped) + require.NoError(t, err) + require.Equal(t, dek, plain) + require.Equal(t, gcpKMSAAD, client.decryptInput.AdditionalAuthenticatedData) + require.True(t, checksumMatches(wrapped, client.decryptInput.CiphertextCrc32C)) + require.Equal(t, "gcp-kms:"+keyName, wrapper.Name()) +} + +func TestGCPKMSWrapperRejectsIntegrityFailures(t *testing.T) { + const keyName = "projects/p/locations/global/keyRings/r/cryptoKeys/k" + dek := bytes.Repeat([]byte{0x52}, fileKEKSize) + client := &fakeGCPKMSClient{encryptOutput: &kmspb.EncryptResponse{ + Ciphertext: []byte("ciphertext"), + CiphertextCrc32C: crc32c([]byte("different")), + VerifiedPlaintextCrc32C: true, + VerifiedAdditionalAuthenticatedDataCrc32C: true, + }} + _, err := newGCPKMSWrapper(client, keyName).Wrap(dek) + require.ErrorIs(t, err, ErrInvalidProviderResponse) + + client.decryptOutput = &kmspb.DecryptResponse{Plaintext: dek, PlaintextCrc32C: crc32c([]byte("different"))} + _, err = newGCPKMSWrapper(client, keyName).Unwrap([]byte("ciphertext")) + require.ErrorIs(t, err, ErrInvalidProviderResponse) +} + +func TestValidGCPKeyName(t *testing.T) { + require.True(t, validGCPKeyName("projects/p/locations/global/keyRings/r/cryptoKeys/k")) + require.False(t, validGCPKeyName("projects/p/keyRings/r/cryptoKeys/k")) +} diff --git a/internal/encryption/kek/kek.go b/internal/encryption/kek/kek.go index a4550af2e..93dc8e8f2 100644 --- a/internal/encryption/kek/kek.go +++ b/internal/encryption/kek/kek.go @@ -6,8 +6,8 @@ // in a KMS, a sealed file, or HashiCorp Vault — and only exercised at // process boot and at DEK rotation. // -// Stage 0 ships only the FileWrapper for tests / single-host clusters. -// AWS KMS, GCP KMS, and Vault providers are added in Stage 9. +// Implementations include AWS KMS, GCP KMS, Vault Transit, a static file, and +// the test/CI-only environment provider described by the design. package kek // Wrapper wraps and unwraps DEK bytes under an externally-held KEK. diff --git a/internal/encryption/kek/provider.go b/internal/encryption/kek/provider.go new file mode 100644 index 000000000..5991bc7ca --- /dev/null +++ b/internal/encryption/kek/provider.go @@ -0,0 +1,66 @@ +package kek + +import ( + "context" + "crypto/rand" + "crypto/subtle" + "time" + + "github.com/cockroachdb/errors" +) + +const providerRequestTimeout = 30 * time.Second + +var ( + // ErrInvalidDEKLength rejects non-AES-256 data keys at every provider + // boundary, including malformed provider responses. + ErrInvalidDEKLength = errors.New("kek: DEK must be exactly 32 bytes") + // ErrInvalidProviderResponse rejects an empty or integrity-invalid response + // before it can be persisted in the encryption sidecar. + ErrInvalidProviderResponse = errors.New("kek: invalid provider response") + // ErrKEKPreflightFailed prevents mutators from opening when the configured + // provider cannot complete a real wrap/unwrap round trip. + ErrKEKPreflightFailed = errors.New("kek: provider preflight failed") +) + +// VerifyWrapper proves credentials, provider reachability, encrypt/decrypt +// permissions, and key binding before any encryption mutator can commit a +// wrapped DEK. Provider constructors alone only validate local configuration. +func VerifyWrapper(wrapper Wrapper) error { + if wrapper == nil { + return errors.Wrap(ErrKEKPreflightFailed, "wrapper is nil") + } + dek := make([]byte, fileKEKSize) + defer clear(dek) + if _, err := rand.Read(dek); err != nil { + return errors.Wrapf(ErrKEKPreflightFailed, "generate probe DEK: %v", err) + } + wrapped, err := wrapper.Wrap(dek) + if err != nil { + return errors.Wrapf(ErrKEKPreflightFailed, "wrap with %s: %v", wrapper.Name(), err) + } + defer clear(wrapped) + plain, err := wrapper.Unwrap(wrapped) + if err != nil { + return errors.Wrapf(ErrKEKPreflightFailed, "unwrap with %s: %v", wrapper.Name(), err) + } + defer clear(plain) + if len(plain) != len(dek) || subtle.ConstantTimeCompare(plain, dek) != 1 { + return errors.Wrapf(ErrKEKPreflightFailed, "%s returned a different DEK", wrapper.Name()) + } + return nil +} + +func validateDEK(dek []byte) error { + if len(dek) != fileKEKSize { + return errors.Wrapf(ErrInvalidDEKLength, "got %d bytes", len(dek)) + } + return nil +} + +func requestContext(timeout time.Duration) (context.Context, context.CancelFunc) { + if timeout <= 0 { + timeout = providerRequestTimeout + } + return context.WithTimeout(context.Background(), timeout) +} diff --git a/internal/encryption/kek/provider_test.go b/internal/encryption/kek/provider_test.go new file mode 100644 index 000000000..af913df00 --- /dev/null +++ b/internal/encryption/kek/provider_test.go @@ -0,0 +1,82 @@ +package kek + +import ( + "bytes" + "testing" + + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +type preflightWrapper struct { + wrapErr error + unwrapErr error + mismatch bool +} + +func (w *preflightWrapper) Wrap(dek []byte) ([]byte, error) { + if w.wrapErr != nil { + return nil, w.wrapErr + } + return append([]byte(nil), dek...), nil +} + +func (w *preflightWrapper) Unwrap(wrapped []byte) ([]byte, error) { + if w.unwrapErr != nil { + return nil, w.unwrapErr + } + plain := append([]byte(nil), wrapped...) + if w.mismatch { + plain[0] ^= 0xFF + } + return plain, nil +} + +func (*preflightWrapper) Name() string { return "test" } + +func TestVerifyWrapper(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + wrapper Wrapper + wantErr bool + }{ + {name: "round trip", wrapper: &preflightWrapper{}}, + {name: "nil wrapper", wrapper: nil, wantErr: true}, + {name: "wrap failure", wrapper: &preflightWrapper{wrapErr: errors.New("denied")}, wantErr: true}, + {name: "unwrap failure", wrapper: &preflightWrapper{unwrapErr: errors.New("denied")}, wantErr: true}, + {name: "mismatched plaintext", wrapper: &preflightWrapper{mismatch: true}, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + err := VerifyWrapper(tc.wrapper) + if tc.wantErr { + require.ErrorIs(t, err, ErrKEKPreflightFailed) + return + } + require.NoError(t, err) + }) + } +} + +func TestVerifyWrapperUsesRandomDEKLength(t *testing.T) { + t.Parallel() + wrapper := &capturingPreflightWrapper{} + require.NoError(t, VerifyWrapper(wrapper)) + require.Len(t, wrapper.seen, fileKEKSize) + require.False(t, bytes.Equal(wrapper.seen, make([]byte, fileKEKSize))) +} + +type capturingPreflightWrapper struct { + seen []byte +} + +func (w *capturingPreflightWrapper) Wrap(dek []byte) ([]byte, error) { + w.seen = append([]byte(nil), dek...) + return append([]byte(nil), dek...), nil +} + +func (*capturingPreflightWrapper) Unwrap(wrapped []byte) ([]byte, error) { + return append([]byte(nil), wrapped...), nil +} + +func (*capturingPreflightWrapper) Name() string { return "capture" } diff --git a/internal/encryption/kek/source.go b/internal/encryption/kek/source.go new file mode 100644 index 000000000..fecf5bc7a --- /dev/null +++ b/internal/encryption/kek/source.go @@ -0,0 +1,91 @@ +package kek + +import ( + "context" + "os" + "strings" + + "github.com/cockroachdb/errors" +) + +const ( + awsKMSScheme = "aws-kms://" + gcpKMSScheme = "gcp-kms://" + vaultTransitScheme = "vault-transit://" +) + +var ( + // ErrMultipleKEKSources rejects ambiguous key configuration rather than + // silently selecting one source by precedence. + ErrMultipleKEKSources = errors.New("kek: configure exactly one of --kekFile, --kekUri, or ELASTICKV_KEK_BASE64") + // ErrInvalidKEKURI rejects unknown providers and malformed provider targets. + ErrInvalidKEKURI = errors.New("kek: invalid KEK URI") +) + +// NewWrapperFromSources resolves exactly one file, URI, or environment-backed +// KEK. No configured source returns nil so encryption-disabled deployments keep +// their existing startup behavior. +func NewWrapperFromSources(ctx context.Context, filePath, uri string) (Wrapper, error) { + _, envSet := os.LookupEnv(EnvVar) + if countConfiguredSources(filePath != "", uri != "", envSet) > 1 { + return nil, rejectSourceConflict(envSet) + } + switch { + case filePath != "": + wrapper, err := NewFileWrapper(filePath) + if err != nil { + return nil, errors.Wrapf(err, "kek: load file %q", filePath) + } + return wrapper, nil + case uri != "": + return newURIWrapper(ctx, uri) + case envSet: + return NewEnvWrapper() + default: + return nil, nil + } +} + +func countConfiguredSources(sources ...bool) int { + configured := 0 + for _, present := range sources { + if present { + configured++ + } + } + return configured +} + +func rejectSourceConflict(envSet bool) error { + if envSet { + if err := os.Unsetenv(EnvVar); err != nil { + return errors.Wrapf(err, "kek: unset %s after source conflict", EnvVar) + } + } + return errors.WithStack(ErrMultipleKEKSources) +} + +func newURIWrapper(ctx context.Context, uri string) (Wrapper, error) { + switch { + case strings.HasPrefix(uri, awsKMSScheme): + target := strings.TrimPrefix(uri, awsKMSScheme) + if target == "" { + return nil, errors.WithStack(ErrInvalidKEKURI) + } + return NewAWSKMSWrapper(ctx, target) + case strings.HasPrefix(uri, gcpKMSScheme): + target := strings.TrimPrefix(uri, gcpKMSScheme) + if target == "" { + return nil, errors.WithStack(ErrInvalidKEKURI) + } + return NewGCPKMSWrapper(ctx, target) + case strings.HasPrefix(uri, vaultTransitScheme): + target := strings.TrimPrefix(uri, vaultTransitScheme) + if target == "" { + return nil, errors.WithStack(ErrInvalidKEKURI) + } + return NewVaultTransitWrapper(target) + default: + return nil, errors.Wrapf(ErrInvalidKEKURI, "unsupported URI %q", uri) + } +} diff --git a/internal/encryption/kek/source_test.go b/internal/encryption/kek/source_test.go new file mode 100644 index 000000000..4bb834f7d --- /dev/null +++ b/internal/encryption/kek/source_test.go @@ -0,0 +1,59 @@ +package kek + +import ( + "bytes" + "context" + "encoding/base64" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNewWrapperFromSourcesNoSource(t *testing.T) { + t.Setenv(EnvVar, "") + require.NoError(t, os.Unsetenv(EnvVar)) + wrapper, err := NewWrapperFromSources(context.Background(), "", "") + require.NoError(t, err) + require.Nil(t, wrapper) +} + +func TestNewWrapperFromSourcesRejectsAmbiguity(t *testing.T) { + t.Setenv(EnvVar, base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{1}, fileKEKSize))) + _, err := NewWrapperFromSources(context.Background(), "/tmp/kek", "") + require.ErrorIs(t, err, ErrMultipleKEKSources) + _, stillSet := os.LookupEnv(EnvVar) + require.False(t, stillSet) +} + +func TestNewWrapperFromSourcesFileAndEnv(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "kek.bin") + require.NoError(t, os.WriteFile(path, bytes.Repeat([]byte{2}, fileKEKSize), 0o600)) + + wrapper, err := NewWrapperFromSources(context.Background(), path, "") + require.NoError(t, err) + require.Contains(t, wrapper.Name(), "file:") + + t.Setenv(EnvVar, base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{3}, fileKEKSize))) + wrapper, err = NewWrapperFromSources(context.Background(), "", "") + require.NoError(t, err) + require.Equal(t, "env", wrapper.Name()) + _, stillSet := os.LookupEnv(EnvVar) + require.False(t, stillSet) +} + +func TestNewURIWrapperRejectsInvalidTargetsWithoutNetwork(t *testing.T) { + for _, uri := range []string{ + "unknown://key", + "aws-kms://not-an-arn", + "gcp-kms://projects/incomplete", + "vault-transit://transit", + } { + t.Run(uri, func(t *testing.T) { + _, err := newURIWrapper(context.Background(), uri) + require.ErrorIsf(t, err, ErrInvalidKEKURI, "uri=%q", uri) + }) + } +} diff --git a/internal/encryption/kek/vault.go b/internal/encryption/kek/vault.go new file mode 100644 index 000000000..9a36cef7d --- /dev/null +++ b/internal/encryption/kek/vault.go @@ -0,0 +1,154 @@ +package kek + +import ( + "context" + "encoding/base64" + "os" + "strconv" + "strings" + "time" + + "github.com/cockroachdb/errors" + vaultapi "github.com/hashicorp/vault/api" +) + +type vaultLogicalClient interface { + ReadWithContext(context.Context, string) (*vaultapi.Secret, error) + WriteWithContext(context.Context, string, map[string]interface{}) (*vaultapi.Secret, error) +} + +var vaultTransitAAD = base64.StdEncoding.EncodeToString([]byte("elastickv-dek-wrap-v1")) + +// VaultTransitWrapper wraps DEKs with a Vault Transit symmetric key. +type VaultTransitWrapper struct { + logical vaultLogicalClient + mount string + keyName string + timeout time.Duration +} + +// NewVaultTransitWrapper uses the standard VAULT_ADDR, VAULT_TOKEN, TLS, and +// namespace environment configuration. target is /. +func NewVaultTransitWrapper(target string) (*VaultTransitWrapper, error) { + mount, keyName, err := parseVaultTarget(target) + if err != nil { + return nil, err + } + config := vaultapi.DefaultConfig() + if err := config.ReadEnvironment(); err != nil { + return nil, errors.Wrap(err, "kek: read Vault environment") + } + client, err := vaultapi.NewClient(config) + if err != nil { + return nil, errors.Wrap(err, "kek: create Vault client") + } + client.SetToken(os.Getenv("VAULT_TOKEN")) + return newVaultTransitWrapper(client.Logical(), mount, keyName), nil +} + +func newVaultTransitWrapper(logical vaultLogicalClient, mount, keyName string) *VaultTransitWrapper { + return &VaultTransitWrapper{logical: logical, mount: mount, keyName: keyName, timeout: providerRequestTimeout} +} + +func parseVaultTarget(target string) (string, string, error) { + parts := strings.Split(target, "/") + if len(parts) < 2 || parts[0] == "" { + return "", "", errors.Wrapf(ErrInvalidKEKURI, "invalid Vault Transit target %q", target) + } + for _, part := range parts { + if part == "" || part == "." || part == ".." { + return "", "", errors.Wrapf(ErrInvalidKEKURI, "invalid Vault Transit target %q", target) + } + } + return strings.Join(parts[:len(parts)-1], "/"), parts[len(parts)-1], nil +} + +// Wrap base64-encodes the binary DEK for Vault's JSON API and stores Vault's +// versioned ciphertext string as the wrapped sidecar bytes. +func (w *VaultTransitWrapper) Wrap(dek []byte) ([]byte, error) { + if w == nil || w.logical == nil { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: Vault client is nil") + } + if err := validateDEK(dek); err != nil { + return nil, err + } + ctx, cancel := requestContext(w.timeout) + defer cancel() + key, err := w.logical.ReadWithContext(ctx, w.mount+"/keys/"+w.keyName) + if err != nil { + return nil, errors.Wrap(err, "kek: read Vault Transit key") + } + if _, ok := vaultString(key, "type"); !ok { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: Vault Transit key does not exist") + } + secret, err := w.logical.WriteWithContext(ctx, w.mount+"/encrypt/"+w.keyName, map[string]interface{}{ + "plaintext": base64.StdEncoding.EncodeToString(dek), + "associated_data": vaultTransitAAD, + }) + if err != nil { + return nil, errors.Wrap(err, "kek: Vault Transit encrypt") + } + ciphertext, ok := vaultString(secret, "ciphertext") + if !ok || !validVaultCiphertext(ciphertext) { + return nil, errors.WithStack(ErrInvalidProviderResponse) + } + return []byte(ciphertext), nil +} + +// Unwrap asks Vault Transit to decrypt its versioned ciphertext and validates +// the returned base64 plaintext as a 32-byte DEK. +func (w *VaultTransitWrapper) Unwrap(wrapped []byte) ([]byte, error) { + if w == nil || w.logical == nil { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: Vault client is nil") + } + if !validVaultCiphertext(string(wrapped)) { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: invalid Vault ciphertext") + } + ctx, cancel := requestContext(w.timeout) + defer cancel() + secret, err := w.logical.WriteWithContext(ctx, w.mount+"/decrypt/"+w.keyName, map[string]interface{}{ + "ciphertext": string(wrapped), + "associated_data": vaultTransitAAD, + }) + if err != nil { + return nil, errors.Wrap(err, "kek: Vault Transit decrypt") + } + encoded, ok := vaultString(secret, "plaintext") + if !ok { + return nil, errors.WithStack(ErrInvalidProviderResponse) + } + plain, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return nil, errors.Wrap(ErrInvalidProviderResponse, "kek: Vault plaintext is not base64") + } + if err := validateDEK(plain); err != nil { + clear(plain) + return nil, errors.Wrap(err, "kek: Vault plaintext") + } + return plain, nil +} + +func validVaultCiphertext(ciphertext string) bool { + const prefix = "vault:v" + if !strings.HasPrefix(ciphertext, prefix) { + return false + } + rest := strings.TrimPrefix(ciphertext, prefix) + separator := strings.IndexByte(rest, ':') + if separator <= 0 || separator == len(rest)-1 { + return false + } + version, err := strconv.ParseUint(rest[:separator], 10, 64) + return err == nil && version > 0 +} + +func vaultString(secret *vaultapi.Secret, field string) (string, bool) { + if secret == nil || secret.Data == nil { + return "", false + } + value, ok := secret.Data[field].(string) + return value, ok && value != "" +} + +// Name returns the provider and Transit mount/key path. +func (w *VaultTransitWrapper) Name() string { return "vault-transit:" + w.mount + "/" + w.keyName } diff --git a/internal/encryption/kek/vault_test.go b/internal/encryption/kek/vault_test.go new file mode 100644 index 000000000..900ea5115 --- /dev/null +++ b/internal/encryption/kek/vault_test.go @@ -0,0 +1,104 @@ +package kek + +import ( + "bytes" + "context" + "encoding/base64" + "testing" + + "github.com/hashicorp/vault/api" + "github.com/stretchr/testify/require" +) + +type fakeVaultLogical struct { + readPath string + writePath string + data map[string]interface{} + dek []byte + ciphertext string + key *api.Secret +} + +func (f *fakeVaultLogical) ReadWithContext(_ context.Context, path string) (*api.Secret, error) { + f.readPath = path + if f.key != nil { + return f.key, nil + } + return &api.Secret{Data: map[string]interface{}{"type": "aes256-gcm96"}}, nil +} + +func (f *fakeVaultLogical) WriteWithContext(_ context.Context, path string, data map[string]interface{}) (*api.Secret, error) { + f.writePath = path + f.data = data + if _, ok := data["plaintext"]; ok { + ciphertext := f.ciphertext + if ciphertext == "" { + ciphertext = "vault:v1:ciphertext" + } + return &api.Secret{Data: map[string]interface{}{"ciphertext": ciphertext}}, nil + } + return &api.Secret{Data: map[string]interface{}{"plaintext": base64.StdEncoding.EncodeToString(f.dek)}}, nil +} + +func TestVaultTransitWrapperRequestBinding(t *testing.T) { + dek := bytes.Repeat([]byte{0x62}, fileKEKSize) + logical := &fakeVaultLogical{dek: dek} + wrapper := newVaultTransitWrapper(logical, "security/transit", "orders") + + wrapped, err := wrapper.Wrap(dek) + require.NoError(t, err) + require.Equal(t, []byte("vault:v1:ciphertext"), wrapped) + require.Equal(t, "security/transit/keys/orders", logical.readPath) + require.Equal(t, "security/transit/encrypt/orders", logical.writePath) + require.Equal(t, base64.StdEncoding.EncodeToString(dek), logical.data["plaintext"]) + require.Equal(t, vaultTransitAAD, logical.data["associated_data"]) + + plain, err := wrapper.Unwrap(wrapped) + require.NoError(t, err) + require.Equal(t, dek, plain) + require.Equal(t, "security/transit/decrypt/orders", logical.writePath) + require.Equal(t, "vault:v1:ciphertext", logical.data["ciphertext"]) + require.Equal(t, vaultTransitAAD, logical.data["associated_data"]) + require.Equal(t, "vault-transit:security/transit/orders", wrapper.Name()) +} + +func TestParseVaultTarget(t *testing.T) { + mount, keyName, err := parseVaultTarget("transit/service/key") + require.NoError(t, err) + require.Equal(t, "transit/service", mount) + require.Equal(t, "key", keyName) + for _, target := range []string{"", "transit", "transit//key", "../key", "transit/../key"} { + t.Run(target, func(t *testing.T) { + _, _, err := parseVaultTarget(target) + require.ErrorIsf(t, err, ErrInvalidKEKURI, "target=%q", target) + }) + } +} + +func TestVaultTransitWrapperRequiresExistingKey(t *testing.T) { + logical := &fakeVaultLogical{key: &api.Secret{Data: map[string]interface{}{}}} + wrapper := newVaultTransitWrapper(logical, "transit", "missing") + + _, err := wrapper.Wrap(bytes.Repeat([]byte{0x42}, fileKEKSize)) + require.ErrorIs(t, err, ErrInvalidProviderResponse) + require.Equal(t, "transit/keys/missing", logical.readPath) + require.Empty(t, logical.writePath) +} + +func TestVaultTransitWrapperRejectsMalformedResponses(t *testing.T) { + logical := &fakeVaultLogical{dek: []byte("short")} + wrapper := newVaultTransitWrapper(logical, "transit", "key") + _, err := wrapper.Unwrap([]byte("vault:v1:ciphertext")) + require.ErrorIs(t, err, ErrInvalidDEKLength) + _, err = wrapper.Unwrap([]byte("not-vault")) + require.ErrorIs(t, err, ErrInvalidProviderResponse) + for _, ciphertext := range []string{"vault:v", "vault:v1:", "vault:v0:data", "vault:vx:data"} { + t.Run(ciphertext, func(t *testing.T) { + invalid := newVaultTransitWrapper(&fakeVaultLogical{dek: bytes.Repeat([]byte{1}, fileKEKSize), ciphertext: ciphertext}, "transit", "key") + _, wrapErr := invalid.Wrap(bytes.Repeat([]byte{2}, fileKEKSize)) + require.ErrorIs(t, wrapErr, ErrInvalidProviderResponse) + _, unwrapErr := invalid.Unwrap([]byte(ciphertext)) + require.ErrorIs(t, unwrapErr, ErrInvalidProviderResponse) + }) + } +} diff --git a/internal/encryption/raft_envelope.go b/internal/encryption/raft_envelope.go index bf317c744..0610917cb 100644 --- a/internal/encryption/raft_envelope.go +++ b/internal/encryption/raft_envelope.go @@ -88,6 +88,7 @@ func WrapRaftPayload(c *Cipher, keyID uint32, nonce, payload []byte) ([]byte, er // // - ErrEnvelopeShort: encoded shorter than HeaderSize+TagSize // - ErrEnvelopeVersion: unknown version byte +// - ErrEnvelopeFlag: a storage-only or unknown flag bit is present // - ErrUnknownKeyID: DEK is not loaded (retired or sidecar missing) // - ErrIntegrity: GCM tag mismatch (tampered envelope, wrong DEK, // or layer confusion with a storage envelope) @@ -104,6 +105,14 @@ func UnwrapRaftPayload(c *Cipher, encoded []byte) ([]byte, error) { if err != nil { return nil, errors.Wrap(err, "encryption: raft envelope decode") } + if env.Flag != 0 { + return nil, errors.Wrapf(ErrEnvelopeFlag, + "encryption: raft envelope flag must be 0x00, got 0x%02x", env.Flag) + } + if env.Version != EnvelopeVersionV1 { + return nil, errors.Wrapf(ErrEnvelopeVersion, + "encryption: raft envelope version must be 0x%02x, got 0x%02x", EnvelopeVersionV1, env.Version) + } aad := BuildRaftAAD(env.Version, env.KeyID) plain, err := c.Decrypt(env.Body, aad, env.KeyID, env.Nonce[:]) if err != nil { diff --git a/internal/encryption/raft_envelope_test.go b/internal/encryption/raft_envelope_test.go index 12915a321..7fddd8c48 100644 --- a/internal/encryption/raft_envelope_test.go +++ b/internal/encryption/raft_envelope_test.go @@ -172,6 +172,33 @@ func TestRaftEnvelope_RejectsRetiredKey(t *testing.T) { } } +func TestRaftEnvelope_RejectsStorageCompressionFlag(t *testing.T) { + t.Parallel() + c, keyID := raftFixture(t) + nonce := newRandomNonce(t) + payload := []byte("raft payload") + aad := encryption.BuildRaftAAD(encryption.EnvelopeVersionV2, keyID) + body, err := c.Encrypt(payload, aad, keyID, nonce) + if err != nil { + t.Fatalf("Encrypt: %v", err) + } + var nonceArray [encryption.NonceSize]byte + copy(nonceArray[:], nonce) + encoded, err := (&encryption.Envelope{ + Version: encryption.EnvelopeVersionV2, + Flag: encryption.FlagCompressed, + KeyID: keyID, + Nonce: nonceArray, + Body: body, + }).Encode() + if err != nil { + t.Fatalf("Envelope.Encode: %v", err) + } + if _, err := encryption.UnwrapRaftPayload(c, encoded); !errors.Is(err, encryption.ErrEnvelopeFlag) { + t.Fatalf("expected ErrEnvelopeFlag, got %v", err) + } +} + // TestRaftEnvelope_ShortInputRejected covers DecodeEnvelope's // length precondition (HeaderSize + TagSize = 34 bytes minimum). func TestRaftEnvelope_ShortInputRejected(t *testing.T) { diff --git a/internal/encryption/startup.go b/internal/encryption/startup.go index 411aa42ef..6f8278bf4 100644 --- a/internal/encryption/startup.go +++ b/internal/encryption/startup.go @@ -28,7 +28,7 @@ type StartupConfig struct { // (downgrade prevention). EncryptionEnabled bool - // KEKConfigured is true iff --kekFile is non-empty. The KEK + // KEKConfigured is true iff one configured KEK source loaded. The KEK // itself is supplied via KEK below; KEKConfigured exists // independently so the helper can distinguish "operator did // not supply a KEK source" from "supplied but failed to load" @@ -203,7 +203,7 @@ func guardSidecarWithoutFlag(cfg StartupConfig, sidecarPresent bool) error { } // guardKEKRequired fires when the operator turned on -// --encryption-enabled without supplying --kekFile. A flag-on / +// --encryption-enabled without supplying a KEK source. A flag-on / // KEK-off node would refuse every mutator at the Stage 6B-2 RPC // gate AND HaltApply if a mutator ever did commit, neither of // which matches the operator's stated intent. Fail fast at startup. @@ -220,7 +220,7 @@ func guardKEKRequired(cfg StartupConfig) error { // guardKEKMatchesSidecar attempts to KEK-unwrap every wrapped DEK // in the sidecar. A single failure fires ErrKEKMismatch with the // offending key_id annotated — the classic operator error here is -// "wrong --kekFile points at a key from a different cluster" and +// "the configured KEK belongs to a different cluster" and // the key_id identifies which DEK could not be unwrapped, which is // almost always enough to root-cause. // diff --git a/main.go b/main.go index 479bcc69b..09d6ef994 100644 --- a/main.go +++ b/main.go @@ -191,22 +191,22 @@ var ( // // Mutating RPCs (BootstrapEncryption / RotateDEK / // RegisterEncryptionWriter) are gated by Stage 6B-2 on the - // AND of --encryption-enabled and --kekFile being non-empty. + // AND of --encryption-enabled and a successfully loaded KEK source. // Setting --encryptionSidecarPath ALONE no longer enables // mutators; the operator must explicitly opt in to encryption // AND supply a KEK source. With either gate condition false, // registerEncryptionAdminServer omits the Proposer + LeaderView // options and every mutator short-circuits at the gRPC boundary // with FailedPrecondition before any Raft proposal is created. - encryptionSidecarPath = flag.String("encryptionSidecarPath", "", "§5.1 keys.json path; enables read-only EncryptionAdmin capability probing. Mutating RPCs (Bootstrap / RotateDEK / RegisterEncryptionWriter) are additionally gated on this flag being non-empty AND --encryption-enabled AND --kekFile being non-empty (all three required so the applier's WithKEK + WithKeystore + WithSidecarPath options are all wired before mutators can commit).") + encryptionSidecarPath = flag.String("encryptionSidecarPath", "", "§5.1 keys.json path; enables read-only EncryptionAdmin capability probing. Mutating RPCs additionally require --encryption-enabled and one loaded KEK source.") // Stage 6B-2: cluster-wide encryption opt-in flag. The mutating // EncryptionAdmin RPCs (BootstrapEncryption, RotateDEK, // RegisterEncryptionWriter) become reachable only when this - // flag is set AND --kekFile points at a valid KEK source. + // flag is set AND exactly one configured KEK source loads successfully. // Default off; pre-Stage-6 clusters and operators who have // not yet committed to encryption are unaffected. - encryptionEnabled = flag.Bool("encryption-enabled", false, "§6.5 opt-in to encryption-mutating EncryptionAdmin RPCs. Requires --kekFile to be set; without that, mutators still refuse with FailedPrecondition. Default off.") + encryptionEnabled = flag.Bool("encryption-enabled", false, "§6.5 opt-in to encryption-mutating EncryptionAdmin RPCs. Requires exactly one KEK source from --kekFile, --kekUri, or ELASTICKV_KEK_BASE64. Default off.") // Stage 6F: operator-requested DEK rotation at boot. The flag is // intentionally a request, not a guarantee: only the leader of the @@ -215,15 +215,14 @@ var ( // leadership during this process uptime. encryptionRotateOnStartup = flag.Bool("encryption-rotate-on-startup", false, "§6.5 request a one-shot DEK rotation after this node becomes leader of the default Raft group. Safe for rolling restarts: followers keep the request in memory and only fire if they acquire leadership during this process uptime.") - // Stage 6B-2: KEK source. The KEK never appears in elastickv's + // KEK sources. The KEK never appears in elastickv's // data dir; it is held externally and exercised only at process - // boot and at DEK bootstrap/rotation per §5.1. Stage 6B-2 ships - // only the file-backed wrapper (kek.FileWrapper); KMS providers - // (--kekUri) land in Stage 9. Empty disables KEK loading; the + // boot and at DEK bootstrap/rotation per §5.1. Empty disables KEK loading; the // applier's ApplyBootstrap and ApplyRotation paths then return // ErrKEKNotConfigured at apply time, which is masked at the // RPC boundary by the mutator gate documented above. kekFile = flag.String("kekFile", "", "§5.1 KEK file path (32 raw bytes, owner-only mode). When set, the file-backed kek.Wrapper is constructed at startup and threaded into the §6.3 EncryptionApplier so ApplyBootstrap and ApplyRotation can KEK-unwrap.") + kekURI = flag.String("kekUri", "", "§5.1 remote KEK URI: aws-kms://, gcp-kms://, or vault-transit:///. Mutually exclusive with --kekFile and ELASTICKV_KEK_BASE64.") // Key visualizer sampler flags. The sampler runs entirely in-memory // on each node, feeds AdminServer.GetKeyVizMatrix, and is disabled @@ -437,7 +436,7 @@ func run() error { // visible to every shard's storage cipher. // // Both are nil-safe in the applier path: WithKEK / WithKeystore - // are only attached to the applier when --kekFile is non-empty + // are only attached to the applier when a KEK source is loaded // (else the applier stays in the Stage 6A posture where // ApplyBootstrap / ApplyRotation return ErrKEKNotConfigured). kekWrapper, err := loadKEKAfterPreNonceStartupGuards(cfg) @@ -599,6 +598,7 @@ func run() error { redisApplyObserver: redisApplyObserver, cleanup: &cleanup, encWiring: encWiring, + kekConfigured: kekWrapper != nil, keyvizSampler: sampler, encryptionConfChangeInterceptor: encryptionConfChangeInterceptor, }); err != nil { @@ -1467,6 +1467,7 @@ func buildShardGroups( _ = st.Close() return nil, nil, errors.Wrapf(err, "failed to start raft group %d", g.id) } + runtime.writerRegistry = reg runtimes = append(runtimes, runtime) // Stage 6E-2c: route every shard group's TransactionManager // through NewLeaderProxyForShardGroup so the proposer chain @@ -1544,6 +1545,22 @@ func postCutoverProposerForRuntime(rt *raftGroupRuntime, shardGroups map[uint64] return proposerForGroup(rt, shardGroups) } +func writerRegistryForEncryptionAdmin(runtimes []*raftGroupRuntime, defaultGroup uint64) encryption.WriterRegistryStore { + defaultRuntime := findDefaultGroupRuntime(runtimes, defaultGroup) + if defaultRuntime == nil { + return nil + } + return defaultRuntime.writerRegistry +} + +func recoveryStateForEncryptionAdmin(runtimes []*raftGroupRuntime, defaultGroup uint64) (raftengine.LeaderView, func() uint64) { + defaultRuntime := findDefaultGroupRuntime(runtimes, defaultGroup) + if defaultRuntime == nil { + return nil, nil + } + return defaultRuntime.engine, appliedIndexForEngine(defaultRuntime.engine) +} + func appliedIndexForEngine(engine raftengine.Engine) func() uint64 { applied, ok := engine.(interface{ AppliedIndex() uint64 }) if !ok { @@ -1567,7 +1584,7 @@ func loadKEKAfterPreNonceStartupGuards(cfg runtimeConfig) (kek.Wrapper, error) { return loadKEKAndRunStartupGuards() } -// loadKEKAndRunStartupGuards loads the file-backed KEK wrapper and +// loadKEKAndRunStartupGuards loads the configured KEK wrapper and // runs the §9.1 startup-refusal guards (Stage 6C-1) BEFORE // buildShardGroups constructs any Raft engine or storage state. The // two operations are paired in a single helper because the guards @@ -1596,29 +1613,39 @@ func loadKEKAndRunStartupGuards() (kek.Wrapper, error) { } if err := encryption.CheckStartupGuards(encryption.StartupConfig{ EncryptionEnabled: *encryptionEnabled, - KEKConfigured: *kekFile != "", + KEKConfigured: kekWrapper != nil, KEK: kekWrapper, SidecarPath: *encryptionSidecarPath, }); err != nil { return nil, errors.Wrap(err, "encryption startup guards refused process start") } + if err := verifyKEKBeforeMutators(kekWrapper); err != nil { + return nil, err + } return kekWrapper, nil } -// loadKEKWrapperFromFlag constructs the file-backed KEK wrapper -// from the --kekFile flag, returning nil if the flag is empty. +func verifyKEKBeforeMutators(wrapper kek.Wrapper) error { + if !*encryptionEnabled || *encryptionSidecarPath == "" || wrapper == nil { + return nil + } + if err := kek.VerifyWrapper(wrapper); err != nil { + return errors.Wrap(err, "encryption KEK preflight refused process start") + } + return nil +} + +// loadKEKWrapperFromFlag constructs the configured KEK wrapper from the +// mutually-exclusive file, URI, or environment source. // Returns the kek.Wrapper interface rather than the concrete // *kek.FileWrapper so the call site (buildShardGroups → applier) // stays decoupled from the file-mode provider — Stage 9 KMS // providers (AWS KMS, GCP KMS, Vault) will satisfy the same // interface and slot in without rewriting the dispatch site. func loadKEKWrapperFromFlag() (kek.Wrapper, error) { - if *kekFile == "" { - return nil, nil - } - w, err := kek.NewFileWrapper(*kekFile) + w, err := kek.NewWrapperFromSources(context.Background(), *kekFile, *kekURI) if err != nil { - return nil, errors.Wrapf(err, "failed to load KEK from %s", *kekFile) + return nil, errors.Wrap(err, "failed to load KEK source") } return w, nil } @@ -1772,6 +1799,7 @@ type serversInput struct { metricsRegistry *monitoring.Registry cfg runtimeConfig encWiring encryptionWriteWiring + kekConfigured bool redisApplyObserver *adapter.RedisApplyObserver cleanup *internalutil.CleanupStack // keyvizSampler is the in-memory key visualizer sampler, or nil @@ -1855,8 +1883,10 @@ func startServersAfterStartupRotation(waitRotateOnStartup startupRotationWaiter, pubsubRelay: adapter.NewRedisPubSubRelay(), readTracker: in.readTracker, encWiring: in.encWiring, + kekConfigured: in.kekConfigured, redisApplyObserver: in.redisApplyObserver, dynamoAddress: *dynamoAddr, + defaultGroup: in.cfg.defaultGroup, leaderDynamo: in.cfg.leaderDynamo, s3Address: *s3Addr, leaderS3: in.cfg.leaderS3, @@ -2502,14 +2532,18 @@ func startRaftServers( forwardDeps adminForwardServerDeps, confChangeInterceptor internalraftadmin.MembershipChangeInterceptor, encWiring encryptionWriteWiring, + kekConfigured bool, + defaultGroup uint64, ) error { forwardLogger := slog.Default().With(slog.String("component", "admin")) // extraOptsCap reserves slots for the unary + stream admin interceptor // options appended below. Sized as a constant so the magic-number // linter does not complain. const extraOptsCap = 2 - enableMutators := encryptionMutatorsEnabled() + enableMutators := encryptionMutatorsEnabled(kekConfigured) encryptionCapabilityFanout := buildEncryptionCapabilityFanout(ctx, eg, runtimes, enableMutators) + adminWriterRegistry := writerRegistryForEncryptionAdmin(runtimes, defaultGroup) + recoveryLeaderView, recoveryAppliedIndex := recoveryStateForEncryptionAdmin(runtimes, defaultGroup) for _, rt := range runtimes { baseOpts := internalutil.GRPCServerOptions() opts := make([]grpc.ServerOption, 0, len(baseOpts)+extraOptsCap) @@ -2554,9 +2588,10 @@ func startRaftServers( // a stable, distinct value. Codex r1 P1 on PR #760. // Stage 6B-2 mutator gate is resolved once above the // per-shard loop. Each shard's own engine remains the raw - // Proposer + LeaderView for the cutover marker, while - // ShardGroup.Proposer() supplies the wrap-aware post-cutover - // path for normal admin entries. + // proposer, while recovery leadership and applied-index evidence + // come from the default group that owns the shared sidecar and + // writer registry. ShardGroup.Proposer() supplies the wrap-aware + // post-cutover path for normal admin entries. registerEncryptionAdminServer( gs, etcdraftengine.DeriveNodeID(*raftId), @@ -2564,7 +2599,9 @@ func startRaftServers( enableMutators, rt.engine, encryptionCapabilityFanout, - adapter.WithEncryptionAdminLatestAppliedIndex(appliedIndexForEngine(rt.engine)), + adapter.WithEncryptionAdminWriterRegistry(adminWriterRegistry), + adapter.WithEncryptionAdminRecoveryLeaderView(recoveryLeaderView), + adapter.WithEncryptionAdminLatestAppliedIndex(recoveryAppliedIndex), adapter.WithEncryptionAdminPostCutoverProposer(proposerForGroup(rt, shardGroups)), adapter.WithEncryptionAdminCutoverBarrier(encWiring.raftEnvelope.barrier()), ) @@ -2885,7 +2922,9 @@ type runtimeServerRunner struct { readTracker *kv.ActiveTimestampTracker redisApplyObserver *adapter.RedisApplyObserver encWiring encryptionWriteWiring + kekConfigured bool dynamoAddress string + defaultGroup uint64 leaderDynamo map[string]string s3Address string leaderS3 map[string]string @@ -2999,6 +3038,8 @@ func (r *runtimeServerRunner) startRaftTransport() error { forwardDeps, r.encryptionConfChangeInterceptor, r.encWiring, + r.kekConfigured, + r.defaultGroup, ); err != nil { return r.startupFailure(err) } diff --git a/main_encryption_admin.go b/main_encryption_admin.go index 7a92976d6..32859dd97 100644 --- a/main_encryption_admin.go +++ b/main_encryption_admin.go @@ -12,7 +12,7 @@ import ( // readback. Returns true iff THIS NODE has all three of: // // - --encryption-enabled (explicit operator opt-in) -// - --kekFile non-empty (KEK source loaded so ApplyBootstrap +// - a KEK source loaded (file, URI, or test/CI environment source, so ApplyBootstrap // / ApplyRotation can KEK-unwrap) // - --encryptionSidecarPath non-empty (so the applier can // crash-durably persist Active.{Storage,Raft} + keys[]) @@ -60,8 +60,8 @@ import ( // Kept in this file (not main.go) so the flag-driven gate logic // is colocated with the registerEncryptionAdminServer helper // that consumes it. -func encryptionMutatorsEnabled() bool { - return *encryptionEnabled && *kekFile != "" && *encryptionSidecarPath != "" +func encryptionMutatorsEnabled(kekConfigured bool) bool { + return *encryptionEnabled && kekConfigured && *encryptionSidecarPath != "" } // encryptionAdminEngine is the subset of raftengine.Engine the @@ -94,7 +94,7 @@ type encryptionAdminEngine interface { // // Stage 6B-2: the mutator wiring (Proposer + LeaderView) is now // gated on the supplied enableMutators boolean, which the caller -// in main.go computes as (--encryption-enabled AND --kekFile +// in main.go computes as (--encryption-enabled AND loaded KEK // non-empty AND engine non-nil). When enableMutators is false, // Proposer + LeaderView stay unwired and EncryptionAdminServer's // BootstrapEncryption / RotateDEK / RegisterEncryptionWriter @@ -113,7 +113,7 @@ type encryptionAdminEngine interface { // means the cluster has explicitly chosen NOT to participate // in the §7.1 rollout, so mutator RPCs MUST refuse even on // a fully-keyed binary. -// - --kekFile being non-empty means a KEK source is loaded; +// - kekConfigured means a file, URI, or environment KEK source is loaded; // without it, a mutator that committed would land in the // applier with no KEK and return ErrKEKNotConfigured from // the §6.3 HaltApply path — that is fail-closed but it diff --git a/main_encryption_admin_test.go b/main_encryption_admin_test.go index df98b0d1c..77186e43f 100644 --- a/main_encryption_admin_test.go +++ b/main_encryption_admin_test.go @@ -5,6 +5,7 @@ import ( "net" "testing" + "github.com/bootjp/elastickv/internal/encryption" "github.com/bootjp/elastickv/internal/raftengine" etcdraftengine "github.com/bootjp/elastickv/internal/raftengine/etcd" pb "github.com/bootjp/elastickv/proto" @@ -41,6 +42,29 @@ func (stubEncryptionAdminEngine) LinearizableRead(context.Context) (uint64, erro return 0, nil } +type stubEncryptionAdminRegistry struct { + name string +} + +type stubEncryptionRecoveryEngine struct { + raftengine.Engine + applied uint64 +} + +func (s *stubEncryptionRecoveryEngine) AppliedIndex() uint64 { + return s.applied +} + +func (stubEncryptionAdminRegistry) GetRegistryRow([]byte) ([]byte, bool, error) { + return nil, false, nil +} + +func (stubEncryptionAdminRegistry) SetRegistryRow(_, _ []byte) error { + return nil +} + +var _ encryption.WriterRegistryStore = stubEncryptionAdminRegistry{} + // TestEncryptionAdminFullNodeID_DistinctPerRaftId pins the // PR760 r1 Codex P1 regression: full_node_id must be derived from // --raftId (per-node-stable) and NOT from the Raft group id @@ -65,6 +89,58 @@ func TestEncryptionAdminFullNodeID_DistinctPerRaftId(t *testing.T) { } } +func TestWriterRegistryForEncryptionAdminUsesDefaultGroup(t *testing.T) { + defaultRegistry := stubEncryptionAdminRegistry{name: "default"} + otherRegistry := stubEncryptionAdminRegistry{name: "other"} + runtimes := []*raftGroupRuntime{ + {spec: groupSpec{id: 2}, writerRegistry: otherRegistry}, + {spec: groupSpec{id: 1}, writerRegistry: defaultRegistry}, + } + + got := writerRegistryForEncryptionAdmin(runtimes, 1) + if got != defaultRegistry { + t.Fatalf("writerRegistryForEncryptionAdmin selected %#v, want default group registry %#v", got, defaultRegistry) + } +} + +func TestWriterRegistryForEncryptionAdminMissingDefaultGroup(t *testing.T) { + got := writerRegistryForEncryptionAdmin([]*raftGroupRuntime{ + {spec: groupSpec{id: 2}, writerRegistry: stubEncryptionAdminRegistry{name: "other"}}, + }, 1) + if got != nil { + t.Fatalf("writerRegistryForEncryptionAdmin selected %#v, want nil when default group runtime is missing", got) + } +} + +func TestRecoveryStateForEncryptionAdminUsesDefaultGroup(t *testing.T) { + defaultEngine := &stubEncryptionRecoveryEngine{applied: 41} + otherEngine := &stubEncryptionRecoveryEngine{applied: 99} + runtimes := []*raftGroupRuntime{ + {spec: groupSpec{id: 2}, engine: otherEngine}, + {spec: groupSpec{id: 1}, engine: defaultEngine}, + } + + leaderView, appliedIndex := recoveryStateForEncryptionAdmin(runtimes, 1) + if leaderView != defaultEngine { + t.Fatalf("recoveryStateForEncryptionAdmin selected %T, want default group engine", leaderView) + } + if appliedIndex == nil { + t.Fatal("recoveryStateForEncryptionAdmin returned a nil applied-index callback") + } + if got := appliedIndex(); got != 41 { + t.Fatalf("recoveryStateForEncryptionAdmin applied index = %d, want 41", got) + } +} + +func TestRecoveryStateForEncryptionAdminMissingDefaultGroup(t *testing.T) { + leaderView, appliedIndex := recoveryStateForEncryptionAdmin([]*raftGroupRuntime{ + {spec: groupSpec{id: 2}, engine: &stubEncryptionRecoveryEngine{applied: 99}}, + }, 1) + if leaderView != nil || appliedIndex != nil { + t.Fatalf("recoveryStateForEncryptionAdmin returned leader=%T callback-present=%t, want both nil", leaderView, appliedIndex != nil) + } +} + // TestEncryptionAdmin_MutatingRPCRefusedWhenGateOff pins the // Stage 6B-2 double-gate. With either condition of the gate false // (enableMutators=false OR engine=nil), Proposer + LeaderView diff --git a/main_encryption_kek_source_test.go b/main_encryption_kek_source_test.go new file mode 100644 index 000000000..dead8a56b --- /dev/null +++ b/main_encryption_kek_source_test.go @@ -0,0 +1,129 @@ +package main + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "testing" + + "github.com/bootjp/elastickv/internal/encryption" + "github.com/bootjp/elastickv/internal/encryption/fsmwire" + "github.com/bootjp/elastickv/internal/encryption/kek" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestEncryptionMutatorsEnabledUsesLoadedKEKState(t *testing.T) { + previousEnabled := *encryptionEnabled + previousSidecar := *encryptionSidecarPath + t.Cleanup(func() { + *encryptionEnabled = previousEnabled + *encryptionSidecarPath = previousSidecar + }) + + *encryptionEnabled = true + *encryptionSidecarPath = "/data/encryption/keys.json" + require.True(t, encryptionMutatorsEnabled(true)) + require.False(t, encryptionMutatorsEnabled(false)) + *encryptionEnabled = false + require.False(t, encryptionMutatorsEnabled(true)) + *encryptionEnabled = true + *encryptionSidecarPath = "" + require.False(t, encryptionMutatorsEnabled(true)) +} + +type failingStartupKEK struct{} + +func (failingStartupKEK) Wrap([]byte) ([]byte, error) { return nil, errors.New("provider unavailable") } +func (failingStartupKEK) Unwrap([]byte) ([]byte, error) { + return nil, errors.New("provider unavailable") +} +func (failingStartupKEK) Name() string { return "failing" } + +func TestVerifyKEKBeforeMutators(t *testing.T) { + previousEnabled := *encryptionEnabled + previousSidecar := *encryptionSidecarPath + t.Cleanup(func() { + *encryptionEnabled = previousEnabled + *encryptionSidecarPath = previousSidecar + }) + + *encryptionEnabled = true + *encryptionSidecarPath = "/data/encryption/keys.json" + require.ErrorIs(t, verifyKEKBeforeMutators(failingStartupKEK{}), kek.ErrKEKPreflightFailed) + + *encryptionEnabled = false + require.NoError(t, verifyKEKBeforeMutators(failingStartupKEK{})) + *encryptionEnabled = true + *encryptionSidecarPath = "" + require.NoError(t, verifyKEKBeforeMutators(failingStartupKEK{})) +} + +func TestEnvKEKBootstrapCutoverSnapshotRestore(t *testing.T) { + staticKEK := bytes.Repeat([]byte{0x71}, encryption.KeySize) + t.Setenv(kek.EnvVar, base64.StdEncoding.EncodeToString(staticKEK)) + wrapper, err := kek.NewWrapperFromSources(context.Background(), "", "") + require.NoError(t, err) + require.NoError(t, kek.VerifyWrapper(wrapper)) + + storageDEK := bytes.Repeat([]byte{0x72}, encryption.KeySize) + raftDEK := bytes.Repeat([]byte{0x73}, encryption.KeySize) + wrappedStorage, err := wrapper.Wrap(storageDEK) + require.NoError(t, err) + wrappedRaft, err := wrapper.Wrap(raftDEK) + require.NoError(t, err) + + dir := t.TempDir() + sidecarPath := dir + "/keys.json" + keystore := encryption.NewKeystore() + wiring, err := buildEncryptionWriteWiring(true, "n1", sidecarPath, wrapper, keystore, []groupSpec{{id: 1}}) + require.NoError(t, err) + source, err := store.NewPebbleStore(dir+"/source", wiring.pebbleOptions()...) + require.NoError(t, err) + t.Cleanup(func() { _ = source.Close() }) + registry, err := store.WriterRegistryFor(source) + require.NoError(t, err) + applier, err := encryption.NewApplier( + registry, + encryption.WithKEK(wrapper), + encryption.WithKeystore(keystore), + encryption.WithSidecarPath(sidecarPath), + encryption.WithStateCache(wiring.cache), + ) + require.NoError(t, err) + + require.NoError(t, applier.ApplyBootstrap(1, fsmwire.BootstrapPayload{ + StorageDEKID: e2eStorageDEKID, + WrappedStorage: wrappedStorage, + RaftDEKID: e2eRaftDEKID, + WrappedRaft: wrappedRaft, + BatchRegistry: []fsmwire.RegistrationPayload{ + {DEKID: e2eStorageDEKID, FullNodeID: e2eNodeID, LocalEpoch: 0}, + }, + })) + require.NoError(t, applier.ApplyRotation(2, fsmwire.RotationPayload{ + SubTag: fsmwire.RotateSubEnableStorageEnvelope, + DEKID: e2eStorageDEKID, + Purpose: fsmwire.PurposeStorage, + Wrapped: []byte{}, + ProposerRegistration: fsmwire.RegistrationPayload{DEKID: e2eStorageDEKID, FullNodeID: e2eNodeID, LocalEpoch: 1}, + })) + wiring.cache.MarkRegistered(e2eStorageDEKID) + require.NoError(t, source.PutAt(context.Background(), []byte("env-kek"), []byte("secret"), 100, 0)) + + snapshot, err := source.Snapshot() + require.NoError(t, err) + var snapshotBytes bytes.Buffer + _, err = snapshot.WriteTo(&snapshotBytes) + require.NoError(t, err) + require.NoError(t, snapshot.Close()) + + target, err := store.NewPebbleStore(dir+"/target", wiring.pebbleOptions()...) + require.NoError(t, err) + t.Cleanup(func() { _ = target.Close() }) + require.NoError(t, target.Restore(bytes.NewReader(snapshotBytes.Bytes()))) + got, err := target.GetAt(context.Background(), []byte("env-kek"), 100) + require.NoError(t, err) + require.Equal(t, []byte("secret"), got) +} diff --git a/multiraft_runtime.go b/multiraft_runtime.go index c46daacfe..e5695ec3e 100644 --- a/multiraft_runtime.go +++ b/multiraft_runtime.go @@ -8,6 +8,7 @@ import ( "strings" "sync" + "github.com/bootjp/elastickv/internal/encryption" "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" @@ -27,8 +28,9 @@ type raftGroupRuntime struct { engineMu sync.RWMutex engine raftengine.Engine - store store.MVCCStore - stateMachine raftengine.StateMachine + store store.MVCCStore + writerRegistry encryption.WriterRegistryStore + stateMachine raftengine.StateMachine registerTransport func(grpc.ServiceRegistrar) closeFactory func() error // releases factory-created resources (transport, stores) diff --git a/store/encryption_compression_test.go b/store/encryption_compression_test.go new file mode 100644 index 000000000..941dc8a2a --- /dev/null +++ b/store/encryption_compression_test.go @@ -0,0 +1,302 @@ +package store + +import ( + "bytes" + "context" + "crypto/rand" + "testing" + + "github.com/bootjp/elastickv/internal/encryption" + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/v2" + "github.com/golang/snappy" + "github.com/stretchr/testify/require" + "pgregory.net/rapid" +) + +func TestEncryption_CompressesOnlyWhenSmaller(t *testing.T) { + t.Parallel() + incompressible := make([]byte, 4096) + if _, err := rand.Read(incompressible); err != nil { + t.Fatalf("rand.Read incompressible value: %v", err) + } + cases := []struct { + name string + value []byte + compressed bool + }{ + {name: "compressible", value: bytes.Repeat([]byte("json-field:"), 512), compressed: true}, + {name: "below threshold", value: bytes.Repeat([]byte("a"), minEncryptionCompressionSize-1), compressed: false}, + {name: "at threshold", value: bytes.Repeat([]byte("a"), minEncryptionCompressionSize), compressed: true}, + {name: "empty", value: []byte{}, compressed: false}, + {name: "small", value: []byte("value"), compressed: false}, + {name: "high entropy", value: incompressible, compressed: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + f := newEncryptedStoreFixture(t, 101) + key := []byte("compression/" + tc.name) + if err := f.mvcc.PutAt(context.Background(), key, tc.value, 100, 0); err != nil { + t.Fatalf("PutAt: %v", err) + } + + var version, flag byte + f.tamperPebbleValue(t, key, 100, func(raw []byte) []byte { + sv, err := decodeValue(raw) + if err != nil { + t.Fatalf("decodeValue: %v", err) + } + env, err := encryption.DecodeEnvelope(sv.Value) + if err != nil { + t.Fatalf("DecodeEnvelope: %v", err) + } + version = env.Version + flag = env.Flag + return raw + }) + require.Equal(t, tc.compressed, flag&encryption.FlagCompressed != 0) + if tc.compressed { + require.Equal(t, encryption.EnvelopeVersionV2, version) + } else { + require.Equal(t, encryption.EnvelopeVersionV1, version) + } + + got, err := f.mvcc.GetAt(context.Background(), key, 100) + require.NoError(t, err) + require.Equal(t, tc.value, got) + }) + } +} + +func TestEncryption_CompressionFlagTamperRejected(t *testing.T) { + t.Parallel() + f := newEncryptedStoreFixture(t, 102) + key := []byte("compressed-tamper") + value := bytes.Repeat([]byte("compress-me"), 512) + if err := f.mvcc.PutAt(context.Background(), key, value, 100, 0); err != nil { + t.Fatalf("PutAt: %v", err) + } + f.tamperPebbleValue(t, key, 100, func(raw []byte) []byte { + // value header (9) + envelope version (1) precede the flag. + raw[valueHeaderSize+1] &^= encryption.FlagCompressed + return raw + }) + if _, err := f.mvcc.GetAt(context.Background(), key, 100); !errors.Is(err, ErrEncryptedReadIntegrity) { + t.Fatalf("compression flag tamper: expected ErrEncryptedReadIntegrity, got %v", err) + } +} + +func TestEncryption_CompressedEnvelopeRebadgeRejected(t *testing.T) { + t.Parallel() + f := newEncryptedStoreFixture(t, 104) + key := []byte("compressed-rebadge") + value := bytes.Repeat([]byte("sensitive-json-field"), 512) + if err := f.mvcc.PutAt(context.Background(), key, value, 100, 0); err != nil { + t.Fatalf("PutAt: %v", err) + } + f.tamperPebbleValue(t, key, 100, func(raw []byte) []byte { + raw[0] &^= encStateMask + return raw + }) + if _, err := f.mvcc.GetAt(context.Background(), key, 100); !errors.Is(err, ErrEncryptedReadIntegrity) { + t.Fatalf("compressed rebadge: expected ErrEncryptedReadIntegrity, got %v", err) + } +} + +func TestEncryption_RejectsAuthenticatedMalformedSnappy(t *testing.T) { + t.Parallel() + f := newEncryptedStoreFixture(t, 103) + key := []byte("malformed-snappy") + const commitTS uint64 = 100 + pebbleKey := encodeKey(key, commitTS) + var header [valueHeaderSize]byte + writeValueHeaderBytes(header[:], false, 0, encStateEncrypted) + aad := buildStorageAAD(encryption.EnvelopeVersionV2, encryption.FlagCompressed, f.keyID, header[:], pebbleKey) + var nonce [encryption.NonceSize]byte + if _, err := rand.Read(nonce[:]); err != nil { + t.Fatalf("rand.Read nonce: %v", err) + } + body, err := f.cipher.Encrypt([]byte{0xff}, aad, f.keyID, nonce[:]) + if err != nil { + t.Fatalf("Encrypt malformed payload: %v", err) + } + envelopeBytes, err := (&encryption.Envelope{ + Version: encryption.EnvelopeVersionV2, + Flag: encryption.FlagCompressed, + KeyID: f.keyID, + Nonce: nonce, + Body: body, + }).Encode() + if err != nil { + t.Fatalf("Envelope.Encode: %v", err) + } + + f.closeIfOpen(t) + pdb, err := pebble.Open(f.dir, &pebble.Options{}) + if err != nil { + t.Fatalf("pebble.Open: %v", err) + } + if err := pdb.Set(pebbleKey, encodeValue(envelopeBytes, false, 0, encStateEncrypted), pebble.Sync); err != nil { + _ = pdb.Close() + t.Fatalf("pdb.Set: %v", err) + } + if err := pdb.Close(); err != nil { + t.Fatalf("pdb.Close: %v", err) + } + f.reopen(t) + + if _, err := f.mvcc.GetAt(context.Background(), key, commitTS); !errors.Is(err, ErrEncryptedReadCompression) { + t.Fatalf("malformed authenticated Snappy: expected ErrEncryptedReadCompression, got %v", err) + } + if err := f.mvcc.DeletePrefixAt(context.Background(), []byte("malformed-"), nil, commitTS+1); err != nil { + t.Fatalf("visibility-only prefix delete expanded malformed Snappy: %v", err) + } +} + +func TestEncryption_RejectsAuthenticatedOversizedSnappyBeforeDecode(t *testing.T) { + previousMax := maxSnapshotValueSize + maxSnapshotValueSize = 64 + t.Cleanup(func() { maxSnapshotValueSize = previousMax }) + + f := newEncryptedStoreFixture(t, 106) + key := []byte("oversized-snappy") + const commitTS uint64 = 100 + pebbleKey := encodeKey(key, commitTS) + var header [valueHeaderSize]byte + writeValueHeaderBytes(header[:], false, 0, encStateEncrypted) + aad := buildStorageAAD(encryption.EnvelopeVersionV2, encryption.FlagCompressed, f.keyID, header[:], pebbleKey) + compressed := snappy.Encode(nil, bytes.Repeat([]byte("a"), maxSnapshotValueSize+1)) + var nonce [encryption.NonceSize]byte + if _, err := rand.Read(nonce[:]); err != nil { + t.Fatalf("rand.Read nonce: %v", err) + } + body, err := f.cipher.Encrypt(compressed, aad, f.keyID, nonce[:]) + if err != nil { + t.Fatalf("Encrypt oversized payload: %v", err) + } + envelopeBytes, err := (&encryption.Envelope{ + Version: encryption.EnvelopeVersionV2, + Flag: encryption.FlagCompressed, + KeyID: f.keyID, + Nonce: nonce, + Body: body, + }).Encode() + if err != nil { + t.Fatalf("Envelope.Encode: %v", err) + } + + f.closeIfOpen(t) + pdb, err := pebble.Open(f.dir, &pebble.Options{}) + if err != nil { + t.Fatalf("pebble.Open: %v", err) + } + if err := pdb.Set(pebbleKey, encodeValue(envelopeBytes, false, 0, encStateEncrypted), pebble.Sync); err != nil { + _ = pdb.Close() + t.Fatalf("pdb.Set: %v", err) + } + if err := pdb.Close(); err != nil { + t.Fatalf("pdb.Close: %v", err) + } + f.reopen(t) + + if _, err := f.mvcc.GetAt(context.Background(), key, commitTS); !errors.Is(err, ErrEncryptedReadCompression) { + t.Fatalf("oversized authenticated Snappy: expected ErrEncryptedReadCompression, got %v", err) + } +} + +func TestCompressionRoundTripProperty(t *testing.T) { + rapid.Check(t, func(t *rapid.T) { + plaintext := rapid.SliceOfN(rapid.Byte(), 0, 16<<10).Draw(t, "plaintext") + payload, flag := compressForEncryption(plaintext) + if flag == 0 { + if !bytes.Equal(payload, plaintext) { + t.Fatalf("uncompressed payload mismatch") + } + return + } + if flag != encryption.FlagCompressed { + t.Fatalf("unexpected compression flag %#x", flag) + } + if len(payload) >= len(plaintext) { + t.Fatalf("compressed payload grew: got=%d original=%d", len(payload), len(plaintext)) + } + got, err := snappy.Decode(nil, payload) + if err != nil { + t.Fatalf("snappy.Decode: %v", err) + } + if !bytes.Equal(got, plaintext) { + t.Fatalf("round-trip mismatch") + } + }) +} + +func TestEncryptedStoreCompressionRoundTripProperty(t *testing.T) { + t.Parallel() + ks := encryption.NewKeystore() + dek := make([]byte, encryption.KeySize) + if _, err := rand.Read(dek); err != nil { + t.Fatalf("rand.Read DEK: %v", err) + } + const keyID uint32 = 105 + if err := ks.Set(keyID, dek); err != nil { + t.Fatalf("Keystore.Set: %v", err) + } + cipher, err := encryption.NewCipher(ks) + if err != nil { + t.Fatalf("NewCipher: %v", err) + } + s := &pebbleStore{ + cipher: cipher, + nonceFactory: NewCounterNonceFactory(0x0102, 0x0304), + activeStorageKeyID: func() (uint32, bool) { return keyID, true }, + } + rapid.Check(t, func(t *rapid.T) { + key := rapid.SliceOfN(rapid.Byte(), 0, 256).Draw(t, "key") + plaintext := rapid.SliceOfN(rapid.Byte(), 0, 16<<10).Draw(t, "plaintext") + expireAt := rapid.Uint64().Draw(t, "expireAt") + pebbleKey := encodeKey(key, 100) + body, encState, err := s.encryptForKey(pebbleKey, plaintext, expireAt, false) + if err != nil { + t.Fatalf("encryptForKey: %v", err) + } + if encState != encStateEncrypted { + t.Fatalf("encryption state=%#x, want encrypted", encState) + } + got, err := s.decryptForKey(pebbleKey, storedValue{ + Value: body, + EncState: encState, + ExpireAt: expireAt, + }, body) + if err != nil { + t.Fatalf("decryptForKey: %v", err) + } + if !bytes.Equal(got, plaintext) { + t.Fatalf("round-trip mismatch") + } + }) +} + +func TestPebbleCompressionPolicy(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + name string + disable bool + wantName string + }{ + {name: "legacy defaults", disable: false, wantName: "Snappy"}, + {name: "encrypted store", disable: true, wantName: "NoCompression"}, + } { + t.Run(tc.name, func(t *testing.T) { + opts, cache := defaultPebbleOptionsWithCache(tc.disable) + t.Cleanup(cache.Unref) + for level, levelOpts := range opts.Levels { + profile := levelOpts.Compression() + if tc.disable { + require.Equalf(t, tc.wantName, profile.Name, "level %d", level) + continue + } + require.NotEqualf(t, "NoCompression", profile.Name, "level %d", level) + } + }) + } +} diff --git a/store/encryption_glue.go b/store/encryption_glue.go index a51740c96..506a7a79e 100644 --- a/store/encryption_glue.go +++ b/store/encryption_glue.go @@ -4,6 +4,7 @@ import ( "github.com/bootjp/elastickv/internal/encryption" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/v2" + "github.com/golang/snappy" ) // ErrUnsupportedStoreForWriterRegistry is returned by @@ -103,6 +104,15 @@ func WriterRegistryFor(s MVCCStore) (encryption.WriterRegistryStore, error) { // Callers can disambiguate it from any other read error with errors.Is. var ErrEncryptedReadIntegrity = errors.New("store: encrypted value failed integrity check (GCM tag mismatch); refusing to surface plaintext") +// ErrEncryptedReadCompression is returned when an authenticated envelope is +// marked as Snappy-compressed but its decrypted body is not a valid Snappy +// stream. Disk corruption normally fails GCM first; reaching this sentinel +// means a writer produced a malformed authenticated payload, so reads fail +// closed instead of surfacing the compressed bytes as user data. +var ErrEncryptedReadCompression = errors.New("store: authenticated encrypted value contains an invalid Snappy payload") + +const minEncryptionCompressionSize = 32 + // NonceFactory produces unique 12-byte AES-GCM nonces for the storage // envelope (§4.1). The factory is responsible for the cluster-wide // uniqueness invariant across `(node_id, local_epoch, write_count)` — @@ -345,17 +355,20 @@ func (s *pebbleStore) encryptForKey(pebbleKey, plaintext []byte, expireAt uint64 return nil, 0, errors.Wrap(err, "store: nonce factory") } nonce := nonceArr[:] - // flag = 0: Snappy compression deferred to Stage 9 per design §4.1. - const envelopeFlag byte = 0 + payload, envelopeFlag := compressForEncryption(plaintext) + envelopeVersion := encryption.EnvelopeVersionV1 + if envelopeFlag&encryption.FlagCompressed != 0 { + envelopeVersion = encryption.EnvelopeVersionV2 + } var hdr [valueHeaderSize]byte writeValueHeaderBytes(hdr[:], false /*tombstone*/, expireAt, encStateEncrypted) - aad := buildStorageAAD(encryption.EnvelopeVersionV1, envelopeFlag, keyID, hdr[:], pebbleKey) - ciphertextAndTag, err := s.cipher.Encrypt(plaintext, aad, keyID, nonce) + aad := buildStorageAAD(envelopeVersion, envelopeFlag, keyID, hdr[:], pebbleKey) + ciphertextAndTag, err := s.cipher.Encrypt(payload, aad, keyID, nonce) if err != nil { return nil, 0, errors.Wrap(err, "store: encrypt value") } env := encryption.Envelope{ - Version: encryption.EnvelopeVersionV1, + Version: envelopeVersion, Flag: envelopeFlag, KeyID: keyID, Nonce: nonceArr, @@ -388,6 +401,18 @@ func (s *pebbleStore) encryptForKey(pebbleKey, plaintext []byte, expireAt uint64 // tombstone / expireAt visibility checks AFTER decrypt succeeds — // the values they observe pre-decrypt are not yet authenticated. func (s *pebbleStore) decryptForKey(pebbleKey []byte, sv storedValue, body []byte) ([]byte, error) { + return s.decryptForKeyMode(pebbleKey, sv, body, true) +} + +// authenticateForKey authenticates the envelope and value header without +// expanding compressed plaintext. Visibility-only callers use this before +// branching on tombstone or expiry fields. +func (s *pebbleStore) authenticateForKey(pebbleKey []byte, sv storedValue, body []byte) error { + _, err := s.decryptForKeyMode(pebbleKey, sv, body, false) + return err +} + +func (s *pebbleStore) decryptForKeyMode(pebbleKey []byte, sv storedValue, body []byte, decompress bool) ([]byte, error) { if sv.EncState == encStateCleartext { if err := s.rejectRebadgedEnvelope(pebbleKey, sv, body); err != nil { return nil, err @@ -413,6 +438,12 @@ func (s *pebbleStore) decryptForKey(pebbleKey []byte, sv storedValue, body []byt } return nil, errors.Wrap(err, "store: decrypt value") } + if decompress { + plain, err = decompressAuthenticatedValue(plain, env.Flag) + if err != nil { + return nil, err + } + } // AES-GCM Open returns a nil dst slice for an empty plaintext; // upstream callers (notably ExistsAt) distinguish "key absent" // from "key present with empty value" via val != nil. Normalize @@ -424,6 +455,50 @@ func (s *pebbleStore) decryptForKey(pebbleKey []byte, sv storedValue, body []byt return plain, nil } +// decompressAuthenticatedValue inspects the Snappy length prefix before any +// output allocation. Callers must authenticate the envelope before reaching +// this helper; the compressed bytes are otherwise attacker-controlled. +func decompressAuthenticatedValue(plain []byte, flag byte) ([]byte, error) { + if flag&encryption.FlagCompressed == 0 { + return plain, nil + } + decodedLen, err := snappy.DecodedLen(plain) + if err != nil { + return nil, errors.Wrap( + errors.WithSecondaryError(ErrEncryptedReadCompression, err), + "store: inspect compressed encrypted value") + } + if decodedLen > maxSnapshotValueSize { + return nil, errors.Wrapf( + ErrEncryptedReadCompression, + "store: compressed encrypted value expands to %d bytes, maximum is %d", + decodedLen, + maxSnapshotValueSize) + } + decoded, err := snappy.Decode(nil, plain) + if err != nil { + return nil, errors.Wrap( + errors.WithSecondaryError(ErrEncryptedReadCompression, err), + "store: decompress encrypted value") + } + return decoded, nil +} + +// compressForEncryption applies §6.4's compress-then-encrypt policy. Snappy +// output is used only when it is strictly smaller than the original bytes; +// already-compressed and small payloads stay uncompressed so encryption never +// increases CPU and storage cost for a larger intermediate representation. +func compressForEncryption(plaintext []byte) ([]byte, byte) { + if len(plaintext) < minEncryptionCompressionSize { + return plaintext, 0 + } + compressed := snappy.Encode(nil, plaintext) + if len(compressed) >= len(plaintext) { + return plaintext, 0 + } + return compressed, encryption.FlagCompressed +} + // rejectRebadgedEnvelope is the cleartext-branch guard for the §4.1 // encryption-state rebadge attack. The on-disk encryption_state bit // is not itself authenticated, so a disk attacker who flips it from @@ -444,11 +519,9 @@ func (s *pebbleStore) decryptForKey(pebbleKey []byte, sv storedValue, body []byt // uses, instead of trusting the on-disk header bytes the attacker // could have flipped: // -// - envelope_version = EnvelopeVersionV1 (the encrypt path's -// fixed value; trusting on-disk would let a corrupted version -// byte force the body through DecodeEnvelope's error path) -// - flag = 0 (Snappy compression is deferred; the -// encrypt path's fixed value) +// - version/flag = V1/uncompressed or V2/Snappy-compressed (the two +// combinations emitted by the encrypt path; trusting on-disk would +// let corrupted format bytes bypass the trial) // - tombstone = false (the encrypt path never wraps // tombstones, so any on-disk tombstone bit on an encrypted // entry is necessarily attacker-supplied) @@ -496,12 +569,20 @@ func (s *pebbleStore) rejectRebadgedEnvelope(pebbleKey []byte, sv storedValue, b } for _, kid := range s.cipher.LoadedKeyIDs() { for _, candidateExpire := range candidateExpireAts { - var hdr [valueHeaderSize]byte - writeValueHeaderBytes(hdr[:], false /*canonical*/, candidateExpire, encStateEncrypted) - aad := buildStorageAAD(encryption.EnvelopeVersionV1, 0 /*flag canonical*/, kid, hdr[:], pebbleKey) - if _, err := s.cipher.Decrypt(ct, aad, kid, nonce); err == nil { - return errors.Wrap(ErrEncryptedReadIntegrity, - "store: cleartext-labelled value verifies as a relabeled envelope under a loaded DEK") + for _, candidate := range []struct { + version byte + flag byte + }{ + {version: encryption.EnvelopeVersionV1}, + {version: encryption.EnvelopeVersionV2, flag: encryption.FlagCompressed}, + } { + var hdr [valueHeaderSize]byte + writeValueHeaderBytes(hdr[:], false /*canonical*/, candidateExpire, encStateEncrypted) + aad := buildStorageAAD(candidate.version, candidate.flag, kid, hdr[:], pebbleKey) + if _, err := s.cipher.Decrypt(ct, aad, kid, nonce); err == nil { + return errors.Wrap(ErrEncryptedReadIntegrity, + "store: cleartext-labelled value verifies as a relabeled envelope under a loaded DEK") + } } } } diff --git a/store/lsm_store.go b/store/lsm_store.go index 4eee52e9a..98a624390 100644 --- a/store/lsm_store.go +++ b/store/lsm_store.go @@ -300,7 +300,7 @@ func WithSSTIngestSnapshots(enabled bool) PebbleStoreOption { // fully release the memory. Callers that only need *pebble.Options should // still take the cache handle and defer its Unref to avoid leaking a // 256 MiB (default) allocation per call. -func defaultPebbleOptionsWithCache() (*pebble.Options, *pebble.Cache) { +func defaultPebbleOptionsWithCache(disableCompression bool) (*pebble.Options, *pebble.Cache) { cache := pebble.NewCache(pebbleCacheBytes) opts := &pebble.Options{ FS: vfs.Default, @@ -310,6 +310,14 @@ func defaultPebbleOptionsWithCache() (*pebble.Options, *pebble.Cache) { // Enable automatic compactions and apply all other Pebble defaults. // EnsureDefaults leaves Cache alone because we already set it. opts.EnsureDefaults() + if disableCompression { + // Storage-envelope payloads are compressed before encryption. The + // resulting ciphertext is high entropy, so Pebble block compression + // only burns CPU and cannot recover additional space. + opts.ApplyCompressionSettings(func() pebble.DBCompressionSettings { + return pebble.DBCompressionNone + }) + } return opts, cache } @@ -334,7 +342,7 @@ func NewPebbleStore(dir string, opts ...PebbleStoreOption) (MVCCStore, error) { return nil, err } - pebbleOpts, cache := defaultPebbleOptionsWithCache() + pebbleOpts, cache := defaultPebbleOptionsWithCache(s.cipher != nil) db, err := pebble.Open(dir, pebbleOpts) if err != nil { cache.Unref() @@ -1207,7 +1215,7 @@ func (s *pebbleStore) foundValueVisible(iter *pebble.Iterator, ts uint64) (bool, // This intentionally also runs for cleartext-labelled rows so the // cleartext rebadge guard rejects encrypted envelopes whose encState bit // was flipped before we branch on tombstone or expireAt. - if _, err := s.decryptForKey(iter.Key(), sv, sv.Value); err != nil { + if err := s.authenticateForKey(iter.Key(), sv, sv.Value); err != nil { return false, err } return !sv.Tombstone && (sv.ExpireAt == 0 || sv.ExpireAt > ts), nil @@ -2526,12 +2534,12 @@ func (s *pebbleStore) isVisibleLiveKey(iter *pebble.Iterator, userKey []byte, ve if err != nil { return false, errors.WithStack(err) } - // decryptForKey authenticates the value-header bytes when the + // authenticateForKey authenticates the value-header bytes when the // entry is encrypted (cleartext entries no-op except for the // rebadge guard). We discard the plaintext — we only need the // authentication side-effect; tombstone / expireAt visibility // is then decided on now-trusted bytes. - if _, err := s.decryptForKey(iter.Key(), sv, sv.Value); err != nil { + if err := s.authenticateForKey(iter.Key(), sv, sv.Value); err != nil { return false, err } if sv.Tombstone || (sv.ExpireAt != 0 && sv.ExpireAt <= commitTS) { @@ -3028,7 +3036,7 @@ func (s *pebbleStore) reopenFreshDB() error { if err := os.MkdirAll(s.dir, dirPerms); err != nil { return errors.WithStack(err) } - opts, cache := defaultPebbleOptionsWithCache() + opts, cache := defaultPebbleOptionsWithCache(s.cipher != nil) db, err := pebble.Open(s.dir, opts) if err != nil { cache.Unref() @@ -3099,7 +3107,7 @@ func (s *pebbleStore) restorePebbleNativeAtomic(r io.Reader) error { return err } - if err := writeNativeSnapshotToTempDir(r, tmpDir, ts); err != nil { + if err := writeNativeSnapshotToTempDir(r, tmpDir, ts, s.cipher != nil); err != nil { _ = os.RemoveAll(tmpDir) return err } @@ -3141,8 +3149,8 @@ func makeSiblingTempDir(dir, tag string) (string, error) { // writeNativeSnapshotToTempDir writes batch-loop entries from r into a fresh // Pebble database at tmpDir, persists ts as the lastCommitTS meta-key, then // closes the database. -func writeNativeSnapshotToTempDir(r io.Reader, tmpDir string, ts uint64) error { - opts, cache := defaultPebbleOptionsWithCache() +func writeNativeSnapshotToTempDir(r io.Reader, tmpDir string, ts uint64, disableCompression bool) error { + opts, cache := defaultPebbleOptionsWithCache(disableCompression) tmpDB, err := pebble.Open(tmpDir, opts) if err != nil { cache.Unref() @@ -3188,12 +3196,12 @@ func readStreamingMVCCRestoreHeader(r io.Reader) (io.Reader, hash.Hash32, uint32 return body, hash, expectedChecksum, lastCommitTS, minRetainedTS, nil } -func writeStreamingMVCCRestoreTempDB(dir string, body io.Reader, hash hash.Hash32, expectedChecksum uint32, lastCommitTS uint64, minRetainedTS uint64) (string, error) { +func writeStreamingMVCCRestoreTempDB(dir string, body io.Reader, hash hash.Hash32, expectedChecksum uint32, lastCommitTS uint64, minRetainedTS uint64, disableCompression bool) (string, error) { tmpDir := filepath.Clean(dir) + ".restore-tmp" if err := os.RemoveAll(tmpDir); err != nil { return "", errors.WithStack(err) } - opts, cache := defaultPebbleOptionsWithCache() + opts, cache := defaultPebbleOptionsWithCache(disableCompression) tmpDB, err := pebble.Open(tmpDir, opts) if err != nil { cache.Unref() @@ -3233,7 +3241,7 @@ func (s *pebbleStore) restoreFromStreamingMVCC(r io.Reader) error { return err } - tmpDir, err := writeStreamingMVCCRestoreTempDB(s.dir, body, hash, expectedChecksum, lastCommitTS, minRetainedTS) + tmpDir, err := writeStreamingMVCCRestoreTempDB(s.dir, body, hash, expectedChecksum, lastCommitTS, minRetainedTS, s.cipher != nil) if err != nil { return err } @@ -3317,7 +3325,7 @@ func (s *pebbleStore) swapInTempDBWithMetadata(tmpDir string, expected *pebbleSn } func (s *pebbleStore) reopenStoreDB(dir string) error { - opts, cache := defaultPebbleOptionsWithCache() + opts, cache := defaultPebbleOptionsWithCache(s.cipher != nil) db, err := pebble.Open(dir, opts) if err != nil { cache.Unref() diff --git a/store/lsm_store_env_test.go b/store/lsm_store_env_test.go index addf7d8de..8a66d0057 100644 --- a/store/lsm_store_env_test.go +++ b/store/lsm_store_env_test.go @@ -83,7 +83,7 @@ func TestSetPebbleCacheBytesForTestRestores(t *testing.T) { // is safe to call after closing the DB (the primary lifecycle path). func TestDefaultPebbleOptionsCarriesCache(t *testing.T) { setPebbleCacheBytesForTest(t, 16<<20) - opts, cache := defaultPebbleOptionsWithCache() + opts, cache := defaultPebbleOptionsWithCache(false) require.NotNil(t, cache) require.Same(t, cache, opts.Cache) require.Equal(t, int64(16)<<20, cache.MaxSize()) diff --git a/store/lsm_store_sync_mode_benchmark_test.go b/store/lsm_store_sync_mode_benchmark_test.go index bb1f02662..bab9be116 100644 --- a/store/lsm_store_sync_mode_benchmark_test.go +++ b/store/lsm_store_sync_mode_benchmark_test.go @@ -2,9 +2,11 @@ package store import ( "context" + "crypto/rand" "fmt" "testing" + "github.com/bootjp/elastickv/internal/encryption" "github.com/cockroachdb/pebble/v2" ) @@ -74,3 +76,74 @@ func BenchmarkApplyMutationsRaft_SyncMode(b *testing.B) { }) } } + +// BenchmarkApplyMutationsRaft_Encryption compares the Stage 9 +// compress-then-encrypt path against the legacy cleartext path with 1 KiB +// values. NoSync isolates encryption/compression CPU from fsync latency; the +// merge gate is evaluated with benchstat on the paired results. +func BenchmarkApplyMutationsRaft_Encryption(b *testing.B) { + value := make([]byte, 1024) + if _, err := rand.Read(value); err != nil { + b.Fatalf("rand.Read value: %v", err) + } + for _, tc := range []struct { + name string + encrypted bool + }{ + {name: "cleartext", encrypted: false}, + {name: "encrypted", encrypted: true}, + } { + b.Run(tc.name, func(b *testing.B) { + s, err := NewPebbleStore(b.TempDir(), benchmarkEncryptionOptions(b, tc.encrypted)...) + if err != nil { + b.Fatalf("NewPebbleStore: %v", err) + } + defer s.Close() + ps, ok := s.(*pebbleStore) + if !ok { + b.Fatalf("NewPebbleStore returned non-*pebbleStore type: %T", s) + } + ps.fsmApplyWriteOpts = pebble.NoSync + ps.fsmApplySyncModeLabel = fsmSyncModeNoSync + b.SetBytes(int64(len(value))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + key := []byte(fmt.Sprintf("enc-bench-%010d", i)) + if i < 0 { + b.Fatalf("unexpected negative iteration counter: %d", i) + } + startTS := uint64(i) * 2 + muts := []*KVPairMutation{{Op: OpTypePut, Key: key, Value: value}} + if err := s.ApplyMutationsRaft(context.Background(), muts, nil, startTS, startTS+1); err != nil { + b.Fatalf("ApplyMutationsRaft: %v", err) + } + } + }) + } +} + +func benchmarkEncryptionOptions(b *testing.B, enabled bool) []PebbleStoreOption { + b.Helper() + if !enabled { + return nil + } + ks := encryption.NewKeystore() + dek := make([]byte, encryption.KeySize) + if _, err := rand.Read(dek); err != nil { + b.Fatalf("rand.Read DEK: %v", err) + } + const keyID uint32 = 1 + if err := ks.Set(keyID, dek); err != nil { + b.Fatalf("Keystore.Set: %v", err) + } + cipher, err := encryption.NewCipher(ks) + if err != nil { + b.Fatalf("NewCipher: %v", err) + } + return []PebbleStoreOption{WithEncryption( + cipher, + NewCounterNonceFactory(1, 1), + func() (uint32, bool) { return keyID, true }, + )} +} diff --git a/store/lsm_store_test.go b/store/lsm_store_test.go index ecea4526a..83a1d6f7c 100644 --- a/store/lsm_store_test.go +++ b/store/lsm_store_test.go @@ -817,7 +817,7 @@ func TestSnapshotBatchShouldFlushOnByteLimit(t *testing.T) { require.NoError(t, err) defer os.RemoveAll(dir) - opts, cache := defaultPebbleOptionsWithCache() + opts, cache := defaultPebbleOptionsWithCache(false) db, err := pebble.Open(dir, opts) require.NoError(t, err) defer func() { diff --git a/store/snapshot_pebble_sst.go b/store/snapshot_pebble_sst.go index c23c972af..500eb0107 100644 --- a/store/snapshot_pebble_sst.go +++ b/store/snapshot_pebble_sst.go @@ -66,13 +66,14 @@ type pebbleSSTIngestFileMeta struct { } type pebbleSSTIngestSnapshot struct { - checkpointDir string - fallback *pebbleSnapshot - metadata pebbleSnapshotMetadata - targetFileBytes uint64 - log *slog.Logger - once sync.Once - err error + checkpointDir string + fallback *pebbleSnapshot + metadata pebbleSnapshotMetadata + targetFileBytes uint64 + disableCompression bool + log *slog.Logger + once sync.Once + err error } type pebbleSSTIngestBundle struct { @@ -231,9 +232,10 @@ func (s *pebbleStore) newSSTIngestSnapshot() (Snapshot, error) { snapshot: pebbleSnap, lastCommitTS: metadata.LastCommitTS, }, - metadata: metadata, - targetFileBytes: targetFileBytes, - log: s.log, + metadata: metadata, + targetFileBytes: targetFileBytes, + disableCompression: s.cipher != nil, + log: s.log, }, nil } @@ -296,7 +298,7 @@ func (s *pebbleSSTIngestSnapshot) WriteTo(w io.Writer) (int64, error) { if s == nil || s.fallback == nil { return 0, errors.New("snapshot is not available") } - bundle, err := buildPebbleSSTIngestBundle(s.checkpointDir, s.metadata, s.targetFileBytes) + bundle, err := buildPebbleSSTIngestBundle(s.checkpointDir, s.metadata, s.targetFileBytes, s.disableCompression) if err != nil { if s.log != nil { s.log.Warn("failed to build SST ingest snapshot; using legacy stream", "error", err) @@ -326,8 +328,8 @@ func (s *pebbleSSTIngestSnapshot) Close() error { return errors.WithStack(s.err) } -func buildPebbleSSTIngestBundle(checkpointDir string, metadata pebbleSnapshotMetadata, targetFileBytes uint64) (*pebbleSSTIngestBundle, error) { - opts, cache := defaultPebbleOptionsWithCache() +func buildPebbleSSTIngestBundle(checkpointDir string, metadata pebbleSnapshotMetadata, targetFileBytes uint64, disableCompression bool) (*pebbleSSTIngestBundle, error) { + opts, cache := defaultPebbleOptionsWithCache(disableCompression) opts.ReadOnly = true db, err := pebble.Open(checkpointDir, opts) if err != nil { @@ -593,7 +595,7 @@ func (s *pebbleStore) restorePebbleSSTIngestAtomic(r io.Reader) error { return err } metadata := metadataFromSSTIngestManifest(manifest) - if err := ingestSSTSnapshotIntoTempDB(tmpDir, paths, metadata); err != nil { + if err := ingestSSTSnapshotIntoTempDB(tmpDir, paths, metadata, s.cipher != nil); err != nil { _ = os.RemoveAll(tmpDir) return err } @@ -760,8 +762,8 @@ func metadataFromSSTIngestManifest(manifest pebbleSSTIngestManifest) pebbleSnaps return metadata } -func ingestSSTSnapshotIntoTempDB(tmpDir string, paths []string, metadata pebbleSnapshotMetadata) error { - opts, cache := defaultPebbleOptionsWithCache() +func ingestSSTSnapshotIntoTempDB(tmpDir string, paths []string, metadata pebbleSnapshotMetadata, disableCompression bool) error { + opts, cache := defaultPebbleOptionsWithCache(disableCompression) db, err := pebble.Open(tmpDir, opts) if err != nil { cache.Unref() diff --git a/store/snapshot_pebble_sst_test.go b/store/snapshot_pebble_sst_test.go index 64277a263..0a2139ea9 100644 --- a/store/snapshot_pebble_sst_test.go +++ b/store/snapshot_pebble_sst_test.go @@ -12,6 +12,7 @@ import ( "testing/iotest" internalutil "github.com/bootjp/elastickv/internal" + "github.com/bootjp/elastickv/internal/encryption" "github.com/stretchr/testify/require" ) @@ -122,6 +123,35 @@ func TestPebbleStoreSSTIngestSnapshotRoundTrip(t *testing.T) { require.NoDirExists(t, checkpointDir) } +func TestPebbleStoreEncryptedSSTIngestSnapshotRoundTrip(t *testing.T) { + setPebbleCacheBytesForTest(t, 8<<20) + const keyID = uint32(7) + ks := encryption.NewKeystore() + require.NoError(t, ks.Set(keyID, bytes.Repeat([]byte{0x47}, encryption.KeySize))) + cipher, err := encryption.NewCipher(ks) + require.NoError(t, err) + encryptionOption := func(epoch uint16) PebbleStoreOption { + return WithEncryption(cipher, NewCounterNonceFactory(1, epoch), func() (uint32, bool) { + return keyID, true + }) + } + + src := openSSTSnapshotTestStore(t, filepath.Join(t.TempDir(), "src"), + WithSSTIngestSnapshots(true), encryptionOption(1)) + writeSSTSnapshotTestData(t, src) + raw, snapshot := snapshotBytesForTest(t, src) + sstSnapshot, ok := snapshot.(*pebbleSSTIngestSnapshot) + require.True(t, ok) + require.True(t, sstSnapshot.disableCompression) + require.NoError(t, snapshot.Close()) + + dst := openSSTSnapshotTestStore(t, filepath.Join(t.TempDir(), "dst"), encryptionOption(2)) + require.NoError(t, dst.Restore(bytes.NewReader(raw))) + value, err := dst.GetAt(context.Background(), []byte("alpha"), 100) + require.NoError(t, err) + require.Equal(t, strings.Repeat("alpha", 32), string(value)) +} + func TestPebbleStoreSSTIngestSnapshotCorruptionPreservesDestination(t *testing.T) { setPebbleCacheBytesForTest(t, 8<<20) src := openSSTSnapshotTestStore(t, filepath.Join(t.TempDir(), "src"), WithSSTIngestSnapshots(true))