From 26d45baebb34ae6afbfaf39394e62deafcddc481 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 20:08:28 +0900 Subject: [PATCH 1/9] backup: add live pin FSM substrate --- adapter/admin_grpc.go | 245 +++++- adapter/admin_grpc_test.go | 438 ++++++++- .../2026_04_29_proposed_logical_backup.md | 12 +- internal/raftadmin/server_test.go | 4 + internal/raftengine/engine.go | 5 + internal/raftengine/etcd/engine.go | 6 + internal/raftengine/etcd/wal_purge_test.go | 5 + kv/active_timestamp_tracker.go | 354 +++++++- kv/active_timestamp_tracker_test.go | 230 +++++ kv/backup_codec.go | 163 ++++ kv/backup_codec_test.go | 93 ++ kv/compactor.go | 10 +- kv/compactor_test.go | 38 + kv/coordinator_retry_test.go | 3 +- kv/fsm.go | 29 +- kv/fsm_backup.go | 38 + kv/fsm_backup_test.go | 205 +++++ kv/leader_proxy_test.go | 6 +- kv/lease_read_test.go | 2 + kv/shard_store_test.go | 2 + kv/sharded_coordinator_txn_test.go | 3 +- main.go | 95 +- main_admin.go | 42 +- main_admin_test.go | 94 +- main_bootstrap_e2e_test.go | 8 +- main_encryption_write_wiring.go | 4 +- main_sqs_leadership_refusal_test.go | 3 +- multiraft_runtime_test.go | 8 +- proto/admin.pb.go | 831 ++++++++++++++++-- proto/admin.proto | 65 ++ proto/admin_grpc.pb.go | 202 ++++- 31 files changed, 3079 insertions(+), 164 deletions(-) create mode 100644 kv/backup_codec.go create mode 100644 kv/backup_codec_test.go create mode 100644 kv/fsm_backup.go create mode 100644 kv/fsm_backup_test.go diff --git a/adapter/admin_grpc.go b/adapter/admin_grpc.go index 1e147330b..99bd3f5ec 100644 --- a/adapter/admin_grpc.go +++ b/adapter/admin_grpc.go @@ -8,6 +8,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/bootjp/elastickv/internal/raftengine" @@ -41,6 +42,7 @@ type KeyVizSampler interface { type AdminGroup interface { Status() raftengine.Status Configuration(ctx context.Context) (raftengine.Configuration, error) + SnapshotEvery() uint64 } // NodeIdentity is the value form of the protobuf NodeIdentity message used for @@ -55,6 +57,55 @@ func (n NodeIdentity) toProto() *pb.NodeIdentity { return &pb.NodeIdentity{NodeId: n.NodeID, GrpcAddress: n.GRPCAddress} } +// LeaderVersionProbe fetches the Admin service version for a peer address. +// Implementations must honor ctx for the 500ms async GetRaftGroups probe +// budget and any auth metadata copied from the inbound Admin request. +type LeaderVersionProbe func(ctx context.Context, grpcAddress string) (string, error) + +// AdminOption adjusts optional AdminServer behavior without changing existing +// test construction call sites. +type AdminOption func(*AdminServer) + +func WithAdminNodeVersion(version string) AdminOption { + return func(s *AdminServer) { + s.nodeVersion = version + } +} + +func WithAdminLeaderVersionProbe(probe LeaderVersionProbe) AdminOption { + return func(s *AdminServer) { + s.leaderVersionProbe = probe + } +} + +func WithAdminLeaderVersionProbeTimeout(timeout time.Duration) AdminOption { + return func(s *AdminServer) { + if timeout > 0 { + s.leaderVersionProbeTimeout = timeout + } + } +} + +func WithAdminLeaderVersionCacheTTL(ttl time.Duration) AdminOption { + return func(s *AdminServer) { + if ttl > 0 { + s.leaderVersionCacheTTL = ttl + } + } +} + +type versionCacheEntry struct { + version string + fetchedAt time.Time + probeID uint64 +} + +const ( + defaultAdminLeaderVersionProbeTimeout = 500 * time.Millisecond + defaultAdminLeaderVersionCacheTTL = 10 * time.Second + minLeaderVersionProbeAttemptTimeout = 100 * time.Millisecond +) + // AdminServer implements the node-side Admin gRPC service described in // docs/admin_ui_key_visualizer_design.md §4 (Layer A). Phase 0 only implements // GetClusterOverview and GetRaftGroups; remaining RPCs return Unimplemented so @@ -79,6 +130,13 @@ type AdminServer struct { // pairs atomically with concurrent RPC reads. sampler KeyVizSampler + nodeVersion string + leaderVersionProbe LeaderVersionProbe + leaderVersionProbeTimeout time.Duration + leaderVersionCacheTTL time.Duration + leaderVersionProbeSeq atomic.Uint64 + versionCache sync.Map + pb.UnimplementedAdminServer } @@ -87,15 +145,23 @@ type AdminServer struct { // snapshot shipped to the admin binary; callers that already have a membership // source may pass nil and let the admin binary's fan-out layer discover peers // by other means. -func NewAdminServer(self NodeIdentity, members []NodeIdentity) *AdminServer { +func NewAdminServer(self NodeIdentity, members []NodeIdentity, opts ...AdminOption) *AdminServer { cloned := append([]NodeIdentity(nil), members...) - return &AdminServer{ - self: self, - members: cloned, - capabilities: make(map[string]bool), - groups: make(map[uint64]AdminGroup), - now: time.Now, + srv := &AdminServer{ + self: self, + members: cloned, + capabilities: make(map[string]bool), + groups: make(map[uint64]AdminGroup), + now: time.Now, + leaderVersionProbeTimeout: defaultAdminLeaderVersionProbeTimeout, + leaderVersionCacheTTL: defaultAdminLeaderVersionCacheTTL, + } + for _, opt := range opts { + if opt != nil { + opt(srv) + } } + return srv } // SetClock overrides the clock used by GetRaftGroups, letting tests inject a @@ -410,16 +476,21 @@ func mergeSeedMembers(seeds []NodeIdentity, selfID string, live *liveMembers) { // GetRaftGroups returns per-group state snapshots. Phase 0 wires commit/applied // indices only; per-follower contact and term history land in later phases. func (s *AdminServer) GetRaftGroups( - _ context.Context, + ctx context.Context, _ *pb.GetRaftGroupsRequest, ) (*pb.GetRaftGroupsResponse, error) { s.groupsMu.RLock() defer s.groupsMu.RUnlock() ids := sortedGroupIDs(s.groups) + statuses := make([]raftengine.Status, len(ids)) + for i, id := range ids { + statuses[i] = s.groups[id].Status() + } + probeAddresses := leaderProbeAddressesByNode(statuses) out := make([]*pb.RaftGroupState, 0, len(ids)) now := s.now() - for _, id := range ids { - st := s.groups[id].Status() + for i, id := range ids { + st := statuses[i] // Translate LastContact (duration since the last contact with the // leader, per raftengine.Status) into an absolute unix-ms so UI // clients can diff against their own clock instead of having to @@ -440,11 +511,165 @@ func (s *AdminServer) GetRaftGroups( CommitIndex: st.CommitIndex, AppliedIndex: st.AppliedIndex, LastContactUnixMs: lastContactUnixMs, + LeaderNodeVersion: s.leaderNodeVersion(ctx, st.Leader, now, probeAddresses[leaderVersionCacheKey(st.Leader)]), }) } return &pb.GetRaftGroupsResponse{Groups: out}, nil } +func (s *AdminServer) GetNodeVersion( + context.Context, + *pb.GetNodeVersionRequest, +) (*pb.GetNodeVersionResponse, error) { + return &pb.GetNodeVersionResponse{NodeVersion: s.nodeVersion}, nil +} + +func (s *AdminServer) leaderNodeVersion(ctx context.Context, leader raftengine.LeaderInfo, now time.Time, addresses []string) string { + if version, ok := s.localLeaderVersion(leader); ok { + return version + } + key := leaderVersionCacheKey(leader) + if version, ok := s.cachedLeaderVersion(key, now); ok { + return version + } + if s.leaderVersionProbe == nil || len(addresses) == 0 { + return "" + } + version, reservation, reserved := s.reserveLeaderVersionProbe(key, now) + if !reserved { + return version + } + s.probeLeaderVersionAsync(ctx, key, addresses, reservation) + return "" +} + +func leaderProbeAddressesByNode(statuses []raftengine.Status) map[string][]string { + addresses := make(map[string][]string) + seen := make(map[string]map[string]struct{}) + for _, st := range statuses { + key := leaderVersionCacheKey(st.Leader) + address := strings.TrimSpace(st.Leader.Address) + if key == "" || address == "" { + continue + } + if seen[key] == nil { + seen[key] = make(map[string]struct{}) + } + if _, ok := seen[key][address]; ok { + continue + } + seen[key][address] = struct{}{} + addresses[key] = append(addresses[key], address) + } + return addresses +} + +func (s *AdminServer) localLeaderVersion(leader raftengine.LeaderInfo) (string, bool) { + if leader.ID == "" && leader.Address == "" { + return "", true + } + if leader.ID == s.self.NodeID || (leader.Address != "" && leader.Address == s.self.GRPCAddress) { + return s.nodeVersion, true + } + return "", false +} + +func leaderVersionCacheKey(leader raftengine.LeaderInfo) string { + if leader.ID != "" { + return leader.ID + } + return leader.Address +} + +func (s *AdminServer) cachedLeaderVersion(key string, now time.Time) (string, bool) { + if key == "" || s.leaderVersionCacheTTL <= 0 { + return "", false + } + actual, ok := s.versionCache.Load(key) + if !ok { + return "", false + } + entry, ok := actual.(versionCacheEntry) + if !ok { + s.versionCache.Delete(key) + return "", false + } + if now.Sub(entry.fetchedAt) > s.leaderVersionCacheTTL { + s.versionCache.CompareAndDelete(key, entry) + return "", false + } + return entry.version, true +} + +func (s *AdminServer) reserveLeaderVersionProbe(key string, now time.Time) (string, versionCacheEntry, bool) { + reservation := versionCacheEntry{ + fetchedAt: now, + probeID: s.leaderVersionProbeSeq.Add(1), + } + for { + actual, loaded := s.versionCache.LoadOrStore(key, reservation) + if !loaded { + return "", reservation, true + } + entry, ok := actual.(versionCacheEntry) + switch { + case !ok: + s.versionCache.Store(key, reservation) + return "", reservation, true + case now.Sub(entry.fetchedAt) <= s.leaderVersionCacheTTL: + return entry.version, versionCacheEntry{}, false + default: + if s.versionCache.CompareAndSwap(key, entry, reservation) { + return "", reservation, true + } + } + } +} + +func (s *AdminServer) probeLeaderVersionAsync(ctx context.Context, key string, addresses []string, reservation versionCacheEntry) { + probe := s.leaderVersionProbe + timeout := s.leaderVersionProbeTimeout + now := s.now + candidates := append([]string(nil), addresses...) + attemptTimeout := leaderVersionProbeAttemptTimeout(timeout, len(candidates)) + var md metadata.MD + if incoming, ok := metadata.FromIncomingContext(ctx); ok { + md = incoming.Copy() + } + go func() { + probeCtx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + version := "" + for _, address := range candidates { + attemptCtx, attemptCancel := context.WithTimeout(probeCtx, attemptTimeout) + if md != nil { + attemptCtx = metadata.NewOutgoingContext(attemptCtx, md) + } + probed, err := probe(attemptCtx, address) + attemptCancel() + if err == nil { + version = probed + break + } + if probeCtx.Err() != nil { + break + } + } + s.versionCache.CompareAndSwap(key, reservation, versionCacheEntry{version: version, fetchedAt: now()}) + }() +} + +func leaderVersionProbeAttemptTimeout(total time.Duration, candidates int) time.Duration { + if total <= 0 || candidates <= 1 { + return total + } + perCandidate := total / time.Duration(candidates) + if perCandidate < minLeaderVersionProbeAttemptTimeout { + return min(total, minLeaderVersionProbeAttemptTimeout) + } + return perCandidate +} + func (s *AdminServer) snapshotLeaders() []*pb.GroupLeader { s.groupsMu.RLock() defer s.groupsMu.RUnlock() diff --git a/adapter/admin_grpc_test.go b/adapter/admin_grpc_test.go index 75fe0b9a2..8671fb137 100644 --- a/adapter/admin_grpc_test.go +++ b/adapter/admin_grpc_test.go @@ -3,11 +3,13 @@ package adapter import ( "context" "sync" + "sync/atomic" "testing" "time" "github.com/bootjp/elastickv/internal/raftengine" pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" @@ -15,17 +17,29 @@ import ( ) type fakeGroup struct { - leaderID string - term uint64 - commit uint64 - applied uint64 - servers []raftengine.Server - cfgErr error + leaderID string + leaderAddr string + term uint64 + commit uint64 + applied uint64 + servers []raftengine.Server + cfgErr error + snapshotEach uint64 +} + +type countingStatusGroup struct { + fakeGroup + calls *atomic.Int64 +} + +func (g countingStatusGroup) Status() raftengine.Status { + g.calls.Add(1) + return g.fakeGroup.Status() } func (f fakeGroup) Status() raftengine.Status { return raftengine.Status{ - Leader: raftengine.LeaderInfo{ID: f.leaderID}, + Leader: raftengine.LeaderInfo{ID: f.leaderID, Address: f.leaderAddr}, Term: f.term, CommitIndex: f.commit, AppliedIndex: f.applied, @@ -39,6 +53,13 @@ func (f fakeGroup) Configuration(context.Context) (raftengine.Configuration, err return raftengine.Configuration{Servers: append([]raftengine.Server(nil), f.servers...)}, nil } +func (f fakeGroup) SnapshotEvery() uint64 { + if f.snapshotEach == 0 { + return 10_000 + } + return f.snapshotEach +} + func TestGetClusterOverviewReturnsSelfAndLeaders(t *testing.T) { t.Parallel() srv := NewAdminServer( @@ -71,6 +92,23 @@ func TestGetClusterOverviewReturnsSelfAndLeaders(t *testing.T) { } } +func TestGetNodeVersionReturnsConfiguredVersion(t *testing.T) { + t.Parallel() + srv := NewAdminServer( + NodeIdentity{NodeID: "n1"}, + nil, + WithAdminNodeVersion("v1.2.3"), + ) + + resp, err := srv.GetNodeVersion(context.Background(), &pb.GetNodeVersionRequest{}) + if err != nil { + t.Fatalf("GetNodeVersion: %v", err) + } + if resp.NodeVersion != "v1.2.3" { + t.Fatalf("NodeVersion = %q, want v1.2.3", resp.NodeVersion) + } +} + func TestGetClusterOverviewReturnsCapabilities(t *testing.T) { t.Parallel() srv := NewAdminServer(NodeIdentity{NodeID: "node-a"}, nil) @@ -117,6 +155,389 @@ func TestGetRaftGroupsExposesCommitApplied(t *testing.T) { } } +func TestGetRaftGroupsReportsLocalLeaderVersion(t *testing.T) { + t.Parallel() + srv := NewAdminServer( + NodeIdentity{NodeID: "n1", GRPCAddress: "10.0.0.11:50051"}, + nil, + WithAdminNodeVersion("v-local"), + ) + srv.RegisterGroup(1, fakeGroup{leaderID: "n1", leaderAddr: "10.0.0.11:50051"}) + + resp, err := srv.GetRaftGroups(context.Background(), &pb.GetRaftGroupsRequest{}) + if err != nil { + t.Fatalf("GetRaftGroups: %v", err) + } + if got := resp.Groups[0].LeaderNodeVersion; got != "v-local" { + t.Fatalf("LeaderNodeVersion = %q, want v-local", got) + } +} + +func TestGetRaftGroupsLeaderVersionAsync(t *testing.T) { + t.Parallel() + probeStarted := make(chan struct{}, 1) + releaseProbe := make(chan struct{}) + var calls int + var callsMu sync.Mutex + srv := NewAdminServer( + NodeIdentity{NodeID: "n1", GRPCAddress: "10.0.0.11:50051"}, + nil, + WithAdminLeaderVersionProbeTimeout(2*time.Second), + WithAdminLeaderVersionCacheTTL(time.Second), + WithAdminLeaderVersionProbe(func(ctx context.Context, addr string) (string, error) { + if addr != "10.0.0.12:50051" { + t.Errorf("probe addr = %q, want 10.0.0.12:50051", addr) + } + callsMu.Lock() + calls++ + callsMu.Unlock() + probeStarted <- struct{}{} + select { + case <-releaseProbe: + return "v-remote", nil + case <-ctx.Done(): + return "", ctx.Err() + } + }), + ) + srv.RegisterGroup(1, fakeGroup{leaderID: "n2", leaderAddr: "10.0.0.12:50051"}) + + resp, err := srv.GetRaftGroups(context.Background(), &pb.GetRaftGroupsRequest{}) + if err != nil { + t.Fatalf("GetRaftGroups first: %v", err) + } + requireLeaderVersion(t, resp, "", "first") + select { + case <-probeStarted: + case <-time.After(time.Second): + t.Fatal("leader version probe was not started") + } + second, err := srv.GetRaftGroups(context.Background(), &pb.GetRaftGroupsRequest{}) + if err != nil { + t.Fatalf("GetRaftGroups second: %v", err) + } + requireLeaderVersion(t, second, "", "second") + close(releaseProbe) + waitForLeaderVersion(t, srv, "v-remote") + callsMu.Lock() + gotCalls := calls + callsMu.Unlock() + if gotCalls != 1 { + t.Fatalf("probe calls = %d, want 1 while cache is fresh", gotCalls) + } +} + +func TestGetRaftGroupsLeaderVersionProbeUsesServerClockForCacheStamp(t *testing.T) { + t.Parallel() + base := time.Unix(1_700_000_000, 0) + var clockMu sync.Mutex + now := base + setNow := func(next time.Time) { + clockMu.Lock() + now = next + clockMu.Unlock() + } + var callsMu sync.Mutex + calls := 0 + srv := NewAdminServer( + NodeIdentity{NodeID: "n1", GRPCAddress: "10.0.0.11:50051"}, + nil, + WithAdminLeaderVersionProbeTimeout(time.Second), + WithAdminLeaderVersionCacheTTL(time.Second), + WithAdminLeaderVersionProbe(func(context.Context, string) (string, error) { + callsMu.Lock() + defer callsMu.Unlock() + calls++ + if calls == 1 { + return "v1", nil + } + return "v2", nil + }), + ) + srv.SetClock(func() time.Time { + clockMu.Lock() + defer clockMu.Unlock() + return now + }) + srv.RegisterGroup(1, fakeGroup{leaderID: "n2", leaderAddr: "10.0.0.12:50051"}) + + _, err := srv.GetRaftGroups(context.Background(), &pb.GetRaftGroupsRequest{}) + if err != nil { + t.Fatalf("GetRaftGroups first: %v", err) + } + waitForLeaderVersion(t, srv, "v1") + + setNow(base.Add(2 * time.Second)) + resp, err := srv.GetRaftGroups(context.Background(), &pb.GetRaftGroupsRequest{}) + if err != nil { + t.Fatalf("GetRaftGroups after TTL: %v", err) + } + requireLeaderVersion(t, resp, "", "expired") + waitForLeaderVersion(t, srv, "v2") +} + +func TestGetRaftGroupsLeaderVersionStaleProbeCannotOverwriteFreshCache(t *testing.T) { + t.Parallel() + base := time.Unix(1_700_000_000, 0) + var clockMu sync.Mutex + now := base + setNow := func(next time.Time) { + clockMu.Lock() + now = next + clockMu.Unlock() + } + probeCalls := make(chan chan leaderVersionProbeResult, 2) + srv := NewAdminServer( + NodeIdentity{NodeID: "n1", GRPCAddress: "10.0.0.11:50051"}, + nil, + WithAdminLeaderVersionProbeTimeout(5*time.Second), + WithAdminLeaderVersionCacheTTL(time.Second), + WithAdminLeaderVersionProbe(blockingLeaderVersionProbe(probeCalls)), + ) + srv.SetClock(func() time.Time { + clockMu.Lock() + defer clockMu.Unlock() + return now + }) + srv.RegisterGroup(1, fakeGroup{leaderID: "n2", leaderAddr: "10.0.0.12:50051"}) + + _, err := srv.GetRaftGroups(context.Background(), &pb.GetRaftGroupsRequest{}) + require.NoError(t, err) + firstProbe := receiveLeaderVersionProbeCall(t, probeCalls, "first") + + setNow(base.Add(2 * time.Second)) + resp, err := srv.GetRaftGroups(context.Background(), &pb.GetRaftGroupsRequest{}) + require.NoError(t, err) + requireLeaderVersion(t, resp, "", "second") + secondProbe := receiveLeaderVersionProbeCall(t, probeCalls, "second") + secondProbe <- leaderVersionProbeResult{version: "v-fresh"} + waitForLeaderVersion(t, srv, "v-fresh") + + firstProbe <- leaderVersionProbeResult{version: "v-stale"} + requireLeaderVersionNever(t, srv, "v-stale", 100*time.Millisecond) + resp, err = srv.GetRaftGroups(context.Background(), &pb.GetRaftGroupsRequest{}) + require.NoError(t, err) + requireLeaderVersion(t, resp, "v-fresh", "after stale completion") +} + +type leaderVersionProbeResult struct { + version string + err error +} + +func blockingLeaderVersionProbe(calls chan<- chan leaderVersionProbeResult) LeaderVersionProbe { + return func(ctx context.Context, _ string) (string, error) { + resultCh := make(chan leaderVersionProbeResult) + select { + case calls <- resultCh: + case <-ctx.Done(): + return "", ctx.Err() + } + select { + case result := <-resultCh: + return result.version, result.err + case <-ctx.Done(): + return "", ctx.Err() + } + } +} + +func requireLeaderVersionNever(t *testing.T, srv *AdminServer, unwanted string, waitFor time.Duration) { + t.Helper() + require.Never(t, func() bool { + resp, err := srv.GetRaftGroups(context.Background(), &pb.GetRaftGroupsRequest{}) + return err == nil && len(resp.Groups) == 1 && resp.Groups[0].LeaderNodeVersion == unwanted + }, waitFor, 10*time.Millisecond) +} + +func receiveLeaderVersionProbeCall( + t *testing.T, + calls <-chan chan leaderVersionProbeResult, + label string, +) chan leaderVersionProbeResult { + t.Helper() + select { + case resultCh := <-calls: + return resultCh + case <-time.After(time.Second): + t.Fatalf("%s leader version probe was not started", label) + return nil + } +} + +func TestGetRaftGroupsLeaderVersionTriesAlternateAddressForSameLeaderID(t *testing.T) { + t.Parallel() + var callsMu sync.Mutex + callsByAddr := map[string]int{} + srv := NewAdminServer( + NodeIdentity{NodeID: "n1", GRPCAddress: "10.0.0.11:50051"}, + nil, + WithAdminLeaderVersionProbeTimeout(time.Second), + WithAdminLeaderVersionCacheTTL(time.Second), + WithAdminLeaderVersionProbe(func(_ context.Context, addr string) (string, error) { + callsMu.Lock() + callsByAddr[addr]++ + callsMu.Unlock() + if addr == "10.0.0.12:50051" { + return "", status.Error(codes.Unavailable, "stale listener") + } + return "v-good", nil + }), + ) + srv.RegisterGroup(1, fakeGroup{leaderID: "n2", leaderAddr: "10.0.0.12:50051"}) + srv.RegisterGroup(2, fakeGroup{leaderID: "n2", leaderAddr: "10.0.0.22:50051"}) + + _, err := srv.GetRaftGroups(context.Background(), &pb.GetRaftGroupsRequest{}) + if err != nil { + t.Fatalf("GetRaftGroups first: %v", err) + } + require.Eventually(t, func() bool { + resp, err := srv.GetRaftGroups(context.Background(), &pb.GetRaftGroupsRequest{}) + if err != nil || len(resp.Groups) != 2 { + return false + } + return resp.Groups[0].LeaderNodeVersion == "v-good" && resp.Groups[1].LeaderNodeVersion == "v-good" + }, time.Second, 10*time.Millisecond) + callsMu.Lock() + defer callsMu.Unlock() + if callsByAddr["10.0.0.12:50051"] == 0 || callsByAddr["10.0.0.22:50051"] == 0 { + t.Fatalf("probe calls by address = %v, want both stale and alternate addresses", callsByAddr) + } + require.Equal(t, 2, callsByAddr["10.0.0.12:50051"]+callsByAddr["10.0.0.22:50051"], + "one per-node probe should try each candidate once") +} + +func TestGetRaftGroupsSnapshotsEachGroupOnce(t *testing.T) { + t.Parallel() + const groupCount = 64 + var calls atomic.Int64 + srv := NewAdminServer( + NodeIdentity{NodeID: "n1", GRPCAddress: "10.0.0.11:50051"}, + nil, + WithAdminLeaderVersionProbe(func(context.Context, string) (string, error) { return "v2", nil }), + ) + for id := uint64(1); id <= groupCount; id++ { + srv.RegisterGroup(id, countingStatusGroup{ + fakeGroup: fakeGroup{leaderID: "n2", leaderAddr: "10.0.0.12:50051"}, + calls: &calls, + }) + } + + resp, err := srv.GetRaftGroups(context.Background(), &pb.GetRaftGroupsRequest{}) + require.NoError(t, err) + require.Len(t, resp.GetGroups(), groupCount) + require.Equal(t, int64(groupCount), calls.Load()) +} + +func TestGetRaftGroupsLeaderVersionBoundsEachCandidateProbe(t *testing.T) { + t.Parallel() + var callsMu sync.Mutex + callsByAddr := map[string]int{} + srv := NewAdminServer( + NodeIdentity{NodeID: "n1", GRPCAddress: "10.0.0.11:50051"}, + nil, + WithAdminLeaderVersionProbeTimeout(200*time.Millisecond), + WithAdminLeaderVersionCacheTTL(time.Second), + WithAdminLeaderVersionProbe(func(ctx context.Context, addr string) (string, error) { + callsMu.Lock() + callsByAddr[addr]++ + callsMu.Unlock() + if addr == "10.0.0.12:50051" { + <-ctx.Done() + return "", ctx.Err() + } + return "v-good", nil + }), + ) + srv.RegisterGroup(1, fakeGroup{leaderID: "n2", leaderAddr: "10.0.0.12:50051"}) + srv.RegisterGroup(2, fakeGroup{leaderID: "n2", leaderAddr: "10.0.0.22:50051"}) + + _, err := srv.GetRaftGroups(context.Background(), &pb.GetRaftGroupsRequest{}) + if err != nil { + t.Fatalf("GetRaftGroups first: %v", err) + } + require.Eventually(t, func() bool { + resp, err := srv.GetRaftGroups(context.Background(), &pb.GetRaftGroupsRequest{}) + if err != nil || len(resp.Groups) != 2 { + return false + } + return resp.Groups[0].LeaderNodeVersion == "v-good" && resp.Groups[1].LeaderNodeVersion == "v-good" + }, time.Second, 10*time.Millisecond) + callsMu.Lock() + defer callsMu.Unlock() + if callsByAddr["10.0.0.12:50051"] == 0 || callsByAddr["10.0.0.22:50051"] == 0 { + t.Fatalf("probe calls by address = %v, want timeout candidate and alternate address", callsByAddr) + } +} + +func TestLeaderVersionProbeAttemptTimeoutHasSaneMinimum(t *testing.T) { + t.Parallel() + require.Equal(t, 100*time.Millisecond, leaderVersionProbeAttemptTimeout(500*time.Millisecond, 100)) + require.Equal(t, 50*time.Millisecond, leaderVersionProbeAttemptTimeout(50*time.Millisecond, 100)) + require.Equal(t, 250*time.Millisecond, leaderVersionProbeAttemptTimeout(500*time.Millisecond, 2)) +} + +func requireLeaderVersion(t *testing.T, resp *pb.GetRaftGroupsResponse, want, label string) { + t.Helper() + if len(resp.Groups) != 1 { + t.Fatalf("%s groups = %d, want 1", label, len(resp.Groups)) + } + if got := resp.Groups[0].LeaderNodeVersion; got != want { + t.Fatalf("%s LeaderNodeVersion = %q, want %q", label, got, want) + } +} + +func waitForLeaderVersion(t *testing.T, srv *AdminServer, want string) { + t.Helper() + deadline := time.After(time.Second) + for { + resp, err := srv.GetRaftGroups(context.Background(), &pb.GetRaftGroupsRequest{}) + if err != nil { + t.Fatalf("GetRaftGroups while waiting for version: %v", err) + } + if len(resp.Groups) == 1 && resp.Groups[0].LeaderNodeVersion == want { + return + } + select { + case <-deadline: + t.Fatalf("leader version cache was not populated with %q", want) + case <-time.After(10 * time.Millisecond): + } + } +} + +func TestGetRaftGroupsLeaderVersionProbePropagatesAuthMetadata(t *testing.T) { + t.Parallel() + gotAuth := make(chan []string, 1) + srv := NewAdminServer( + NodeIdentity{NodeID: "n1", GRPCAddress: "10.0.0.11:50051"}, + nil, + WithAdminLeaderVersionProbeTimeout(time.Second), + WithAdminLeaderVersionProbe(func(ctx context.Context, _ string) (string, error) { + md, _ := metadata.FromOutgoingContext(ctx) + gotAuth <- md.Get("authorization") + return "v-remote", nil + }), + ) + srv.RegisterGroup(1, fakeGroup{leaderID: "n2", leaderAddr: "10.0.0.12:50051"}) + + ctx := metadata.NewIncomingContext( + context.Background(), + metadata.Pairs("authorization", "Bearer admin-token"), + ) + if _, err := srv.GetRaftGroups(ctx, &pb.GetRaftGroupsRequest{}); err != nil { + t.Fatalf("GetRaftGroups: %v", err) + } + select { + case got := <-gotAuth: + if len(got) != 1 || got[0] != "Bearer admin-token" { + t.Fatalf("authorization metadata = %v, want bearer token", got) + } + case <-time.After(time.Second): + t.Fatal("leader version probe did not run") + } +} + // TestGetClusterOverviewUnionsSeedsAndLiveConfig asserts that // GetClusterOverview picks up a node that was added to a Raft group after the // admin server was constructed (scale-out). Without Configuration polling, @@ -345,6 +766,7 @@ func (t testProbeGroup) Configuration(context.Context) (raftengine.Configuration } return raftengine.Configuration{}, nil } +func (t testProbeGroup) SnapshotEvery() uint64 { return 10_000 } // TestGetClusterOverviewKeepsSeedOnPartialConfigFailure asserts that a // bootstrap seed is NOT pruned when one group's Configuration succeeds and @@ -488,6 +910,8 @@ func (f fakeGroupWithContact) Configuration(context.Context) (raftengine.Configu return raftengine.Configuration{}, nil } +func (f fakeGroupWithContact) SnapshotEvery() uint64 { return 10_000 } + // TestGroupOrderingIsStable locks in deterministic ascending-by-RaftGroupId // ordering so admin UIs and diff-based tests do not see rows jump around. func TestGroupOrderingIsStable(t *testing.T) { diff --git a/docs/design/2026_04_29_proposed_logical_backup.md b/docs/design/2026_04_29_proposed_logical_backup.md index 39350d188..f3929a085 100644 --- a/docs/design/2026_04_29_proposed_logical_backup.md +++ b/docs/design/2026_04_29_proposed_logical_backup.md @@ -1045,7 +1045,7 @@ dumps with retention pressure. `BackupScanner.Next(at_ts=read_ts)`. 5. **Renew on long dumps**: the producer calls `RenewBackup(pin_token, ttl_ms)` every `ttl_ms / 3`. The admin - server proposes `BackupExtend{pin_id, deadline}` on every group + server proposes `BackupExtend{pin_id, read_ts, deadline}` on every group recorded in `pin_token`. The `read_ts` is preserved across renewals; only the deadline shifts. A multi-hour dump never relies on a single 30-minute pin. Renewals are cheap (one Raft entry per @@ -1500,13 +1500,15 @@ written. types. Hand-coded fixed-layout binary (matching the HLC lease style): ``` - BackupPin : [tag:1][pin_id:16][read_ts:8][deadline_ms:8] = 33 bytes - BackupExtend : [tag:1][pin_id:16][deadline_ms:8] = 25 bytes - BackupRelease : [tag:1][pin_id:16] = 17 bytes + BackupPin : [envelope:2][pin_id:16][read_ts:8][deadline_ms:8] = 34 bytes + BackupExtend : [envelope:2][pin_id:16][read_ts:8][deadline_ms:8] = 34 bytes + BackupRelease : [envelope:2][pin_id:16] = 18 bytes ``` `pin_id` is a UUIDv4 generated by the admin server at `BeginBackup` time and echoed in every subsequent `BackupExtend` / `BackupRelease` so the FSM can target the right tracker entry. + `BackupExtend` also carries `read_ts`, allowing committed renewal + apply to reconstruct a fence that a replica-local expiry sweep removed. Hand-coded binary (vs. proto) keeps the entry small enough to stay well within the `MaxSizePerMsg` limit (default 1 MiB, `internal/raftengine/etcd/engine.go:55`) and avoids @@ -1588,7 +1590,7 @@ Scope: out of this proposal; mentioned only to draw the boundary. | `TestBeginBackupPinFanOutAllNodes` | A 3-node cluster: `BeginBackup` issued to node A; verify nodes B and C have applied the `BackupPin` Raft entry and their compactors retain MVCC versions at `read_ts`. Compactor on B forced to run mid-dump must not retire pinned versions | | `TestBeginBackupPinSurvivesLeaderChange` | After `BeginBackup` on node A, force a leadership change on a group; the new leader still honors the pin (its FSM applied the same entry); subsequent `BackupScanner.Next` calls succeed | | `TestBeginBackupGroupUnreachable` | If one group cannot commit `BackupPin` within `--begin-backup-deadline`, `BeginBackup` returns `Unavailable` and proposes `BackupRelease` on every group that did commit; no stranded pins remain | -| `TestBackupPinFSMCodecRoundTrip` | `BackupPin` / `BackupExtend` / `BackupRelease` byte layouts (33 / 25 / 17 bytes) round-trip through the FSM apply path; unknown tag bytes return `ErrUnknownRequestType` rather than panicking | +| `TestBackupPinFSMCodecRoundTrip` | `BackupPin` / `BackupExtend` / `BackupRelease` byte layouts (34 / 34 / 18 bytes) round-trip through the FSM apply path; unknown tag bytes return `ErrUnknownRequestType` rather than panicking | | `TestRestoreWipesLocalPins` | A replica that installs a Raft snapshot during a backup loses its `BackupPin`; the producer's per-scope expected-keys baseline detects the resulting `ScanAt` shortfall (count below `99% × baseline ± sqrt(baseline)`) and fails the dump with `ErrCompactionDuringDump` rather than emitting a corrupted artifact | | `TestBeginBackupRefusesNearSnapshotThreshold` | When any group's `SnapshotEvery - (AppliedIndex - LastSnapshotIndex) < --snapshot-headroom-entries`, `BeginBackup` returns `FailedPrecondition` rather than starting a dump that risks the snapshot-installation path. Verify a freshly-snapshotted cluster (largest remaining headroom) is allowed | | `TestExpectedKeysBaselineToleratesTTLExpiry` | Routine TTL expiry between baseline and dump (1% of keys gone) does NOT trigger `ErrCompactionDuringDump`; a 5% drop DOES | diff --git a/internal/raftadmin/server_test.go b/internal/raftadmin/server_test.go index 4ec02aba6..f99bba9a7 100644 --- a/internal/raftadmin/server_test.go +++ b/internal/raftadmin/server_test.go @@ -108,6 +108,8 @@ func (f *fakeEngine) Configuration(context.Context) (raftengine.Configuration, e return f.config, nil } +func (f *fakeEngine) SnapshotEvery() uint64 { return 10_000 } + func (f *fakeEngine) CheckServing(context.Context) error { f.mu.Lock() defer f.mu.Unlock() @@ -390,6 +392,8 @@ func (s stateOnlyEngine) Configuration(context.Context) (raftengine.Configuratio return raftengine.Configuration{}, nil } +func (s stateOnlyEngine) SnapshotEvery() uint64 { return 10_000 } + func TestCurrentHealthStatusFallsBackToLocalState(t *testing.T) { t.Parallel() diff --git a/internal/raftengine/engine.go b/internal/raftengine/engine.go index 75ceacc8c..4b67eb458 100644 --- a/internal/raftengine/engine.go +++ b/internal/raftengine/engine.go @@ -235,6 +235,10 @@ type ConfigReader interface { Configuration(ctx context.Context) (Configuration, error) } +type SnapshotReader interface { + SnapshotEvery() uint64 +} + type HealthReader interface { CheckServing(ctx context.Context) error } @@ -303,5 +307,6 @@ type Engine interface { LeaderView StatusReader ConfigReader + SnapshotReader io.Closer } diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index 3fa139c8e..c857d204d 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -4379,6 +4379,12 @@ func (e *Engine) snapshotThreshold() uint64 { return e.snapshotEvery } +// SnapshotEvery returns the configured FSM-snapshot trigger threshold for +// admin/control-plane code that needs to reason about snapshot headroom. +func (e *Engine) SnapshotEvery() uint64 { + return e.snapshotThreshold() +} + // snapshotEveryFromEnv returns the FSM-snapshot trigger threshold (in applied // raft entries past the last snapshot). Operators can override via // ELASTICKV_RAFT_SNAPSHOT_COUNT; invalid or missing values fall back to diff --git a/internal/raftengine/etcd/wal_purge_test.go b/internal/raftengine/etcd/wal_purge_test.go index 2e2bf8d97..dcb46d685 100644 --- a/internal/raftengine/etcd/wal_purge_test.go +++ b/internal/raftengine/etcd/wal_purge_test.go @@ -213,3 +213,8 @@ func TestSnapshotEveryFromEnv_FallsBackOnInvalid(t *testing.T) { t.Setenv(snapshotEveryEnvVar, "not-a-number") require.Equal(t, uint64(defaultSnapshotEvery), snapshotEveryFromEnv()) } + +func TestEngineSnapshotEveryReadsConfiguredThreshold(t *testing.T) { + require.Equal(t, uint64(defaultSnapshotEvery), (&Engine{}).SnapshotEvery()) + require.Equal(t, uint64(5000), (&Engine{snapshotEvery: 5000}).SnapshotEvery()) +} diff --git a/kv/active_timestamp_tracker.go b/kv/active_timestamp_tracker.go index 18f387695..2d751fea3 100644 --- a/kv/active_timestamp_tracker.go +++ b/kv/active_timestamp_tracker.go @@ -1,13 +1,85 @@ package kv -import "sync" +import ( + "encoding/hex" + "log/slog" + "sync" + "time" + + "github.com/cockroachdb/errors" +) + +const ( + defaultMaxActiveBackupPins = 64 + defaultBackupPinSweepEvery = time.Second +) + +var ( + ErrInvalidBackupPin = errors.New("backup pin is invalid") + ErrTooManyActiveBackups = errors.New("too many active backup pins") +) + +type BackupPinID [16]byte + +func (id BackupPinID) IsZero() bool { + return id == BackupPinID{} +} + +func (id BackupPinID) String() string { + return hex.EncodeToString(id[:]) +} + +type backupDeadlinePin struct { + readTS uint64 + deadline time.Time +} + +type backupPinKey struct { + id BackupPinID + groupID uint64 +} + +func newBackupPinKey(pinID BackupPinID, groupID uint64) backupPinKey { + return backupPinKey{id: pinID, groupID: groupID} +} + +type ActiveTimestampTrackerOption func(*ActiveTimestampTracker) + +func WithActiveTimestampTrackerMaxBackupPins(maxPins int) ActiveTimestampTrackerOption { + return func(t *ActiveTimestampTracker) { + if maxPins > 0 { + t.maxBackupPins = maxPins + } + } +} + +func WithActiveTimestampTrackerSweepInterval(interval time.Duration) ActiveTimestampTrackerOption { + return func(t *ActiveTimestampTracker) { + t.sweepEvery = interval + } +} + +func WithActiveTimestampTrackerLogger(logger *slog.Logger) ActiveTimestampTrackerOption { + return func(t *ActiveTimestampTracker) { + if logger != nil { + t.logger = logger + } + } +} // ActiveTimestampTracker tracks in-flight read or transaction timestamps that // must remain readable while background compaction is running. type ActiveTimestampTracker struct { - mu sync.Mutex - nextID uint64 - active map[uint64]uint64 + mu sync.Mutex + nextID uint64 + active map[uint64]uint64 + backupPins map[backupPinKey]backupDeadlinePin + maxBackupPins int + sweepEvery time.Duration + sweepOnce sync.Once + stopCh chan struct{} + closeOnce sync.Once + logger *slog.Logger } // ActiveTimestampToken releases one tracked timestamp when the owning @@ -18,10 +90,21 @@ type ActiveTimestampToken struct { once sync.Once } -func NewActiveTimestampTracker() *ActiveTimestampTracker { - return &ActiveTimestampTracker{ - active: make(map[uint64]uint64), +func NewActiveTimestampTracker(opts ...ActiveTimestampTrackerOption) *ActiveTimestampTracker { + t := &ActiveTimestampTracker{ + active: make(map[uint64]uint64), + backupPins: make(map[backupPinKey]backupDeadlinePin), + maxBackupPins: defaultMaxActiveBackupPins, + sweepEvery: defaultBackupPinSweepEvery, + stopCh: make(chan struct{}), + logger: slog.Default(), } + for _, opt := range opts { + if opt != nil { + opt(t) + } + } + return t } func (t *ActiveTimestampTracker) Pin(ts uint64) *ActiveTimestampToken { @@ -40,13 +123,29 @@ func (t *ActiveTimestampTracker) Pin(ts uint64) *ActiveTimestampToken { } func (t *ActiveTimestampTracker) Oldest() uint64 { + return t.oldestForGroup(0, false) +} + +// OldestForGroup returns the oldest process-wide read pin or backup pin for +// groupID. Backup pins for other Raft groups do not constrain this group. +func (t *ActiveTimestampTracker) OldestForGroup(groupID uint64) uint64 { + return t.oldestForGroup(groupID, true) +} + +func (t *ActiveTimestampTracker) oldestForGroup(groupID uint64, scoped bool) uint64 { if t == nil { return 0 } t.mu.Lock() defer t.mu.Unlock() + readOldest := oldestReadTimestamp(t.active) + backupOldest := oldestBackupTimestamp(t.backupPins, groupID, scoped, time.Now()) + return oldestNonZeroTimestamp(readOldest, backupOldest) +} + +func oldestReadTimestamp(active map[uint64]uint64) uint64 { var oldest uint64 - for _, ts := range t.active { + for _, ts := range active { if oldest == 0 || ts < oldest { oldest = ts } @@ -54,6 +153,245 @@ func (t *ActiveTimestampTracker) Oldest() uint64 { return oldest } +func oldestBackupTimestamp(pins map[backupPinKey]backupDeadlinePin, groupID uint64, scoped bool, now time.Time) uint64 { + var oldest uint64 + for key, pin := range pins { + if scoped && key.groupID != groupID { + continue + } + if !pin.deadline.After(now) { + continue + } + if oldest == 0 || pin.readTS < oldest { + oldest = pin.readTS + } + } + return oldest +} + +func oldestNonZeroTimestamp(a, b uint64) uint64 { + if a == 0 || (b != 0 && b < a) { + return b + } + return a +} + +func (t *ActiveTimestampTracker) PinWithDeadline(pinID BackupPinID, readTS uint64, deadline time.Time) error { + return t.PinWithDeadlineForGroup(pinID, 0, readTS, deadline) +} + +func (t *ActiveTimestampTracker) PinWithDeadlineForGroup(pinID BackupPinID, groupID uint64, readTS uint64, deadline time.Time) error { + return t.pinWithDeadlineForGroup(pinID, groupID, readTS, deadline, true) +} + +func (t *ActiveTimestampTracker) ApplyPinWithDeadlineForGroup(pinID BackupPinID, groupID uint64, readTS uint64, deadline time.Time) error { + return t.pinWithDeadlineForGroup(pinID, groupID, readTS, deadline, false) +} + +func (t *ActiveTimestampTracker) pinWithDeadlineForGroup(pinID BackupPinID, groupID uint64, readTS uint64, deadline time.Time, enforceLimit bool) error { + if t == nil { + return nil + } + if !validBackupDeadlinePin(pinID, readTS, deadline) { + return errors.WithStack(ErrInvalidBackupPin) + } + t.mu.Lock() + expired := t.reapExpiredBackupPinsLocked(time.Now()) + key := newBackupPinKey(pinID, groupID) + if enforceLimit && !t.hasBackupPinIDLocked(pinID) && t.activeBackupPinIDCountLocked() >= t.maxBackupPins { + t.mu.Unlock() + t.logExpiredBackupPins(expired) + return errors.WithStack(ErrTooManyActiveBackups) + } + t.backupPins[key] = mergeBackupDeadlinePin( + t.backupPins[key], backupDeadlinePin{readTS: readTS, deadline: deadline}, + ) + t.startBackupPinSweeperLocked() + t.mu.Unlock() + t.logExpiredBackupPins(expired) + return nil +} + +func validBackupDeadlinePin(pinID BackupPinID, readTS uint64, deadline time.Time) bool { + return !pinID.IsZero() && readTS != 0 && readTS != ^uint64(0) && !deadline.IsZero() +} + +func mergeBackupDeadlinePin(existing, requested backupDeadlinePin) backupDeadlinePin { + if existing.readTS != 0 && existing.readTS < requested.readTS { + requested.readTS = existing.readTS + } + if existing.deadline.After(requested.deadline) { + requested.deadline = existing.deadline + } + return requested +} + +func (t *ActiveTimestampTracker) Extend(pinID BackupPinID, deadline time.Time) error { + return t.ExtendForGroup(pinID, 0, deadline) +} + +func (t *ActiveTimestampTracker) ExtendForGroup(pinID BackupPinID, groupID uint64, deadline time.Time) error { + return t.extendForGroup(pinID, groupID, deadline) +} + +func (t *ActiveTimestampTracker) ApplyExtendForGroup( + pinID BackupPinID, + groupID uint64, + readTS uint64, + deadline time.Time, +) error { + return t.pinWithDeadlineForGroup(pinID, groupID, readTS, deadline, false) +} + +func (t *ActiveTimestampTracker) extendForGroup(pinID BackupPinID, groupID uint64, deadline time.Time) error { + if t == nil { + return nil + } + if pinID.IsZero() || deadline.IsZero() { + return errors.WithStack(ErrInvalidBackupPin) + } + t.mu.Lock() + key := newBackupPinKey(pinID, groupID) + pin, exists := t.backupPins[key] + if !exists { + t.mu.Unlock() + return errors.WithStack(ErrInvalidBackupPin) + } + if !pin.deadline.After(time.Now()) { + delete(t.backupPins, key) + t.mu.Unlock() + t.logExpiredBackupPins([]expiredBackupPin{{key: key, ts: pin.readTS}}) + return errors.WithStack(ErrInvalidBackupPin) + } + if deadline.After(pin.deadline) { + pin.deadline = deadline + } + t.backupPins[key] = pin + t.mu.Unlock() + return nil +} + +func (t *ActiveTimestampTracker) ReleaseBackupPin(pinID BackupPinID) { + t.ReleaseBackupPinForGroup(pinID, 0) +} + +func (t *ActiveTimestampTracker) ReleaseBackupPinForGroup(pinID BackupPinID, groupID uint64) { + if t == nil || pinID.IsZero() { + return + } + t.mu.Lock() + defer t.mu.Unlock() + delete(t.backupPins, newBackupPinKey(pinID, groupID)) +} + +func (t *ActiveTimestampTracker) ActiveBackupPinCount() int { + if t == nil { + return 0 + } + t.mu.Lock() + defer t.mu.Unlock() + return len(t.backupPins) +} + +func (t *ActiveTimestampTracker) BackupPinDeadline(pinID BackupPinID) (time.Time, bool) { + return t.BackupPinDeadlineForGroup(pinID, 0) +} + +func (t *ActiveTimestampTracker) BackupPinDeadlineForGroup(pinID BackupPinID, groupID uint64) (time.Time, bool) { + if t == nil || pinID.IsZero() { + return time.Time{}, false + } + t.mu.Lock() + defer t.mu.Unlock() + pin, ok := t.backupPins[newBackupPinKey(pinID, groupID)] + return pin.deadline, ok +} + +func (t *ActiveTimestampTracker) startBackupPinSweeperLocked() { + if t.sweepEvery <= 0 { + return + } + t.sweepOnce.Do(func() { + go t.sweepBackupPins() + }) +} + +func (t *ActiveTimestampTracker) sweepBackupPins() { + if t.sweepEvery <= 0 { + return + } + ticker := time.NewTicker(t.sweepEvery) + defer ticker.Stop() + for { + select { + case now := <-ticker.C: + t.reapExpiredBackupPins(now) + case <-t.stopCh: + return + } + } +} + +// Close stops the backup-pin sweeper goroutine. It is safe to call more than +// once and on trackers that never started the sweeper. +func (t *ActiveTimestampTracker) Close() { + if t == nil { + return + } + t.closeOnce.Do(func() { + close(t.stopCh) + }) +} + +func (t *ActiveTimestampTracker) reapExpiredBackupPins(now time.Time) { + if t == nil { + return + } + t.mu.Lock() + expired := t.reapExpiredBackupPinsLocked(now) + t.mu.Unlock() + t.logExpiredBackupPins(expired) +} + +type expiredBackupPin struct { + key backupPinKey + ts uint64 +} + +func (t *ActiveTimestampTracker) reapExpiredBackupPinsLocked(now time.Time) []expiredBackupPin { + expired := make([]expiredBackupPin, 0) + for key, pin := range t.backupPins { + if !pin.deadline.After(now) { + expired = append(expired, expiredBackupPin{key: key, ts: pin.readTS}) + delete(t.backupPins, key) + } + } + return expired +} + +func (t *ActiveTimestampTracker) hasBackupPinIDLocked(pinID BackupPinID) bool { + for key := range t.backupPins { + if key.id == pinID { + return true + } + } + return false +} + +func (t *ActiveTimestampTracker) activeBackupPinIDCountLocked() int { + seen := make(map[BackupPinID]struct{}, len(t.backupPins)) + for key := range t.backupPins { + seen[key.id] = struct{}{} + } + return len(seen) +} + +func (t *ActiveTimestampTracker) logExpiredBackupPins(expired []expiredBackupPin) { + for _, pin := range expired { + t.logger.Warn("backup_pin_expired", "pin_id", pin.key.id.String(), "raft_group_id", pin.key.groupID, "read_ts", pin.ts) + } +} + func (t *ActiveTimestampToken) Release() { if t == nil || t.tracker == nil || t.id == 0 { return diff --git a/kv/active_timestamp_tracker_test.go b/kv/active_timestamp_tracker_test.go index cd35f0648..8d29d7aee 100644 --- a/kv/active_timestamp_tracker_test.go +++ b/kv/active_timestamp_tracker_test.go @@ -2,6 +2,7 @@ package kv import ( "testing" + "time" "github.com/stretchr/testify/require" ) @@ -21,3 +22,232 @@ func TestActiveTimestampTrackerOldest(t *testing.T) { second.Release() require.Equal(t, uint64(30), tracker.Oldest()) } + +func TestActiveTimestampTrackerOldestIncludesBackupPins(t *testing.T) { + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + + token := tracker.Pin(30) + defer token.Release() + + pinID := backupTrackerTestPinID(1) + require.NoError(t, tracker.PinWithDeadline(pinID, 20, time.Now().Add(time.Hour))) + require.Equal(t, uint64(20), tracker.Oldest()) + + tracker.ReleaseBackupPin(pinID) + require.Equal(t, uint64(30), tracker.Oldest()) +} + +func TestActiveTimestampTrackerBackupPinExpiry(t *testing.T) { + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + now := time.UnixMilli(3000) + + pinID := backupTrackerTestPinID(1) + require.NoError(t, tracker.PinWithDeadline(pinID, 20, now.Add(-time.Millisecond))) + require.Equal(t, uint64(0), tracker.Oldest()) + require.Equal(t, 1, tracker.ActiveBackupPinCount()) + + tracker.reapExpiredBackupPins(now) + require.Equal(t, 0, tracker.ActiveBackupPinCount()) + require.Equal(t, uint64(0), tracker.Oldest()) +} + +func TestActiveTimestampTrackerBackupPinExtendMovesDeadline(t *testing.T) { + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + pinID := backupTrackerTestPinID(1) + now := time.Now() + firstDeadline := now.Add(time.Hour) + secondDeadline := now.Add(2 * time.Hour) + + require.NoError(t, tracker.PinWithDeadline(pinID, 20, firstDeadline)) + require.NoError(t, tracker.Extend(pinID, secondDeadline)) + + got, ok := tracker.BackupPinDeadline(pinID) + require.True(t, ok) + require.Equal(t, secondDeadline, got) + tracker.reapExpiredBackupPins(firstDeadline.Add(time.Millisecond)) + require.Equal(t, 1, tracker.ActiveBackupPinCount()) +} + +func TestActiveTimestampTrackerBackupPinExtendKeepsLaterDeadline(t *testing.T) { + t.Parallel() + + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + defer tracker.Close() + pinID := backupTrackerTestPinID(1) + laterDeadline := time.Now().Add(time.Hour) + retriedDeadline := laterDeadline.Add(-time.Minute) + require.NoError(t, tracker.PinWithDeadline(pinID, 42, laterDeadline)) + + require.NoError(t, tracker.Extend(pinID, retriedDeadline)) + + got, ok := tracker.BackupPinDeadline(pinID) + require.True(t, ok) + require.Equal(t, laterDeadline, got) +} + +func TestActiveTimestampTrackerBackupPinExtendMissingIsInvalid(t *testing.T) { + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + + require.ErrorIs(t, tracker.Extend(backupTrackerTestPinID(1), time.UnixMilli(5000)), ErrInvalidBackupPin) + require.Equal(t, 0, tracker.ActiveBackupPinCount()) +} + +func TestActiveTimestampTrackerBackupPinExtendExpiredIsInvalid(t *testing.T) { + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + pinID := backupTrackerTestPinID(1) + now := time.Now() + + require.NoError(t, tracker.PinWithDeadline(pinID, 20, now.Add(-time.Millisecond))) + require.ErrorIs(t, tracker.Extend(pinID, now.Add(time.Hour)), ErrInvalidBackupPin) + require.Equal(t, 0, tracker.ActiveBackupPinCount()) + require.Equal(t, uint64(0), tracker.Oldest()) +} + +func TestActiveTimestampTrackerApplyExtendReplaysAfterLocalExpiry(t *testing.T) { + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + pinID := backupTrackerTestPinID(1) + now := time.Now() + + require.NoError(t, tracker.ApplyPinWithDeadlineForGroup(pinID, 7, 20, now.Add(-time.Millisecond))) + tracker.reapExpiredBackupPins(now) + require.Equal(t, 0, tracker.ActiveBackupPinCount()) + require.NoError(t, tracker.ApplyExtendForGroup(pinID, 7, 20, now.Add(time.Hour))) + + deadline, ok := tracker.BackupPinDeadlineForGroup(pinID, 7) + require.True(t, ok) + require.Equal(t, now.Add(time.Hour), deadline) + require.Equal(t, uint64(20), tracker.OldestForGroup(7)) +} + +func TestActiveTimestampTrackerBackupPinReleaseIsIdempotent(t *testing.T) { + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + pinID := backupTrackerTestPinID(1) + + require.NoError(t, tracker.PinWithDeadline(pinID, 20, time.UnixMilli(5000))) + tracker.ReleaseBackupPin(pinID) + tracker.ReleaseBackupPin(pinID) + + require.Equal(t, 0, tracker.ActiveBackupPinCount()) + require.Equal(t, uint64(0), tracker.Oldest()) +} + +func TestActiveTimestampTrackerBackupPinLimit(t *testing.T) { + tracker := NewActiveTimestampTracker( + WithActiveTimestampTrackerSweepInterval(0), + WithActiveTimestampTrackerMaxBackupPins(1), + ) + first := backupTrackerTestPinID(1) + second := backupTrackerTestPinID(2) + + now := time.Now() + require.NoError(t, tracker.PinWithDeadline(first, 20, now.Add(time.Hour))) + require.NoError(t, tracker.PinWithDeadline(first, 25, now.Add(2*time.Hour))) + require.ErrorIs(t, tracker.PinWithDeadline(second, 30, now.Add(3*time.Hour)), ErrTooManyActiveBackups) + require.Equal(t, 1, tracker.ActiveBackupPinCount()) + require.Equal(t, uint64(20), tracker.Oldest()) +} + +func TestActiveTimestampTrackerBackupPinLimitCountsLogicalPinIDs(t *testing.T) { + tracker := NewActiveTimestampTracker( + WithActiveTimestampTrackerSweepInterval(0), + WithActiveTimestampTrackerMaxBackupPins(1), + ) + first := backupTrackerTestPinID(1) + second := backupTrackerTestPinID(2) + deadline := time.Now().Add(time.Hour) + + require.NoError(t, tracker.PinWithDeadlineForGroup(first, 1, 20, deadline)) + require.NoError(t, tracker.PinWithDeadlineForGroup(first, 2, 20, deadline)) + require.Equal(t, 2, tracker.ActiveBackupPinCount()) + require.ErrorIs(t, tracker.PinWithDeadlineForGroup(second, 3, 30, deadline), ErrTooManyActiveBackups) +} + +func TestActiveTimestampTrackerBackupPinLimitReapsExpiredPinsFirst(t *testing.T) { + tracker := NewActiveTimestampTracker( + WithActiveTimestampTrackerSweepInterval(0), + WithActiveTimestampTrackerMaxBackupPins(1), + ) + first := backupTrackerTestPinID(1) + second := backupTrackerTestPinID(2) + now := time.Now() + + require.NoError(t, tracker.PinWithDeadline(first, 20, now.Add(-time.Millisecond))) + require.NoError(t, tracker.PinWithDeadline(second, 30, now.Add(time.Hour))) + require.Equal(t, 1, tracker.ActiveBackupPinCount()) + require.Equal(t, uint64(30), tracker.Oldest()) +} + +func TestActiveTimestampTrackerBackupPinsAreScopedByRaftGroup(t *testing.T) { + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + pinID := backupTrackerTestPinID(1) + deadline := time.Now().Add(time.Hour) + + require.NoError(t, tracker.PinWithDeadlineForGroup(pinID, 1, 20, deadline)) + require.NoError(t, tracker.PinWithDeadlineForGroup(pinID, 2, 20, deadline)) + require.Equal(t, 2, tracker.ActiveBackupPinCount()) + + tracker.ReleaseBackupPinForGroup(pinID, 1) + _, ok := tracker.BackupPinDeadlineForGroup(pinID, 1) + require.False(t, ok) + got, ok := tracker.BackupPinDeadlineForGroup(pinID, 2) + require.True(t, ok) + require.Equal(t, deadline, got) + require.Equal(t, 1, tracker.ActiveBackupPinCount()) +} + +func TestActiveTimestampTrackerDuplicatePinApplyIsMonotonic(t *testing.T) { + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + pinID := backupTrackerTestPinID(1) + now := time.Now() + laterDeadline := now.Add(2 * time.Hour) + + require.NoError(t, tracker.ApplyPinWithDeadlineForGroup(pinID, 7, 40, laterDeadline)) + require.NoError(t, tracker.ApplyPinWithDeadlineForGroup(pinID, 7, 50, now.Add(time.Hour))) + require.NoError(t, tracker.ApplyPinWithDeadlineForGroup(pinID, 7, 30, now.Add(time.Hour))) + + deadline, ok := tracker.BackupPinDeadlineForGroup(pinID, 7) + require.True(t, ok) + require.Equal(t, laterDeadline, deadline) + require.Equal(t, uint64(30), tracker.OldestForGroup(7)) +} + +func TestActiveTimestampTrackerOldestForGroupKeepsReadPinsGlobal(t *testing.T) { + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + readPin := tracker.Pin(25) + defer readPin.Release() + deadline := time.Now().Add(time.Hour) + + require.NoError(t, tracker.PinWithDeadlineForGroup(backupTrackerTestPinID(1), 1, 10, deadline)) + require.NoError(t, tracker.PinWithDeadlineForGroup(backupTrackerTestPinID(2), 2, 20, deadline)) + + require.Equal(t, uint64(10), tracker.OldestForGroup(1)) + require.Equal(t, uint64(20), tracker.OldestForGroup(2)) + require.Equal(t, uint64(25), tracker.OldestForGroup(3)) +} + +func TestActiveTimestampTrackerRejectsInvalidBackupPins(t *testing.T) { + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + validID := backupTrackerTestPinID(1) + + require.ErrorIs(t, tracker.PinWithDeadline(BackupPinID{}, 20, time.UnixMilli(5000)), ErrInvalidBackupPin) + require.ErrorIs(t, tracker.PinWithDeadline(validID, 0, time.UnixMilli(5000)), ErrInvalidBackupPin) + require.ErrorIs(t, tracker.PinWithDeadline(validID, ^uint64(0), time.UnixMilli(5000)), ErrInvalidBackupPin) + require.ErrorIs(t, tracker.PinWithDeadline(validID, 20, time.Time{}), ErrInvalidBackupPin) + require.ErrorIs(t, tracker.Extend(BackupPinID{}, time.UnixMilli(5000)), ErrInvalidBackupPin) + require.ErrorIs(t, tracker.Extend(validID, time.Time{}), ErrInvalidBackupPin) +} + +func TestActiveTimestampTrackerCloseIsIdempotent(t *testing.T) { + tracker := NewActiveTimestampTracker() + + tracker.Close() + tracker.Close() +} + +func backupTrackerTestPinID(seed byte) BackupPinID { + var id BackupPinID + for i := range id { + id[i] = seed + byte(i) + } + return id +} diff --git a/kv/backup_codec.go b/kv/backup_codec.go new file mode 100644 index 000000000..acfb27f68 --- /dev/null +++ b/kv/backup_codec.go @@ -0,0 +1,163 @@ +package kv + +import ( + "encoding/binary" + "math" + "time" + + "github.com/cockroachdb/errors" +) + +const ( + raftEncodeBackup byte = 0x0e + + backupSubtypePin byte = 0x01 + backupSubtypeExtend byte = 0x02 + backupSubtypeRelease byte = 0x03 + + backupPinIDBytes = 16 + backupUint64Size = 8 + + backupEnvelopeHeaderLen = 2 + backupPinEntryLen = backupEnvelopeHeaderLen + backupPinIDBytes + backupUint64Size + backupUint64Size + backupExtendEntryLen = backupPinEntryLen + backupReleaseEntryLen = backupEnvelopeHeaderLen + backupPinIDBytes + + backupPinIDStart = backupEnvelopeHeaderLen + backupPinIDEnd = backupPinIDStart + backupPinIDBytes + backupReadTSStart = backupPinIDEnd + backupReadTSEnd = backupReadTSStart + backupUint64Size + backupDeadlineStart = backupReadTSEnd + backupDeadlineEnd = backupDeadlineStart + backupUint64Size +) + +var ( + ErrBackupWireMalformed = errors.New("backup fsm wire payload is malformed") + ErrBackupWireSubtype = errors.New("backup fsm wire subtype is unknown") +) + +type BackupPinEntry struct { + PinID BackupPinID + ReadTS uint64 + Deadline time.Time +} + +type BackupExtendEntry struct { + PinID BackupPinID + ReadTS uint64 + Deadline time.Time +} + +type BackupReleaseEntry struct { + PinID BackupPinID +} + +type backupEntry struct { + subtype byte + pin BackupPinEntry + extend BackupExtendEntry + release BackupReleaseEntry +} + +func EncodeBackupPinEntry(entry BackupPinEntry) []byte { + out := make([]byte, backupPinEntryLen) + out[0] = raftEncodeBackup + out[1] = backupSubtypePin + copy(out[backupPinIDStart:backupPinIDEnd], entry.PinID[:]) + binary.BigEndian.PutUint64(out[backupReadTSStart:backupReadTSEnd], entry.ReadTS) + binary.BigEndian.PutUint64(out[backupDeadlineStart:backupDeadlineEnd], backupDeadlineMillis(entry.Deadline)) + return out +} + +func EncodeBackupExtendEntry(entry BackupExtendEntry) []byte { + out := make([]byte, backupExtendEntryLen) + out[0] = raftEncodeBackup + out[1] = backupSubtypeExtend + copy(out[backupPinIDStart:backupPinIDEnd], entry.PinID[:]) + binary.BigEndian.PutUint64(out[backupReadTSStart:backupReadTSEnd], entry.ReadTS) + binary.BigEndian.PutUint64(out[backupDeadlineStart:backupDeadlineEnd], backupDeadlineMillis(entry.Deadline)) + return out +} + +func EncodeBackupReleaseEntry(entry BackupReleaseEntry) []byte { + out := make([]byte, backupReleaseEntryLen) + out[0] = raftEncodeBackup + out[1] = backupSubtypeRelease + copy(out[backupPinIDStart:backupPinIDEnd], entry.PinID[:]) + return out +} + +func decodeBackupEntry(data []byte) (backupEntry, error) { + if len(data) < backupEnvelopeHeaderLen || data[0] != raftEncodeBackup { + return backupEntry{}, errors.WithStack(ErrBackupWireMalformed) + } + return decodeBackupPayload(data[1:]) +} + +func decodeBackupPayload(data []byte) (backupEntry, error) { + if len(data) < backupEnvelopeHeaderLen-1 { + return backupEntry{}, errors.WithStack(ErrBackupWireMalformed) + } + switch data[0] { + case backupSubtypePin: + if len(data) != backupPinEntryLen-1 { + return backupEntry{}, errors.WithStack(ErrBackupWireMalformed) + } + var id BackupPinID + copy(id[:], data[backupPinIDStart-1:backupPinIDEnd-1]) + return backupEntry{ + subtype: backupSubtypePin, + pin: BackupPinEntry{ + PinID: id, + ReadTS: binary.BigEndian.Uint64(data[backupReadTSStart-1 : backupReadTSEnd-1]), + Deadline: backupDeadlineFromMillis(binary.BigEndian.Uint64(data[backupDeadlineStart-1 : backupDeadlineEnd-1])), + }, + }, nil + case backupSubtypeExtend: + if len(data) != backupExtendEntryLen-1 { + return backupEntry{}, errors.WithStack(ErrBackupWireMalformed) + } + var id BackupPinID + copy(id[:], data[backupPinIDStart-1:backupPinIDEnd-1]) + return backupEntry{ + subtype: backupSubtypeExtend, + extend: BackupExtendEntry{ + PinID: id, + ReadTS: binary.BigEndian.Uint64(data[backupReadTSStart-1 : backupReadTSEnd-1]), + Deadline: backupDeadlineFromMillis(binary.BigEndian.Uint64(data[backupDeadlineStart-1 : backupDeadlineEnd-1])), + }, + }, nil + case backupSubtypeRelease: + if len(data) != backupReleaseEntryLen-1 { + return backupEntry{}, errors.WithStack(ErrBackupWireMalformed) + } + var id BackupPinID + copy(id[:], data[backupPinIDStart-1:backupPinIDEnd-1]) + return backupEntry{ + subtype: backupSubtypeRelease, + release: BackupReleaseEntry{ + PinID: id, + }, + }, nil + default: + return backupEntry{}, errors.WithStack(ErrBackupWireSubtype) + } +} + +func backupDeadlineMillis(deadline time.Time) uint64 { + ms := deadline.UnixMilli() + if ms <= 0 { + return 0 + } + return uint64(ms) +} + +func backupDeadlineFromMillis(ms uint64) time.Time { + if ms == 0 { + return time.Time{} + } + if ms > math.MaxInt64 { + ms = math.MaxInt64 + } + return time.UnixMilli(int64(ms)) //nolint:gosec // clamped to MaxInt64 above. +} diff --git a/kv/backup_codec_test.go b/kv/backup_codec_test.go new file mode 100644 index 000000000..87003450c --- /dev/null +++ b/kv/backup_codec_test.go @@ -0,0 +1,93 @@ +package kv + +import ( + "testing" + "time" + + "github.com/bootjp/elastickv/internal/encryption/fsmwire" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +func TestBackupCodecRoundTrip(t *testing.T) { + pinID := backupTrackerTestPinID(7) + deadline := time.UnixMilli(1780000000123) + + pinWire := EncodeBackupPinEntry(BackupPinEntry{ + PinID: pinID, + ReadTS: 42, + Deadline: deadline, + }) + require.Len(t, pinWire, backupPinEntryLen) + gotPin, err := decodeBackupEntry(pinWire) + require.NoError(t, err) + require.Equal(t, backupSubtypePin, gotPin.subtype) + require.Equal(t, BackupPinEntry{PinID: pinID, ReadTS: 42, Deadline: deadline}, gotPin.pin) + + extendWire := EncodeBackupExtendEntry(BackupExtendEntry{ + PinID: pinID, + ReadTS: 42, + Deadline: deadline.Add(time.Second), + }) + require.Len(t, extendWire, backupExtendEntryLen) + gotExtend, err := decodeBackupEntry(extendWire) + require.NoError(t, err) + require.Equal(t, backupSubtypeExtend, gotExtend.subtype) + require.Equal(t, BackupExtendEntry{PinID: pinID, ReadTS: 42, Deadline: deadline.Add(time.Second)}, gotExtend.extend) + + releaseWire := EncodeBackupReleaseEntry(BackupReleaseEntry{PinID: pinID}) + require.Len(t, releaseWire, backupReleaseEntryLen) + gotRelease, err := decodeBackupEntry(releaseWire) + require.NoError(t, err) + require.Equal(t, backupSubtypeRelease, gotRelease.subtype) + require.Equal(t, BackupReleaseEntry{PinID: pinID}, gotRelease.release) +} + +func TestBackupCodecRejectsMalformedWire(t *testing.T) { + pinID := backupTrackerTestPinID(1) + valid := EncodeBackupPinEntry(BackupPinEntry{ + PinID: pinID, + ReadTS: 42, + Deadline: time.UnixMilli(5000), + }) + + _, err := decodeBackupEntry(nil) + require.ErrorIs(t, err, ErrBackupWireMalformed) + _, err = decodeBackupEntry([]byte{raftEncodeBatch, backupSubtypeRelease}) + require.ErrorIs(t, err, ErrBackupWireMalformed) + _, err = decodeBackupEntry(valid[:len(valid)-1]) + require.ErrorIs(t, err, ErrBackupWireMalformed) + _, err = decodeBackupEntry([]byte{raftEncodeBackup, 0xff}) + require.ErrorIs(t, err, ErrBackupWireSubtype) +} + +func TestBackupCodecZeroDeadlineDecodesToZeroTime(t *testing.T) { + pinID := backupTrackerTestPinID(1) + + gotPin, err := decodeBackupEntry(EncodeBackupPinEntry(BackupPinEntry{ + PinID: pinID, + ReadTS: 42, + Deadline: time.Time{}, + })) + require.NoError(t, err) + require.True(t, gotPin.pin.Deadline.IsZero()) + + gotExtend, err := decodeBackupEntry(EncodeBackupExtendEntry(BackupExtendEntry{ + PinID: pinID, + ReadTS: 42, + Deadline: time.Time{}, + })) + require.NoError(t, err) + require.True(t, gotExtend.extend.Deadline.IsZero()) +} + +func TestBackupEnvelopeOpcodeDoesNotCollide(t *testing.T) { + require.Equal(t, byte(0x0e), raftEncodeBackup) + require.Greater(t, raftEncodeBackup, fsmwire.OpEncryptionMax) + require.NotContains(t, []byte{raftEncodeSingle, raftEncodeBatch, raftEncodeHLCLease}, raftEncodeBackup) + require.NotContains(t, []byte{0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d}, raftEncodeBackup) + + err := proto.Unmarshal([]byte{raftEncodeBackup, 0x00}, &pb.Request{}) + require.Error(t, err, "0x0e must remain an invalid proto wire start byte") +} diff --git a/kv/compactor.go b/kv/compactor.go index c13c7d836..c021a5b7c 100644 --- a/kv/compactor.go +++ b/kv/compactor.go @@ -230,7 +230,7 @@ func (c *FSMCompactor) compactRuntime(ctx context.Context, runtime FSMCompactRun } lastCommitTS := runtime.Store.LastCommitTS() - safeMinTS, ok := c.targetMinTS(lastCommitTS, retention.MinRetainedTS(), now) + safeMinTS, ok := c.targetMinTS(runtime.GroupID, lastCommitTS, retention.MinRetainedTS(), now) if !ok { return nil } @@ -325,11 +325,11 @@ func (c *FSMCompactor) handleCompactError(ctx context.Context, groupID uint64, s ) } -func (c *FSMCompactor) targetMinTS(lastCommitTS, minRetainedTS uint64, now time.Time) (uint64, bool) { +func (c *FSMCompactor) targetMinTS(groupID, lastCommitTS, minRetainedTS uint64, now time.Time) (uint64, bool) { if lastCommitTS == 0 { return 0, false } - safeMinTS := c.safeMinTS(now) + safeMinTS := c.safeMinTS(groupID, now) if safeMinTS == 0 { return 0, false } @@ -461,14 +461,14 @@ func fsmCompactionBudgetExhausted(err error, workCtx, parentCtx context.Context) return workCtx.Err() != nil } -func (c *FSMCompactor) safeMinTS(now time.Time) uint64 { +func (c *FSMCompactor) safeMinTS(groupID uint64, now time.Time) uint64 { cutoff := hlcTimestampFromTime(now.Add(-c.retentionWindow)) if cutoff == 0 { return 0 } oldest := uint64(0) if c.tracker != nil { - oldest = c.tracker.Oldest() + oldest = c.tracker.OldestForGroup(groupID) } if oldest != 0 && oldest <= cutoff { return oldest - 1 diff --git a/kv/compactor_test.go b/kv/compactor_test.go index b6417a8e0..7fdb1b3c2 100644 --- a/kv/compactor_test.go +++ b/kv/compactor_test.go @@ -146,6 +146,44 @@ func TestFSMCompactorRespectsPinnedTimestamp(t *testing.T) { require.Equal(t, []byte("v20"), val) } +func TestFSMCompactorScopesBackupPinsByGroup(t *testing.T) { + ctx := context.Background() + stores := map[uint64]store.MVCCStore{1: store.NewMVCCStore(), 2: store.NewMVCCStore()} + for _, st := range stores { + require.NoError(t, st.PutAt(ctx, []byte("k"), []byte("v20"), 20, 0)) + require.NoError(t, st.PutAt(ctx, []byte("k"), []byte("v30"), 30, 0)) + } + + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + require.NoError(t, tracker.PinWithDeadlineForGroup( + backupTrackerTestPinID(1), 1, 20, time.Now().Add(time.Hour), + )) + runtimes := make([]FSMCompactRuntime, 0, len(stores)) + for groupID, st := range stores { + runtimes = append(runtimes, FSMCompactRuntime{ + GroupID: groupID, + StatusReader: fakeRaftStatus{status: raftengine.Status{ + State: raftengine.StateFollower, AppliedIndex: 10, CommitIndex: 10, + }}, + Store: st, + }) + } + compactor := NewFSMCompactor( + runtimes, + WithFSMCompactorActiveTimestampTracker(tracker), + WithFSMCompactorInterval(time.Hour), + WithFSMCompactorRetentionWindow(time.Millisecond), + ) + + require.NoError(t, compactor.SyncOnce(ctx)) + + value, err := stores[1].GetAt(ctx, []byte("k"), 20) + require.NoError(t, err) + require.Equal(t, []byte("v20"), value) + _, err = stores[2].GetAt(ctx, []byte("k"), 20) + require.ErrorIs(t, err, store.ErrReadTSCompacted) +} + func TestFSMCompactorSkipsLaggingRuntime(t *testing.T) { st := store.NewMVCCStore() ctx := context.Background() diff --git a/kv/coordinator_retry_test.go b/kv/coordinator_retry_test.go index 56f0ef361..222d68390 100644 --- a/kv/coordinator_retry_test.go +++ b/kv/coordinator_retry_test.go @@ -40,7 +40,8 @@ func (stubLeaderEngine) Status() raftengine.Status { func (stubLeaderEngine) Configuration(context.Context) (raftengine.Configuration, error) { return raftengine.Configuration{}, nil } -func (stubLeaderEngine) Close() error { return nil } +func (stubLeaderEngine) SnapshotEvery() uint64 { return 10_000 } +func (stubLeaderEngine) Close() error { return nil } // scriptedTransactional returns the error registered in errs for the // 0-indexed call that triggered Commit; calls without a registered diff --git a/kv/fsm.go b/kv/fsm.go index 6ed81f076..8d0f52cfb 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -58,6 +58,11 @@ type kvFSM struct { // write", preserving Stage 6A behavior for backends that did // not opt in). pendingApplyIdx uint64 + // readTracker is shared with the local FSM compactor. BackupPin + // FSM entries mutate this tracker so compaction retains versions + // at the live-backup read timestamp until the pin is released or + // its deadline expires. + readTracker *ActiveTimestampTracker // cutoverSource provides the writer-side view of the Phase-2 // envelope cutover index for snapshot v1/v2 selection (Stage // 8a §3.3). nil = always v1 output. @@ -241,6 +246,12 @@ func WithRouteHistory(routes RouteHistory, shardGroupID uint64) FSMOption { } } +func WithActiveTimestampTracker(tracker *ActiveTimestampTracker) FSMOption { + return func(f *kvFSM) { + f.readTracker = tracker + } +} + // NewKvFSMWithHLC creates a KV FSM that updates hlc.physicalCeiling whenever // a HLC lease entry is applied. The caller must pass the same *HLC instance to // the coordinator so both sides share the agreed physical ceiling. @@ -264,6 +275,13 @@ func NewKvFSMWithHLC(store store.MVCCStore, hlc *HLC, opts ...FSMOption) FSM { return f } +func NewKvFSMWithHLCAndTracker(store store.MVCCStore, hlc *HLC, tracker *ActiveTimestampTracker, opts ...FSMOption) FSM { + all := make([]FSMOption, 0, len(opts)+1) + all = append(all, WithActiveTimestampTracker(tracker)) + all = append(all, opts...) + return NewKvFSMWithHLC(store, hlc, all...) +} + var _ FSM = (*kvFSM)(nil) var _ raftengine.StateMachine = (*kvFSM)(nil) @@ -359,6 +377,8 @@ func (f *kvFSM) applyReservedOpcode(data []byte) (any, bool) { switch { case data[0] == raftEncodeHLCLease: return f.applyHLCLease(data[1:]), true + case data[0] == raftEncodeBackup: + return f.applyBackup(data[1:]), true case data[0] >= fsmwire.OpEncryptionMin && data[0] <= fsmwire.OpEncryptionMax: return f.applyEncryption(f.pendingApplyIdx, data[0], data[1:]), true default: @@ -642,18 +662,19 @@ func (f *kvFSM) ApplySnapshotHeader(ceiling, cutover uint64) { // is monotonic and lives purely in memory. After the cold-start skip // gate fires, the engine still delivers WAL committed-tail entries // past snapshot.Metadata.Index; without this classifier those -// volatile entries get dropped along with KV/MVCC duplicates and the -// post-snapshot ceiling raise is lost. Codex P1 #934 round 7. +// volatile entries get dropped along with KV/MVCC duplicates. HLC +// would lose the post-snapshot ceiling raise; backup pins would lose +// a post-snapshot retention fence. // // Re-applying KV/MVCC entries would re-execute OCC validation against // store state that has already moved past commit_ts, surfacing -// spurious conflicts. Returning false for any non-HLC payload tag +// spurious conflicts. Returning false for persistent internal tags // preserves that idempotency. Encryption opcodes (0x03..0x07) MUST // also return false — they persist DEK state in the encryption // sidecar and re-applying would diverge the sidecar's // RaftAppliedIndex from the engine's appliedIndex. func (f *kvFSM) IsVolatileOnlyPayload(payload []byte) bool { - return len(payload) > 0 && payload[0] == raftEncodeHLCLease + return len(payload) > 0 && (payload[0] == raftEncodeHLCLease || payload[0] == raftEncodeBackup) } func (f *kvFSM) handleTxnRequest(ctx context.Context, r *pb.Request, commitTS uint64) error { diff --git a/kv/fsm_backup.go b/kv/fsm_backup.go new file mode 100644 index 000000000..6b2c151a3 --- /dev/null +++ b/kv/fsm_backup.go @@ -0,0 +1,38 @@ +package kv + +import "github.com/cockroachdb/errors" + +var ErrBackupApply = errors.New("backup fsm apply failed") + +func (f *kvFSM) applyBackup(data []byte) any { + if f.readTracker == nil { + return haltErr(errors.Wrap(ErrBackupApply, "kv/fsm: backup entry arrived but no ActiveTimestampTracker is wired")) + } + entry, err := decodeBackupPayload(data) + if err != nil { + return haltErr(errors.Wrap(errors.Mark(err, ErrBackupApply), "kv/fsm: decode backup entry")) + } + switch entry.subtype { + case backupSubtypePin: + err = f.readTracker.ApplyPinWithDeadlineForGroup(entry.pin.PinID, f.shardGroupID, entry.pin.ReadTS, entry.pin.Deadline) + case backupSubtypeExtend: + err = f.readTracker.ApplyExtendForGroup( + entry.extend.PinID, + f.shardGroupID, + entry.extend.ReadTS, + entry.extend.Deadline, + ) + case backupSubtypeRelease: + f.readTracker.ReleaseBackupPinForGroup(entry.release.PinID, f.shardGroupID) + return nil + default: + err = ErrBackupWireSubtype + } + if err != nil { + if errors.Is(err, ErrInvalidBackupPin) { + return err + } + return haltErr(errors.Wrap(errors.Mark(err, ErrBackupApply), "kv/fsm: apply backup entry")) + } + return nil +} diff --git a/kv/fsm_backup_test.go b/kv/fsm_backup_test.go new file mode 100644 index 000000000..f6915c326 --- /dev/null +++ b/kv/fsm_backup_test.go @@ -0,0 +1,205 @@ +package kv + +import ( + "testing" + "time" + + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +func TestApplyBackupUsesSharedTracker(t *testing.T) { + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + fsm := newBackupTestFSM(t, tracker) + pinID := backupTrackerTestPinID(1) + now := time.Now() + firstDeadline := time.UnixMilli(now.Add(time.Hour).UnixMilli()) + secondDeadline := time.UnixMilli(now.Add(2 * time.Hour).UnixMilli()) + + require.NoError(t, haltApplyOf(fsm.Apply(EncodeBackupPinEntry(BackupPinEntry{ + PinID: pinID, + ReadTS: 42, + Deadline: firstDeadline, + })))) + require.Equal(t, 1, tracker.ActiveBackupPinCount()) + require.Equal(t, uint64(42), tracker.Oldest()) + + require.NoError(t, haltApplyOf(fsm.Apply(EncodeBackupExtendEntry(BackupExtendEntry{ + PinID: pinID, + ReadTS: 42, + Deadline: secondDeadline, + })))) + gotDeadline, ok := tracker.BackupPinDeadline(pinID) + require.True(t, ok) + require.Equal(t, secondDeadline, gotDeadline) + + require.NoError(t, haltApplyOf(fsm.Apply(EncodeBackupReleaseEntry(BackupReleaseEntry{PinID: pinID})))) + require.Equal(t, 0, tracker.ActiveBackupPinCount()) + require.Equal(t, uint64(0), tracker.Oldest()) +} + +func TestApplyBackupWithoutTrackerHalts(t *testing.T) { + fsm, ok := NewKvFSMWithHLC(store.NewMVCCStore(), NewHLC()).(*kvFSM) + require.True(t, ok) + + err := haltApplyOf(fsm.Apply(EncodeBackupPinEntry(BackupPinEntry{ + PinID: backupTrackerTestPinID(1), + ReadTS: 42, + Deadline: time.UnixMilli(5000), + }))) + require.True(t, errors.Is(err, ErrBackupApply), "err = %v", err) +} + +func TestApplyBackupUnknownSubtypeHalts(t *testing.T) { + fsm := newBackupTestFSM(t, NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0))) + + err := haltApplyOf(fsm.Apply([]byte{raftEncodeBackup, 0xff})) + require.True(t, errors.Is(err, ErrBackupApply), "err = %v", err) + require.True(t, errors.Is(err, ErrBackupWireSubtype), "err = %v", err) +} + +func TestApplyBackupInvalidPinReturnsNonFatalError(t *testing.T) { + fsm := newBackupTestFSM(t, NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0))) + + resp := fsm.Apply(EncodeBackupPinEntry(BackupPinEntry{ + PinID: BackupPinID{}, + ReadTS: 42, + Deadline: time.UnixMilli(5000), + })) + require.NoError(t, haltApplyOf(resp)) + respErr, ok := resp.(error) + require.True(t, ok) + require.ErrorIs(t, respErr, ErrInvalidBackupPin) +} + +func TestApplyBackupLimitDoesNotDropCommittedPins(t *testing.T) { + tracker := NewActiveTimestampTracker( + WithActiveTimestampTrackerSweepInterval(0), + WithActiveTimestampTrackerMaxBackupPins(1), + ) + fsm := newBackupTestFSM(t, tracker) + require.NoError(t, haltApplyOf(fsm.Apply(EncodeBackupPinEntry(BackupPinEntry{ + PinID: backupTrackerTestPinID(1), + ReadTS: 42, + Deadline: time.Now().Add(time.Hour), + })))) + + resp := fsm.Apply(EncodeBackupPinEntry(BackupPinEntry{ + PinID: backupTrackerTestPinID(2), + ReadTS: 43, + Deadline: time.Now().Add(2 * time.Hour), + })) + + require.NoError(t, haltApplyOf(resp)) + require.Nil(t, resp) + require.Equal(t, 2, tracker.ActiveBackupPinCount()) + require.Equal(t, uint64(42), tracker.Oldest()) +} + +func TestApplyBackupMissingExtendRestoresCommittedFence(t *testing.T) { + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + fsm := newBackupTestFSM(t, tracker) + + resp := fsm.Apply(EncodeBackupExtendEntry(BackupExtendEntry{ + PinID: backupTrackerTestPinID(1), + ReadTS: 42, + Deadline: time.Now().Add(time.Hour), + })) + + require.NoError(t, haltApplyOf(resp)) + require.Nil(t, resp) + require.Equal(t, 1, tracker.ActiveBackupPinCount()) + require.Equal(t, uint64(42), tracker.Oldest()) +} + +func TestApplyBackupExpiredExtendRestoresCommittedFence(t *testing.T) { + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + fsm := newBackupTestFSM(t, tracker) + pinID := backupTrackerTestPinID(1) + require.NoError(t, tracker.PinWithDeadline(pinID, 42, time.Now().Add(-time.Millisecond))) + tracker.reapExpiredBackupPins(time.Now()) + require.Equal(t, 0, tracker.ActiveBackupPinCount()) + + resp := fsm.Apply(EncodeBackupExtendEntry(BackupExtendEntry{ + PinID: pinID, + ReadTS: 42, + Deadline: time.Now().Add(time.Hour), + })) + + require.NoError(t, haltApplyOf(resp)) + require.Nil(t, resp) + require.Equal(t, 1, tracker.ActiveBackupPinCount()) + require.Equal(t, uint64(42), tracker.Oldest()) +} + +func TestApplyBackupZeroDeadlineReturnsNonFatalError(t *testing.T) { + fsm := newBackupTestFSM(t, NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0))) + + resp := fsm.Apply(EncodeBackupPinEntry(BackupPinEntry{ + PinID: backupTrackerTestPinID(1), + ReadTS: 42, + Deadline: time.Time{}, + })) + + require.NoError(t, haltApplyOf(resp)) + respErr, ok := resp.(error) + require.True(t, ok) + require.ErrorIs(t, respErr, ErrInvalidBackupPin) +} + +func TestApplyBackupPinsAreScopedByRaftGroup(t *testing.T) { + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + fsm1 := newBackupTestFSMWithGroup(t, tracker, 1) + fsm2 := newBackupTestFSMWithGroup(t, tracker, 2) + pinID := backupTrackerTestPinID(1) + entry := BackupPinEntry{ + PinID: pinID, + ReadTS: 42, + Deadline: time.Now().Add(time.Hour), + } + + require.NoError(t, haltApplyOf(fsm1.Apply(EncodeBackupPinEntry(entry)))) + require.NoError(t, haltApplyOf(fsm2.Apply(EncodeBackupPinEntry(entry)))) + require.Equal(t, 2, tracker.ActiveBackupPinCount()) + + require.NoError(t, haltApplyOf(fsm1.Apply(EncodeBackupReleaseEntry(BackupReleaseEntry{PinID: pinID})))) + require.Equal(t, 1, tracker.ActiveBackupPinCount()) + _, ok := tracker.BackupPinDeadlineForGroup(pinID, 1) + require.False(t, ok) + _, ok = tracker.BackupPinDeadlineForGroup(pinID, 2) + require.True(t, ok) +} + +func TestBackupPayloadIsVolatileOnly(t *testing.T) { + fsm := &kvFSM{} + pinID := backupTrackerTestPinID(1) + + require.True(t, fsm.IsVolatileOnlyPayload(EncodeBackupPinEntry(BackupPinEntry{ + PinID: pinID, + ReadTS: 42, + Deadline: time.UnixMilli(5000), + }))) + require.False(t, fsm.IsVolatileOnlyPayload(nil)) + require.False(t, fsm.IsVolatileOnlyPayload([]byte{raftEncodeSingle})) + require.False(t, fsm.IsVolatileOnlyPayload([]byte{0x03})) +} + +func newBackupTestFSM(t *testing.T, tracker *ActiveTimestampTracker) *kvFSM { + t.Helper() + fsm, ok := NewKvFSMWithHLCAndTracker(store.NewMVCCStore(), NewHLC(), tracker).(*kvFSM) + require.True(t, ok) + return fsm +} + +func newBackupTestFSMWithGroup(t *testing.T, tracker *ActiveTimestampTracker, groupID uint64) *kvFSM { + t.Helper() + fsm, ok := NewKvFSMWithHLCAndTracker( + store.NewMVCCStore(), + NewHLC(), + tracker, + WithRouteHistory(nil, groupID), + ).(*kvFSM) + require.True(t, ok) + return fsm +} diff --git a/kv/leader_proxy_test.go b/kv/leader_proxy_test.go index ea02d6c2a..7f0d17320 100644 --- a/kv/leader_proxy_test.go +++ b/kv/leader_proxy_test.go @@ -63,7 +63,8 @@ func (s *stubFollowerEngine) Status() raftengine.Status { func (s *stubFollowerEngine) Configuration(context.Context) (raftengine.Configuration, error) { return raftengine.Configuration{}, nil } -func (s *stubFollowerEngine) Close() error { return nil } +func (s *stubFollowerEngine) SnapshotEvery() uint64 { return 10_000 } +func (s *stubFollowerEngine) Close() error { return nil } func TestLeaderProxy_CommitLocalWhenLeader(t *testing.T) { t.Parallel() @@ -186,7 +187,8 @@ func (e *togglingFollowerEngine) Status() raftengine.Status { func (e *togglingFollowerEngine) Configuration(context.Context) (raftengine.Configuration, error) { return raftengine.Configuration{}, nil } -func (e *togglingFollowerEngine) Close() error { return nil } +func (e *togglingFollowerEngine) SnapshotEvery() uint64 { return 10_000 } +func (e *togglingFollowerEngine) Close() error { return nil } func TestLeaderProxy_ForwardsAfterLeaderPublishes(t *testing.T) { t.Parallel() diff --git a/kv/lease_read_test.go b/kv/lease_read_test.go index d62c103d2..05fe41ac2 100644 --- a/kv/lease_read_test.go +++ b/kv/lease_read_test.go @@ -62,6 +62,7 @@ func (e *fakeLeaseEngine) Status() raftengine.Status { func (e *fakeLeaseEngine) Configuration(context.Context) (raftengine.Configuration, error) { return raftengine.Configuration{}, nil } +func (e *fakeLeaseEngine) SnapshotEvery() uint64 { return 10_000 } func (e *fakeLeaseEngine) Propose(context.Context, []byte) (*raftengine.ProposalResult, error) { e.proposeCalls.Add(1) if e.proposeHook != nil { @@ -163,6 +164,7 @@ func (e *nonLeaseEngine) Status() raftengine.Status { func (e *nonLeaseEngine) Configuration(context.Context) (raftengine.Configuration, error) { return raftengine.Configuration{}, nil } +func (e *nonLeaseEngine) SnapshotEvery() uint64 { return 10_000 } func (e *nonLeaseEngine) Propose(context.Context, []byte) (*raftengine.ProposalResult, error) { return &raftengine.ProposalResult{}, nil } diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 5587661a6..ba9c16944 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -49,6 +49,8 @@ func (e *followerProxyEngine) Status() raftengine.Status { } } +func (e *followerProxyEngine) SnapshotEvery() uint64 { return 0 } + func (e *followerProxyEngine) Configuration(context.Context) (raftengine.Configuration, error) { return raftengine.Configuration{}, nil } diff --git a/kv/sharded_coordinator_txn_test.go b/kv/sharded_coordinator_txn_test.go index 42b65f328..36a59297f 100644 --- a/kv/sharded_coordinator_txn_test.go +++ b/kv/sharded_coordinator_txn_test.go @@ -447,7 +447,8 @@ func (noopEngine) Status() raftengine.Status { return r func (noopEngine) Configuration(_ context.Context) (raftengine.Configuration, error) { return raftengine.Configuration{}, nil } -func (noopEngine) Close() error { return nil } +func (noopEngine) SnapshotEvery() uint64 { return 10_000 } +func (noopEngine) Close() error { return nil } func TestValidateReadOnlyShards_DetectsConflictOnReadOnlyShard(t *testing.T) { t.Parallel() diff --git a/main.go b/main.go index 479bcc69b..1d05aa288 100644 --- a/main.go +++ b/main.go @@ -446,6 +446,7 @@ func run() error { } keystore := encryption.NewKeystore() redisApplyObserver := adapter.NewRedisApplyObserver() + readTracker := kv.NewActiveTimestampTracker() // Stage 6D-6c: buildShardGroupsWithEncryptionWiring assembles the // storage-envelope write-path wiring (cipher + deterministic nonce @@ -472,6 +473,7 @@ func run() error { keystore, *encryptionSidecarPath, *encryptionEnabled, + readTracker, cfg.engine, redisApplyObserver, ) @@ -494,9 +496,9 @@ func run() error { cleanup := internalutil.CleanupStack{} defer cleanup.Run() + cleanup.Add(readTracker.Close) ctx, cancel := context.WithCancel(context.Background()) - readTracker := kv.NewActiveTimestampTracker() shardStore := kv.NewShardStore(cfg.engine, shardGroups) cleanup.Add(func() { _ = shardStore.Close() @@ -1376,6 +1378,7 @@ func buildShardGroups( factory raftengine.Factory, proposalObserverForGroup func(uint64) kv.ProposalObserver, clock *kv.HLC, + readTracker *kv.ActiveTimestampTracker, kekWrapper kek.Wrapper, keystore *encryption.Keystore, sidecarPath string, @@ -1454,7 +1457,8 @@ func buildShardGroups( // work. At M2 the FSM stores both but does not consult them; // see docs/design/2026_05_29_implemented_composed1_cross_group_commit_guard.md // §M2. - sm := kv.NewKvFSMWithHLC(st, clock, fsmOptionsForGroup(applier, routeEngine, g.id, encWiring, applyObservers...)...) + sm := kv.NewKvFSMWithHLCAndTracker(st, clock, readTracker, + fsmOptionsForGroup(applier, routeEngine, g.id, encWiring, applyObservers...)...) groupBootstrap, groupBootstrapServers, groupBootstrapSeed := bootstrapSettingsForGroup(bootstrapCfg, g.id, bootstrap) runtime, err := buildRuntimeForGroup( raftID, g, raftDir, multi, groupBootstrap, @@ -1794,43 +1798,27 @@ type serversInput struct { // to catch up, prepares the public listeners, waits for any requested startup // rotation, then starts serving public traffic. func startServersAfterStartupRotation(waitRotateOnStartup startupRotationWaiter, in serversInput) error { - adminServer, adminGRPCOpts, err := setupAdminService(*raftId, *myAddr, in.runtimes, in.bootstrapServers, in.keyvizSampler) - if err != nil { - return err - } - // roleStore + connCache are gated on *adminEnabled. With admin - // disabled, building either is wasted work AND a security - // regression risk: a non-empty -adminFullAccessKeys flag would - // otherwise still flip forwardDeps.readyForRegistration() to - // true, registering the leader-side gRPC AdminForward service - // and re-exposing the table-write surface a follower-direct - // admin call could reach (P1/Major review on #648). - // The HTTP admin listener already short-circuits in - // prepareAdminFromFlags when *adminEnabled is false; the gRPC path - // must do the same. var ( roleStore admin.RoleStore connCache *kv.GRPCConnCache ) + adminGRPCEnabled := *adminTokenFile != "" || *adminInsecureNoAuth + // roleStore is gated on *adminEnabled. With admin HTTP disabled, building + // it is wasted work AND a security regression risk: a non-empty + // -adminFullAccessKeys flag would otherwise still flip + // forwardDeps.readyForRegistration() to true, registering the leader-side + // gRPC AdminForward service and re-exposing the table-write surface a + // follower-direct admin call could reach (P1/Major review on #648). + // + // connCache is also used by the gRPC Admin GetNodeVersion probe, so create + // it when either admin HTTP or gRPC Admin is enabled. + connCache = prepareAdminConnCache(in.ctx, in.eg, *adminEnabled || adminGRPCEnabled) + adminServer, adminGRPCOpts, err := setupAdminService(*raftId, *myAddr, in.runtimes, in.bootstrapServers, in.keyvizSampler, connCache) + if err != nil { + return err + } if *adminEnabled { roleStore = roleStoreFromFlags(parseCSV(*adminFullAccessKeys), parseCSV(*adminReadOnlyAccessKeys)) - // connCache is shared between the follower-side LeaderForwarder - // (built inside prepareAdminFromFlags) and any future bridge that - // dials the leader's gRPC ports. Keeping a single instance per - // process means the two paths re-use TLS / HTTP/2 connections - // rather than each maintaining a parallel pool. The shutdown - // goroutine drains the cache on context cancellation so the - // accumulated HTTP/2 connections are not leaked when the - // process exits gracefully (Claude review on #648). - connCache = &kv.GRPCConnCache{} - cache := connCache - in.eg.Go(func() error { - <-in.ctx.Done() - if err := cache.Close(); err != nil { - return errors.Wrap(err, "close admin gRPC connection cache") - } - return nil - }) } publicKVGate := &startupPublicKVGate{} installHLCLeaseRenewalBlocker(in.coordinate, waitRotateOnStartup.BlockMutators) @@ -2065,6 +2053,7 @@ func setupAdminService( runtimes []*raftGroupRuntime, bootstrapServers []raftengine.Server, keyvizSampler *keyviz.MemSampler, + connCache *kv.GRPCConnCache, ) (*adapter.AdminServer, adminGRPCInterceptors, error) { members := adminMembersFromBootstrap(nodeID, bootstrapServers) // In multi-group mode the process does not listen on *myAddr — each group @@ -2078,6 +2067,7 @@ func setupAdminService( *adminInsecureNoAuth, adapter.NodeIdentity{NodeID: nodeID, GRPCAddress: selfAddr}, members, + connCache, ) if err != nil { return nil, adminGRPCInterceptors{}, err @@ -2299,6 +2289,7 @@ func configureAdminService( insecureNoAuth bool, self adapter.NodeIdentity, members []adapter.NodeIdentity, + connCache *kv.GRPCConnCache, ) (*adapter.AdminServer, adminGRPCInterceptors, error) { if tokenPath == "" && !insecureNoAuth { return nil, adminGRPCInterceptors{}, nil @@ -2314,7 +2305,11 @@ func configureAdminService( } token = loaded } - srv := adapter.NewAdminServer(self, members) + opts := []adapter.AdminOption{adapter.WithAdminNodeVersion(buildVersion())} + if probe := adminLeaderVersionProbe(connCache); probe != nil { + opts = append(opts, adapter.WithAdminLeaderVersionProbe(probe)) + } + srv := adapter.NewAdminServer(self, members, opts...) srv.SetCapability(adapter.S3BlobOffloadCapabilityName, adapter.S3BlobOffloadLocalCapability()) unary, stream := adapter.AdminTokenAuth(token) var icept adminGRPCInterceptors @@ -2327,6 +2322,23 @@ func configureAdminService( return srv, icept, nil } +func adminLeaderVersionProbe(connCache *kv.GRPCConnCache) adapter.LeaderVersionProbe { + if connCache == nil { + return nil + } + return func(ctx context.Context, address string) (string, error) { + conn, err := connCache.ConnFor(address) + if err != nil { + return "", errors.Wrap(err, "admin leader version probe: dial peer") + } + resp, err := pb.NewAdminClient(conn).GetNodeVersion(ctx, &pb.GetNodeVersionRequest{}) + if err != nil { + return "", errors.Wrap(err, "admin leader version probe: get node version") + } + return resp.GetNodeVersion(), nil + } +} + // loadAdminTokenFile materialises --adminTokenFile with a strict upper bound // so a misconfigured path (for example a log file) cannot force an arbitrary // allocation before the bearer-token check. Delegates to the shared helper in @@ -3005,6 +3017,21 @@ func (r *runtimeServerRunner) startRaftTransport() error { return nil } +func prepareAdminConnCache(ctx context.Context, eg *errgroup.Group, enabled bool) *kv.GRPCConnCache { + if !enabled { + return nil + } + connCache := &kv.GRPCConnCache{} + eg.Go(func() error { + <-ctx.Done() + if err := connCache.Close(); err != nil { + return errors.Wrap(err, "close admin gRPC connection cache") + } + return nil + }) + return connCache +} + func (r *runtimeServerRunner) prepareAdminForwardServers() error { r.dynamoServer = newDynamoDBServer(r.shardStore, r.coordinate, r.leaderDynamo, r.metricsRegistry, r.readTracker) s3Server, err := newS3Server( diff --git a/main_admin.go b/main_admin.go index 6fdce1785..6083ccfa4 100644 --- a/main_admin.go +++ b/main_admin.go @@ -8,6 +8,7 @@ import ( "net" "net/http" "os" + "runtime/debug" "strings" "time" @@ -33,16 +34,43 @@ const ( adminWriteTimeout = 10 * time.Second adminIdleTimeout = 30 * time.Second adminShutdownTimeout = 5 * time.Second +) + +// adminBuildVersion is surfaced by both GET /admin/api/v1/cluster and the +// Admin gRPC GetNodeVersion RPC. Release builds may stamp it with: +// +// -ldflags "-X main.adminBuildVersion=" +var adminBuildVersion = "dev" - // adminBuildVersion is surfaced in GET /admin/api/v1/cluster. Until - // we wire real ldflags-injected build info, a placeholder is fine. - adminBuildVersion = "dev" +const ( + buildInfoVCSRevisionKey = "vcs.revision" + buildInfoVCSModifiedKey = "vcs.modified" ) -// buildVersion returns the elastickv binary version for admin purposes. -// It is intentionally a function, not a constant, so build tooling can -// link-replace it via -ldflags in the future. -func buildVersion() string { return adminBuildVersion } +func buildVersion() string { + if version := strings.TrimSpace(adminBuildVersion); version != "" && version != "dev" { + return version + } + if info, ok := debug.ReadBuildInfo(); ok { + revision := "" + modified := false + for _, setting := range info.Settings { + switch setting.Key { + case buildInfoVCSRevisionKey: + revision = strings.TrimSpace(setting.Value) + case buildInfoVCSModifiedKey: + modified = setting.Value == "true" + } + } + if revision != "" { + if modified { + return revision + "-modified" + } + return revision + } + } + return "dev" +} // adminListenerConfig is the subset of startup inputs that goes into the // admin listener. Collecting them in a struct keeps the main.go call site diff --git a/main_admin_test.go b/main_admin_test.go index f12d7314d..98d52f857 100644 --- a/main_admin_test.go +++ b/main_admin_test.go @@ -26,13 +26,15 @@ import ( "github.com/bootjp/elastickv/internal/admin" "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" + "google.golang.org/grpc" ) func TestConfigureAdminServiceDisabledByDefault(t *testing.T) { t.Parallel() - srv, icept, err := configureAdminService("", false, adapter.NodeIdentity{NodeID: "n1"}, nil) + srv, icept, err := configureAdminService("", false, adapter.NodeIdentity{NodeID: "n1"}, nil, nil) if err != nil { t.Fatalf("disabled-by-default should not error: %v", err) } @@ -48,7 +50,7 @@ func TestConfigureAdminServiceRejectsMutualExclusion(t *testing.T) { if err := os.WriteFile(tokPath, []byte("x"), 0o600); err != nil { t.Fatal(err) } - if _, _, err := configureAdminService(tokPath, true, adapter.NodeIdentity{}, nil); err == nil { + if _, _, err := configureAdminService(tokPath, true, adapter.NodeIdentity{}, nil, nil); err == nil { t.Fatal("expected mutual-exclusion error") } } @@ -60,7 +62,7 @@ func TestConfigureAdminServiceTokenFile(t *testing.T) { if err := os.WriteFile(tokPath, []byte("hunter2\n"), 0o600); err != nil { t.Fatal(err) } - srv, icept, err := configureAdminService(tokPath, false, adapter.NodeIdentity{NodeID: "n1"}, nil) + srv, icept, err := configureAdminService(tokPath, false, adapter.NodeIdentity{NodeID: "n1"}, nil, nil) if err != nil { t.Fatalf("configureAdminService: %v", err) } @@ -75,7 +77,7 @@ func TestConfigureAdminServiceTokenFile(t *testing.T) { func TestConfigureAdminServiceInsecureNoAuth(t *testing.T) { t.Parallel() - srv, icept, err := configureAdminService("", true, adapter.NodeIdentity{NodeID: "n1"}, nil) + srv, icept, err := configureAdminService("", true, adapter.NodeIdentity{NodeID: "n1"}, nil, nil) if err != nil { t.Fatalf("insecure mode should succeed: %v", err) } @@ -87,6 +89,90 @@ func TestConfigureAdminServiceInsecureNoAuth(t *testing.T) { } } +func TestConfigureAdminServiceWiresLeaderVersionProbe(t *testing.T) { + t.Parallel() + var lc net.ListenConfig + lis, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + remote := adapter.NewAdminServer( + adapter.NodeIdentity{NodeID: "n2", GRPCAddress: lis.Addr().String()}, + nil, + adapter.WithAdminNodeVersion("v-remote"), + ) + gs := grpc.NewServer() + pb.RegisterAdminServer(gs, remote) + serveErr := make(chan error, 1) + go func() { + serveErr <- gs.Serve(lis) + }() + t.Cleanup(func() { + gs.Stop() + _ = lis.Close() + <-serveErr + }) + + var cache kv.GRPCConnCache + t.Cleanup(func() { require.NoError(t, cache.Close()) }) + srv, _, err := configureAdminService( + "", + true, + adapter.NodeIdentity{NodeID: "n1", GRPCAddress: "127.0.0.1:50051"}, + nil, + &cache, + ) + require.NoError(t, err) + require.NotNil(t, srv) + srv.RegisterGroup(1, adminVersionProbeGroup{leader: raftengine.LeaderInfo{ + ID: "n2", + Address: lis.Addr().String(), + }}) + + resp, err := srv.GetRaftGroups(context.Background(), &pb.GetRaftGroupsRequest{}) + require.NoError(t, err) + require.Len(t, resp.Groups, 1) + require.Empty(t, resp.Groups[0].LeaderNodeVersion) + require.Eventually(t, func() bool { + resp, err := srv.GetRaftGroups(context.Background(), &pb.GetRaftGroupsRequest{}) + return err == nil && len(resp.Groups) == 1 && resp.Groups[0].LeaderNodeVersion == "v-remote" + }, time.Second, 10*time.Millisecond) +} + +func TestConfigureAdminServiceUsesStampedBuildVersion(t *testing.T) { + old := adminBuildVersion + adminBuildVersion = "v-admin-test" + t.Cleanup(func() { adminBuildVersion = old }) + + if got := buildVersion(); got != "v-admin-test" { + t.Fatalf("buildVersion = %q, want v-admin-test", got) + } + srv, _, err := configureAdminService( + "", + true, + adapter.NodeIdentity{NodeID: "n1", GRPCAddress: "127.0.0.1:50051"}, + nil, + nil, + ) + require.NoError(t, err) + require.NotNil(t, srv) + resp, err := srv.GetNodeVersion(context.Background(), &pb.GetNodeVersionRequest{}) + require.NoError(t, err) + require.Equal(t, "v-admin-test", resp.GetNodeVersion()) +} + +type adminVersionProbeGroup struct { + leader raftengine.LeaderInfo +} + +func (g adminVersionProbeGroup) Status() raftengine.Status { + return raftengine.Status{Leader: g.leader} +} + +func (adminVersionProbeGroup) Configuration(context.Context) (raftengine.Configuration, error) { + return raftengine.Configuration{}, nil +} + +func (adminVersionProbeGroup) SnapshotEvery() uint64 { return 0 } + func TestAdminMembersFromBootstrapExcludesSelf(t *testing.T) { t.Parallel() servers := []raftengine.Server{ diff --git a/main_bootstrap_e2e_test.go b/main_bootstrap_e2e_test.go index d962c8105..1e101daae 100644 --- a/main_bootstrap_e2e_test.go +++ b/main_bootstrap_e2e_test.go @@ -540,7 +540,9 @@ func startBootstrapE2ENode( return nil, err } clock := kv.NewHLC() - runtimes, shardGroups, err := buildShardGroups(ep.id, baseDir, cfg.groups, cfg.multi, bootstrap, raftBootstrapConfig{legacyServers: bootstrapServers}, factory, nil, clock, nil, nil, "", encryptionWriteWiring{}, cfg.engine) + runtimes, shardGroups, err := buildShardGroups(ep.id, baseDir, cfg.groups, cfg.multi, bootstrap, + raftBootstrapConfig{legacyServers: bootstrapServers}, factory, nil, clock, kv.NewActiveTimestampTracker(), + nil, nil, "", encryptionWriteWiring{}, cfg.engine) if err != nil { return nil, err } @@ -623,7 +625,9 @@ func startBootstrapE2EMultiGroupNode( return nil, err } clock := kv.NewHLC() - runtimes, shardGroups, err := buildShardGroups(ep.id, baseDir, cfg.groups, cfg.multi, bootstrap, bootstrapCfg, factory, nil, clock, nil, nil, "", encryptionWriteWiring{}, cfg.engine) + runtimes, shardGroups, err := buildShardGroups(ep.id, baseDir, cfg.groups, cfg.multi, bootstrap, + bootstrapCfg, factory, nil, clock, kv.NewActiveTimestampTracker(), nil, nil, "", + encryptionWriteWiring{}, cfg.engine) if err != nil { return nil, err } diff --git a/main_encryption_write_wiring.go b/main_encryption_write_wiring.go index b3f124e71..165a03b18 100644 --- a/main_encryption_write_wiring.go +++ b/main_encryption_write_wiring.go @@ -36,6 +36,7 @@ func buildShardGroupsWithEncryptionWiring( keystore *encryption.Keystore, sidecarPath string, encryptionEnabled bool, + readTracker *kv.ActiveTimestampTracker, routeEngine *distribution.Engine, applyObserver kv.ApplyObserver, ) ([]*raftGroupRuntime, map[uint64]*kv.ShardGroup, encryptionWriteWiring, error) { @@ -57,7 +58,8 @@ func buildShardGroupsWithEncryptionWiring( } configureRaftEnvelopeFactory(factory, encWiring) runtimes, shardGroups, err := buildShardGroups(raftID, raftDir, groups, multi, bootstrap, bootstrapCfg, - factory, proposalObserverForGroup, clock, kekWrapper, keystore, sidecarPath, encWiring, routeEngine, applyObserver) + factory, proposalObserverForGroup, clock, readTracker, kekWrapper, keystore, sidecarPath, encWiring, + routeEngine, applyObserver) if err != nil { return runtimes, shardGroups, encWiring, err } diff --git a/main_sqs_leadership_refusal_test.go b/main_sqs_leadership_refusal_test.go index c7889c0a1..7f7731423 100644 --- a/main_sqs_leadership_refusal_test.go +++ b/main_sqs_leadership_refusal_test.go @@ -33,7 +33,8 @@ func (f *fakeLeadershipController) Status() raftengine.Status { return raftengin func (f *fakeLeadershipController) Configuration(_ context.Context) (raftengine.Configuration, error) { return raftengine.Configuration{}, nil } -func (f *fakeLeadershipController) Close() error { return nil } +func (f *fakeLeadershipController) SnapshotEvery() uint64 { return 10_000 } +func (f *fakeLeadershipController) Close() error { return nil } // fakeLeadershipController is a sqsLeadershipController test // double. It records TransferLeadership invocations and exposes diff --git a/multiraft_runtime_test.go b/multiraft_runtime_test.go index 11670c9e9..de43e6fc3 100644 --- a/multiraft_runtime_test.go +++ b/multiraft_runtime_test.go @@ -62,6 +62,7 @@ func TestBuildShardGroupsWithDedicatedTSOPreservesSingleDataGroupDir(t *testing. kv.NewHLC(), nil, nil, + nil, "", encryptionWriteWiring{}, engine, @@ -111,7 +112,8 @@ func TestBuildShardGroupsWithEtcdEngineRoutesAcrossGroups(t *testing.T) { factory, err := newRaftFactory(raftEngineEtcd, nil) require.NoError(t, err) clock := kv.NewHLC() - runtimes, shardGroups, err := buildShardGroups("n1", baseDir, groups, true, true, raftBootstrapConfig{}, factory, nil, clock, nil, nil, "", encryptionWriteWiring{}, nil) + runtimes, shardGroups, err := buildShardGroups("n1", baseDir, groups, true, true, raftBootstrapConfig{}, + factory, nil, clock, kv.NewActiveTimestampTracker(), nil, nil, "", encryptionWriteWiring{}, nil) require.NoError(t, err) engine := distribution.NewEngine() @@ -165,7 +167,9 @@ func TestBuildShardGroupsWithEtcdEngineRestartsAcrossGroups(t *testing.T) { openShardStore := func(bootstrap bool) ([]*raftGroupRuntime, map[uint64]*kv.ShardGroup, *kv.ShardStore) { factory, err := newRaftFactory(raftEngineEtcd, nil) require.NoError(t, err) - runtimes, shardGroups, err := buildShardGroups("n1", baseDir, groups, true, bootstrap, raftBootstrapConfig{}, factory, nil, sharedClock, nil, nil, "", encryptionWriteWiring{}, nil) + runtimes, shardGroups, err := buildShardGroups("n1", baseDir, groups, true, bootstrap, + raftBootstrapConfig{}, factory, nil, sharedClock, kv.NewActiveTimestampTracker(), nil, nil, "", + encryptionWriteWiring{}, nil) require.NoError(t, err) shardStore := kv.NewShardStore(engine, shardGroups) return runtimes, shardGroups, shardStore diff --git a/proto/admin.pb.go b/proto/admin.pb.go index e6b087967..58165ee74 100644 --- a/proto/admin.pb.go +++ b/proto/admin.pb.go @@ -364,6 +364,9 @@ type RaftGroupState struct { // engine is a follower that has never heard from a leader); UIs should // render that case as "unknown" rather than "contacted at epoch". LastContactUnixMs int64 `protobuf:"varint,6,opt,name=last_contact_unix_ms,json=lastContactUnixMs,proto3" json:"last_contact_unix_ms,omitempty"` + // leader_node_version is populated asynchronously from the leader's Admin + // GetNodeVersion RPC. Empty means unknown/unreachable/cache miss. + LeaderNodeVersion string `protobuf:"bytes,7,opt,name=leader_node_version,json=leaderNodeVersion,proto3" json:"leader_node_version,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -440,6 +443,13 @@ func (x *RaftGroupState) GetLastContactUnixMs() int64 { return 0 } +func (x *RaftGroupState) GetLeaderNodeVersion() string { + if x != nil { + return x.LeaderNodeVersion + } + return "" +} + type GetRaftGroupsRequest struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -1115,6 +1125,634 @@ func (x *GetRouteDetailResponse) GetPerAdapter() []*AdapterSummary { return nil } +type BackupShardApplied struct { + state protoimpl.MessageState `protogen:"open.v1"` + RaftGroupId uint64 `protobuf:"varint,1,opt,name=raft_group_id,json=raftGroupId,proto3" json:"raft_group_id,omitempty"` + AppliedIndex uint64 `protobuf:"varint,2,opt,name=applied_index,json=appliedIndex,proto3" json:"applied_index,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BackupShardApplied) Reset() { + *x = BackupShardApplied{} + mi := &file_admin_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BackupShardApplied) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BackupShardApplied) ProtoMessage() {} + +func (x *BackupShardApplied) ProtoReflect() protoreflect.Message { + mi := &file_admin_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BackupShardApplied.ProtoReflect.Descriptor instead. +func (*BackupShardApplied) Descriptor() ([]byte, []int) { + return file_admin_proto_rawDescGZIP(), []int{15} +} + +func (x *BackupShardApplied) GetRaftGroupId() uint64 { + if x != nil { + return x.RaftGroupId + } + return 0 +} + +func (x *BackupShardApplied) GetAppliedIndex() uint64 { + if x != nil { + return x.AppliedIndex + } + return 0 +} + +type BackupExpectedKeys struct { + state protoimpl.MessageState `protogen:"open.v1"` + Adapter string `protobuf:"bytes,1,opt,name=adapter,proto3" json:"adapter,omitempty"` + Scope string `protobuf:"bytes,2,opt,name=scope,proto3" json:"scope,omitempty"` + KeyCount uint64 `protobuf:"varint,3,opt,name=key_count,json=keyCount,proto3" json:"key_count,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BackupExpectedKeys) Reset() { + *x = BackupExpectedKeys{} + mi := &file_admin_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BackupExpectedKeys) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BackupExpectedKeys) ProtoMessage() {} + +func (x *BackupExpectedKeys) ProtoReflect() protoreflect.Message { + mi := &file_admin_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BackupExpectedKeys.ProtoReflect.Descriptor instead. +func (*BackupExpectedKeys) Descriptor() ([]byte, []int) { + return file_admin_proto_rawDescGZIP(), []int{16} +} + +func (x *BackupExpectedKeys) GetAdapter() string { + if x != nil { + return x.Adapter + } + return "" +} + +func (x *BackupExpectedKeys) GetScope() string { + if x != nil { + return x.Scope + } + return "" +} + +func (x *BackupExpectedKeys) GetKeyCount() uint64 { + if x != nil { + return x.KeyCount + } + return 0 +} + +type BeginBackupRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + TtlMs uint64 `protobuf:"varint,1,opt,name=ttl_ms,json=ttlMs,proto3" json:"ttl_ms,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BeginBackupRequest) Reset() { + *x = BeginBackupRequest{} + mi := &file_admin_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BeginBackupRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BeginBackupRequest) ProtoMessage() {} + +func (x *BeginBackupRequest) ProtoReflect() protoreflect.Message { + mi := &file_admin_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BeginBackupRequest.ProtoReflect.Descriptor instead. +func (*BeginBackupRequest) Descriptor() ([]byte, []int) { + return file_admin_proto_rawDescGZIP(), []int{17} +} + +func (x *BeginBackupRequest) GetTtlMs() uint64 { + if x != nil { + return x.TtlMs + } + return 0 +} + +type BeginBackupResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + ReadTs uint64 `protobuf:"varint,1,opt,name=read_ts,json=readTs,proto3" json:"read_ts,omitempty"` + PinToken []byte `protobuf:"bytes,2,opt,name=pin_token,json=pinToken,proto3" json:"pin_token,omitempty"` + TtlMsEffective uint64 `protobuf:"varint,3,opt,name=ttl_ms_effective,json=ttlMsEffective,proto3" json:"ttl_ms_effective,omitempty"` + Shards []*BackupShardApplied `protobuf:"bytes,4,rep,name=shards,proto3" json:"shards,omitempty"` + ExpectedKeys []*BackupExpectedKeys `protobuf:"bytes,5,rep,name=expected_keys,json=expectedKeys,proto3" json:"expected_keys,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BeginBackupResponse) Reset() { + *x = BeginBackupResponse{} + mi := &file_admin_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BeginBackupResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BeginBackupResponse) ProtoMessage() {} + +func (x *BeginBackupResponse) ProtoReflect() protoreflect.Message { + mi := &file_admin_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BeginBackupResponse.ProtoReflect.Descriptor instead. +func (*BeginBackupResponse) Descriptor() ([]byte, []int) { + return file_admin_proto_rawDescGZIP(), []int{18} +} + +func (x *BeginBackupResponse) GetReadTs() uint64 { + if x != nil { + return x.ReadTs + } + return 0 +} + +func (x *BeginBackupResponse) GetPinToken() []byte { + if x != nil { + return x.PinToken + } + return nil +} + +func (x *BeginBackupResponse) GetTtlMsEffective() uint64 { + if x != nil { + return x.TtlMsEffective + } + return 0 +} + +func (x *BeginBackupResponse) GetShards() []*BackupShardApplied { + if x != nil { + return x.Shards + } + return nil +} + +func (x *BeginBackupResponse) GetExpectedKeys() []*BackupExpectedKeys { + if x != nil { + return x.ExpectedKeys + } + return nil +} + +type RenewBackupRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + PinToken []byte `protobuf:"bytes,1,opt,name=pin_token,json=pinToken,proto3" json:"pin_token,omitempty"` + TtlMs uint64 `protobuf:"varint,2,opt,name=ttl_ms,json=ttlMs,proto3" json:"ttl_ms,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RenewBackupRequest) Reset() { + *x = RenewBackupRequest{} + mi := &file_admin_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RenewBackupRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RenewBackupRequest) ProtoMessage() {} + +func (x *RenewBackupRequest) ProtoReflect() protoreflect.Message { + mi := &file_admin_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RenewBackupRequest.ProtoReflect.Descriptor instead. +func (*RenewBackupRequest) Descriptor() ([]byte, []int) { + return file_admin_proto_rawDescGZIP(), []int{19} +} + +func (x *RenewBackupRequest) GetPinToken() []byte { + if x != nil { + return x.PinToken + } + return nil +} + +func (x *RenewBackupRequest) GetTtlMs() uint64 { + if x != nil { + return x.TtlMs + } + return 0 +} + +type RenewBackupResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + TtlMsEffective uint64 `protobuf:"varint,1,opt,name=ttl_ms_effective,json=ttlMsEffective,proto3" json:"ttl_ms_effective,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RenewBackupResponse) Reset() { + *x = RenewBackupResponse{} + mi := &file_admin_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RenewBackupResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RenewBackupResponse) ProtoMessage() {} + +func (x *RenewBackupResponse) ProtoReflect() protoreflect.Message { + mi := &file_admin_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RenewBackupResponse.ProtoReflect.Descriptor instead. +func (*RenewBackupResponse) Descriptor() ([]byte, []int) { + return file_admin_proto_rawDescGZIP(), []int{20} +} + +func (x *RenewBackupResponse) GetTtlMsEffective() uint64 { + if x != nil { + return x.TtlMsEffective + } + return 0 +} + +type EndBackupRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + PinToken []byte `protobuf:"bytes,1,opt,name=pin_token,json=pinToken,proto3" json:"pin_token,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EndBackupRequest) Reset() { + *x = EndBackupRequest{} + mi := &file_admin_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EndBackupRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EndBackupRequest) ProtoMessage() {} + +func (x *EndBackupRequest) ProtoReflect() protoreflect.Message { + mi := &file_admin_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EndBackupRequest.ProtoReflect.Descriptor instead. +func (*EndBackupRequest) Descriptor() ([]byte, []int) { + return file_admin_proto_rawDescGZIP(), []int{21} +} + +func (x *EndBackupRequest) GetPinToken() []byte { + if x != nil { + return x.PinToken + } + return nil +} + +type EndBackupResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EndBackupResponse) Reset() { + *x = EndBackupResponse{} + mi := &file_admin_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EndBackupResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EndBackupResponse) ProtoMessage() {} + +func (x *EndBackupResponse) ProtoReflect() protoreflect.Message { + mi := &file_admin_proto_msgTypes[22] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EndBackupResponse.ProtoReflect.Descriptor instead. +func (*EndBackupResponse) Descriptor() ([]byte, []int) { + return file_admin_proto_rawDescGZIP(), []int{22} +} + +type ListAdaptersAndScopesRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + PinToken []byte `protobuf:"bytes,1,opt,name=pin_token,json=pinToken,proto3" json:"pin_token,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListAdaptersAndScopesRequest) Reset() { + *x = ListAdaptersAndScopesRequest{} + mi := &file_admin_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListAdaptersAndScopesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListAdaptersAndScopesRequest) ProtoMessage() {} + +func (x *ListAdaptersAndScopesRequest) ProtoReflect() protoreflect.Message { + mi := &file_admin_proto_msgTypes[23] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListAdaptersAndScopesRequest.ProtoReflect.Descriptor instead. +func (*ListAdaptersAndScopesRequest) Descriptor() ([]byte, []int) { + return file_admin_proto_rawDescGZIP(), []int{23} +} + +func (x *ListAdaptersAndScopesRequest) GetPinToken() []byte { + if x != nil { + return x.PinToken + } + return nil +} + +type BackupScope struct { + state protoimpl.MessageState `protogen:"open.v1"` + Adapter string `protobuf:"bytes,1,opt,name=adapter,proto3" json:"adapter,omitempty"` + Scope string `protobuf:"bytes,2,opt,name=scope,proto3" json:"scope,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BackupScope) Reset() { + *x = BackupScope{} + mi := &file_admin_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BackupScope) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BackupScope) ProtoMessage() {} + +func (x *BackupScope) ProtoReflect() protoreflect.Message { + mi := &file_admin_proto_msgTypes[24] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BackupScope.ProtoReflect.Descriptor instead. +func (*BackupScope) Descriptor() ([]byte, []int) { + return file_admin_proto_rawDescGZIP(), []int{24} +} + +func (x *BackupScope) GetAdapter() string { + if x != nil { + return x.Adapter + } + return "" +} + +func (x *BackupScope) GetScope() string { + if x != nil { + return x.Scope + } + return "" +} + +type ListAdaptersAndScopesResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Scopes []*BackupScope `protobuf:"bytes,1,rep,name=scopes,proto3" json:"scopes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListAdaptersAndScopesResponse) Reset() { + *x = ListAdaptersAndScopesResponse{} + mi := &file_admin_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListAdaptersAndScopesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListAdaptersAndScopesResponse) ProtoMessage() {} + +func (x *ListAdaptersAndScopesResponse) ProtoReflect() protoreflect.Message { + mi := &file_admin_proto_msgTypes[25] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListAdaptersAndScopesResponse.ProtoReflect.Descriptor instead. +func (*ListAdaptersAndScopesResponse) Descriptor() ([]byte, []int) { + return file_admin_proto_rawDescGZIP(), []int{25} +} + +func (x *ListAdaptersAndScopesResponse) GetScopes() []*BackupScope { + if x != nil { + return x.Scopes + } + return nil +} + +type GetNodeVersionRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetNodeVersionRequest) Reset() { + *x = GetNodeVersionRequest{} + mi := &file_admin_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetNodeVersionRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetNodeVersionRequest) ProtoMessage() {} + +func (x *GetNodeVersionRequest) ProtoReflect() protoreflect.Message { + mi := &file_admin_proto_msgTypes[26] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetNodeVersionRequest.ProtoReflect.Descriptor instead. +func (*GetNodeVersionRequest) Descriptor() ([]byte, []int) { + return file_admin_proto_rawDescGZIP(), []int{26} +} + +type GetNodeVersionResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + NodeVersion string `protobuf:"bytes,1,opt,name=node_version,json=nodeVersion,proto3" json:"node_version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetNodeVersionResponse) Reset() { + *x = GetNodeVersionResponse{} + mi := &file_admin_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetNodeVersionResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetNodeVersionResponse) ProtoMessage() {} + +func (x *GetNodeVersionResponse) ProtoReflect() protoreflect.Message { + mi := &file_admin_proto_msgTypes[27] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetNodeVersionResponse.ProtoReflect.Descriptor instead. +func (*GetNodeVersionResponse) Descriptor() ([]byte, []int) { + return file_admin_proto_rawDescGZIP(), []int{27} +} + +func (x *GetNodeVersionResponse) GetNodeVersion() string { + if x != nil { + return x.NodeVersion + } + return "" +} + type StreamEventsRequest struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -1123,7 +1761,7 @@ type StreamEventsRequest struct { func (x *StreamEventsRequest) Reset() { *x = StreamEventsRequest{} - mi := &file_admin_proto_msgTypes[15] + mi := &file_admin_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1135,7 +1773,7 @@ func (x *StreamEventsRequest) String() string { func (*StreamEventsRequest) ProtoMessage() {} func (x *StreamEventsRequest) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[15] + mi := &file_admin_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1148,7 +1786,7 @@ func (x *StreamEventsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use StreamEventsRequest.ProtoReflect.Descriptor instead. func (*StreamEventsRequest) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{15} + return file_admin_proto_rawDescGZIP(), []int{28} } type StreamEventsEvent struct { @@ -1164,7 +1802,7 @@ type StreamEventsEvent struct { func (x *StreamEventsEvent) Reset() { *x = StreamEventsEvent{} - mi := &file_admin_proto_msgTypes[16] + mi := &file_admin_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1176,7 +1814,7 @@ func (x *StreamEventsEvent) String() string { func (*StreamEventsEvent) ProtoMessage() {} func (x *StreamEventsEvent) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[16] + mi := &file_admin_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1189,7 +1827,7 @@ func (x *StreamEventsEvent) ProtoReflect() protoreflect.Message { // Deprecated: Use StreamEventsEvent.ProtoReflect.Descriptor instead. func (*StreamEventsEvent) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{16} + return file_admin_proto_rawDescGZIP(), []int{29} } func (x *StreamEventsEvent) GetEvent() isStreamEventsEvent_Event { @@ -1245,7 +1883,7 @@ type RouteTransition struct { func (x *RouteTransition) Reset() { *x = RouteTransition{} - mi := &file_admin_proto_msgTypes[17] + mi := &file_admin_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1257,7 +1895,7 @@ func (x *RouteTransition) String() string { func (*RouteTransition) ProtoMessage() {} func (x *RouteTransition) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[17] + mi := &file_admin_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1270,7 +1908,7 @@ func (x *RouteTransition) ProtoReflect() protoreflect.Message { // Deprecated: Use RouteTransition.ProtoReflect.Descriptor instead. func (*RouteTransition) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{17} + return file_admin_proto_rawDescGZIP(), []int{30} } func (x *RouteTransition) GetParentRouteId() uint64 { @@ -1312,7 +1950,7 @@ type KeyVizColumn struct { func (x *KeyVizColumn) Reset() { *x = KeyVizColumn{} - mi := &file_admin_proto_msgTypes[18] + mi := &file_admin_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1324,7 +1962,7 @@ func (x *KeyVizColumn) String() string { func (*KeyVizColumn) ProtoMessage() {} func (x *KeyVizColumn) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[18] + mi := &file_admin_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1337,7 +1975,7 @@ func (x *KeyVizColumn) ProtoReflect() protoreflect.Message { // Deprecated: Use KeyVizColumn.ProtoReflect.Descriptor instead. func (*KeyVizColumn) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{18} + return file_admin_proto_rawDescGZIP(), []int{31} } func (x *KeyVizColumn) GetColumnUnixMs() int64 { @@ -1383,7 +2021,7 @@ const file_admin_proto_rawDesc = "" + "\fcapabilities\x18\x05 \x03(\v2-.GetClusterOverviewResponse.CapabilitiesEntryR\fcapabilities\x1a?\n" + "\x11CapabilitiesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + - "\x05value\x18\x02 \x01(\bR\x05value:\x028\x01\"\xf4\x01\n" + + "\x05value\x18\x02 \x01(\bR\x05value:\x028\x01\"\xa4\x02\n" + "\x0eRaftGroupState\x12\"\n" + "\rraft_group_id\x18\x01 \x01(\x04R\vraftGroupId\x12$\n" + "\x0eleader_node_id\x18\x02 \x01(\tR\fleaderNodeId\x12\x1f\n" + @@ -1391,7 +2029,8 @@ const file_admin_proto_rawDesc = "" + "leaderTerm\x12!\n" + "\fcommit_index\x18\x04 \x01(\x04R\vcommitIndex\x12#\n" + "\rapplied_index\x18\x05 \x01(\x04R\fappliedIndex\x12/\n" + - "\x14last_contact_unix_ms\x18\x06 \x01(\x03R\x11lastContactUnixMs\"\x16\n" + + "\x14last_contact_unix_ms\x18\x06 \x01(\x03R\x11lastContactUnixMs\x12.\n" + + "\x13leader_node_version\x18\a \x01(\tR\x11leaderNodeVersion\"\x16\n" + "\x14GetRaftGroupsRequest\"@\n" + "\x15GetRaftGroupsResponse\x12'\n" + "\x06groups\x18\x01 \x03(\v2\x0f.RaftGroupStateR\x06groups\"\xfe\x01\n" + @@ -1447,7 +2086,40 @@ const file_admin_proto_rawDesc = "" + "\x03row\x18\x01 \x01(\v2\n" + ".KeyVizRowR\x03row\x120\n" + "\vper_adapter\x18\x02 \x03(\v2\x0f.AdapterSummaryR\n" + - "perAdapter\"\x15\n" + + "perAdapter\"]\n" + + "\x12BackupShardApplied\x12\"\n" + + "\rraft_group_id\x18\x01 \x01(\x04R\vraftGroupId\x12#\n" + + "\rapplied_index\x18\x02 \x01(\x04R\fappliedIndex\"a\n" + + "\x12BackupExpectedKeys\x12\x18\n" + + "\aadapter\x18\x01 \x01(\tR\aadapter\x12\x14\n" + + "\x05scope\x18\x02 \x01(\tR\x05scope\x12\x1b\n" + + "\tkey_count\x18\x03 \x01(\x04R\bkeyCount\"+\n" + + "\x12BeginBackupRequest\x12\x15\n" + + "\x06ttl_ms\x18\x01 \x01(\x04R\x05ttlMs\"\xdc\x01\n" + + "\x13BeginBackupResponse\x12\x17\n" + + "\aread_ts\x18\x01 \x01(\x04R\x06readTs\x12\x1b\n" + + "\tpin_token\x18\x02 \x01(\fR\bpinToken\x12(\n" + + "\x10ttl_ms_effective\x18\x03 \x01(\x04R\x0ettlMsEffective\x12+\n" + + "\x06shards\x18\x04 \x03(\v2\x13.BackupShardAppliedR\x06shards\x128\n" + + "\rexpected_keys\x18\x05 \x03(\v2\x13.BackupExpectedKeysR\fexpectedKeys\"H\n" + + "\x12RenewBackupRequest\x12\x1b\n" + + "\tpin_token\x18\x01 \x01(\fR\bpinToken\x12\x15\n" + + "\x06ttl_ms\x18\x02 \x01(\x04R\x05ttlMs\"?\n" + + "\x13RenewBackupResponse\x12(\n" + + "\x10ttl_ms_effective\x18\x01 \x01(\x04R\x0ettlMsEffective\"/\n" + + "\x10EndBackupRequest\x12\x1b\n" + + "\tpin_token\x18\x01 \x01(\fR\bpinToken\"\x13\n" + + "\x11EndBackupResponse\";\n" + + "\x1cListAdaptersAndScopesRequest\x12\x1b\n" + + "\tpin_token\x18\x01 \x01(\fR\bpinToken\"=\n" + + "\vBackupScope\x12\x18\n" + + "\aadapter\x18\x01 \x01(\tR\aadapter\x12\x14\n" + + "\x05scope\x18\x02 \x01(\tR\x05scope\"E\n" + + "\x1dListAdaptersAndScopesResponse\x12$\n" + + "\x06scopes\x18\x01 \x03(\v2\f.BackupScopeR\x06scopes\"\x17\n" + + "\x15GetNodeVersionRequest\";\n" + + "\x16GetNodeVersionResponse\x12!\n" + + "\fnode_version\x18\x01 \x01(\tR\vnodeVersion\"\x15\n" + "\x13StreamEventsRequest\"\x91\x01\n" + "\x11StreamEventsEvent\x12=\n" + "\x10route_transition\x18\x01 \x01(\v2\x10.RouteTransitionH\x00R\x0frouteTransition\x124\n" + @@ -1475,13 +2147,18 @@ const file_admin_proto_rawDesc = "" + "\x17SAMPLE_ROLE_UNSPECIFIED\x10\x00\x12\x1c\n" + "\x18SAMPLE_ROLE_LEADER_WRITE\x10\x01\x12\x1b\n" + "\x17SAMPLE_ROLE_LEADER_READ\x10\x02\x12\x1d\n" + - "\x19SAMPLE_ROLE_FOLLOWER_READ\x10\x032\xb3\x03\n" + + "\x19SAMPLE_ROLE_FOLLOWER_READ\x10\x032\x80\x06\n" + "\x05Admin\x12O\n" + "\x12GetClusterOverview\x12\x1a.GetClusterOverviewRequest\x1a\x1b.GetClusterOverviewResponse\"\x00\x12@\n" + "\rGetRaftGroups\x12\x15.GetRaftGroupsRequest\x1a\x16.GetRaftGroupsResponse\"\x00\x12L\n" + "\x11GetAdapterSummary\x12\x19.GetAdapterSummaryRequest\x1a\x1a.GetAdapterSummaryResponse\"\x00\x12F\n" + "\x0fGetKeyVizMatrix\x12\x17.GetKeyVizMatrixRequest\x1a\x18.GetKeyVizMatrixResponse\"\x00\x12C\n" + - "\x0eGetRouteDetail\x12\x16.GetRouteDetailRequest\x1a\x17.GetRouteDetailResponse\"\x00\x12<\n" + + "\x0eGetRouteDetail\x12\x16.GetRouteDetailRequest\x1a\x17.GetRouteDetailResponse\"\x00\x12:\n" + + "\vBeginBackup\x12\x13.BeginBackupRequest\x1a\x14.BeginBackupResponse\"\x00\x12:\n" + + "\vRenewBackup\x12\x13.RenewBackupRequest\x1a\x14.RenewBackupResponse\"\x00\x124\n" + + "\tEndBackup\x12\x11.EndBackupRequest\x1a\x12.EndBackupResponse\"\x00\x12X\n" + + "\x15ListAdaptersAndScopes\x12\x1d.ListAdaptersAndScopesRequest\x1a\x1e.ListAdaptersAndScopesResponse\"\x00\x12C\n" + + "\x0eGetNodeVersion\x12\x16.GetNodeVersionRequest\x1a\x17.GetNodeVersionResponse\"\x00\x12<\n" + "\fStreamEvents\x12\x14.StreamEventsRequest\x1a\x12.StreamEventsEvent\"\x000\x01B#Z!github.com/bootjp/elastickv/protob\x06proto3" var ( @@ -1497,36 +2174,49 @@ func file_admin_proto_rawDescGZIP() []byte { } var file_admin_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_admin_proto_msgTypes = make([]protoimpl.MessageInfo, 20) +var file_admin_proto_msgTypes = make([]protoimpl.MessageInfo, 33) var file_admin_proto_goTypes = []any{ - (KeyVizSeries)(0), // 0: KeyVizSeries - (SampleRole)(0), // 1: SampleRole - (*NodeIdentity)(nil), // 2: NodeIdentity - (*GroupLeader)(nil), // 3: GroupLeader - (*GetClusterOverviewRequest)(nil), // 4: GetClusterOverviewRequest - (*GetClusterOverviewResponse)(nil), // 5: GetClusterOverviewResponse - (*RaftGroupState)(nil), // 6: RaftGroupState - (*GetRaftGroupsRequest)(nil), // 7: GetRaftGroupsRequest - (*GetRaftGroupsResponse)(nil), // 8: GetRaftGroupsResponse - (*AdapterSummary)(nil), // 9: AdapterSummary - (*GetAdapterSummaryRequest)(nil), // 10: GetAdapterSummaryRequest - (*GetAdapterSummaryResponse)(nil), // 11: GetAdapterSummaryResponse - (*KeyVizRow)(nil), // 12: KeyVizRow - (*GetKeyVizMatrixRequest)(nil), // 13: GetKeyVizMatrixRequest - (*GetKeyVizMatrixResponse)(nil), // 14: GetKeyVizMatrixResponse - (*GetRouteDetailRequest)(nil), // 15: GetRouteDetailRequest - (*GetRouteDetailResponse)(nil), // 16: GetRouteDetailResponse - (*StreamEventsRequest)(nil), // 17: StreamEventsRequest - (*StreamEventsEvent)(nil), // 18: StreamEventsEvent - (*RouteTransition)(nil), // 19: RouteTransition - (*KeyVizColumn)(nil), // 20: KeyVizColumn - nil, // 21: GetClusterOverviewResponse.CapabilitiesEntry + (KeyVizSeries)(0), // 0: KeyVizSeries + (SampleRole)(0), // 1: SampleRole + (*NodeIdentity)(nil), // 2: NodeIdentity + (*GroupLeader)(nil), // 3: GroupLeader + (*GetClusterOverviewRequest)(nil), // 4: GetClusterOverviewRequest + (*GetClusterOverviewResponse)(nil), // 5: GetClusterOverviewResponse + (*RaftGroupState)(nil), // 6: RaftGroupState + (*GetRaftGroupsRequest)(nil), // 7: GetRaftGroupsRequest + (*GetRaftGroupsResponse)(nil), // 8: GetRaftGroupsResponse + (*AdapterSummary)(nil), // 9: AdapterSummary + (*GetAdapterSummaryRequest)(nil), // 10: GetAdapterSummaryRequest + (*GetAdapterSummaryResponse)(nil), // 11: GetAdapterSummaryResponse + (*KeyVizRow)(nil), // 12: KeyVizRow + (*GetKeyVizMatrixRequest)(nil), // 13: GetKeyVizMatrixRequest + (*GetKeyVizMatrixResponse)(nil), // 14: GetKeyVizMatrixResponse + (*GetRouteDetailRequest)(nil), // 15: GetRouteDetailRequest + (*GetRouteDetailResponse)(nil), // 16: GetRouteDetailResponse + (*BackupShardApplied)(nil), // 17: BackupShardApplied + (*BackupExpectedKeys)(nil), // 18: BackupExpectedKeys + (*BeginBackupRequest)(nil), // 19: BeginBackupRequest + (*BeginBackupResponse)(nil), // 20: BeginBackupResponse + (*RenewBackupRequest)(nil), // 21: RenewBackupRequest + (*RenewBackupResponse)(nil), // 22: RenewBackupResponse + (*EndBackupRequest)(nil), // 23: EndBackupRequest + (*EndBackupResponse)(nil), // 24: EndBackupResponse + (*ListAdaptersAndScopesRequest)(nil), // 25: ListAdaptersAndScopesRequest + (*BackupScope)(nil), // 26: BackupScope + (*ListAdaptersAndScopesResponse)(nil), // 27: ListAdaptersAndScopesResponse + (*GetNodeVersionRequest)(nil), // 28: GetNodeVersionRequest + (*GetNodeVersionResponse)(nil), // 29: GetNodeVersionResponse + (*StreamEventsRequest)(nil), // 30: StreamEventsRequest + (*StreamEventsEvent)(nil), // 31: StreamEventsEvent + (*RouteTransition)(nil), // 32: RouteTransition + (*KeyVizColumn)(nil), // 33: KeyVizColumn + nil, // 34: GetClusterOverviewResponse.CapabilitiesEntry } var file_admin_proto_depIdxs = []int32{ 2, // 0: GetClusterOverviewResponse.self:type_name -> NodeIdentity 2, // 1: GetClusterOverviewResponse.members:type_name -> NodeIdentity 3, // 2: GetClusterOverviewResponse.group_leaders:type_name -> GroupLeader - 21, // 3: GetClusterOverviewResponse.capabilities:type_name -> GetClusterOverviewResponse.CapabilitiesEntry + 34, // 3: GetClusterOverviewResponse.capabilities:type_name -> GetClusterOverviewResponse.CapabilitiesEntry 6, // 4: GetRaftGroupsResponse.groups:type_name -> RaftGroupState 9, // 5: GetAdapterSummaryResponse.summaries:type_name -> AdapterSummary 1, // 6: KeyVizRow.sample_roles:type_name -> SampleRole @@ -1534,27 +2224,40 @@ var file_admin_proto_depIdxs = []int32{ 12, // 8: GetKeyVizMatrixResponse.rows:type_name -> KeyVizRow 12, // 9: GetRouteDetailResponse.row:type_name -> KeyVizRow 9, // 10: GetRouteDetailResponse.per_adapter:type_name -> AdapterSummary - 19, // 11: StreamEventsEvent.route_transition:type_name -> RouteTransition - 20, // 12: StreamEventsEvent.keyviz_column:type_name -> KeyVizColumn - 0, // 13: KeyVizColumn.series:type_name -> KeyVizSeries - 12, // 14: KeyVizColumn.rows:type_name -> KeyVizRow - 4, // 15: Admin.GetClusterOverview:input_type -> GetClusterOverviewRequest - 7, // 16: Admin.GetRaftGroups:input_type -> GetRaftGroupsRequest - 10, // 17: Admin.GetAdapterSummary:input_type -> GetAdapterSummaryRequest - 13, // 18: Admin.GetKeyVizMatrix:input_type -> GetKeyVizMatrixRequest - 15, // 19: Admin.GetRouteDetail:input_type -> GetRouteDetailRequest - 17, // 20: Admin.StreamEvents:input_type -> StreamEventsRequest - 5, // 21: Admin.GetClusterOverview:output_type -> GetClusterOverviewResponse - 8, // 22: Admin.GetRaftGroups:output_type -> GetRaftGroupsResponse - 11, // 23: Admin.GetAdapterSummary:output_type -> GetAdapterSummaryResponse - 14, // 24: Admin.GetKeyVizMatrix:output_type -> GetKeyVizMatrixResponse - 16, // 25: Admin.GetRouteDetail:output_type -> GetRouteDetailResponse - 18, // 26: Admin.StreamEvents:output_type -> StreamEventsEvent - 21, // [21:27] is the sub-list for method output_type - 15, // [15:21] is the sub-list for method input_type - 15, // [15:15] is the sub-list for extension type_name - 15, // [15:15] is the sub-list for extension extendee - 0, // [0:15] is the sub-list for field type_name + 17, // 11: BeginBackupResponse.shards:type_name -> BackupShardApplied + 18, // 12: BeginBackupResponse.expected_keys:type_name -> BackupExpectedKeys + 26, // 13: ListAdaptersAndScopesResponse.scopes:type_name -> BackupScope + 32, // 14: StreamEventsEvent.route_transition:type_name -> RouteTransition + 33, // 15: StreamEventsEvent.keyviz_column:type_name -> KeyVizColumn + 0, // 16: KeyVizColumn.series:type_name -> KeyVizSeries + 12, // 17: KeyVizColumn.rows:type_name -> KeyVizRow + 4, // 18: Admin.GetClusterOverview:input_type -> GetClusterOverviewRequest + 7, // 19: Admin.GetRaftGroups:input_type -> GetRaftGroupsRequest + 10, // 20: Admin.GetAdapterSummary:input_type -> GetAdapterSummaryRequest + 13, // 21: Admin.GetKeyVizMatrix:input_type -> GetKeyVizMatrixRequest + 15, // 22: Admin.GetRouteDetail:input_type -> GetRouteDetailRequest + 19, // 23: Admin.BeginBackup:input_type -> BeginBackupRequest + 21, // 24: Admin.RenewBackup:input_type -> RenewBackupRequest + 23, // 25: Admin.EndBackup:input_type -> EndBackupRequest + 25, // 26: Admin.ListAdaptersAndScopes:input_type -> ListAdaptersAndScopesRequest + 28, // 27: Admin.GetNodeVersion:input_type -> GetNodeVersionRequest + 30, // 28: Admin.StreamEvents:input_type -> StreamEventsRequest + 5, // 29: Admin.GetClusterOverview:output_type -> GetClusterOverviewResponse + 8, // 30: Admin.GetRaftGroups:output_type -> GetRaftGroupsResponse + 11, // 31: Admin.GetAdapterSummary:output_type -> GetAdapterSummaryResponse + 14, // 32: Admin.GetKeyVizMatrix:output_type -> GetKeyVizMatrixResponse + 16, // 33: Admin.GetRouteDetail:output_type -> GetRouteDetailResponse + 20, // 34: Admin.BeginBackup:output_type -> BeginBackupResponse + 22, // 35: Admin.RenewBackup:output_type -> RenewBackupResponse + 24, // 36: Admin.EndBackup:output_type -> EndBackupResponse + 27, // 37: Admin.ListAdaptersAndScopes:output_type -> ListAdaptersAndScopesResponse + 29, // 38: Admin.GetNodeVersion:output_type -> GetNodeVersionResponse + 31, // 39: Admin.StreamEvents:output_type -> StreamEventsEvent + 29, // [29:40] is the sub-list for method output_type + 18, // [18:29] is the sub-list for method input_type + 18, // [18:18] is the sub-list for extension type_name + 18, // [18:18] is the sub-list for extension extendee + 0, // [0:18] is the sub-list for field type_name } func init() { file_admin_proto_init() } @@ -1562,7 +2265,7 @@ func file_admin_proto_init() { if File_admin_proto != nil { return } - file_admin_proto_msgTypes[16].OneofWrappers = []any{ + file_admin_proto_msgTypes[29].OneofWrappers = []any{ (*StreamEventsEvent_RouteTransition)(nil), (*StreamEventsEvent_KeyvizColumn)(nil), } @@ -1572,7 +2275,7 @@ func file_admin_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_admin_proto_rawDesc), len(file_admin_proto_rawDesc)), NumEnums: 2, - NumMessages: 20, + NumMessages: 33, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/admin.proto b/proto/admin.proto index 96d111020..5d9fefa77 100644 --- a/proto/admin.proto +++ b/proto/admin.proto @@ -12,6 +12,11 @@ service Admin { rpc GetAdapterSummary (GetAdapterSummaryRequest) returns (GetAdapterSummaryResponse) {} rpc GetKeyVizMatrix (GetKeyVizMatrixRequest) returns (GetKeyVizMatrixResponse) {} rpc GetRouteDetail (GetRouteDetailRequest) returns (GetRouteDetailResponse) {} + rpc BeginBackup (BeginBackupRequest) returns (BeginBackupResponse) {} + rpc RenewBackup (RenewBackupRequest) returns (RenewBackupResponse) {} + rpc EndBackup (EndBackupRequest) returns (EndBackupResponse) {} + rpc ListAdaptersAndScopes (ListAdaptersAndScopesRequest) returns (ListAdaptersAndScopesResponse) {} + rpc GetNodeVersion (GetNodeVersionRequest) returns (GetNodeVersionResponse) {} rpc StreamEvents (StreamEventsRequest) returns (stream StreamEventsEvent) {} } @@ -47,6 +52,9 @@ message RaftGroupState { // engine is a follower that has never heard from a leader); UIs should // render that case as "unknown" rather than "contacted at epoch". int64 last_contact_unix_ms = 6; + // leader_node_version is populated asynchronously from the leader's Admin + // GetNodeVersion RPC. Empty means unknown/unreachable/cache miss. + string leader_node_version = 7; } message GetRaftGroupsRequest {} @@ -155,6 +163,63 @@ message GetRouteDetailResponse { repeated AdapterSummary per_adapter = 2; } +message BackupShardApplied { + uint64 raft_group_id = 1; + uint64 applied_index = 2; +} + +message BackupExpectedKeys { + string adapter = 1; + string scope = 2; + uint64 key_count = 3; +} + +message BeginBackupRequest { + uint64 ttl_ms = 1; +} + +message BeginBackupResponse { + uint64 read_ts = 1; + bytes pin_token = 2; + uint64 ttl_ms_effective = 3; + repeated BackupShardApplied shards = 4; + repeated BackupExpectedKeys expected_keys = 5; +} + +message RenewBackupRequest { + bytes pin_token = 1; + uint64 ttl_ms = 2; +} + +message RenewBackupResponse { + uint64 ttl_ms_effective = 1; +} + +message EndBackupRequest { + bytes pin_token = 1; +} + +message EndBackupResponse {} + +message ListAdaptersAndScopesRequest { + bytes pin_token = 1; +} + +message BackupScope { + string adapter = 1; + string scope = 2; +} + +message ListAdaptersAndScopesResponse { + repeated BackupScope scopes = 1; +} + +message GetNodeVersionRequest {} + +message GetNodeVersionResponse { + string node_version = 1; +} + message StreamEventsRequest {} message StreamEventsEvent { diff --git a/proto/admin_grpc.pb.go b/proto/admin_grpc.pb.go index 021b1e834..06829837d 100644 --- a/proto/admin_grpc.pb.go +++ b/proto/admin_grpc.pb.go @@ -19,12 +19,17 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - Admin_GetClusterOverview_FullMethodName = "/Admin/GetClusterOverview" - Admin_GetRaftGroups_FullMethodName = "/Admin/GetRaftGroups" - Admin_GetAdapterSummary_FullMethodName = "/Admin/GetAdapterSummary" - Admin_GetKeyVizMatrix_FullMethodName = "/Admin/GetKeyVizMatrix" - Admin_GetRouteDetail_FullMethodName = "/Admin/GetRouteDetail" - Admin_StreamEvents_FullMethodName = "/Admin/StreamEvents" + Admin_GetClusterOverview_FullMethodName = "/Admin/GetClusterOverview" + Admin_GetRaftGroups_FullMethodName = "/Admin/GetRaftGroups" + Admin_GetAdapterSummary_FullMethodName = "/Admin/GetAdapterSummary" + Admin_GetKeyVizMatrix_FullMethodName = "/Admin/GetKeyVizMatrix" + Admin_GetRouteDetail_FullMethodName = "/Admin/GetRouteDetail" + Admin_BeginBackup_FullMethodName = "/Admin/BeginBackup" + Admin_RenewBackup_FullMethodName = "/Admin/RenewBackup" + Admin_EndBackup_FullMethodName = "/Admin/EndBackup" + Admin_ListAdaptersAndScopes_FullMethodName = "/Admin/ListAdaptersAndScopes" + Admin_GetNodeVersion_FullMethodName = "/Admin/GetNodeVersion" + Admin_StreamEvents_FullMethodName = "/Admin/StreamEvents" ) // AdminClient is the client API for Admin service. @@ -41,6 +46,11 @@ type AdminClient interface { GetAdapterSummary(ctx context.Context, in *GetAdapterSummaryRequest, opts ...grpc.CallOption) (*GetAdapterSummaryResponse, error) GetKeyVizMatrix(ctx context.Context, in *GetKeyVizMatrixRequest, opts ...grpc.CallOption) (*GetKeyVizMatrixResponse, error) GetRouteDetail(ctx context.Context, in *GetRouteDetailRequest, opts ...grpc.CallOption) (*GetRouteDetailResponse, error) + BeginBackup(ctx context.Context, in *BeginBackupRequest, opts ...grpc.CallOption) (*BeginBackupResponse, error) + RenewBackup(ctx context.Context, in *RenewBackupRequest, opts ...grpc.CallOption) (*RenewBackupResponse, error) + EndBackup(ctx context.Context, in *EndBackupRequest, opts ...grpc.CallOption) (*EndBackupResponse, error) + ListAdaptersAndScopes(ctx context.Context, in *ListAdaptersAndScopesRequest, opts ...grpc.CallOption) (*ListAdaptersAndScopesResponse, error) + GetNodeVersion(ctx context.Context, in *GetNodeVersionRequest, opts ...grpc.CallOption) (*GetNodeVersionResponse, error) StreamEvents(ctx context.Context, in *StreamEventsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[StreamEventsEvent], error) } @@ -102,6 +112,56 @@ func (c *adminClient) GetRouteDetail(ctx context.Context, in *GetRouteDetailRequ return out, nil } +func (c *adminClient) BeginBackup(ctx context.Context, in *BeginBackupRequest, opts ...grpc.CallOption) (*BeginBackupResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(BeginBackupResponse) + err := c.cc.Invoke(ctx, Admin_BeginBackup_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *adminClient) RenewBackup(ctx context.Context, in *RenewBackupRequest, opts ...grpc.CallOption) (*RenewBackupResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(RenewBackupResponse) + err := c.cc.Invoke(ctx, Admin_RenewBackup_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *adminClient) EndBackup(ctx context.Context, in *EndBackupRequest, opts ...grpc.CallOption) (*EndBackupResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(EndBackupResponse) + err := c.cc.Invoke(ctx, Admin_EndBackup_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *adminClient) ListAdaptersAndScopes(ctx context.Context, in *ListAdaptersAndScopesRequest, opts ...grpc.CallOption) (*ListAdaptersAndScopesResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListAdaptersAndScopesResponse) + err := c.cc.Invoke(ctx, Admin_ListAdaptersAndScopes_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *adminClient) GetNodeVersion(ctx context.Context, in *GetNodeVersionRequest, opts ...grpc.CallOption) (*GetNodeVersionResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetNodeVersionResponse) + err := c.cc.Invoke(ctx, Admin_GetNodeVersion_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *adminClient) StreamEvents(ctx context.Context, in *StreamEventsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[StreamEventsEvent], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &Admin_ServiceDesc.Streams[0], Admin_StreamEvents_FullMethodName, cOpts...) @@ -135,6 +195,11 @@ type AdminServer interface { GetAdapterSummary(context.Context, *GetAdapterSummaryRequest) (*GetAdapterSummaryResponse, error) GetKeyVizMatrix(context.Context, *GetKeyVizMatrixRequest) (*GetKeyVizMatrixResponse, error) GetRouteDetail(context.Context, *GetRouteDetailRequest) (*GetRouteDetailResponse, error) + BeginBackup(context.Context, *BeginBackupRequest) (*BeginBackupResponse, error) + RenewBackup(context.Context, *RenewBackupRequest) (*RenewBackupResponse, error) + EndBackup(context.Context, *EndBackupRequest) (*EndBackupResponse, error) + ListAdaptersAndScopes(context.Context, *ListAdaptersAndScopesRequest) (*ListAdaptersAndScopesResponse, error) + GetNodeVersion(context.Context, *GetNodeVersionRequest) (*GetNodeVersionResponse, error) StreamEvents(*StreamEventsRequest, grpc.ServerStreamingServer[StreamEventsEvent]) error mustEmbedUnimplementedAdminServer() } @@ -161,6 +226,21 @@ func (UnimplementedAdminServer) GetKeyVizMatrix(context.Context, *GetKeyVizMatri func (UnimplementedAdminServer) GetRouteDetail(context.Context, *GetRouteDetailRequest) (*GetRouteDetailResponse, error) { return nil, status.Error(codes.Unimplemented, "method GetRouteDetail not implemented") } +func (UnimplementedAdminServer) BeginBackup(context.Context, *BeginBackupRequest) (*BeginBackupResponse, error) { + return nil, status.Error(codes.Unimplemented, "method BeginBackup not implemented") +} +func (UnimplementedAdminServer) RenewBackup(context.Context, *RenewBackupRequest) (*RenewBackupResponse, error) { + return nil, status.Error(codes.Unimplemented, "method RenewBackup not implemented") +} +func (UnimplementedAdminServer) EndBackup(context.Context, *EndBackupRequest) (*EndBackupResponse, error) { + return nil, status.Error(codes.Unimplemented, "method EndBackup not implemented") +} +func (UnimplementedAdminServer) ListAdaptersAndScopes(context.Context, *ListAdaptersAndScopesRequest) (*ListAdaptersAndScopesResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ListAdaptersAndScopes not implemented") +} +func (UnimplementedAdminServer) GetNodeVersion(context.Context, *GetNodeVersionRequest) (*GetNodeVersionResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetNodeVersion not implemented") +} func (UnimplementedAdminServer) StreamEvents(*StreamEventsRequest, grpc.ServerStreamingServer[StreamEventsEvent]) error { return status.Error(codes.Unimplemented, "method StreamEvents not implemented") } @@ -275,6 +355,96 @@ func _Admin_GetRouteDetail_Handler(srv interface{}, ctx context.Context, dec fun return interceptor(ctx, in, info, handler) } +func _Admin_BeginBackup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(BeginBackupRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AdminServer).BeginBackup(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Admin_BeginBackup_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AdminServer).BeginBackup(ctx, req.(*BeginBackupRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Admin_RenewBackup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RenewBackupRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AdminServer).RenewBackup(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Admin_RenewBackup_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AdminServer).RenewBackup(ctx, req.(*RenewBackupRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Admin_EndBackup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(EndBackupRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AdminServer).EndBackup(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Admin_EndBackup_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AdminServer).EndBackup(ctx, req.(*EndBackupRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Admin_ListAdaptersAndScopes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListAdaptersAndScopesRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AdminServer).ListAdaptersAndScopes(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Admin_ListAdaptersAndScopes_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AdminServer).ListAdaptersAndScopes(ctx, req.(*ListAdaptersAndScopesRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Admin_GetNodeVersion_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetNodeVersionRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AdminServer).GetNodeVersion(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Admin_GetNodeVersion_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AdminServer).GetNodeVersion(ctx, req.(*GetNodeVersionRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Admin_StreamEvents_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(StreamEventsRequest) if err := stream.RecvMsg(m); err != nil { @@ -313,6 +483,26 @@ var Admin_ServiceDesc = grpc.ServiceDesc{ MethodName: "GetRouteDetail", Handler: _Admin_GetRouteDetail_Handler, }, + { + MethodName: "BeginBackup", + Handler: _Admin_BeginBackup_Handler, + }, + { + MethodName: "RenewBackup", + Handler: _Admin_RenewBackup_Handler, + }, + { + MethodName: "EndBackup", + Handler: _Admin_EndBackup_Handler, + }, + { + MethodName: "ListAdaptersAndScopes", + Handler: _Admin_ListAdaptersAndScopes_Handler, + }, + { + MethodName: "GetNodeVersion", + Handler: _Admin_GetNodeVersion_Handler, + }, }, Streams: []grpc.StreamDesc{ { From 6a85349d2deec210799349b420d35c9563666e55 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 20:08:29 +0900 Subject: [PATCH 2/9] backup: add live logical backup control plane --- adapter/admin_backup.go | 1151 +++++++++++++++++ adapter/admin_backup_test.go | 688 ++++++++++ adapter/admin_grpc.go | 21 +- adapter/internal.go | 43 + adapter/internal_admin_proposal_test.go | 97 ++ .../2026_04_29_proposed_logical_backup.md | 12 +- internal/backup/live.go | 253 ++++ internal/backup/live_test.go | 71 + internal/backup/s3.go | 17 +- internal/backup/sqs.go | 20 +- kv/active_timestamp_tracker.go | 20 +- kv/active_timestamp_tracker_test.go | 4 +- kv/backup_codec.go | 58 +- kv/backup_codec_test.go | 23 +- kv/backup_scan.go | 210 ++- kv/compactor_test.go | 32 + kv/fsm_backup.go | 43 +- kv/fsm_backup_test.go | 52 +- kv/leader_admin_proposer.go | 135 ++ kv/leader_admin_proposer_test.go | 109 ++ kv/shard_store_test.go | 129 +- main.go | 200 ++- main_admin_test.go | 64 + proto/admin.pb.go | 311 +++-- proto/admin.proto | 22 +- proto/admin_grpc.pb.go | 47 +- proto/internal.pb.go | 165 ++- proto/internal.proto | 9 + proto/internal_grpc.pb.go | 42 +- 29 files changed, 3803 insertions(+), 245 deletions(-) create mode 100644 adapter/admin_backup.go create mode 100644 adapter/admin_backup_test.go create mode 100644 adapter/internal_admin_proposal_test.go create mode 100644 internal/backup/live.go create mode 100644 internal/backup/live_test.go create mode 100644 kv/leader_admin_proposer.go create mode 100644 kv/leader_admin_proposer_test.go diff --git a/adapter/admin_backup.go b/adapter/admin_backup.go new file mode 100644 index 000000000..51b24d72e --- /dev/null +++ b/adapter/admin_backup.go @@ -0,0 +1,1151 @@ +package adapter + +import ( + "context" + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "fmt" + "log/slog" + "math" + "sort" + "time" + + logicalbackup "github.com/bootjp/elastickv/internal/backup" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +const ( + backupProtocolVersionV1 uint32 = 1 + backupTokenVersion byte = 2 + backupTokenHeaderLen = 1 + 16 + 8 + 8 + 4 + backupTokenMACLen = sha256.Size + maxBackupTokenGroups = 4096 + defaultLiveBackupTTL = 30 * time.Minute + defaultLiveBackupMinTTL = time.Minute + defaultLiveBackupMaxTTL = time.Hour + defaultLiveBackupBeginDeadline = 5 * time.Second + defaultLiveBackupHeadroom = 1000 + defaultLiveBackupScanPageSize = 1024 + defaultLiveBackupRenewAttempts = 3 + defaultLiveBackupRenewBackoff = 500 * time.Millisecond +) + +var ( + ErrBackupUnavailable = errors.New("backup control plane is unavailable") + ErrBackupToken = errors.New("backup pin token is invalid") +) + +type BackupReadFence func(context.Context) (uint64, error) + +type BackupPeerVersion struct { + NodeVersion string + BackupProtocolVersion uint32 +} + +type BackupPeerProbe func(context.Context, string) (BackupPeerVersion, error) + +type BackupStore interface { + CaptureBackupRouteSnapshot(start, end []byte) kv.BackupRouteSnapshot + NewBackupKeyScannerAtSnapshot(snapshot kv.BackupRouteSnapshot, ts uint64, pageSize int) kv.BackupKeyScanner + NewBackupScannerAtSnapshot(snapshot kv.BackupRouteSnapshot, ts uint64, pageSize int) kv.BackupScanner +} + +type BackupPinLimiter interface { + PinWithDeadline(pinID kv.BackupPinID, readTS uint64, deadline time.Time) error + ReleaseBackupPin(pinID kv.BackupPinID) +} + +type AdminBackupConfig struct { + DefaultTTL time.Duration + MinTTL time.Duration + MaxTTL time.Duration + BeginDeadline time.Duration + SnapshotHeadroomEntries uint64 + ScanPageSize int + RenewAttempts int + RenewBackoff time.Duration +} + +type backupConfig struct { + defaultTTL time.Duration + minTTL time.Duration + maxTTL time.Duration + beginDeadline time.Duration + snapshotHeadroomEntries uint64 + scanPageSize int + renewAttempts int + renewBackoff time.Duration +} + +func defaultBackupConfig() backupConfig { + return backupConfig{ + defaultTTL: defaultLiveBackupTTL, + minTTL: defaultLiveBackupMinTTL, + maxTTL: defaultLiveBackupMaxTTL, + beginDeadline: defaultLiveBackupBeginDeadline, + snapshotHeadroomEntries: defaultLiveBackupHeadroom, + scanPageSize: defaultLiveBackupScanPageSize, + renewAttempts: defaultLiveBackupRenewAttempts, + renewBackoff: defaultLiveBackupRenewBackoff, + } +} + +func WithAdminBackupControl( + store BackupStore, + readFence BackupReadFence, + peerProbe BackupPeerProbe, + limiter BackupPinLimiter, + tokenKey []byte, +) AdminOption { + return func(s *AdminServer) { + if store == nil || readFence == nil || peerProbe == nil || limiter == nil || len(tokenKey) == 0 { + return + } + s.backupStore = store + s.backupReadFence = readFence + s.backupPeerProbe = peerProbe + s.backupLimiter = limiter + s.backupTokenKey = sha256.Sum256(tokenKey) + s.backupProtocolVersion = backupProtocolVersionV1 + } +} + +func WithAdminBackupConfig(cfg AdminBackupConfig) AdminOption { + return func(s *AdminServer) { + if cfg.DefaultTTL > 0 { + s.backupConfig.defaultTTL = cfg.DefaultTTL + } + if cfg.MinTTL > 0 { + s.backupConfig.minTTL = cfg.MinTTL + } + if cfg.MaxTTL > 0 { + s.backupConfig.maxTTL = cfg.MaxTTL + } + if cfg.BeginDeadline > 0 { + s.backupConfig.beginDeadline = cfg.BeginDeadline + } + if cfg.SnapshotHeadroomEntries > 0 { + s.backupConfig.snapshotHeadroomEntries = cfg.SnapshotHeadroomEntries + } + if cfg.ScanPageSize > 0 { + s.backupConfig.scanPageSize = cfg.ScanPageSize + } + if cfg.RenewAttempts > 0 { + s.backupConfig.renewAttempts = cfg.RenewAttempts + } + if cfg.RenewBackoff > 0 { + s.backupConfig.renewBackoff = cfg.RenewBackoff + } + } +} + +// RegisterBackupProposer binds the wrap-aware proposer for one group. The +// ordinary AdminGroup remains read-only so existing status-only fakes do not +// gain mutation methods. +func (s *AdminServer) RegisterBackupProposer(groupID uint64, proposer raftengine.Proposer) { + if s == nil || proposer == nil { + return + } + s.groupsMu.Lock() + s.backupProposers[groupID] = proposer + s.groupsMu.Unlock() +} + +type backupGroup struct { + id uint64 + reader AdminGroup + status raftengine.Status + every uint64 + proposer raftengine.Proposer +} + +type backupToken struct { + pinID kv.BackupPinID + readTS uint64 + deadline time.Time + groupIDs []uint64 +} + +type backupSession struct { + routes kv.BackupRouteSnapshot + readTS uint64 + deadline time.Time +} + +type preparedBackup struct { + groups []backupGroup + commits map[uint64]uint64 + controlGroup backupGroup + pinID kv.BackupPinID + readTS uint64 + ttl time.Duration + routes kv.BackupRouteSnapshot +} + +func (s *AdminServer) BeginBackup(ctx context.Context, req *pb.BeginBackupRequest) (*pb.BeginBackupResponse, error) { + if err := s.requireBackupControl(); err != nil { + return nil, err + } + ttl, err := s.effectiveBackupTTL(req.GetTtlMs()) + if err != nil { + return nil, err + } + + // Serialize BeginBackup on one admin endpoint so local preflight capacity + // and proposal compensation cannot interleave. + s.backupMu.Lock() + defer s.backupMu.Unlock() + + beginCtx, cancel := context.WithTimeout(ctx, s.backupConfig.beginDeadline) + defer cancel() + prepared, err := s.prepareBackup(beginCtx, ttl) + if err != nil { + return nil, err + } + counts, appliedAtCount, err := s.buildExpectedBackupBaseline(ctx, prepared) + if err != nil { + s.compensateBackupRelease(prepared.controlGroup, prepared.groups, prepared.pinID) + return nil, err + } + deadline, err := s.renewBackupGroups(ctx, prepared.groups, prepared.pinID, prepared.readTS, prepared.ttl) + if err != nil { + s.compensateBackupRelease(prepared.controlGroup, prepared.groups, prepared.pinID) + return nil, status.Errorf(codes.Unavailable, "refresh backup pin after baseline: %v", err) + } + tok := backupToken{ + pinID: prepared.pinID, readTS: prepared.readTS, deadline: deadline, + groupIDs: backupGroupIDs(prepared.groups), + } + encodedToken, err := s.encodeBackupToken(tok) + if err != nil { + s.compensateBackupRelease(prepared.controlGroup, prepared.groups, prepared.pinID) + return nil, status.Errorf(codes.Internal, "encode backup token: %v", err) + } + s.rememberBackupSession(tok, prepared.routes) + + return &pb.BeginBackupResponse{ + ReadTs: prepared.readTS, + PinToken: encodedToken, + TtlMsEffective: uint64(prepared.ttl / time.Millisecond), //nolint:gosec // validated positive. + Shards: backupShardResponses(prepared.groups, prepared.commits), + ExpectedKeys: backupExpectedResponses(counts, appliedAtCount), + }, nil +} + +func (s *AdminServer) prepareBackup(ctx context.Context, ttl time.Duration) (preparedBackup, error) { + if err := s.gateBackupPeerVersions(ctx); err != nil { + return preparedBackup{}, err + } + groups, err := s.snapshotBackupGroups() + if err != nil { + return preparedBackup{}, err + } + if err := s.checkBackupSnapshotHeadroom(groups); err != nil { + return preparedBackup{}, err + } + readTS, err := s.backupReadFence(ctx) + if err != nil { + return preparedBackup{}, status.Errorf(codes.FailedPrecondition, "backup read fence failed: %v", err) + } + if readTS == 0 || readTS == ^uint64(0) { + return preparedBackup{}, status.Errorf(codes.FailedPrecondition, "%s", "backup read fence returned an invalid timestamp") + } + routes := s.backupStore.CaptureBackupRouteSnapshot(nil, nil) + pinID, err := newBackupPinID() + if err != nil { + return preparedBackup{}, status.Errorf(codes.Internal, "generate backup pin id: %v", err) + } + deadline := s.nowSnapshot().Add(ttl) + controlGroup := groups[0] + commits, err := s.pinBackupGroups(ctx, groups, controlGroup, pinID, readTS, deadline) + if err != nil { + return preparedBackup{}, err + } + return preparedBackup{ + groups: groups, commits: commits, controlGroup: controlGroup, + pinID: pinID, readTS: readTS, ttl: ttl, routes: routes, + }, nil +} + +func (s *AdminServer) pinBackupGroups( + ctx context.Context, + groups []backupGroup, + controlGroup backupGroup, + pinID kv.BackupPinID, + readTS uint64, + deadline time.Time, +) (map[uint64]uint64, error) { + reserveEntry := kv.EncodeBackupReserveEntry(kv.BackupReserveEntry{PinID: pinID, ReadTS: readTS, Deadline: deadline}) + if _, _, err := proposeBackupAll(ctx, []backupGroup{controlGroup}, reserveEntry); err != nil { + if errors.Is(err, kv.ErrTooManyActiveBackups) { + return nil, status.Errorf(codes.ResourceExhausted, "%s", kv.ErrTooManyActiveBackups) + } + return nil, status.Errorf(codes.Unavailable, "reserve backup capacity: %v", err) + } + + entry := kv.EncodeBackupPinEntry(kv.BackupPinEntry{PinID: pinID, ReadTS: readTS, Deadline: deadline}) + commits, committed, err := proposeBackupAll(ctx, groups, entry) + if err != nil { + s.compensateBackupRelease(controlGroup, committed, pinID) + return nil, status.Errorf(codes.Unavailable, "commit backup pin: %v", err) + } + return commits, nil +} + +func (s *AdminServer) buildExpectedBackupBaseline( + ctx context.Context, + prepared preparedBackup, +) (map[logicalbackup.Scope]uint64, uint64, error) { + stopRenew := make(chan struct{}) + renewDone := make(chan error, 1) + go func() { + renewDone <- s.renewBackupLoop( + ctx, stopRenew, prepared.groups, prepared.pinID, prepared.readTS, prepared.ttl, + ) + }() + counts, appliedAtCount, scanErr := s.scanBackupScopeCounts(ctx, prepared.routes, prepared.readTS, prepared.groups) + close(stopRenew) + renewErr := <-renewDone + if scanErr != nil { + return nil, 0, status.Errorf(codes.FailedPrecondition, "build expected-key baseline: %v", scanErr) + } + if renewErr != nil { + return nil, 0, status.Errorf(codes.Unavailable, "renew backup pin while building baseline: %v", renewErr) + } + return counts, appliedAtCount, nil +} + +func (s *AdminServer) RenewBackup(ctx context.Context, req *pb.RenewBackupRequest) (*pb.RenewBackupResponse, error) { + if err := s.requireBackupControl(); err != nil { + return nil, err + } + ttl, err := s.effectiveBackupTTL(req.GetTtlMs()) + if err != nil { + return nil, err + } + tok, err := s.decodeBackupToken(req.GetPinToken()) + if err != nil { + return nil, err + } + if err := s.requireUnexpiredBackupToken(tok); err != nil { + return nil, err + } + if _, err := s.backupRouteSnapshotForToken(tok); err != nil { + return nil, err + } + groups, err := s.backupGroupsForToken(tok) + if err != nil { + return nil, err + } + deadline, err := s.renewBackupGroups(ctx, groups, tok.pinID, tok.readTS, ttl) + if err != nil { + s.compensateBackupRelease(groups[0], groups, tok.pinID) + s.forgetBackupSession(tok.pinID) + return nil, status.Errorf(codes.Unavailable, "renew backup pin: %v", err) + } + if err := s.requireUnexpiredBackupToken(tok); err != nil { + s.compensateBackupRelease(groups[0], groups, tok.pinID) + s.forgetBackupSession(tok.pinID) + return nil, err + } + tok.deadline = deadline + encodedToken, err := s.encodeBackupToken(tok) + if err != nil { + return nil, status.Errorf(codes.Internal, "encode renewed backup token: %v", err) + } + s.extendBackupSession(tok) + return &pb.RenewBackupResponse{ + TtlMsEffective: uint64(ttl / time.Millisecond), //nolint:gosec // validated positive. + PinToken: encodedToken, + }, nil +} + +func (s *AdminServer) EndBackup(ctx context.Context, req *pb.EndBackupRequest) (*pb.EndBackupResponse, error) { + if err := s.requireBackupControl(); err != nil { + return nil, err + } + tok, err := s.decodeBackupToken(req.GetPinToken()) + if err != nil { + return nil, err + } + defer s.forgetBackupSession(tok.pinID) + groups, err := s.backupGroupsForToken(tok) + if err != nil { + return nil, err + } + entry := kv.EncodeBackupReleaseEntry(kv.BackupReleaseEntry{PinID: tok.pinID}) + _, _, pinErr := proposeBackupAll(ctx, groups, entry) + unreserve := kv.EncodeBackupUnreserveEntry(kv.BackupUnreserveEntry{PinID: tok.pinID}) + _, _, reserveErr := proposeBackupAll(ctx, groups[:1], unreserve) + if pinErr != nil { + return nil, status.Errorf(codes.Unavailable, "release backup pin: %v", pinErr) + } + if reserveErr != nil { + return nil, status.Errorf(codes.Unavailable, "release backup capacity reservation: %v", reserveErr) + } + return &pb.EndBackupResponse{}, nil +} + +func (s *AdminServer) ListAdaptersAndScopes( + ctx context.Context, + req *pb.ListAdaptersAndScopesRequest, +) (*pb.ListAdaptersAndScopesResponse, error) { + if err := s.requireBackupControl(); err != nil { + return nil, err + } + tok, err := s.decodeBackupToken(req.GetPinToken()) + if err != nil { + return nil, err + } + if err := s.requireUnexpiredBackupToken(tok); err != nil { + return nil, err + } + groups, err := s.backupGroupsForToken(tok) + if err != nil { + return nil, err + } + routes, err := s.backupRouteSnapshotForToken(tok) + if err != nil { + return nil, err + } + counts, _, err := s.scanBackupScopeCounts(ctx, routes, tok.readTS, groups) + if err != nil { + return nil, status.Errorf(codes.FailedPrecondition, "list backup scopes: %v", err) + } + scopes := logicalbackup.SortedScopes(counts) + resp := &pb.ListAdaptersAndScopesResponse{Scopes: make([]*pb.BackupScope, 0, len(scopes))} + for _, scope := range scopes { + resp.Scopes = append(resp.Scopes, &pb.BackupScope{Adapter: scope.Adapter, Scope: scope.Name}) + } + return resp, nil +} + +func (s *AdminServer) StreamBackup( + req *pb.StreamBackupRequest, + stream grpc.ServerStreamingServer[pb.BackupKV], +) error { + if err := s.requireBackupControl(); err != nil { + return err + } + tok, err := s.decodeBackupToken(req.GetPinToken()) + if err != nil { + return err + } + if err := s.requireUnexpiredBackupToken(tok); err != nil { + return err + } + if _, err := s.backupGroupsForToken(tok); err != nil { + return err + } + routes, err := s.backupRouteSnapshotForToken(tok) + if err != nil { + return err + } + selected, err := selectedBackupScopes(req.GetScopes()) + if err != nil { + return err + } + scanner := s.backupStore.NewBackupScannerAtSnapshot(routes, tok.readTS, s.backupConfig.scanPageSize) + if scanner == nil { + return status.Errorf(codes.Unavailable, "%s", "backup scanner is nil") + } + scanErr := streamBackupRecords(stream, scanner, selected) + if err := finishBackupScan(stream.Context(), scanner, scanErr); err != nil { + if scanErr != nil { + return err + } + return status.Errorf(codes.Internal, "close backup scanner: %v", err) + } + return nil +} + +func streamBackupRecords( + stream grpc.ServerStreamingServer[pb.BackupKV], + scanner kv.BackupScanner, + selected map[logicalbackup.Scope]bool, +) error { + for { + pair, ok, err := scanner.Next(stream.Context()) + if err != nil { + return backupScanStreamError(err) + } + if !ok { + return nil + } + if pair == nil { + return status.Errorf(codes.Internal, "%s", "backup scanner returned a nil record") + } + scope, scoped, err := logicalbackup.ScopeForKey(pair.Key) + if err != nil { + return status.Errorf(codes.FailedPrecondition, "classify backup key: %v", err) + } + if !scoped || (len(selected) > 0 && !selected[scope]) { + continue + } + if err := stream.Send(&pb.BackupKV{Key: pair.Key, Value: pair.Value}); err != nil { + return backupSendStreamError(err) + } + } +} + +func backupScanStreamError(err error) error { + if contextErr := backupContextStreamError(err); contextErr != nil { + return contextErr + } + return status.Errorf(codes.FailedPrecondition, "scan backup at read_ts: %v", err) +} + +func backupSendStreamError(err error) error { + if contextErr := backupContextStreamError(err); contextErr != nil { + return contextErr + } + return errors.WithStack(err) +} + +func backupContextStreamError(err error) error { + switch { + case errors.Is(err, context.Canceled): + return status.Errorf(codes.Canceled, "%s", err) + case errors.Is(err, context.DeadlineExceeded): + return status.Errorf(codes.DeadlineExceeded, "%s", err) + default: + return nil + } +} + +func selectedBackupScopes(scopes []*pb.BackupScope) (map[logicalbackup.Scope]bool, error) { + selected := make(map[logicalbackup.Scope]bool, len(scopes)) + for _, scope := range scopes { + if scope == nil || scope.GetAdapter() == "" || scope.GetScope() == "" { + return nil, status.Errorf(codes.InvalidArgument, "%s", "backup scope requires adapter and scope") + } + switch scope.GetAdapter() { + case "dynamodb", "s3", "redis", "sqs": + default: + return nil, status.Errorf(codes.InvalidArgument, "unknown backup adapter %q", scope.GetAdapter()) + } + selected[logicalbackup.Scope{Adapter: scope.GetAdapter(), Name: scope.GetScope()}] = true + } + return selected, nil +} + +func (s *AdminServer) requireBackupControl() error { + if s == nil || s.backupStore == nil || s.backupReadFence == nil || s.backupPeerProbe == nil || s.backupLimiter == nil || s.backupTokenKey == ([32]byte{}) { + return status.Errorf(codes.Unavailable, "%s", ErrBackupUnavailable) + } + return nil +} + +func (s *AdminServer) effectiveBackupTTL(ttlMS uint64) (time.Duration, error) { + if ttlMS == 0 { + return s.backupConfig.defaultTTL, nil + } + if ttlMS > uint64(math.MaxInt64/int64(time.Millisecond)) { //nolint:gosec // positive constant ratio. + return 0, status.Errorf(codes.InvalidArgument, "%s", "backup ttl_ms overflows duration") + } + ttl := time.Duration(ttlMS) * time.Millisecond //nolint:gosec // bounded above before conversion. + if ttl < s.backupConfig.minTTL || ttl > s.backupConfig.maxTTL { + return 0, status.Errorf(codes.InvalidArgument, "backup ttl must be between %s and %s", s.backupConfig.minTTL, s.backupConfig.maxTTL) + } + return ttl, nil +} + +func (s *AdminServer) nowSnapshot() time.Time { + s.groupsMu.RLock() + now := s.now + s.groupsMu.RUnlock() + return now() +} + +func (s *AdminServer) snapshotBackupGroups() ([]backupGroup, error) { + s.groupsMu.RLock() + groups := make([]backupGroup, 0, len(s.groups)) + for id, group := range s.groups { + proposer := s.backupProposers[id] + if group == nil || proposer == nil { + s.groupsMu.RUnlock() + return nil, status.Errorf(codes.Unavailable, "backup proposer unavailable for raft group %d", id) + } + groups = append(groups, backupGroup{id: id, reader: group, status: group.Status(), every: group.SnapshotEvery(), proposer: proposer}) + } + s.groupsMu.RUnlock() + if len(groups) == 0 { + return nil, status.Errorf(codes.Unavailable, "%s", "no raft groups registered for backup") + } + sort.Slice(groups, func(i, j int) bool { return groups[i].id < groups[j].id }) + return groups, nil +} + +func (s *AdminServer) checkBackupSnapshotHeadroom(groups []backupGroup) error { + for _, group := range groups { + if group.every == 0 { + continue + } + if group.status.AppliedIndex < group.status.LastSnapshotIndex { + return status.Errorf(codes.FailedPrecondition, "raft group %d reports applied index below snapshot index", group.id) + } + used := group.status.AppliedIndex - group.status.LastSnapshotIndex + remaining := uint64(0) + if used < group.every { + remaining = group.every - used + } + if remaining < s.backupConfig.snapshotHeadroomEntries { + return status.Errorf(codes.FailedPrecondition, "raft group %d has %d snapshot entries remaining; need %d", group.id, remaining, s.backupConfig.snapshotHeadroomEntries) + } + } + return nil +} + +func (s *AdminServer) gateBackupPeerVersions(ctx context.Context) error { + members, err := s.snapshotBackupMembers(ctx) + if err != nil { + return err + } + type result struct { + nodeID string + version BackupPeerVersion + err error + } + results := make(chan result, len(members)) + md, _ := metadata.FromIncomingContext(ctx) + launched := 0 + for _, member := range members { + if member == nil { + continue + } + launched++ + if member.GetNodeId() == s.self.NodeID { + results <- result{nodeID: member.GetNodeId(), version: BackupPeerVersion{ + NodeVersion: s.nodeVersion, BackupProtocolVersion: s.backupProtocolVersion, + }} + continue + } + go func() { + peerCtx, cancel := context.WithTimeout(ctx, s.backupConfig.beginDeadline) + defer cancel() + peerCtx = metadata.NewOutgoingContext(peerCtx, md.Copy()) + version, err := s.backupPeerProbe(peerCtx, member.GetGrpcAddress()) + results <- result{nodeID: member.GetNodeId(), version: version, err: err} + }() + } + failures := make([]string, 0) + for i := 0; i < launched; i++ { + result := <-results + switch { + case result.err != nil: + failures = append(failures, fmt.Sprintf("node %s did not respond within %s: %v", result.nodeID, s.backupConfig.beginDeadline, result.err)) + case result.version.BackupProtocolVersion < backupProtocolVersionV1: + failures = append(failures, fmt.Sprintf("node %s reports version %s with backup protocol %d; minimum protocol is %d", result.nodeID, result.version.NodeVersion, result.version.BackupProtocolVersion, backupProtocolVersionV1)) + } + } + if len(failures) > 0 { + sort.Strings(failures) + return status.Errorf(codes.FailedPrecondition, "%s", failures[0]) + } + return nil +} + +// snapshotBackupMembers is the fail-closed counterpart to snapshotMembers. +// Capability gating must observe every Raft Configuration before proposing a +// backup FSM entry; the overview RPC intentionally retains its tolerant view. +func (s *AdminServer) snapshotBackupMembers(ctx context.Context) ([]*pb.NodeIdentity, error) { + groups := s.cloneGroupsSorted() + if len(groups) == 0 { + return nil, status.Errorf(codes.FailedPrecondition, "%s", "backup membership has no registered raft groups") + } + results := fanoutConfigurationCalls(ctx, groups) + if len(results) != len(groups) { + return nil, status.Errorf(codes.FailedPrecondition, "backup membership incomplete: received %d of %d raft configurations", len(results), len(groups)) + } + sort.Slice(results, func(i, j int) bool { return results[i].i < results[j].i }) + live := liveMembers{ + addrByID: make(map[string]string), + seenID: make(map[string]struct{}), + order: make([]string, 0), + } + for _, result := range results { + if result.i < 0 || result.i >= len(groups) { + return nil, status.Errorf(codes.FailedPrecondition, "%s", "backup membership returned an invalid raft group result") + } + if result.err != nil { + return nil, status.Errorf(codes.FailedPrecondition, "backup membership for raft group %d: %v", groups[result.i].id, result.err) + } + mergeBackupConfiguration(result.cfg, s.self.NodeID, &live) + } + live.authoritative = true + mergeSeedMembers(s.members, s.self.NodeID, &live) + appendMissingBackupMembers(&live) + + out := make([]*pb.NodeIdentity, 0, len(live.order)) + for _, id := range live.order { + address := live.addrByID[id] + if address == "" { + return nil, status.Errorf(codes.FailedPrecondition, "backup membership for node %s has no gRPC address", id) + } + out = append(out, &pb.NodeIdentity{NodeId: id, GrpcAddress: address}) + } + return out, nil +} + +func appendMissingBackupMembers(live *liveMembers) { + missingAddresses := make([]string, 0) + for id := range live.seenID { + if _, hasAddress := live.addrByID[id]; !hasAddress { + missingAddresses = append(missingAddresses, id) + } + } + sort.Strings(missingAddresses) + live.order = append(live.order, missingAddresses...) +} + +func mergeBackupConfiguration(cfg raftengine.Configuration, selfID string, live *liveMembers) { + for _, server := range cfg.Servers { + if server.ID == "" || server.ID == selfID { + continue + } + live.seenID[server.ID] = struct{}{} + if server.Address == "" { + continue + } + if _, exists := live.addrByID[server.ID]; exists { + continue + } + live.addrByID[server.ID] = server.Address + live.order = append(live.order, server.ID) + } +} + +func proposeBackupAll(ctx context.Context, groups []backupGroup, entry []byte) (map[uint64]uint64, []backupGroup, error) { + type result struct { + group backupGroup + index uint64 + err error + } + results := make(chan result, len(groups)) + for _, group := range groups { + go func() { + proposal, err := group.proposer.ProposeAdmin(ctx, entry) + if err == nil { + err = backupProposalResponseError(proposal) + } + index := uint64(0) + if proposal != nil { + index = proposal.CommitIndex + } + results <- result{group: group, index: index, err: err} + }() + } + commits := make(map[uint64]uint64, len(groups)) + committed := make([]backupGroup, 0, len(groups)) + var firstErr error + for range groups { + result := <-results + if result.err != nil { + if firstErr == nil { + firstErr = errors.Wrapf(result.err, "raft group %d", result.group.id) + } + continue + } + commits[result.group.id] = result.index + committed = append(committed, result.group) + } + return commits, committed, firstErr +} + +func backupProposalResponseError(result *raftengine.ProposalResult) error { + if result == nil { + return errors.New("raft proposal returned nil result") + } + if result.Response == nil { + return nil + } + if err, ok := result.Response.(error); ok { + return errors.WithStack(err) + } + return errors.Wrapf(ErrBackupUnavailable, "unexpected backup apply response %T", result.Response) +} + +func (s *AdminServer) compensateBackupRelease(control backupGroup, groups []backupGroup, pinID kv.BackupPinID) { + ctx, cancel := context.WithTimeout(context.Background(), s.backupConfig.beginDeadline) + defer cancel() + if len(groups) > 0 { + entry := kv.EncodeBackupReleaseEntry(kv.BackupReleaseEntry{PinID: pinID}) + _, _, _ = proposeBackupAll(ctx, groups, entry) + } + unreserve := kv.EncodeBackupUnreserveEntry(kv.BackupUnreserveEntry{PinID: pinID}) + _, _, _ = proposeBackupAll(ctx, []backupGroup{control}, unreserve) +} + +func (s *AdminServer) proposeBackupWithRetry(ctx context.Context, groups []backupGroup, entry []byte) error { + pending := append([]backupGroup(nil), groups...) + var firstErr error + for attempt := 0; attempt < s.backupConfig.renewAttempts && len(pending) > 0; attempt++ { + _, committed, err := proposeBackupAll(ctx, pending, entry) + if err == nil { + return nil + } + firstErr = err + pending = remainingBackupGroups(pending, committed) + if len(pending) > 0 && attempt+1 < s.backupConfig.renewAttempts { + if err := waitBackupRetry(ctx, s.backupConfig.renewBackoff); err != nil { + return err + } + } + } + return firstErr +} + +func remainingBackupGroups(pending, committed []backupGroup) []backupGroup { + committedIDs := make(map[uint64]struct{}, len(committed)) + for _, group := range committed { + committedIDs[group.id] = struct{}{} + } + next := pending[:0] + for _, group := range pending { + if _, ok := committedIDs[group.id]; !ok { + next = append(next, group) + } + } + return next +} + +func waitBackupRetry(ctx context.Context, delay time.Duration) error { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return errors.WithStack(ctx.Err()) + case <-timer.C: + return nil + } +} + +func (s *AdminServer) renewBackupGroups( + ctx context.Context, + groups []backupGroup, + pinID kv.BackupPinID, + readTS uint64, + ttl time.Duration, +) (time.Time, error) { + if len(groups) == 0 { + return time.Time{}, errors.New("no raft groups available for backup renewal") + } + deadline := s.nowSnapshot().Add(ttl) + reserveEntry := kv.EncodeBackupReserveEntry(kv.BackupReserveEntry{PinID: pinID, ReadTS: readTS, Deadline: deadline}) + if err := s.proposeBackupWithRetry(ctx, groups[:1], reserveEntry); err != nil { + return time.Time{}, errors.Wrap(err, "capacity reservation") + } + // Reapply the complete pin rather than only its deadline. This restores a + // missing replica-local fence after partial delivery while duplicate Pin + // apply preserves the earliest read timestamp and latest deadline. + pinEntry := kv.EncodeBackupPinEntry(kv.BackupPinEntry{PinID: pinID, ReadTS: readTS, Deadline: deadline}) + if err := s.proposeBackupWithRetry(ctx, groups, pinEntry); err != nil { + return time.Time{}, errors.Wrap(err, "group pins") + } + return deadline, nil +} + +func (s *AdminServer) renewBackupLoop( + ctx context.Context, + stop <-chan struct{}, + groups []backupGroup, + pinID kv.BackupPinID, + readTS uint64, + ttl time.Duration, +) error { + interval := ttl / 3 //nolint:mnd // live-backup contract renews at one third of TTL. + if interval <= 0 { + return errors.New("backup renewal interval is not positive") + } + timer := time.NewTimer(interval) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + return errors.WithStack(ctx.Err()) + case <-stop: + return nil + case <-timer.C: + if _, err := s.renewBackupGroups(ctx, groups, pinID, readTS, ttl); err != nil { + return err + } + timer.Reset(interval) + } + } +} + +func (s *AdminServer) scanBackupScopeCounts( + ctx context.Context, + routes kv.BackupRouteSnapshot, + readTS uint64, + groups []backupGroup, +) (counts map[logicalbackup.Scope]uint64, applied uint64, retErr error) { + scanner := s.backupStore.NewBackupKeyScannerAtSnapshot(routes, readTS, s.backupConfig.scanPageSize) + if scanner == nil { + return nil, 0, errors.New("backup key scanner is nil") + } + defer func() { + retErr = finishBackupScan(ctx, scanner, retErr) + }() + counts = make(map[logicalbackup.Scope]uint64) + for { + key, ok, err := scanner.Next(ctx) + if err != nil { + return nil, 0, errors.Wrap(err, "scan backup key baseline") + } + if !ok { + break + } + scope, scoped, err := logicalbackup.ScopeForKey(key) + if err != nil { + return nil, 0, errors.Wrap(err, "classify backup key baseline") + } + if scoped { + counts[scope]++ + } + } + applied, err := currentMinBackupAppliedIndex(groups) + if err != nil { + return nil, 0, err + } + return counts, applied, nil +} + +func (s *AdminServer) rememberBackupSession(tok backupToken, routes kv.BackupRouteSnapshot) { + now := s.nowSnapshot() + s.backupStateMu.Lock() + defer s.backupStateMu.Unlock() + s.reapBackupSessionsLocked(now) + if s.backupSessions == nil { + s.backupSessions = make(map[kv.BackupPinID]backupSession) + } + s.backupSessions[tok.pinID] = backupSession{routes: routes, readTS: tok.readTS, deadline: tok.deadline} +} + +func (s *AdminServer) backupRouteSnapshotForToken(tok backupToken) (kv.BackupRouteSnapshot, error) { + now := s.nowSnapshot() + s.backupStateMu.Lock() + defer s.backupStateMu.Unlock() + s.reapBackupSessionsLocked(now) + session, ok := s.backupSessions[tok.pinID] + if !ok || session.readTS != tok.readTS { + return kv.BackupRouteSnapshot{}, status.Errorf(codes.FailedPrecondition, "%s", "backup route snapshot is unavailable on this endpoint") + } + return session.routes, nil +} + +func (s *AdminServer) extendBackupSession(tok backupToken) { + s.backupStateMu.Lock() + defer s.backupStateMu.Unlock() + session, ok := s.backupSessions[tok.pinID] + if !ok || session.readTS != tok.readTS { + return + } + if tok.deadline.After(session.deadline) { + session.deadline = tok.deadline + s.backupSessions[tok.pinID] = session + } +} + +func (s *AdminServer) forgetBackupSession(pinID kv.BackupPinID) { + s.backupStateMu.Lock() + delete(s.backupSessions, pinID) + s.backupStateMu.Unlock() +} + +func (s *AdminServer) reapBackupSessionsLocked(now time.Time) { + for pinID, session := range s.backupSessions { + if !now.Before(session.deadline) { + delete(s.backupSessions, pinID) + } + } +} + +func finishBackupScan(ctx context.Context, scanner interface{ Close() error }, scanErr error) error { + closeErr := scanner.Close() + if closeErr == nil { + return scanErr + } + closeErr = errors.Wrap(closeErr, "close backup scanner") + if scanErr == nil { + return closeErr + } + slog.ErrorContext(ctx, "backup scanner cleanup failed after scan error", "err", closeErr) + return scanErr +} + +func currentMinBackupAppliedIndex(groups []backupGroup) (uint64, error) { + var min uint64 + for _, group := range groups { + if group.reader == nil { + return 0, errors.Wrapf(ErrBackupUnavailable, "raft group %d status reader is nil", group.id) + } + idx := group.reader.Status().AppliedIndex + if min == 0 || idx < min { + min = idx + } + } + return min, nil +} + +func backupExpectedResponses(counts map[logicalbackup.Scope]uint64, applied uint64) []*pb.BackupExpectedKeys { + scopes := logicalbackup.SortedScopes(counts) + out := make([]*pb.BackupExpectedKeys, 0, len(scopes)) + for _, scope := range scopes { + out = append(out, &pb.BackupExpectedKeys{ + Adapter: scope.Adapter, Scope: scope.Name, KeyCount: counts[scope], AppliedIndexAtCount: applied, + }) + } + return out +} + +func backupShardResponses(groups []backupGroup, commits map[uint64]uint64) []*pb.BackupShardApplied { + out := make([]*pb.BackupShardApplied, 0, len(groups)) + for _, group := range groups { + idx := commits[group.id] + if idx == 0 { + idx = group.status.AppliedIndex + } + out = append(out, &pb.BackupShardApplied{RaftGroupId: group.id, AppliedIndex: idx}) + } + return out +} + +func backupGroupIDs(groups []backupGroup) []uint64 { + ids := make([]uint64, 0, len(groups)) + for _, group := range groups { + ids = append(ids, group.id) + } + return ids +} + +func newBackupPinID() (kv.BackupPinID, error) { + var id kv.BackupPinID + if _, err := rand.Read(id[:]); err != nil { + return kv.BackupPinID{}, errors.WithStack(err) + } + if id.IsZero() { + return kv.BackupPinID{}, errors.New("random backup pin id is zero") + } + return id, nil +} + +func (s *AdminServer) encodeBackupToken(tok backupToken) ([]byte, error) { + deadlineMS := tok.deadline.UnixMilli() + if tok.pinID.IsZero() || tok.readTS == 0 || deadlineMS <= 0 || len(tok.groupIDs) == 0 || len(tok.groupIDs) > maxBackupTokenGroups { + return nil, errors.WithStack(ErrBackupToken) + } + buf := make([]byte, backupTokenHeaderLen+len(tok.groupIDs)*8+backupTokenMACLen) //nolint:mnd // uint64 group IDs. + buf[0] = backupTokenVersion + copy(buf[1:17], tok.pinID[:]) + binary.BigEndian.PutUint64(buf[17:25], tok.readTS) + binary.BigEndian.PutUint64(buf[25:33], uint64(deadlineMS)) //nolint:gosec // positive above. + binary.BigEndian.PutUint32(buf[33:37], uint32(len(tok.groupIDs))) //nolint:gosec // capped above. + offset := backupTokenHeaderLen + var previous uint64 + for i, groupID := range tok.groupIDs { + if groupID == 0 || (i > 0 && groupID <= previous) { + return nil, errors.WithStack(ErrBackupToken) + } + binary.BigEndian.PutUint64(buf[offset:offset+8], groupID) //nolint:mnd // uint64 width. + offset += 8 + previous = groupID + } + mac := hmac.New(sha256.New, s.backupTokenKey[:]) + _, _ = mac.Write(buf[:offset]) + copy(buf[offset:], mac.Sum(nil)) + return buf, nil +} + +func (s *AdminServer) decodeBackupToken(raw []byte) (backupToken, error) { + count, payloadEnd, ok := backupTokenEnvelope(raw) + if !ok { + return backupToken{}, invalidBackupTokenError() + } + mac := hmac.New(sha256.New, s.backupTokenKey[:]) + _, _ = mac.Write(raw[:payloadEnd]) + if !hmac.Equal(raw[payloadEnd:], mac.Sum(nil)) { + return backupToken{}, invalidBackupTokenError() + } + var tok backupToken + copy(tok.pinID[:], raw[1:17]) + tok.readTS = binary.BigEndian.Uint64(raw[17:25]) + deadlineMS := binary.BigEndian.Uint64(raw[25:33]) + if tok.pinID.IsZero() || tok.readTS == 0 || deadlineMS == 0 || deadlineMS > math.MaxInt64 { + return backupToken{}, invalidBackupTokenError() + } + tok.deadline = time.UnixMilli(int64(deadlineMS)) //nolint:gosec // bounded above. + tok.groupIDs, ok = decodeBackupTokenGroupIDs(raw, count) + if !ok { + return backupToken{}, invalidBackupTokenError() + } + return tok, nil +} + +func backupTokenEnvelope(raw []byte) (int, int, bool) { + if len(raw) < backupTokenHeaderLen+8+backupTokenMACLen || raw[0] != backupTokenVersion { //nolint:mnd // one group minimum. + return 0, 0, false + } + count := int(binary.BigEndian.Uint32(raw[33:37])) + expectedLen := backupTokenHeaderLen + count*8 + backupTokenMACLen //nolint:mnd // uint64 group IDs. + if count <= 0 || count > maxBackupTokenGroups || len(raw) != expectedLen { + return 0, 0, false + } + return count, len(raw) - backupTokenMACLen, true +} + +func decodeBackupTokenGroupIDs(raw []byte, count int) ([]uint64, bool) { + groupIDs := make([]uint64, 0, count) + offset := backupTokenHeaderLen + var previous uint64 + for i := 0; i < count; i++ { + groupID := binary.BigEndian.Uint64(raw[offset : offset+8]) //nolint:mnd // uint64 width. + if groupID == 0 || (i > 0 && groupID <= previous) { + return nil, false + } + groupIDs = append(groupIDs, groupID) + previous = groupID + offset += 8 + } + return groupIDs, true +} + +func invalidBackupTokenError() error { + return status.Errorf(codes.InvalidArgument, "%s", ErrBackupToken) +} + +func (s *AdminServer) requireUnexpiredBackupToken(tok backupToken) error { + if tok.deadline.IsZero() || !s.nowSnapshot().Before(tok.deadline) { + return status.Errorf(codes.FailedPrecondition, "%s", "backup pin token has expired") + } + return nil +} + +func (s *AdminServer) backupGroupsForToken(tok backupToken) ([]backupGroup, error) { + groups, err := s.snapshotBackupGroups() + if err != nil { + return nil, err + } + byID := make(map[uint64]backupGroup, len(groups)) + for _, group := range groups { + byID[group.id] = group + } + out := make([]backupGroup, 0, len(tok.groupIDs)) + for _, id := range tok.groupIDs { + group, ok := byID[id] + if !ok { + return nil, status.Errorf(codes.FailedPrecondition, "backup token references unavailable raft group %d", id) + } + out = append(out, group) + } + return out, nil +} diff --git a/adapter/admin_backup_test.go b/adapter/admin_backup_test.go new file mode 100644 index 000000000..2df70a6e3 --- /dev/null +++ b/adapter/admin_backup_test.go @@ -0,0 +1,688 @@ +package adapter + +import ( + "context" + "encoding/base64" + stderrors "errors" + "sync" + "sync/atomic" + "testing" + "time" + + logicalbackup "github.com/bootjp/elastickv/internal/backup" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + kvstore "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +const ( + backupSubtypePin byte = 0x01 + backupSubtypeExtend byte = 0x02 + backupSubtypeRelease byte = 0x03 + backupSubtypeReserve byte = 0x04 + backupSubtypeUnreserve byte = 0x05 +) + +type backupTestGroup struct { + mu sync.Mutex + status raftengine.Status + servers []raftengine.Server + every uint64 + cfgErr error +} + +func (g *backupTestGroup) Status() raftengine.Status { + g.mu.Lock() + defer g.mu.Unlock() + return g.status +} + +func (g *backupTestGroup) Configuration(context.Context) (raftengine.Configuration, error) { + g.mu.Lock() + defer g.mu.Unlock() + if g.cfgErr != nil { + return raftengine.Configuration{}, g.cfgErr + } + return raftengine.Configuration{Servers: append([]raftengine.Server(nil), g.servers...)}, nil +} + +func (g *backupTestGroup) SnapshotEvery() uint64 { return g.every } + +func (g *backupTestGroup) setApplied(applied uint64) { + g.mu.Lock() + g.status.AppliedIndex = applied + g.mu.Unlock() +} + +type backupTestProposer struct { + mu sync.Mutex + entries [][]byte + commit uint64 + failures map[byte]int + transportError map[byte]error + responseError map[byte]error + onPropose func(byte) +} + +func newBackupTestProposer() *backupTestProposer { + return &backupTestProposer{ + failures: make(map[byte]int), transportError: make(map[byte]error), responseError: make(map[byte]error), + } +} + +func (p *backupTestProposer) Propose(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) { + return p.ProposeAdmin(ctx, data) +} + +func (p *backupTestProposer) ProposeAdmin(ctx context.Context, data []byte) (*raftengine.ProposalResult, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + p.mu.Lock() + defer p.mu.Unlock() + entry := append([]byte(nil), data...) + p.entries = append(p.entries, entry) + p.commit++ + subtype := entry[1] + if p.failures[subtype] > 0 { + p.failures[subtype]-- + return nil, p.transportError[subtype] + } + if p.onPropose != nil { + p.onPropose(subtype) + } + return &raftengine.ProposalResult{CommitIndex: p.commit, Response: p.responseError[subtype]}, nil +} + +func (p *backupTestProposer) subtypes() []byte { + p.mu.Lock() + defer p.mu.Unlock() + out := make([]byte, 0, len(p.entries)) + for _, entry := range p.entries { + out = append(out, entry[1]) + } + return out +} + +type backupTestStore struct { + mu sync.Mutex + keys [][]byte + readTS []uint64 + onCapture func() + onExhaust func() + scanDelay time.Duration + keyCloseErr error + pairCloseErr error +} + +func (s *backupTestStore) CaptureBackupRouteSnapshot(_, _ []byte) kv.BackupRouteSnapshot { + s.mu.Lock() + onCapture := s.onCapture + s.mu.Unlock() + if onCapture != nil { + onCapture() + } + return kv.BackupRouteSnapshot{} +} + +func (s *backupTestStore) NewBackupKeyScannerAtSnapshot(_ kv.BackupRouteSnapshot, ts uint64, _ int) kv.BackupKeyScanner { + s.mu.Lock() + s.readTS = append(s.readTS, ts) + keys := make([][]byte, len(s.keys)) + for i := range s.keys { + keys[i] = append([]byte(nil), s.keys[i]...) + } + onExhaust := s.onExhaust + delay := s.scanDelay + closeErr := s.keyCloseErr + s.mu.Unlock() + return &backupSliceScanner{keys: keys, onExhaust: onExhaust, delay: delay, closeErr: closeErr} +} + +func (s *backupTestStore) NewBackupScannerAtSnapshot(_ kv.BackupRouteSnapshot, ts uint64, _ int) kv.BackupScanner { + s.mu.Lock() + s.readTS = append(s.readTS, ts) + pairs := make([]*kvstore.KVPair, 0, len(s.keys)) + for _, key := range s.keys { + pairs = append(pairs, &kvstore.KVPair{Key: append([]byte(nil), key...), Value: []byte("value")}) + } + closeErr := s.pairCloseErr + s.mu.Unlock() + return &backupPairScanner{pairs: pairs, closeErr: closeErr} +} + +type backupSliceScanner struct { + keys [][]byte + index int + onExhaust func() + once sync.Once + delay time.Duration + closeErr error +} + +func (s *backupSliceScanner) Next(ctx context.Context) ([]byte, bool, error) { + if err := ctx.Err(); err != nil { + return nil, false, err + } + if s.delay > 0 && s.index < len(s.keys) { + timer := time.NewTimer(s.delay) + select { + case <-ctx.Done(): + timer.Stop() + return nil, false, ctx.Err() + case <-timer.C: + } + } + if s.index >= len(s.keys) { + s.once.Do(func() { + if s.onExhaust != nil { + s.onExhaust() + } + }) + return nil, false, nil + } + key := append([]byte(nil), s.keys[s.index]...) + s.index++ + return key, true, nil +} + +func (s *backupSliceScanner) Close() error { return s.closeErr } + +type backupPairScanner struct { + pairs []*kvstore.KVPair + index int + closeErr error +} + +func (s *backupPairScanner) Next(ctx context.Context) (*kvstore.KVPair, bool, error) { + if err := ctx.Err(); err != nil { + return nil, false, err + } + if s.index >= len(s.pairs) { + return nil, false, nil + } + pair := s.pairs[s.index] + s.index++ + return pair, true, nil +} + +func (s *backupPairScanner) Close() error { return s.closeErr } + +type backupTestStream struct { + grpc.ServerStream + ctx context.Context + got []*pb.BackupKV + sendErr error +} + +func (s *backupTestStream) Context() context.Context { return s.ctx } + +func (s *backupTestStream) Send(pair *pb.BackupKV) error { + if s.sendErr != nil { + return s.sendErr + } + s.got = append(s.got, pair) + return nil +} + +func newBackupControlTestServer( + t *testing.T, + store *backupTestStore, + groups map[uint64]*backupTestGroup, + proposers map[uint64]*backupTestProposer, + probe BackupPeerProbe, + opts ...AdminOption, +) *AdminServer { + t.Helper() + tracker := kv.NewActiveTimestampTracker(kv.WithActiveTimestampTrackerSweepInterval(0)) + t.Cleanup(tracker.Close) + if probe == nil { + probe = func(context.Context, string) (BackupPeerVersion, error) { + return BackupPeerVersion{NodeVersion: "test", BackupProtocolVersion: backupProtocolVersionV1}, nil + } + } + base := []AdminOption{ + WithAdminNodeVersion("test"), + WithAdminBackupControl(store, func(context.Context) (uint64, error) { return 42, nil }, probe, tracker, []byte("test-token-key")), + WithAdminBackupConfig(AdminBackupConfig{SnapshotHeadroomEntries: 10, RenewAttempts: 3, RenewBackoff: time.Millisecond}), + } + base = append(base, opts...) + srv := NewAdminServer(NodeIdentity{NodeID: "n1", GRPCAddress: "n1:50051"}, nil, base...) + for id, group := range groups { + srv.RegisterGroup(id, group) + srv.RegisterBackupProposer(id, proposers[id]) + } + return srv +} + +func TestBeginBackupLifecycleAndBaselineAtPinnedTimestamp(t *testing.T) { + t.Parallel() + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + store := &backupTestStore{ + keys: [][]byte{ + []byte(logicalbackup.DDBTableMetaPrefix + base64.RawURLEncoding.EncodeToString([]byte("orders"))), + []byte(logicalbackup.RedisStringPrefix + "key"), + }, + onExhaust: func() { group.setApplied(125) }, + } + proposer := newBackupTestProposer() + srv := newBackupControlTestServer(t, store, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil) + + begin, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.NoError(t, err) + require.Equal(t, uint64(42), begin.GetReadTs()) + require.NotEmpty(t, begin.GetPinToken()) + require.Equal(t, []byte{ + backupSubtypeReserve, backupSubtypePin, backupSubtypeReserve, backupSubtypePin, + }, proposer.subtypes()) + require.Len(t, begin.GetExpectedKeys(), 2) + for _, expected := range begin.GetExpectedKeys() { + require.Equal(t, uint64(125), expected.GetAppliedIndexAtCount()) + require.Equal(t, uint64(1), expected.GetKeyCount()) + } + + scopes, err := srv.ListAdaptersAndScopes(context.Background(), &pb.ListAdaptersAndScopesRequest{PinToken: begin.GetPinToken()}) + require.NoError(t, err) + require.Len(t, scopes.GetScopes(), 2) + + renewed, err := srv.RenewBackup(context.Background(), &pb.RenewBackupRequest{PinToken: begin.GetPinToken()}) + require.NoError(t, err) + require.NotEmpty(t, renewed.GetPinToken()) + _, err = srv.EndBackup(context.Background(), &pb.EndBackupRequest{PinToken: begin.GetPinToken()}) + require.NoError(t, err) + require.Equal(t, []byte{ + backupSubtypeReserve, backupSubtypePin, backupSubtypeReserve, backupSubtypePin, + backupSubtypeReserve, backupSubtypePin, + backupSubtypeRelease, backupSubtypeUnreserve, + }, proposer.subtypes()) + for _, ts := range store.readTS { + require.Equal(t, uint64(42), ts) + } +} + +func TestBeginBackupCapturesRouteSnapshotAfterReadFence(t *testing.T) { + t.Parallel() + stage := atomic.Int32{} + store := &backupTestStore{} + store.onCapture = func() { + require.Equal(t, int32(1), stage.Load()) + stage.Store(2) + } + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + proposer := newBackupTestProposer() + probe := func(context.Context, string) (BackupPeerVersion, error) { + return BackupPeerVersion{NodeVersion: "test", BackupProtocolVersion: backupProtocolVersionV1}, nil + } + tracker := kv.NewActiveTimestampTracker(kv.WithActiveTimestampTrackerSweepInterval(0)) + t.Cleanup(tracker.Close) + readFence := func(context.Context) (uint64, error) { + require.True(t, stage.CompareAndSwap(0, 1)) + return 42, nil + } + srv := newBackupControlTestServer( + t, + store, + map[uint64]*backupTestGroup{1: group}, + map[uint64]*backupTestProposer{1: proposer}, + probe, + WithAdminBackupControl(store, readFence, probe, tracker, []byte("test-token-key")), + ) + + _, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.NoError(t, err) + require.Equal(t, int32(2), stage.Load()) +} + +func TestBeginBackupRenewsWhileBaselineRuns(t *testing.T) { + t.Parallel() + store := &backupTestStore{ + keys: [][]byte{[]byte(logicalbackup.RedisStringPrefix + "key")}, + scanDelay: 80 * time.Millisecond, + } + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + proposer := newBackupTestProposer() + srv := newBackupControlTestServer(t, store, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil, + WithAdminBackupConfig(AdminBackupConfig{DefaultTTL: 30 * time.Millisecond, MinTTL: time.Millisecond, MaxTTL: time.Second}), + ) + + _, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.NoError(t, err) + var reservations int + for _, subtype := range proposer.subtypes() { + if subtype == backupSubtypeReserve { + reservations++ + } + } + require.GreaterOrEqual(t, reservations, 3, "initial, in-baseline and final reservation renewals") +} + +func TestStreamBackupUsesPinTimestampAndScopeFilter(t *testing.T) { + t.Parallel() + ddbKey := []byte(logicalbackup.DDBTableMetaPrefix + base64.RawURLEncoding.EncodeToString([]byte("orders"))) + redisKey := []byte(logicalbackup.RedisStringPrefix + "key") + store := &backupTestStore{keys: [][]byte{ddbKey, redisKey}} + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + proposer := newBackupTestProposer() + srv := newBackupControlTestServer(t, store, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil) + begin, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.NoError(t, err) + + stream := &backupTestStream{ctx: context.Background()} + err = srv.StreamBackup(&pb.StreamBackupRequest{ + PinToken: begin.GetPinToken(), + Scopes: []*pb.BackupScope{{Adapter: "redis", Scope: "db_0"}}, + }, stream) + require.NoError(t, err) + require.Len(t, stream.got, 1) + require.Equal(t, redisKey, stream.got[0].GetKey()) + require.Equal(t, uint64(42), store.readTS[len(store.readTS)-1]) +} + +func TestStreamBackupPreservesContextStatusAndReportsCloseErrors(t *testing.T) { + t.Parallel() + store := &backupTestStore{keys: [][]byte{[]byte(logicalbackup.RedisStringPrefix + "key")}} + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + proposer := newBackupTestProposer() + srv := newBackupControlTestServer(t, store, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil) + begin, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.NoError(t, err) + + canceled, cancel := context.WithCancel(context.Background()) + cancel() + err = srv.StreamBackup(&pb.StreamBackupRequest{PinToken: begin.GetPinToken()}, &backupTestStream{ctx: canceled}) + require.Equal(t, codes.Canceled, status.Code(err)) + + err = srv.StreamBackup(&pb.StreamBackupRequest{PinToken: begin.GetPinToken()}, &backupTestStream{ + ctx: context.Background(), sendErr: context.DeadlineExceeded, + }) + require.Equal(t, codes.DeadlineExceeded, status.Code(err)) + + store.mu.Lock() + store.pairCloseErr = stderrors.New("close failed") + store.mu.Unlock() + err = srv.StreamBackup(&pb.StreamBackupRequest{PinToken: begin.GetPinToken()}, &backupTestStream{ctx: context.Background()}) + require.Equal(t, codes.Internal, status.Code(err)) +} + +func TestStreamBackupFailsClosedWithoutPinnedRouteSnapshot(t *testing.T) { + t.Parallel() + store := &backupTestStore{keys: [][]byte{[]byte(logicalbackup.RedisStringPrefix + "key")}} + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + proposer := newBackupTestProposer() + srv := newBackupControlTestServer(t, store, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil) + begin, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.NoError(t, err) + decoded, err := srv.decodeBackupToken(begin.GetPinToken()) + require.NoError(t, err) + srv.forgetBackupSession(decoded.pinID) + + err = srv.StreamBackup(&pb.StreamBackupRequest{PinToken: begin.GetPinToken()}, &backupTestStream{ctx: context.Background()}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Contains(t, err.Error(), "route snapshot is unavailable") +} + +func TestListBackupScopesReportsScannerCloseError(t *testing.T) { + t.Parallel() + store := &backupTestStore{keys: [][]byte{[]byte(logicalbackup.RedisStringPrefix + "key")}} + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + proposer := newBackupTestProposer() + srv := newBackupControlTestServer(t, store, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil) + begin, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.NoError(t, err) + + store.mu.Lock() + store.keyCloseErr = stderrors.New("close failed") + store.mu.Unlock() + _, err = srv.ListAdaptersAndScopes(context.Background(), &pb.ListAdaptersAndScopesRequest{PinToken: begin.GetPinToken()}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) +} + +func TestBackupProtocolVersionRequiresCompleteControlWiring(t *testing.T) { + t.Parallel() + srv := NewAdminServer(NodeIdentity{NodeID: "n1"}, nil, + WithAdminBackupControl(nil, nil, nil, nil, nil), + ) + resp, err := srv.GetNodeVersion(context.Background(), &pb.GetNodeVersionRequest{}) + require.NoError(t, err) + require.Zero(t, resp.GetBackupProtocolVersion()) +} + +func TestBeginBackupCompensatesPartialFanout(t *testing.T) { + t.Parallel() + groups := map[uint64]*backupTestGroup{ + 1: {status: raftengine.Status{AppliedIndex: 100}, every: 10_000}, + 2: {status: raftengine.Status{AppliedIndex: 100}, every: 10_000}, + } + p1 := newBackupTestProposer() + p2 := newBackupTestProposer() + p2.failures[backupSubtypePin] = 1 + p2.transportError[backupSubtypePin] = stderrors.New("leader unavailable") + srv := newBackupControlTestServer(t, &backupTestStore{}, groups, map[uint64]*backupTestProposer{1: p1, 2: p2}, nil) + + _, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.Equal(t, codes.Unavailable, status.Code(err)) + require.Contains(t, p1.subtypes(), backupSubtypeRelease) + require.Contains(t, p1.subtypes(), backupSubtypeUnreserve) + require.NotContains(t, p2.subtypes(), backupSubtypeRelease) +} + +func TestBeginBackupGatesOnDynamicPeerVersionAndPropagatesAuth(t *testing.T) { + t.Parallel() + group := &backupTestGroup{ + status: raftengine.Status{AppliedIndex: 100}, every: 10_000, + servers: []raftengine.Server{{ID: "n1", Address: "n1:50051"}, {ID: "n4", Address: "n4:50051"}}, + } + proposer := newBackupTestProposer() + var gotAuthorization []string + probe := func(ctx context.Context, address string) (BackupPeerVersion, error) { + require.Equal(t, "n4:50051", address) + md, _ := metadata.FromOutgoingContext(ctx) + gotAuthorization = md.Get("authorization") + return BackupPeerVersion{NodeVersion: "old", BackupProtocolVersion: 0}, nil + } + srv := newBackupControlTestServer(t, &backupTestStore{}, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, probe) + ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer secret")) + + _, err := srv.BeginBackup(ctx, &pb.BeginBackupRequest{}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Contains(t, err.Error(), "node n4 reports version old") + require.Equal(t, []string{"Bearer secret"}, gotAuthorization) + require.Empty(t, proposer.subtypes()) +} + +func TestBeginBackupFailsClosedOnIncompleteMembership(t *testing.T) { + t.Parallel() + group := &backupTestGroup{ + status: raftengine.Status{AppliedIndex: 100}, every: 10_000, + cfgErr: stderrors.New("configuration timeout"), + } + proposer := newBackupTestProposer() + srv := newBackupControlTestServer(t, &backupTestStore{}, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil) + + _, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Contains(t, err.Error(), "backup membership for raft group 1") + require.Empty(t, proposer.subtypes()) +} + +func TestBeginBackupFailsClosedOnMemberWithoutProbeAddress(t *testing.T) { + t.Parallel() + group := &backupTestGroup{ + status: raftengine.Status{AppliedIndex: 100}, every: 10_000, + servers: []raftengine.Server{{ID: "n1", Address: "n1:50051"}, {ID: "n4"}}, + } + proposer := newBackupTestProposer() + srv := newBackupControlTestServer( + t, + &backupTestStore{}, + map[uint64]*backupTestGroup{1: group}, + map[uint64]*backupTestProposer{1: proposer}, + nil, + ) + + _, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Contains(t, err.Error(), "backup membership for node n4 has no gRPC address") + require.Empty(t, proposer.subtypes()) +} + +func TestBeginBackupRefusesNearSnapshotThreshold(t *testing.T) { + t.Parallel() + group := &backupTestGroup{ + status: raftengine.Status{AppliedIndex: 950, LastSnapshotIndex: 0}, every: 1000, + } + proposer := newBackupTestProposer() + srv := newBackupControlTestServer(t, &backupTestStore{}, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil, + WithAdminBackupConfig(AdminBackupConfig{SnapshotHeadroomEntries: 100}), + ) + + _, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Contains(t, err.Error(), "50 snapshot entries remaining") + require.Empty(t, proposer.subtypes()) +} + +func TestRenewBackupRetriesAndRejectsTamperedToken(t *testing.T) { + t.Parallel() + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + proposer := newBackupTestProposer() + srv := newBackupControlTestServer(t, &backupTestStore{}, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil) + begin, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.NoError(t, err) + + proposer.mu.Lock() + proposer.failures[backupSubtypePin] = 2 + proposer.transportError[backupSubtypePin] = stderrors.New("election") + proposer.mu.Unlock() + renewed, err := srv.RenewBackup(context.Background(), &pb.RenewBackupRequest{PinToken: begin.GetPinToken()}) + require.NoError(t, err) + require.NotEmpty(t, renewed.GetPinToken()) + + tampered := append([]byte(nil), begin.GetPinToken()...) + tampered[len(tampered)-1] ^= 0xff + _, err = srv.RenewBackup(context.Background(), &pb.RenewBackupRequest{PinToken: tampered}) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + _, err = srv.RenewBackup(context.Background(), &pb.RenewBackupRequest{PinToken: begin.GetPinToken(), TtlMs: 999}) + require.Equal(t, codes.InvalidArgument, status.Code(err)) +} + +func TestBackupTokenDeadlineRotatesAndFailsClosed(t *testing.T) { + t.Parallel() + const ttl = 30 * time.Millisecond + nowMS := atomic.Int64{} + base := time.Unix(1_000_000, 0) + nowMS.Store(base.UnixMilli()) + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + proposer := newBackupTestProposer() + srv := newBackupControlTestServer(t, &backupTestStore{}, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil, + WithAdminBackupConfig(AdminBackupConfig{DefaultTTL: ttl, MinTTL: time.Millisecond, MaxTTL: time.Second}), + ) + srv.SetClock(func() time.Time { return time.UnixMilli(nowMS.Load()) }) + + begin, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.NoError(t, err) + decoded, err := srv.decodeBackupToken(begin.GetPinToken()) + require.NoError(t, err) + require.Equal(t, base.Add(ttl), decoded.deadline) + require.True(t, srv.nowSnapshot().Before(decoded.deadline)) + nowMS.Add((10 * time.Millisecond).Milliseconds()) + renewed, err := srv.RenewBackup(context.Background(), &pb.RenewBackupRequest{PinToken: begin.GetPinToken()}) + require.NoError(t, err) + require.NotEqual(t, begin.GetPinToken(), renewed.GetPinToken()) + + nowMS.Add((20 * time.Millisecond).Milliseconds()) + _, err = srv.RenewBackup(context.Background(), &pb.RenewBackupRequest{PinToken: begin.GetPinToken()}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + _, err = srv.ListAdaptersAndScopes(context.Background(), &pb.ListAdaptersAndScopesRequest{PinToken: begin.GetPinToken()}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + err = srv.StreamBackup(&pb.StreamBackupRequest{PinToken: begin.GetPinToken()}, &backupTestStream{ctx: context.Background()}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + + _, err = srv.RenewBackup(context.Background(), &pb.RenewBackupRequest{PinToken: renewed.GetPinToken()}) + require.NoError(t, err) + _, err = srv.EndBackup(context.Background(), &pb.EndBackupRequest{PinToken: begin.GetPinToken()}) + require.NoError(t, err, "expired tokens must remain usable for cleanup") +} + +func TestRenewBackupReleasesResourcesWhenTokenExpiresDuringRenewal(t *testing.T) { + t.Parallel() + const ttl = 30 * time.Millisecond + nowMS := atomic.Int64{} + base := time.Unix(1_000_000, 0) + nowMS.Store(base.UnixMilli()) + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + proposer := newBackupTestProposer() + srv := newBackupControlTestServer(t, &backupTestStore{}, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil, + WithAdminBackupConfig(AdminBackupConfig{DefaultTTL: ttl, MinTTL: time.Millisecond, MaxTTL: time.Second}), + ) + srv.SetClock(func() time.Time { return time.UnixMilli(nowMS.Load()) }) + + begin, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.NoError(t, err) + decoded, err := srv.decodeBackupToken(begin.GetPinToken()) + require.NoError(t, err) + proposer.mu.Lock() + proposer.onPropose = func(subtype byte) { + if subtype == backupSubtypePin { + nowMS.Store(base.Add(ttl).UnixMilli()) + } + } + proposer.mu.Unlock() + + _, err = srv.RenewBackup(context.Background(), &pb.RenewBackupRequest{PinToken: begin.GetPinToken()}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Equal(t, []byte{ + backupSubtypeReserve, backupSubtypePin, backupSubtypeReserve, backupSubtypePin, + backupSubtypeReserve, backupSubtypePin, backupSubtypeRelease, backupSubtypeUnreserve, + }, proposer.subtypes()) + _, err = srv.backupRouteSnapshotForToken(decoded) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) +} + +func TestRenewBackupReleasesResourcesAfterPinFanoutFailure(t *testing.T) { + t.Parallel() + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + proposer := newBackupTestProposer() + srv := newBackupControlTestServer( + t, + &backupTestStore{}, + map[uint64]*backupTestGroup{1: group}, + map[uint64]*backupTestProposer{1: proposer}, + nil, + ) + begin, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.NoError(t, err) + decoded, err := srv.decodeBackupToken(begin.GetPinToken()) + require.NoError(t, err) + proposer.mu.Lock() + proposer.failures[backupSubtypePin] = 3 + proposer.transportError[backupSubtypePin] = stderrors.New("leader unavailable") + proposer.mu.Unlock() + + _, err = srv.RenewBackup(context.Background(), &pb.RenewBackupRequest{PinToken: begin.GetPinToken()}) + require.Equal(t, codes.Unavailable, status.Code(err)) + require.Equal(t, []byte{ + backupSubtypeReserve, backupSubtypePin, backupSubtypeReserve, backupSubtypePin, + backupSubtypeReserve, backupSubtypePin, backupSubtypePin, backupSubtypePin, + backupSubtypeRelease, backupSubtypeUnreserve, + }, proposer.subtypes()) + _, err = srv.backupRouteSnapshotForToken(decoded) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) +} + +func TestBeginBackupMapsCapacityReservationToResourceExhausted(t *testing.T) { + t.Parallel() + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + proposer := newBackupTestProposer() + proposer.responseError[backupSubtypeReserve] = kv.ErrTooManyActiveBackups + srv := newBackupControlTestServer(t, &backupTestStore{}, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil) + + _, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) +} diff --git a/adapter/admin_grpc.go b/adapter/admin_grpc.go index 99bd3f5ec..58d2e5601 100644 --- a/adapter/admin_grpc.go +++ b/adapter/admin_grpc.go @@ -13,6 +13,7 @@ import ( "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/keyviz" + "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/cockroachdb/errors" "google.golang.org/grpc" @@ -137,6 +138,18 @@ type AdminServer struct { leaderVersionProbeSeq atomic.Uint64 versionCache sync.Map + backupMu sync.Mutex + backupStateMu sync.Mutex + backupStore BackupStore + backupReadFence BackupReadFence + backupPeerProbe BackupPeerProbe + backupLimiter BackupPinLimiter + backupTokenKey [32]byte + backupProtocolVersion uint32 + backupConfig backupConfig + backupProposers map[uint64]raftengine.Proposer + backupSessions map[kv.BackupPinID]backupSession + pb.UnimplementedAdminServer } @@ -155,6 +168,9 @@ func NewAdminServer(self NodeIdentity, members []NodeIdentity, opts ...AdminOpti now: time.Now, leaderVersionProbeTimeout: defaultAdminLeaderVersionProbeTimeout, leaderVersionCacheTTL: defaultAdminLeaderVersionCacheTTL, + backupConfig: defaultBackupConfig(), + backupProposers: make(map[uint64]raftengine.Proposer), + backupSessions: make(map[kv.BackupPinID]backupSession), } for _, opt := range opts { if opt != nil { @@ -521,7 +537,10 @@ func (s *AdminServer) GetNodeVersion( context.Context, *pb.GetNodeVersionRequest, ) (*pb.GetNodeVersionResponse, error) { - return &pb.GetNodeVersionResponse{NodeVersion: s.nodeVersion}, nil + return &pb.GetNodeVersionResponse{ + NodeVersion: s.nodeVersion, + BackupProtocolVersion: s.backupProtocolVersion, + }, nil } func (s *AdminServer) leaderNodeVersion(ctx context.Context, leader raftengine.LeaderInfo, now time.Time, addresses []string) string { diff --git a/adapter/internal.go b/adapter/internal.go index bf4279e31..b3e828d93 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -18,6 +18,12 @@ func WithInternalTimestampAllocator(alloc kv.TimestampAllocator) InternalOption } } +func WithInternalAdminProposer(proposer raftengine.Proposer) InternalOption { + return func(i *Internal) { + i.adminProposer = proposer + } +} + func NewInternalWithEngine(txm kv.Transactional, leader raftengine.LeaderView, clock *kv.HLC, relay *RedisPubSubRelay, opts ...InternalOption) *Internal { i := &Internal{ leader: leader, @@ -36,6 +42,7 @@ type Internal struct { transactionManager kv.Transactional clock *kv.HLC tsAllocator kv.TimestampAllocator + adminProposer raftengine.Proposer relay *RedisPubSubRelay pb.UnimplementedInternalServer @@ -78,6 +85,42 @@ func (i *Internal) Forward(ctx context.Context, req *pb.ForwardRequest) (*pb.For }, nil } +func (i *Internal) ForwardAdminProposal( + ctx context.Context, + req *pb.ForwardAdminProposalRequest, +) (*pb.ForwardAdminProposalResponse, error) { + if i.leader == nil || i.leader.State() != raftengine.StateLeader { + return nil, errors.WithStack(ErrNotLeader) + } + if err := i.leader.VerifyLeader(ctx); err != nil { + return nil, errors.WithStack(ErrNotLeader) + } + if i.adminProposer == nil { + return nil, errors.New("admin proposer is unavailable") + } + result, err := i.adminProposer.ProposeAdmin(ctx, req.GetPayload()) + if err != nil { + return nil, errors.WithStack(err) + } + if err := forwardedAdminProposalResponseError(result); err != nil { + return nil, err + } + return &pb.ForwardAdminProposalResponse{CommitIndex: result.CommitIndex}, nil +} + +func forwardedAdminProposalResponseError(result *raftengine.ProposalResult) error { + if result == nil { + return errors.New("admin proposal returned nil result") + } + if result.Response == nil { + return nil + } + if err, ok := result.Response.(error); ok { + return errors.WithStack(err) + } + return errors.Errorf("unexpected admin proposal response %T", result.Response) +} + func (i *Internal) RelayPublish(_ context.Context, req *pb.RelayPublishRequest) (*pb.RelayPublishResponse, error) { if req == nil || i.relay == nil { return &pb.RelayPublishResponse{}, nil diff --git a/adapter/internal_admin_proposal_test.go b/adapter/internal_admin_proposal_test.go new file mode 100644 index 000000000..ef2f87e97 --- /dev/null +++ b/adapter/internal_admin_proposal_test.go @@ -0,0 +1,97 @@ +package adapter + +import ( + "context" + stderrors "errors" + "testing" + + "github.com/bootjp/elastickv/internal/raftengine" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" +) + +type internalAdminLeaderView struct { + state raftengine.State +} + +func (v internalAdminLeaderView) State() raftengine.State { return v.state } +func (internalAdminLeaderView) Leader() raftengine.LeaderInfo { + return raftengine.LeaderInfo{ID: "leader", Address: "leader:50051"} +} +func (internalAdminLeaderView) VerifyLeader(context.Context) error { return nil } +func (internalAdminLeaderView) LinearizableRead(context.Context) (uint64, error) { + return 0, nil +} + +type internalAdminProposer struct { + payload []byte + response any +} + +func (p *internalAdminProposer) Propose(context.Context, []byte) (*raftengine.ProposalResult, error) { + return nil, stderrors.New("unexpected user proposal") +} + +func (p *internalAdminProposer) ProposeAdmin( + _ context.Context, + payload []byte, +) (*raftengine.ProposalResult, error) { + p.payload = append([]byte(nil), payload...) + return &raftengine.ProposalResult{CommitIndex: 17, Response: p.response}, nil +} + +func TestInternalForwardAdminProposalUsesLeaderProposer(t *testing.T) { + t.Parallel() + proposer := &internalAdminProposer{} + internal := NewInternalWithEngine( + nil, + internalAdminLeaderView{state: raftengine.StateLeader}, + nil, + nil, + WithInternalAdminProposer(proposer), + ) + + resp, err := internal.ForwardAdminProposal( + context.Background(), + &pb.ForwardAdminProposalRequest{Payload: []byte("pin")}, + ) + require.NoError(t, err) + require.Equal(t, uint64(17), resp.GetCommitIndex()) + require.Equal(t, []byte("pin"), proposer.payload) +} + +func TestInternalForwardAdminProposalFailsClosed(t *testing.T) { + t.Parallel() + tests := []struct { + name string + state raftengine.State + proposer raftengine.Proposer + errTarget error + errText string + }{ + {name: "follower", state: raftengine.StateFollower, errTarget: ErrNotLeader}, + { + name: "apply response", state: raftengine.StateLeader, + proposer: &internalAdminProposer{response: stderrors.New("apply failed")}, + errText: "apply failed", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + internal := NewInternalWithEngine( + nil, + internalAdminLeaderView{state: tc.state}, + nil, + nil, + WithInternalAdminProposer(tc.proposer), + ) + _, err := internal.ForwardAdminProposal(context.Background(), &pb.ForwardAdminProposalRequest{}) + if tc.errTarget != nil { + require.ErrorIs(t, err, tc.errTarget) + } + if tc.errText != "" { + require.ErrorContains(t, err, tc.errText) + } + }) + } +} diff --git a/docs/design/2026_04_29_proposed_logical_backup.md b/docs/design/2026_04_29_proposed_logical_backup.md index f3929a085..39350d188 100644 --- a/docs/design/2026_04_29_proposed_logical_backup.md +++ b/docs/design/2026_04_29_proposed_logical_backup.md @@ -1045,7 +1045,7 @@ dumps with retention pressure. `BackupScanner.Next(at_ts=read_ts)`. 5. **Renew on long dumps**: the producer calls `RenewBackup(pin_token, ttl_ms)` every `ttl_ms / 3`. The admin - server proposes `BackupExtend{pin_id, read_ts, deadline}` on every group + server proposes `BackupExtend{pin_id, deadline}` on every group recorded in `pin_token`. The `read_ts` is preserved across renewals; only the deadline shifts. A multi-hour dump never relies on a single 30-minute pin. Renewals are cheap (one Raft entry per @@ -1500,15 +1500,13 @@ written. types. Hand-coded fixed-layout binary (matching the HLC lease style): ``` - BackupPin : [envelope:2][pin_id:16][read_ts:8][deadline_ms:8] = 34 bytes - BackupExtend : [envelope:2][pin_id:16][read_ts:8][deadline_ms:8] = 34 bytes - BackupRelease : [envelope:2][pin_id:16] = 18 bytes + BackupPin : [tag:1][pin_id:16][read_ts:8][deadline_ms:8] = 33 bytes + BackupExtend : [tag:1][pin_id:16][deadline_ms:8] = 25 bytes + BackupRelease : [tag:1][pin_id:16] = 17 bytes ``` `pin_id` is a UUIDv4 generated by the admin server at `BeginBackup` time and echoed in every subsequent `BackupExtend` / `BackupRelease` so the FSM can target the right tracker entry. - `BackupExtend` also carries `read_ts`, allowing committed renewal - apply to reconstruct a fence that a replica-local expiry sweep removed. Hand-coded binary (vs. proto) keeps the entry small enough to stay well within the `MaxSizePerMsg` limit (default 1 MiB, `internal/raftengine/etcd/engine.go:55`) and avoids @@ -1590,7 +1588,7 @@ Scope: out of this proposal; mentioned only to draw the boundary. | `TestBeginBackupPinFanOutAllNodes` | A 3-node cluster: `BeginBackup` issued to node A; verify nodes B and C have applied the `BackupPin` Raft entry and their compactors retain MVCC versions at `read_ts`. Compactor on B forced to run mid-dump must not retire pinned versions | | `TestBeginBackupPinSurvivesLeaderChange` | After `BeginBackup` on node A, force a leadership change on a group; the new leader still honors the pin (its FSM applied the same entry); subsequent `BackupScanner.Next` calls succeed | | `TestBeginBackupGroupUnreachable` | If one group cannot commit `BackupPin` within `--begin-backup-deadline`, `BeginBackup` returns `Unavailable` and proposes `BackupRelease` on every group that did commit; no stranded pins remain | -| `TestBackupPinFSMCodecRoundTrip` | `BackupPin` / `BackupExtend` / `BackupRelease` byte layouts (34 / 34 / 18 bytes) round-trip through the FSM apply path; unknown tag bytes return `ErrUnknownRequestType` rather than panicking | +| `TestBackupPinFSMCodecRoundTrip` | `BackupPin` / `BackupExtend` / `BackupRelease` byte layouts (33 / 25 / 17 bytes) round-trip through the FSM apply path; unknown tag bytes return `ErrUnknownRequestType` rather than panicking | | `TestRestoreWipesLocalPins` | A replica that installs a Raft snapshot during a backup loses its `BackupPin`; the producer's per-scope expected-keys baseline detects the resulting `ScanAt` shortfall (count below `99% × baseline ± sqrt(baseline)`) and fails the dump with `ErrCompactionDuringDump` rather than emitting a corrupted artifact | | `TestBeginBackupRefusesNearSnapshotThreshold` | When any group's `SnapshotEvery - (AppliedIndex - LastSnapshotIndex) < --snapshot-headroom-entries`, `BeginBackup` returns `FailedPrecondition` rather than starting a dump that risks the snapshot-installation path. Verify a freshly-snapshotted cluster (largest remaining headroom) is allowed | | `TestExpectedKeysBaselineToleratesTTLExpiry` | Routine TTL expiry between baseline and dump (1% of keys gone) does NOT trigger `ErrCompactionDuringDump`; a 5% drop DOES | diff --git a/internal/backup/live.go b/internal/backup/live.go new file mode 100644 index 000000000..f033e7028 --- /dev/null +++ b/internal/backup/live.go @@ -0,0 +1,253 @@ +package backup + +import ( + "bytes" + "encoding/base64" + "fmt" + "sort" + "strings" + + "github.com/bootjp/elastickv/internal/s3keys" + "github.com/cockroachdb/errors" +) + +// Scope identifies one user-visible adapter namespace in a live backup. +type Scope struct { + Adapter string + Name string +} + +func (s Scope) ID() string { + if s.Adapter == "redis" { + return "redis/" + s.Name + } + return s.Adapter + "/" + s.Name +} + +// ErrScopeKeyMalformed marks a recognized adapter key whose scope cannot be +// recovered. Live backups fail closed on these keys instead of silently +// producing an incomplete expected-key baseline. +var ErrScopeKeyMalformed = errors.New("backup: malformed scoped key") + +// ScopeForKey maps an internal user-data key to its logical backup scope. +// Internal control-plane keys and derivable indexes return (_, false, nil). +func ScopeForKey(key []byte) (Scope, bool, error) { + switch { + case hasAnyBackupPrefix(key, DDBTableMetaPrefix, DDBTableGenPrefix, DDBItemPrefix, DDBGSIPrefix): + return scopeForDDBKey(key) + case hasAnyBackupPrefix(key, + S3BucketMetaPrefix, S3BucketGenPrefix, S3ObjectManifestPrefix, + S3UploadMetaPrefix, S3UploadPartPrefix, S3BlobPrefix, S3GCUploadPrefix, S3RoutePrefix, + ): + return scopeForS3Key(key) + case hasAnyBackupPrefix(key, + SQSQueueMetaPrefix, SQSQueueGenPrefix, SQSMsgDataPrefix, SQSQueueSeqPrefix, + SQSQueueTombstonePrefix, SQSMsgVisPrefix, SQSMsgByAgePrefix, SQSMsgDedupPrefix, SQSMsgGroupPrefix, + ): + return scopeForSQSKey(key) + case isRedisBackupKey(key): + return Scope{Adapter: "redis", Name: "db_0"}, true, nil + default: + return Scope{}, false, nil + } +} + +func scopeForDDBKey(key []byte) (Scope, bool, error) { + switch { + case bytes.HasPrefix(key, []byte(DDBTableMetaPrefix)): + return ddbScopeFromDirectSegment(key, DDBTableMetaPrefix) + case bytes.HasPrefix(key, []byte(DDBTableGenPrefix)): + return ddbScopeFromDirectSegment(key, DDBTableGenPrefix) + case bytes.HasPrefix(key, []byte(DDBItemPrefix)): + encoded, _, err := parseDDBItemKey(key) + if err != nil { + return Scope{}, false, err + } + return decodedScope("dynamodb", encoded) + case bytes.HasPrefix(key, []byte(DDBGSIPrefix)): + return Scope{}, false, nil + default: + return Scope{}, false, nil + } +} + +func scopeForS3Key(key []byte) (Scope, bool, error) { + switch { + case bytes.HasPrefix(key, []byte(S3BucketMetaPrefix)): + bucket, ok := s3keys.ParseBucketMetaKey(key) + return parsedS3Scope(bucket, ok, key) + case bytes.HasPrefix(key, []byte(S3BucketGenPrefix)): + metaKey := append([]byte(S3BucketMetaPrefix), key[len(S3BucketGenPrefix):]...) + bucket, ok := s3keys.ParseBucketMetaKey(metaKey) + return parsedS3Scope(bucket, ok, key) + case bytes.HasPrefix(key, []byte(S3ObjectManifestPrefix)): + bucket, _, _, ok := s3keys.ParseObjectManifestKey(key) + return parsedS3Scope(bucket, ok, key) + case bytes.HasPrefix(key, []byte(S3UploadMetaPrefix)): + bucket, ok := parseUploadFamily(S3UploadMetaPrefix, key) + return parsedS3Scope(bucket, ok, key) + case bytes.HasPrefix(key, []byte(S3UploadPartPrefix)): + bucket, _, _, _, _, ok := s3keys.ParseUploadPartKey(key) + return parsedS3Scope(bucket, ok, key) + case bytes.HasPrefix(key, []byte(S3BlobPrefix)): + bucket, _, _, _, _, _, _, ok := s3keys.ParseBlobKey(key) + return parsedS3Scope(bucket, ok, key) + case bytes.HasPrefix(key, []byte(S3GCUploadPrefix)), bytes.HasPrefix(key, []byte(S3RoutePrefix)): + return Scope{}, false, nil + default: + return Scope{}, false, nil + } +} + +func scopeForSQSKey(key []byte) (Scope, bool, error) { + switch { + case bytes.HasPrefix(key, []byte(SQSQueueMetaPrefix)): + return sqsScopeFromDirectSegment(key, SQSQueueMetaPrefix) + case bytes.HasPrefix(key, []byte(SQSQueueGenPrefix)): + return sqsScopeFromDirectSegment(key, SQSQueueGenPrefix) + case bytes.HasPrefix(key, []byte(SQSMsgDataPrefix)): + encoded, _, _, err := parseSQSMessageDataKey(key) + if err != nil { + return Scope{}, false, err + } + return decodedScope("sqs", encoded) + case bytes.HasPrefix(key, []byte(SQSQueueSeqPrefix)): + return sqsScopeFromDirectSegment(key, SQSQueueSeqPrefix) + default: + prefix, ok := sqsDerivedBackupPrefix(key) + if !ok { + return Scope{}, false, nil + } + return sqsScopeFromGenericKey(key, prefix) + } +} + +func sqsDerivedBackupPrefix(key []byte) (string, bool) { + for _, prefix := range [...]string{ + SQSQueueTombstonePrefix, SQSMsgVisPrefix, SQSMsgByAgePrefix, SQSMsgDedupPrefix, SQSMsgGroupPrefix, + } { + if bytes.HasPrefix(key, []byte(prefix)) { + return prefix, true + } + } + return "", false +} + +func hasAnyBackupPrefix(key []byte, prefixes ...string) bool { + for _, prefix := range prefixes { + if bytes.HasPrefix(key, []byte(prefix)) { + return true + } + } + return false +} + +func ddbScopeFromDirectSegment(key []byte, prefix string) (Scope, bool, error) { + encoded := string(key[len(prefix):]) + if encoded == "" || strings.ContainsRune(encoded, '|') { + return Scope{}, false, errors.Wrapf(ErrScopeKeyMalformed, "dynamodb key %q", key) + } + return decodedScope("dynamodb", encoded) +} + +func sqsScopeFromDirectSegment(key []byte, prefix string) (Scope, bool, error) { + encoded := string(key[len(prefix):]) + if encoded == "" { + return Scope{}, false, errors.Wrapf(ErrScopeKeyMalformed, "sqs key %q", key) + } + return decodedScope("sqs", encoded) +} + +func sqsScopeFromGenericKey(key []byte, prefix string) (Scope, bool, error) { + encoded, err := parseSQSGenericKey(key, prefix) + if err != nil { + return Scope{}, false, err + } + return decodedScope("sqs", encoded) +} + +func decodedScope(adapter, encoded string) (Scope, bool, error) { + name, err := base64.RawURLEncoding.DecodeString(encoded) + if err != nil || len(name) == 0 { + return Scope{}, false, errors.Wrapf(ErrScopeKeyMalformed, "%s segment %q", adapter, encoded) + } + return Scope{Adapter: adapter, Name: string(name)}, true, nil +} + +func parsedS3Scope(name string, ok bool, key []byte) (Scope, bool, error) { + if !ok || name == "" { + return Scope{}, false, errors.Wrapf(ErrScopeKeyMalformed, "s3 key %q", key) + } + return Scope{Adapter: "s3", Name: name}, true, nil +} + +func isRedisBackupKey(key []byte) bool { + prefixes := [...]string{ + RedisHashMetaDeltaPrefix, RedisHashMetaPrefix, RedisHashFieldPrefix, + ListMetaDeltaPrefix, ListMetaPrefix, ListItemPrefix, ListClaimPrefix, + RedisSetMetaDeltaPrefix, RedisSetMetaPrefix, RedisSetMemberPrefix, + RedisZSetMetaDeltaPrefix, RedisZSetMetaPrefix, RedisZSetMemberPrefix, + RedisZSetScorePrefix, RedisZSetLegacyBlobPrefix, + RedisStreamMetaPrefix, RedisStreamEntryPrefix, + RedisStringPrefix, RedisHLLPrefix, RedisTTLPrefix, + } + for _, prefix := range prefixes { + if bytes.HasPrefix(key, []byte(prefix)) { + return true + } + } + return false +} + +// SortedScopes returns a deterministic adapter/name ordering. +func SortedScopes(scopes map[Scope]uint64) []Scope { + out := make([]Scope, 0, len(scopes)) + for scope := range scopes { + out = append(out, scope) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Adapter != out[j].Adapter { + return out[i].Adapter < out[j].Adapter + } + return out[i].Name < out[j].Name + }) + return out +} + +// LiveDecoder feeds key/value pairs read at one pinned timestamp through the +// Phase 0 adapter encoders, producing the same logical directory format. +type LiveDecoder struct { + d *dispatcher + finalized bool +} + +func NewLiveDecoder(opts DecodeOptions) (*LiveDecoder, error) { + d, err := newDispatcher(opts) + if err != nil { + return nil, err + } + return &LiveDecoder{d: d}, nil +} + +func (d *LiveDecoder) Add(key, value []byte) error { + if d == nil || d.d == nil || d.finalized { + return errors.Wrap(ErrDecodeOptionsInvalid, "live decoder is unavailable or finalized") + } + d.d.counters.Total++ + return d.d.route(key, value) +} + +func (d *LiveDecoder) Finalize() (DecodeCounters, error) { + if d == nil || d.d == nil || d.finalized { + return DecodeCounters{}, errors.Wrap(ErrDecodeOptionsInvalid, "live decoder is unavailable or finalized") + } + d.finalized = true + if err := d.d.finalize(); err != nil { + return DecodeCounters{}, err + } + return d.d.counters, nil +} + +func (s Scope) String() string { + return fmt.Sprintf("%s/%s", s.Adapter, s.Name) +} diff --git a/internal/backup/live_test.go b/internal/backup/live_test.go new file mode 100644 index 000000000..cc4b2ed11 --- /dev/null +++ b/internal/backup/live_test.go @@ -0,0 +1,71 @@ +package backup + +import ( + "encoding/base64" + "testing" + + "github.com/bootjp/elastickv/internal/s3keys" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +func TestScopeForKey(t *testing.T) { + t.Parallel() + enc := func(s string) string { return base64.RawURLEncoding.EncodeToString([]byte(s)) } + cases := []struct { + name string + key []byte + want Scope + scoped bool + }{ + {name: "dynamodb meta", key: []byte(DDBTableMetaPrefix + enc("orders")), want: Scope{Adapter: "dynamodb", Name: "orders"}, scoped: true}, + {name: "s3 bucket", key: s3keys.BucketMetaKey("photos"), want: Scope{Adapter: "s3", Name: "photos"}, scoped: true}, + {name: "sqs metadata", key: []byte(SQSQueueMetaPrefix + enc("jobs")), want: Scope{Adapter: "sqs", Name: "jobs"}, scoped: true}, + {name: "sqs sequence", key: []byte(SQSQueueSeqPrefix + enc("jobs")), want: Scope{Adapter: "sqs", Name: "jobs"}, scoped: true}, + {name: "redis", key: []byte(RedisStringPrefix + "key"), want: Scope{Adapter: "redis", Name: "db_0"}, scoped: true}, + {name: "ddb derived gsi", key: []byte(DDBGSIPrefix + "ignored"), scoped: false}, + {name: "transaction", key: []byte("!txn|lock|ignored"), scoped: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, scoped, err := ScopeForKey(tc.key) + require.NoError(t, err) + require.Equal(t, tc.scoped, scoped) + require.Equal(t, tc.want, got) + }) + } +} + +func TestScopeForKeyRejectsMalformedRecognizedKey(t *testing.T) { + t.Parallel() + _, _, err := ScopeForKey([]byte(DDBTableMetaPrefix + "%%%")) + require.True(t, errors.Is(err, ErrScopeKeyMalformed), "err = %v", err) +} + +func TestLiveDecoderReusesPhase0Encoders(t *testing.T) { + t.Parallel() + out := t.TempDir() + decoder, err := NewLiveDecoder(DecodeOptions{OutRoot: out, Adapters: AdapterSet{Redis: true}}) + require.NoError(t, err) + require.NoError(t, decoder.Add([]byte(RedisStringPrefix+"greeting"), []byte("hello"))) + counters, err := decoder.Finalize() + require.NoError(t, err) + require.Equal(t, uint64(1), counters.Total) + require.Equal(t, uint64(1), counters.Redis) + require.FileExists(t, out+"/redis/db_0/strings/greeting.bin") +} + +func TestLiveDecoderAcceptsSQSQueueSequenceSideRecord(t *testing.T) { + t.Parallel() + out := t.TempDir() + encodedQueue := base64.RawURLEncoding.EncodeToString([]byte("orders.fifo")) + decoder, err := NewLiveDecoder(DecodeOptions{ + OutRoot: out, Adapters: AdapterSet{SQS: true}, IncludeSQSSideRecords: true, + }) + require.NoError(t, err) + require.NoError(t, decoder.Add([]byte(SQSQueueSeqPrefix+encodedQueue), []byte("42"))) + counters, err := decoder.Finalize() + require.NoError(t, err) + require.Equal(t, uint64(1), counters.SQS) + require.FileExists(t, out+"/sqs/"+encodedQueue+".orphan/_internals/side_records.jsonl") +} diff --git a/internal/backup/s3.go b/internal/backup/s3.go index 84608f393..504e2bec4 100644 --- a/internal/backup/s3.go +++ b/internal/backup/s3.go @@ -485,7 +485,7 @@ func (s *S3Encoder) HandleIncompleteUpload(prefix string, key, value []byte) err if !s.includeIncompleteUploads { return nil } - bucket, _, _, _, _, ok := parseUploadFamily(prefix, key) + bucket, ok := parseUploadFamily(prefix, key) if !ok { return errors.Wrapf(ErrS3MalformedKey, "upload-family key: %q", key) } @@ -1066,10 +1066,11 @@ func ensureChunkPaths(m map[s3ChunkKey]string) map[s3ChunkKey]string { return m } -func parseUploadFamily(prefix string, key []byte) (bucket string, generation uint64, object string, uploadID string, partNo uint64, ok bool) { +func parseUploadFamily(prefix string, key []byte) (string, bool) { switch prefix { case S3UploadPartPrefix: - return s3keys.ParseUploadPartKey(key) + bucket, _, _, _, _, ok := s3keys.ParseUploadPartKey(key) + return bucket, ok case S3UploadMetaPrefix: // Parse via prefix arithmetic: same shape as upload-part minus // the partNo trailer. ParseUploadPartKey would reject the @@ -1077,22 +1078,22 @@ func parseUploadFamily(prefix string, key []byte) (bucket string, generation uin // only needs the bucket for routing. out := key[len(S3UploadMetaPrefix):] if len(out) == 0 { - return "", 0, "", "", 0, false + return "", false } return decodeBucketSegmentForRouting(out) } - return "", 0, "", "", 0, false + return "", false } -func decodeBucketSegmentForRouting(rest []byte) (string, uint64, string, string, uint64, bool) { +func decodeBucketSegmentForRouting(rest []byte) (string, bool) { // We only need the bucket for routing; the rest is passed through // as opaque bytes. for i := 0; i < len(rest); i++ { if rest[i] == 0x00 && i+1 < len(rest) && rest[i+1] == 0x01 { - return string(rest[:i]), 0, "", "", 0, true + return string(rest[:i]), true } } - return "", 0, "", "", 0, false + return "", false } func uint64Hex(v uint64) string { diff --git a/internal/backup/sqs.go b/internal/backup/sqs.go index bcad8ab24..8045eec68 100644 --- a/internal/backup/sqs.go +++ b/internal/backup/sqs.go @@ -379,7 +379,13 @@ func (s *SQSEncoder) HandleSideRecord(prefix string, key, value []byte) error { if !s.includeSideRecords { return nil } - encQueue, err := parseSQSGenericKey(key, prefix) + var encQueue string + var err error + if prefix == SQSQueueSeqPrefix { + encQueue, err = parseSQSQueueSequenceKey(key) + } else { + encQueue, err = parseSQSGenericKey(key, prefix) + } if err != nil { // Tombstones include a fixed-width gen but no msg ID; the // generic parser tolerates the empty trailer. @@ -394,6 +400,18 @@ func (s *SQSEncoder) HandleSideRecord(prefix string, key, value []byte) error { return nil } +func parseSQSQueueSequenceKey(key []byte) (string, error) { + encoded, err := stripPrefixSegment(key, []byte(SQSQueueSeqPrefix)) + if err != nil { + return "", err + } + decoded, err := base64.RawURLEncoding.DecodeString(encoded) + if err != nil || len(decoded) == 0 { + return "", errors.Wrapf(ErrSQSMalformedKey, "invalid queue sequence key %q", key) + } + return encoded, nil +} + // Finalize flushes every queue's _queue.json and messages.jsonl. Queues // with buffered messages but no meta record (orphans) emit a warning // and have their messages dropped — restoring orphan messages without diff --git a/kv/active_timestamp_tracker.go b/kv/active_timestamp_tracker.go index 2d751fea3..26d68be3a 100644 --- a/kv/active_timestamp_tracker.go +++ b/kv/active_timestamp_tracker.go @@ -10,7 +10,7 @@ import ( ) const ( - defaultMaxActiveBackupPins = 64 + defaultMaxActiveBackupPins = 4 defaultBackupPinSweepEvery = time.Second ) @@ -231,19 +231,14 @@ func (t *ActiveTimestampTracker) Extend(pinID BackupPinID, deadline time.Time) e } func (t *ActiveTimestampTracker) ExtendForGroup(pinID BackupPinID, groupID uint64, deadline time.Time) error { - return t.extendForGroup(pinID, groupID, deadline) + return t.extendForGroup(pinID, groupID, deadline, true) } -func (t *ActiveTimestampTracker) ApplyExtendForGroup( - pinID BackupPinID, - groupID uint64, - readTS uint64, - deadline time.Time, -) error { - return t.pinWithDeadlineForGroup(pinID, groupID, readTS, deadline, false) +func (t *ActiveTimestampTracker) ApplyExtendForGroup(pinID BackupPinID, groupID uint64, deadline time.Time) error { + return t.extendForGroup(pinID, groupID, deadline, false) } -func (t *ActiveTimestampTracker) extendForGroup(pinID BackupPinID, groupID uint64, deadline time.Time) error { +func (t *ActiveTimestampTracker) extendForGroup(pinID BackupPinID, groupID uint64, deadline time.Time, returnMissingExpired bool) error { if t == nil { return nil } @@ -255,9 +250,12 @@ func (t *ActiveTimestampTracker) extendForGroup(pinID BackupPinID, groupID uint6 pin, exists := t.backupPins[key] if !exists { t.mu.Unlock() + if !returnMissingExpired { + return nil + } return errors.WithStack(ErrInvalidBackupPin) } - if !pin.deadline.After(time.Now()) { + if returnMissingExpired && !pin.deadline.After(time.Now()) { delete(t.backupPins, key) t.mu.Unlock() t.logExpiredBackupPins([]expiredBackupPin{{key: key, ts: pin.readTS}}) diff --git a/kv/active_timestamp_tracker_test.go b/kv/active_timestamp_tracker_test.go index 8d29d7aee..aa4d5701d 100644 --- a/kv/active_timestamp_tracker_test.go +++ b/kv/active_timestamp_tracker_test.go @@ -109,9 +109,7 @@ func TestActiveTimestampTrackerApplyExtendReplaysAfterLocalExpiry(t *testing.T) now := time.Now() require.NoError(t, tracker.ApplyPinWithDeadlineForGroup(pinID, 7, 20, now.Add(-time.Millisecond))) - tracker.reapExpiredBackupPins(now) - require.Equal(t, 0, tracker.ActiveBackupPinCount()) - require.NoError(t, tracker.ApplyExtendForGroup(pinID, 7, 20, now.Add(time.Hour))) + require.NoError(t, tracker.ApplyExtendForGroup(pinID, 7, now.Add(time.Hour))) deadline, ok := tracker.BackupPinDeadlineForGroup(pinID, 7) require.True(t, ok) diff --git a/kv/backup_codec.go b/kv/backup_codec.go index acfb27f68..ad0edb016 100644 --- a/kv/backup_codec.go +++ b/kv/backup_codec.go @@ -11,24 +11,28 @@ import ( const ( raftEncodeBackup byte = 0x0e - backupSubtypePin byte = 0x01 - backupSubtypeExtend byte = 0x02 - backupSubtypeRelease byte = 0x03 + backupSubtypePin byte = 0x01 + backupSubtypeExtend byte = 0x02 + backupSubtypeRelease byte = 0x03 + backupSubtypeReserve byte = 0x04 + backupSubtypeUnreserve byte = 0x05 backupPinIDBytes = 16 backupUint64Size = 8 backupEnvelopeHeaderLen = 2 backupPinEntryLen = backupEnvelopeHeaderLen + backupPinIDBytes + backupUint64Size + backupUint64Size - backupExtendEntryLen = backupPinEntryLen + backupExtendEntryLen = backupEnvelopeHeaderLen + backupPinIDBytes + backupUint64Size backupReleaseEntryLen = backupEnvelopeHeaderLen + backupPinIDBytes - backupPinIDStart = backupEnvelopeHeaderLen - backupPinIDEnd = backupPinIDStart + backupPinIDBytes - backupReadTSStart = backupPinIDEnd - backupReadTSEnd = backupReadTSStart + backupUint64Size - backupDeadlineStart = backupReadTSEnd - backupDeadlineEnd = backupDeadlineStart + backupUint64Size + backupPinIDStart = backupEnvelopeHeaderLen + backupPinIDEnd = backupPinIDStart + backupPinIDBytes + backupReadTSStart = backupPinIDEnd + backupReadTSEnd = backupReadTSStart + backupUint64Size + backupDeadlineStart = backupReadTSEnd + backupDeadlineEnd = backupDeadlineStart + backupUint64Size + backupExtendMillisStart = backupPinIDEnd + backupExtendMillisEnd = backupExtendMillisStart + backupUint64Size ) var ( @@ -44,7 +48,6 @@ type BackupPinEntry struct { type BackupExtendEntry struct { PinID BackupPinID - ReadTS uint64 Deadline time.Time } @@ -52,6 +55,13 @@ type BackupReleaseEntry struct { PinID BackupPinID } +// BackupReserveEntry is committed through one deterministic Raft group before +// fan-out. Its group-zero tracker record serializes the cluster-wide pin cap. +type BackupReserveEntry = BackupPinEntry + +// BackupUnreserveEntry releases the group-zero capacity reservation. +type BackupUnreserveEntry = BackupReleaseEntry + type backupEntry struct { subtype byte pin BackupPinEntry @@ -74,8 +84,7 @@ func EncodeBackupExtendEntry(entry BackupExtendEntry) []byte { out[0] = raftEncodeBackup out[1] = backupSubtypeExtend copy(out[backupPinIDStart:backupPinIDEnd], entry.PinID[:]) - binary.BigEndian.PutUint64(out[backupReadTSStart:backupReadTSEnd], entry.ReadTS) - binary.BigEndian.PutUint64(out[backupDeadlineStart:backupDeadlineEnd], backupDeadlineMillis(entry.Deadline)) + binary.BigEndian.PutUint64(out[backupExtendMillisStart:backupExtendMillisEnd], backupDeadlineMillis(entry.Deadline)) return out } @@ -87,6 +96,18 @@ func EncodeBackupReleaseEntry(entry BackupReleaseEntry) []byte { return out } +func EncodeBackupReserveEntry(entry BackupReserveEntry) []byte { + out := EncodeBackupPinEntry(entry) + out[1] = backupSubtypeReserve + return out +} + +func EncodeBackupUnreserveEntry(entry BackupUnreserveEntry) []byte { + out := EncodeBackupReleaseEntry(entry) + out[1] = backupSubtypeUnreserve + return out +} + func decodeBackupEntry(data []byte) (backupEntry, error) { if len(data) < backupEnvelopeHeaderLen || data[0] != raftEncodeBackup { return backupEntry{}, errors.WithStack(ErrBackupWireMalformed) @@ -99,14 +120,14 @@ func decodeBackupPayload(data []byte) (backupEntry, error) { return backupEntry{}, errors.WithStack(ErrBackupWireMalformed) } switch data[0] { - case backupSubtypePin: + case backupSubtypePin, backupSubtypeReserve: if len(data) != backupPinEntryLen-1 { return backupEntry{}, errors.WithStack(ErrBackupWireMalformed) } var id BackupPinID copy(id[:], data[backupPinIDStart-1:backupPinIDEnd-1]) return backupEntry{ - subtype: backupSubtypePin, + subtype: data[0], pin: BackupPinEntry{ PinID: id, ReadTS: binary.BigEndian.Uint64(data[backupReadTSStart-1 : backupReadTSEnd-1]), @@ -123,18 +144,17 @@ func decodeBackupPayload(data []byte) (backupEntry, error) { subtype: backupSubtypeExtend, extend: BackupExtendEntry{ PinID: id, - ReadTS: binary.BigEndian.Uint64(data[backupReadTSStart-1 : backupReadTSEnd-1]), - Deadline: backupDeadlineFromMillis(binary.BigEndian.Uint64(data[backupDeadlineStart-1 : backupDeadlineEnd-1])), + Deadline: backupDeadlineFromMillis(binary.BigEndian.Uint64(data[backupExtendMillisStart-1 : backupExtendMillisEnd-1])), }, }, nil - case backupSubtypeRelease: + case backupSubtypeRelease, backupSubtypeUnreserve: if len(data) != backupReleaseEntryLen-1 { return backupEntry{}, errors.WithStack(ErrBackupWireMalformed) } var id BackupPinID copy(id[:], data[backupPinIDStart-1:backupPinIDEnd-1]) return backupEntry{ - subtype: backupSubtypeRelease, + subtype: data[0], release: BackupReleaseEntry{ PinID: id, }, diff --git a/kv/backup_codec_test.go b/kv/backup_codec_test.go index 87003450c..7d5def7cc 100644 --- a/kv/backup_codec_test.go +++ b/kv/backup_codec_test.go @@ -27,14 +27,13 @@ func TestBackupCodecRoundTrip(t *testing.T) { extendWire := EncodeBackupExtendEntry(BackupExtendEntry{ PinID: pinID, - ReadTS: 42, Deadline: deadline.Add(time.Second), }) require.Len(t, extendWire, backupExtendEntryLen) gotExtend, err := decodeBackupEntry(extendWire) require.NoError(t, err) require.Equal(t, backupSubtypeExtend, gotExtend.subtype) - require.Equal(t, BackupExtendEntry{PinID: pinID, ReadTS: 42, Deadline: deadline.Add(time.Second)}, gotExtend.extend) + require.Equal(t, BackupExtendEntry{PinID: pinID, Deadline: deadline.Add(time.Second)}, gotExtend.extend) releaseWire := EncodeBackupReleaseEntry(BackupReleaseEntry{PinID: pinID}) require.Len(t, releaseWire, backupReleaseEntryLen) @@ -44,6 +43,25 @@ func TestBackupCodecRoundTrip(t *testing.T) { require.Equal(t, BackupReleaseEntry{PinID: pinID}, gotRelease.release) } +func TestBackupCodecReserveRoundTrip(t *testing.T) { + pinID := backupTrackerTestPinID(4) + deadline := time.UnixMilli(9000) + + reserve, err := decodeBackupEntry(EncodeBackupReserveEntry(BackupReserveEntry{ + PinID: pinID, ReadTS: 88, Deadline: deadline, + })) + require.NoError(t, err) + require.Equal(t, backupSubtypeReserve, reserve.subtype) + require.Equal(t, pinID, reserve.pin.PinID) + require.Equal(t, uint64(88), reserve.pin.ReadTS) + require.Equal(t, deadline, reserve.pin.Deadline) + + unreserve, err := decodeBackupEntry(EncodeBackupUnreserveEntry(BackupUnreserveEntry{PinID: pinID})) + require.NoError(t, err) + require.Equal(t, backupSubtypeUnreserve, unreserve.subtype) + require.Equal(t, pinID, unreserve.release.PinID) +} + func TestBackupCodecRejectsMalformedWire(t *testing.T) { pinID := backupTrackerTestPinID(1) valid := EncodeBackupPinEntry(BackupPinEntry{ @@ -75,7 +93,6 @@ func TestBackupCodecZeroDeadlineDecodesToZeroTime(t *testing.T) { gotExtend, err := decodeBackupEntry(EncodeBackupExtendEntry(BackupExtendEntry{ PinID: pinID, - ReadTS: 42, Deadline: time.Time{}, })) require.NoError(t, err) diff --git a/kv/backup_scan.go b/kv/backup_scan.go index 10352718b..ea1d46b2a 100644 --- a/kv/backup_scan.go +++ b/kv/backup_scan.go @@ -6,7 +6,6 @@ import ( "sort" "github.com/bootjp/elastickv/distribution" - "github.com/bootjp/elastickv/internal/fskeys" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" ) @@ -20,6 +19,24 @@ type BackupScanner interface { Close() error } +// BackupKeyScanner is the count-only counterpart to BackupScanner. It pages +// through the same captured route set without materializing values. +type BackupKeyScanner interface { + Next(ctx context.Context) ([]byte, bool, error) + Close() error +} + +// BackupRouteSnapshot is an immutable route view shared by every scan in one +// logical backup. Keeping it separate from a scanner lets BeginBackup count +// keys and StreamBackup materialize values from the same ownership view even +// when the live route catalog changes between those RPCs. +type BackupRouteSnapshot struct { + routes []distribution.Route + clampToRoutes bool + start []byte + end []byte +} + type backupScanner struct { store *ShardStore routes []distribution.Route @@ -39,22 +56,51 @@ type routedScanKey struct { route distribution.Route } +type backupKeyScanner struct { + store *ShardStore + routes []distribution.Route + clampToRoutes bool + end []byte + ts uint64 + pageSize int + cursor []byte + page []routedScanKey + index int + closed bool + exhausted bool +} + func NewBackupScanner(st *ShardStore, start []byte, end []byte, ts uint64, pageSize int) BackupScanner { + snapshot := st.CaptureBackupRouteSnapshot(start, end) + return NewBackupScannerAtSnapshot(st, snapshot, ts, pageSize) +} + +// CaptureBackupRouteSnapshot captures route ownership and scan bounds once. +func (s *ShardStore) CaptureBackupRouteSnapshot(start []byte, end []byte) BackupRouteSnapshot { + if s == nil { + return BackupRouteSnapshot{start: bytes.Clone(start), end: bytes.Clone(end)} + } + routes, clampToRoutes := s.routesForForwardScan(start, end) + return BackupRouteSnapshot{ + routes: cloneBackupRoutes(routes), + clampToRoutes: clampToRoutes, + start: bytes.Clone(start), + end: bytes.Clone(end), + } +} + +// NewBackupScannerAtSnapshot creates a value scanner from a captured route view. +func NewBackupScannerAtSnapshot(st *ShardStore, snapshot BackupRouteSnapshot, ts uint64, pageSize int) BackupScanner { if pageSize <= 0 { pageSize = defaultBackupScanPageSize } - var routes []distribution.Route - var clampToRoutes bool - if st != nil { - routes, clampToRoutes = st.routesForForwardScan(start, end) - routes = append([]distribution.Route(nil), routes...) - } + snapshot = cloneBackupRouteSnapshot(snapshot) return &backupScanner{ store: st, - routes: routes, - clampToRoutes: clampToRoutes, - cursor: bytes.Clone(start), - end: bytes.Clone(end), + routes: snapshot.routes, + clampToRoutes: snapshot.clampToRoutes, + cursor: snapshot.start, + end: snapshot.end, ts: ts, pageSize: pageSize, } @@ -64,6 +110,40 @@ func (s *ShardStore) NewBackupScanner(start []byte, end []byte, ts uint64, pageS return NewBackupScanner(s, start, end, ts, pageSize) } +func (s *ShardStore) NewBackupScannerAtSnapshot(snapshot BackupRouteSnapshot, ts uint64, pageSize int) BackupScanner { + return NewBackupScannerAtSnapshot(s, snapshot, ts, pageSize) +} + +func NewBackupKeyScanner(st *ShardStore, start []byte, end []byte, ts uint64, pageSize int) BackupKeyScanner { + snapshot := st.CaptureBackupRouteSnapshot(start, end) + return NewBackupKeyScannerAtSnapshot(st, snapshot, ts, pageSize) +} + +// NewBackupKeyScannerAtSnapshot creates a key-only scanner from a captured route view. +func NewBackupKeyScannerAtSnapshot(st *ShardStore, snapshot BackupRouteSnapshot, ts uint64, pageSize int) BackupKeyScanner { + if pageSize <= 0 { + pageSize = defaultBackupScanPageSize + } + snapshot = cloneBackupRouteSnapshot(snapshot) + return &backupKeyScanner{ + store: st, + routes: snapshot.routes, + clampToRoutes: snapshot.clampToRoutes, + cursor: snapshot.start, + end: snapshot.end, + ts: ts, + pageSize: pageSize, + } +} + +func (s *ShardStore) NewBackupKeyScanner(start []byte, end []byte, ts uint64, pageSize int) BackupKeyScanner { + return NewBackupKeyScanner(s, start, end, ts, pageSize) +} + +func (s *ShardStore) NewBackupKeyScannerAtSnapshot(snapshot BackupRouteSnapshot, ts uint64, pageSize int) BackupKeyScanner { + return NewBackupKeyScannerAtSnapshot(s, snapshot, ts, pageSize) +} + func (s *backupScanner) Next(ctx context.Context) (*store.KVPair, bool, error) { if s.closed || s.store == nil { return nil, false, nil @@ -94,6 +174,63 @@ func (s *backupScanner) Close() error { return nil } +func (s *backupKeyScanner) Next(ctx context.Context) ([]byte, bool, error) { + if s.closed || s.store == nil { + return nil, false, nil + } + for s.index >= len(s.page) { + if err := s.loadNextPage(ctx); err != nil { + return nil, false, err + } + if len(s.page) == 0 { + if s.exhausted { + return nil, false, nil + } + continue + } + } + key := bytes.Clone(s.page[s.index].key) + s.index++ + return key, true, nil +} + +func (s *backupKeyScanner) Close() error { + s.closed = true + s.exhausted = true + s.page = nil + return nil +} + +func (s *backupKeyScanner) loadNextPage(ctx context.Context) error { + if s.exhausted { + s.page = nil + s.index = 0 + return nil + } + keys, err := s.store.scanKeyRoutesWithSourceAt(ctx, s.routes, s.cursor, s.end, s.pageSize, s.ts, s.clampToRoutes) + if err != nil { + return err + } + s.page = s.page[:0] + for _, item := range keys { + if _, ok := routeForRoutedKey(item, s.routes); ok { + s.page = append(s.page, item) + } + } + s.index = 0 + if len(keys) == 0 { + s.exhausted = true + return nil + } + last := lastRoutedScanKey(keys) + if last == nil { + s.exhausted = true + return nil + } + s.cursor = nextScanCursor(last) + return nil +} + func (s *backupScanner) loadNextPage(ctx context.Context) error { if s.exhausted { s.page = nil @@ -134,11 +271,15 @@ func (s *backupScanner) loadNextPage(ctx context.Context) error { } func (s *backupScanner) materializeRouteForKey(item routedScanKey) (distribution.Route, bool) { + return routeForRoutedKey(item, s.routes) +} + +func routeForRoutedKey(item routedScanKey, routes []distribution.Route) (distribution.Route, bool) { key := routeKey(item.key) if routeContainsKey(item.route, key) { return item.route, true } - for _, route := range s.routes { + for _, route := range routes { if route.GroupID != item.route.GroupID { continue } @@ -149,6 +290,25 @@ func (s *backupScanner) materializeRouteForKey(item routedScanKey) (distribution return distribution.Route{}, false } +func cloneBackupRouteSnapshot(snapshot BackupRouteSnapshot) BackupRouteSnapshot { + return BackupRouteSnapshot{ + routes: cloneBackupRoutes(snapshot.routes), + clampToRoutes: snapshot.clampToRoutes, + start: bytes.Clone(snapshot.start), + end: bytes.Clone(snapshot.end), + } +} + +func cloneBackupRoutes(routes []distribution.Route) []distribution.Route { + out := make([]distribution.Route, len(routes)) + for i, route := range routes { + out[i] = route + out[i].Start = bytes.Clone(route.Start) + out[i].End = bytes.Clone(route.End) + } + return out +} + func (s *ShardStore) scanKeyRoutesWithSourceAt( ctx context.Context, routes []distribution.Route, @@ -177,7 +337,7 @@ func (s *ShardStore) scanKeyRoutesWithSourceAt( if err != nil { return nil, err } - out = s.mergeAndTrimRoutedScanKeys(out, routedScanKeys(route, keys), limit) + out = mergeAndTrimRoutedScanKeys(out, routedScanKeys(route, keys), routes, limit) if clampToRoutes && len(out) >= limit { break } @@ -196,7 +356,12 @@ func routedScanKeys(route distribution.Route, keys [][]byte) []routedScanKey { return items } -func (s *ShardStore) mergeAndTrimRoutedScanKeys(out []routedScanKey, keys []routedScanKey, limit int) []routedScanKey { +func mergeAndTrimRoutedScanKeys( + out []routedScanKey, + keys []routedScanKey, + routes []distribution.Route, + limit int, +) []routedScanKey { if len(keys) == 0 { return out } @@ -210,7 +375,7 @@ func (s *ShardStore) mergeAndTrimRoutedScanKeys(out []routedScanKey, keys []rout continue } if write > 0 && bytes.Equal(out[write-1].key, item.key) { - out[write-1] = s.preferredRoutedScanKey(out[write-1], item) + out[write-1] = preferredRoutedScanKey(out[write-1], item, routes) continue } out[write] = item @@ -225,18 +390,13 @@ func (s *ShardStore) mergeAndTrimRoutedScanKeys(out []routedScanKey, keys []rout return out[:limit] } -func (s *ShardStore) preferredRoutedScanKey(current routedScanKey, candidate routedScanKey) routedScanKey { - if !fskeys.IsUsageRouteKey(candidate.key) { +func preferredRoutedScanKey(current, candidate routedScanKey, routes []distribution.Route) routedScanKey { + _, currentOwned := routeForRoutedKey(current, routes) + _, candidateOwned := routeForRoutedKey(candidate, routes) + if candidateOwned && !currentOwned { return candidate } - owner, ok := s.engine.GetRoute(routeKey(candidate.key)) - if !ok { - return candidate - } - if current.route.GroupID == owner.GroupID { - return current - } - return candidate + return current } func lastRoutedScanKey(keys []routedScanKey) []byte { diff --git a/kv/compactor_test.go b/kv/compactor_test.go index 7fdb1b3c2..3c13f1b8a 100644 --- a/kv/compactor_test.go +++ b/kv/compactor_test.go @@ -146,6 +146,38 @@ func TestFSMCompactorRespectsPinnedTimestamp(t *testing.T) { require.Equal(t, []byte("v20"), val) } +func TestBeginBackupBlocksCompactor(t *testing.T) { + st := store.NewMVCCStore() + ctx := context.Background() + require.NoError(t, st.PutAt(ctx, []byte("k"), []byte("v10"), 10, 0)) + require.NoError(t, st.PutAt(ctx, []byte("k"), []byte("v20"), 20, 0)) + require.NoError(t, st.PutAt(ctx, []byte("k"), []byte("v30"), 30, 0)) + + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + fsm, ok := NewKvFSMWithHLCAndTracker(st, NewHLC(), tracker, WithRouteHistory(nil, 1)).(*kvFSM) + require.True(t, ok) + require.NoError(t, haltApplyOf(fsm.Apply(EncodeBackupPinEntry(BackupPinEntry{ + PinID: backupTrackerTestPinID(7), ReadTS: 20, Deadline: time.Now().Add(time.Hour), + })))) + + compactor := NewFSMCompactor( + []FSMCompactRuntime{{ + GroupID: 1, + StatusReader: fakeRaftStatus{status: raftengine.Status{ + State: raftengine.StateFollower, AppliedIndex: 10, CommitIndex: 10, + }}, + Store: st, + }}, + WithFSMCompactorInterval(time.Hour), + WithFSMCompactorRetentionWindow(time.Millisecond), + WithFSMCompactorActiveTimestampTracker(tracker), + ) + require.NoError(t, compactor.SyncOnce(ctx)) + value, err := st.GetAt(ctx, []byte("k"), 20) + require.NoError(t, err) + require.Equal(t, []byte("v20"), value) +} + func TestFSMCompactorScopesBackupPinsByGroup(t *testing.T) { ctx := context.Background() stores := map[uint64]store.MVCCStore{1: store.NewMVCCStore(), 2: store.NewMVCCStore()} diff --git a/kv/fsm_backup.go b/kv/fsm_backup.go index 6b2c151a3..aee16aaed 100644 --- a/kv/fsm_backup.go +++ b/kv/fsm_backup.go @@ -12,27 +12,42 @@ func (f *kvFSM) applyBackup(data []byte) any { if err != nil { return haltErr(errors.Wrap(errors.Mark(err, ErrBackupApply), "kv/fsm: decode backup entry")) } + err = f.applyDecodedBackup(entry) + if err == nil { + return nil + } + if errors.Is(err, ErrInvalidBackupPin) || errors.Is(err, ErrTooManyActiveBackups) { + return err + } + return haltErr(errors.Wrap(errors.Mark(err, ErrBackupApply), "kv/fsm: apply backup entry")) +} + +func (f *kvFSM) applyDecodedBackup(entry backupEntry) error { switch entry.subtype { case backupSubtypePin: - err = f.readTracker.ApplyPinWithDeadlineForGroup(entry.pin.PinID, f.shardGroupID, entry.pin.ReadTS, entry.pin.Deadline) + err := f.readTracker.ApplyPinWithDeadlineForGroup(entry.pin.PinID, f.shardGroupID, entry.pin.ReadTS, entry.pin.Deadline) + return f.observeBackupReadTimestamp(entry.pin.ReadTS, err) case backupSubtypeExtend: - err = f.readTracker.ApplyExtendForGroup( - entry.extend.PinID, - f.shardGroupID, - entry.extend.ReadTS, - entry.extend.Deadline, - ) + return f.readTracker.ApplyExtendForGroup(entry.extend.PinID, f.shardGroupID, entry.extend.Deadline) case backupSubtypeRelease: f.readTracker.ReleaseBackupPinForGroup(entry.release.PinID, f.shardGroupID) return nil + case backupSubtypeReserve: + err := f.readTracker.PinWithDeadline(entry.pin.PinID, entry.pin.ReadTS, entry.pin.Deadline) + return f.observeBackupReadTimestamp(entry.pin.ReadTS, err) + case backupSubtypeUnreserve: + f.readTracker.ReleaseBackupPin(entry.release.PinID) + return nil default: - err = ErrBackupWireSubtype + return ErrBackupWireSubtype } - if err != nil { - if errors.Is(err, ErrInvalidBackupPin) { - return err - } - return haltErr(errors.Wrap(errors.Mark(err, ErrBackupApply), "kv/fsm: apply backup entry")) +} + +func (f *kvFSM) observeBackupReadTimestamp(readTS uint64, applyErr error) error { + if applyErr == nil && f.hlc != nil { + // The pin entry is also the per-group timestamp barrier. Once it + // applies, later writes on this replica must issue above read_ts. + f.hlc.Observe(readTS) } - return nil + return applyErr } diff --git a/kv/fsm_backup_test.go b/kv/fsm_backup_test.go index f6915c326..d6b584ff9 100644 --- a/kv/fsm_backup_test.go +++ b/kv/fsm_backup_test.go @@ -27,7 +27,6 @@ func TestApplyBackupUsesSharedTracker(t *testing.T) { require.NoError(t, haltApplyOf(fsm.Apply(EncodeBackupExtendEntry(BackupExtendEntry{ PinID: pinID, - ReadTS: 42, Deadline: secondDeadline, })))) gotDeadline, ok := tracker.BackupPinDeadline(pinID) @@ -97,20 +96,18 @@ func TestApplyBackupLimitDoesNotDropCommittedPins(t *testing.T) { require.Equal(t, uint64(42), tracker.Oldest()) } -func TestApplyBackupMissingExtendRestoresCommittedFence(t *testing.T) { +func TestApplyBackupMissingExtendIsNoop(t *testing.T) { tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) fsm := newBackupTestFSM(t, tracker) resp := fsm.Apply(EncodeBackupExtendEntry(BackupExtendEntry{ PinID: backupTrackerTestPinID(1), - ReadTS: 42, Deadline: time.Now().Add(time.Hour), })) require.NoError(t, haltApplyOf(resp)) require.Nil(t, resp) - require.Equal(t, 1, tracker.ActiveBackupPinCount()) - require.Equal(t, uint64(42), tracker.Oldest()) + require.Equal(t, 0, tracker.ActiveBackupPinCount()) } func TestApplyBackupExpiredExtendRestoresCommittedFence(t *testing.T) { @@ -118,12 +115,9 @@ func TestApplyBackupExpiredExtendRestoresCommittedFence(t *testing.T) { fsm := newBackupTestFSM(t, tracker) pinID := backupTrackerTestPinID(1) require.NoError(t, tracker.PinWithDeadline(pinID, 42, time.Now().Add(-time.Millisecond))) - tracker.reapExpiredBackupPins(time.Now()) - require.Equal(t, 0, tracker.ActiveBackupPinCount()) resp := fsm.Apply(EncodeBackupExtendEntry(BackupExtendEntry{ PinID: pinID, - ReadTS: 42, Deadline: time.Now().Add(time.Hour), })) @@ -133,6 +127,48 @@ func TestApplyBackupExpiredExtendRestoresCommittedFence(t *testing.T) { require.Equal(t, uint64(42), tracker.Oldest()) } +func TestApplyBackupObservesPinnedReadTimestamp(t *testing.T) { + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + hlc := NewHLC() + fsm, ok := NewKvFSMWithHLCAndTracker( + store.NewMVCCStore(), hlc, tracker, WithRouteHistory(nil, 1), + ).(*kvFSM) + require.True(t, ok) + readTS := hlc.Next() + 10_000 + + require.NoError(t, haltApplyOf(fsm.Apply(EncodeBackupPinEntry(BackupPinEntry{ + PinID: backupTrackerTestPinID(1), ReadTS: readTS, Deadline: time.Now().Add(time.Hour), + })))) + require.GreaterOrEqual(t, hlc.Current(), readTS) +} + +func TestApplyBackupReserveEnforcesCapacityAndUnreserveReleases(t *testing.T) { + tracker := NewActiveTimestampTracker( + WithActiveTimestampTrackerSweepInterval(0), + WithActiveTimestampTrackerMaxBackupPins(1), + ) + fsm := newBackupTestFSM(t, tracker) + deadline := time.Now().Add(time.Hour) + first := backupTrackerTestPinID(1) + second := backupTrackerTestPinID(2) + + require.NoError(t, haltApplyOf(fsm.Apply(EncodeBackupReserveEntry(BackupReserveEntry{ + PinID: first, ReadTS: 42, Deadline: deadline, + })))) + resp := fsm.Apply(EncodeBackupReserveEntry(BackupReserveEntry{ + PinID: second, ReadTS: 43, Deadline: deadline, + })) + require.NoError(t, haltApplyOf(resp)) + respErr, ok := resp.(error) + require.True(t, ok) + require.ErrorIs(t, respErr, ErrTooManyActiveBackups) + + require.NoError(t, haltApplyOf(fsm.Apply(EncodeBackupUnreserveEntry(BackupUnreserveEntry{PinID: first})))) + require.NoError(t, haltApplyOf(fsm.Apply(EncodeBackupReserveEntry(BackupReserveEntry{ + PinID: second, ReadTS: 43, Deadline: deadline, + })))) +} + func TestApplyBackupZeroDeadlineReturnsNonFatalError(t *testing.T) { fsm := newBackupTestFSM(t, NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0))) diff --git a/kv/leader_admin_proposer.go b/kv/leader_admin_proposer.go new file mode 100644 index 000000000..8b6d41334 --- /dev/null +++ b/kv/leader_admin_proposer.go @@ -0,0 +1,135 @@ +package kv + +import ( + "context" + "time" + + "github.com/bootjp/elastickv/internal/raftengine" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" +) + +// leaderAdminProposer forwards ProposeAdmin calls to the current group leader. +// Propose remains local because only barrier-exempt, idempotent admin entries +// are safe to retry across a leadership change. +type leaderAdminProposer struct { + leader raftengine.LeaderView + local raftengine.Proposer + connCache *GRPCConnCache +} + +func NewLeaderAdminProposer( + leader raftengine.LeaderView, + local raftengine.Proposer, + connCache *GRPCConnCache, +) raftengine.Proposer { + return &leaderAdminProposer{leader: leader, local: local, connCache: connCache} +} + +func (p *leaderAdminProposer) Propose( + ctx context.Context, + data []byte, +) (*raftengine.ProposalResult, error) { + if p.local == nil { + return nil, errors.New("local proposer is unavailable") + } + result, err := p.local.Propose(ctx, data) + return result, errors.WithStack(err) +} + +func (p *leaderAdminProposer) ProposeAdmin( + ctx context.Context, + data []byte, +) (*raftengine.ProposalResult, error) { + if p.local == nil || p.leader == nil { + return nil, errors.New("leader admin proposer is unavailable") + } + if p.leader.State() == raftengine.StateLeader { + if err := p.leader.VerifyLeader(ctx); err == nil { + result, err := p.local.ProposeAdmin(ctx, data) + if err == nil || !isTransientLeaderError(err) { + return result, errors.WithStack(err) + } + } + } + return p.forwardAdminWithRetry(ctx, data) +} + +func (p *leaderAdminProposer) forwardAdminWithRetry( + callerCtx context.Context, + data []byte, +) (*raftengine.ProposalResult, error) { + deadline := time.Now().Add(leaderProxyRetryBudget) + parentCtx, cancel := context.WithDeadline(callerCtx, deadline) + defer cancel() + + var lastErr error + for { + result, err, done := p.runAdminForwardCycle(parentCtx, data, deadline) + if done { + return result, err + } + lastErr = err + if lastErr == nil { + return nil, errors.WithStack(ErrLeaderNotFound) + } + if !time.Now().Before(deadline) { + return nil, lastErr + } + waitLeaderProxyBackoff(parentCtx, leaderProxyRetryInterval, deadline) + if !time.Now().Before(deadline) { + return nil, lastErr + } + } +} + +func (p *leaderAdminProposer) runAdminForwardCycle( + ctx context.Context, + data []byte, + deadline time.Time, +) (*raftengine.ProposalResult, error, bool) { + var lastErr error + for attempt := 0; attempt < maxForwardRetries; attempt++ { + if !time.Now().Before(deadline) { + break + } + result, err := p.forwardAdmin(ctx, data) + if err == nil { + return result, nil, true + } + lastErr = err + if isTransientLeaderError(err) { + break + } + } + if lastErr != nil && !isTransientLeaderError(lastErr) { + return nil, errors.Wrapf(lastErr, "leader admin proposal failed after %d retries", maxForwardRetries), true + } + return nil, lastErr, false +} + +func (p *leaderAdminProposer) forwardAdmin( + parentCtx context.Context, + data []byte, +) (*raftengine.ProposalResult, error) { + addr := leaderAddrFromEngine(p.leader) + if addr == "" { + return nil, errors.WithStack(ErrLeaderNotFound) + } + if p.connCache == nil { + return nil, errors.New("leader admin proposer connection cache is unavailable") + } + conn, err := p.connCache.ConnFor(addr) + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(parentCtx, leaderForwardTimeout) + defer cancel() + resp, err := pb.NewInternalClient(conn).ForwardAdminProposal(ctx, &pb.ForwardAdminProposalRequest{Payload: data}) + if err != nil { + return nil, errors.WithStack(err) + } + return &raftengine.ProposalResult{CommitIndex: resp.GetCommitIndex()}, nil +} + +var _ raftengine.Proposer = (*leaderAdminProposer)(nil) diff --git a/kv/leader_admin_proposer_test.go b/kv/leader_admin_proposer_test.go new file mode 100644 index 000000000..13c7d1c77 --- /dev/null +++ b/kv/leader_admin_proposer_test.go @@ -0,0 +1,109 @@ +package kv + +import ( + "context" + "net" + "sync" + "testing" + + "github.com/bootjp/elastickv/internal/raftengine" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +type adminProposalLeaderView struct { + state raftengine.State + addr string +} + +func (v *adminProposalLeaderView) State() raftengine.State { return v.state } +func (v *adminProposalLeaderView) Leader() raftengine.LeaderInfo { + return raftengine.LeaderInfo{ID: "leader", Address: v.addr} +} +func (v *adminProposalLeaderView) VerifyLeader(context.Context) error { return nil } +func (v *adminProposalLeaderView) LinearizableRead(context.Context) (uint64, error) { + return 0, nil +} + +type recordingAdminProposer struct { + mu sync.Mutex + adminCalls int + payload []byte +} + +func (p *recordingAdminProposer) Propose(context.Context, []byte) (*raftengine.ProposalResult, error) { + return &raftengine.ProposalResult{CommitIndex: 1}, nil +} + +func (p *recordingAdminProposer) ProposeAdmin(_ context.Context, data []byte) (*raftengine.ProposalResult, error) { + p.mu.Lock() + defer p.mu.Unlock() + p.adminCalls++ + p.payload = append([]byte(nil), data...) + return &raftengine.ProposalResult{CommitIndex: 7}, nil +} + +type forwardingAdminServer struct { + pb.UnimplementedInternalServer + + mu sync.Mutex + payload []byte +} + +func (s *forwardingAdminServer) ForwardAdminProposal( + _ context.Context, + req *pb.ForwardAdminProposalRequest, +) (*pb.ForwardAdminProposalResponse, error) { + s.mu.Lock() + s.payload = append([]byte(nil), req.GetPayload()...) + s.mu.Unlock() + return &pb.ForwardAdminProposalResponse{CommitIndex: 123}, nil +} + +func TestLeaderAdminProposerUsesLocalVerifiedLeader(t *testing.T) { + t.Parallel() + local := &recordingAdminProposer{} + proposer := NewLeaderAdminProposer( + &adminProposalLeaderView{state: raftengine.StateLeader}, + local, + &GRPCConnCache{}, + ) + + result, err := proposer.ProposeAdmin(context.Background(), []byte("pin")) + require.NoError(t, err) + require.Equal(t, uint64(7), result.CommitIndex) + require.Equal(t, 1, local.adminCalls) + require.Equal(t, []byte("pin"), local.payload) +} + +func TestLeaderAdminProposerForwardsFollowerProposal(t *testing.T) { + t.Parallel() + var lc net.ListenConfig + lis, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + service := &forwardingAdminServer{} + server := grpc.NewServer() + pb.RegisterInternalServer(server, service) + go func() { _ = server.Serve(lis) }() + t.Cleanup(func() { + server.Stop() + _ = lis.Close() + }) + + cache := &GRPCConnCache{} + t.Cleanup(func() { require.NoError(t, cache.Close()) }) + local := &recordingAdminProposer{} + proposer := NewLeaderAdminProposer( + &adminProposalLeaderView{state: raftengine.StateFollower, addr: lis.Addr().String()}, + local, + cache, + ) + result, err := proposer.ProposeAdmin(context.Background(), []byte("pin")) + require.NoError(t, err) + require.Equal(t, uint64(123), result.CommitIndex) + require.Zero(t, local.adminCalls) + service.mu.Lock() + require.Equal(t, []byte("pin"), service.payload) + service.mu.Unlock() +} diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index ba9c16944..ae0302f36 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -585,6 +585,90 @@ func TestBackupScannerPaging(t *testing.T) { require.Equal(t, [][]byte{[]byte("a"), []byte("b"), []byte("c"), []byte("x"), []byte("z")}, got) } +func TestBackupKeyScannerPaging(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + engine.UpdateRoute([]byte(""), []byte("m"), 1) + engine.UpdateRoute([]byte("m"), nil, 2) + groups := map[uint64]*ShardGroup{ + 1: {Store: store.NewMVCCStore()}, + 2: {Store: store.NewMVCCStore()}, + } + st := NewShardStore(engine, groups) + commitTS := uint64(1) + for _, key := range [][]byte{[]byte("a"), []byte("b"), []byte("x"), []byte("z")} { + require.NoError(t, st.PutAt(ctx, key, []byte("value-not-materialized"), commitTS, 0)) + commitTS++ + } + + sc := st.NewBackupKeyScanner([]byte(""), nil, ^uint64(0), 2) + defer sc.Close() + var got [][]byte + for { + key, ok, err := sc.Next(ctx) + require.NoError(t, err) + if !ok { + break + } + got = append(got, key) + } + require.Equal(t, [][]byte{[]byte("a"), []byte("b"), []byte("x"), []byte("z")}, got) +} + +func TestBackupKeyScannerFiltersStaleOffRouteKeys(t *testing.T) { + t.Parallel() + + ctx := context.Background() + groups := map[uint64]*ShardGroup{1: {Store: store.NewMVCCStore()}} + st := NewShardStore(distribution.NewEngine(), groups) + require.NoError(t, groups[1].Store.PutAt(ctx, []byte("a"), []byte("stale"), 1, 0)) + require.NoError(t, groups[1].Store.PutAt(ctx, []byte("x"), []byte("owned"), 2, 0)) + scanner := &backupKeyScanner{ + store: st, routes: []distribution.Route{{Start: []byte("m"), GroupID: 1}}, + cursor: []byte(""), ts: ^uint64(0), pageSize: 1, + } + defer scanner.Close() + + key, ok, err := scanner.Next(ctx) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, []byte("x"), key) + _, ok, err = scanner.Next(ctx) + require.NoError(t, err) + require.False(t, ok) +} + +func TestBackupScannerReusesCapturedRouteSnapshot(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + engine.UpdateRoute([]byte(""), nil, 1) + groups := map[uint64]*ShardGroup{ + 1: {Store: store.NewMVCCStore()}, + 2: {Store: store.NewMVCCStore()}, + } + st := NewShardStore(engine, groups) + require.NoError(t, groups[1].Store.PutAt(ctx, []byte("a"), []byte("pinned-owner"), 1, 0)) + snapshot := st.CaptureBackupRouteSnapshot(nil, nil) + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), GroupID: 2, State: distribution.RouteStateActive}, + }, + })) + require.NoError(t, groups[2].Store.PutAt(ctx, []byte("a"), []byte("live-owner"), 2, 0)) + + scanner := st.NewBackupScannerAtSnapshot(snapshot, ^uint64(0), 1) + defer scanner.Close() + pair, ok, err := scanner.Next(ctx) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, []byte("pinned-owner"), pair.Value) +} + func TestBackupScannerMaterializesFromCapturedRoute(t *testing.T) { t.Parallel() @@ -631,7 +715,7 @@ func TestBackupScannerMaterializesFromEnumeratedRoute(t *testing.T) { 2: {Store: store.NewMVCCStore()}, } st := NewShardStore(engine, groups) - require.NoError(t, groups[1].Store.PutAt(ctx, []byte("a"), []byte("stale-first-route"), 1, 0)) + require.NoError(t, groups[1].Store.PutAt(ctx, []byte("a"), []byte("first-enumerated-route"), 1, 0)) require.NoError(t, groups[2].Store.PutAt(ctx, []byte("a"), []byte("enumerated-route"), 2, 0)) sc := &backupScanner{ @@ -648,7 +732,48 @@ func TestBackupScannerMaterializesFromEnumeratedRoute(t *testing.T) { require.NoError(t, err) require.True(t, ok) require.Equal(t, []byte("a"), kvp.Key) - require.Equal(t, []byte("enumerated-route"), kvp.Value) + require.Equal(t, []byte("first-enumerated-route"), kvp.Value) +} + +func TestBackupScannersPreferCapturedOwnerForDuplicateKey(t *testing.T) { + t.Parallel() + + ctx := context.Background() + groups := map[uint64]*ShardGroup{ + 1: {Store: store.NewMVCCStore()}, + 2: {Store: store.NewMVCCStore()}, + } + st := NewShardStore(distribution.NewEngine(), groups) + require.NoError(t, groups[2].Store.PutAt(ctx, []byte("a"), []byte("captured-owner"), 1, 0)) + require.NoError(t, groups[1].Store.PutAt(ctx, []byte("a"), []byte("stale-owner"), 1, 0)) + routes := []distribution.Route{ + {RouteID: 2, Start: []byte(""), End: []byte("m"), GroupID: 2}, + {RouteID: 1, Start: []byte("m"), GroupID: 1}, + } + + keyScanner := &backupKeyScanner{ + store: st, routes: routes, clampToRoutes: false, + cursor: []byte(""), ts: ^uint64(0), pageSize: 1, + } + defer keyScanner.Close() + key, ok, err := keyScanner.Next(ctx) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, []byte("a"), key) + _, ok, err = keyScanner.Next(ctx) + require.NoError(t, err) + require.False(t, ok) + + valueScanner := &backupScanner{ + store: st, routes: routes, clampToRoutes: false, + cursor: []byte(""), ts: ^uint64(0), pageSize: 1, + } + defer valueScanner.Close() + pair, ok, err := valueScanner.Next(ctx) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, []byte("a"), pair.Key) + require.Equal(t, []byte("captured-owner"), pair.Value) } func TestBackupScannerPreservesFullRoutingAfterListKeyCursor(t *testing.T) { diff --git a/main.go b/main.go index 1d05aa288..3eeddc69a 100644 --- a/main.go +++ b/main.go @@ -55,12 +55,20 @@ const ( etcdMaxSizePerMsg = 1 << 20 etcdMaxInflightMsg = 1024 defaultTSOBatchSize = 256 + defaultBackupTTL = 30 * time.Minute + defaultBackupMaxTTL = time.Hour + defaultBackupDeadline = 5 * time.Second + defaultBackupHeadroom = 1000 + defaultBackupPageSize = 1024 + defaultBackupPinLimit = 4 defaultFilesystemRootMode = 0o755 defaultFilesystemPlacementScanInterval = 30 * time.Second defaultFilesystemLeaseReapInterval = 30 * time.Second ) +var errInvalidLiveBackupConfig = errors.New("invalid live backup configuration") + func newRaftFactory(engineType raftEngineType, coldStartObs raftengine.ColdStartObserver) (raftengine.Factory, error) { switch engineType { case raftEngineEtcd: @@ -160,8 +168,17 @@ var ( // below — both can be enabled simultaneously, and operators can pick // whichever auth path they need (gRPC bearer token vs. HTTP cookies + // SigV4 access keys). - adminTokenFile = flag.String("adminTokenFile", "", "Path to a file containing the read-only bearer token required on the Admin gRPC service (leave blank with --adminInsecureNoAuth off to disable the Admin service)") - adminInsecureNoAuth = flag.Bool("adminInsecureNoAuth", false, "Register the Admin gRPC service without bearer-token authentication; development only") + adminTokenFile = flag.String("adminTokenFile", "", "Path to a file containing the bearer token required on the Admin gRPC service (leave blank with --adminInsecureNoAuth off to disable the Admin service)") + adminInsecureNoAuth = flag.Bool("adminInsecureNoAuth", false, "Register the Admin gRPC service without bearer-token authentication; development only") + backupDefaultTTL = flag.Duration("backupDefaultTTL", defaultBackupTTL, "Default live-backup pin TTL") + backupMaxTTL = flag.Duration("backupMaxTTL", defaultBackupMaxTTL, "Maximum live-backup pin TTL per renewal") + backupBeginDeadline = flag.Duration("backupBeginDeadline", defaultBackupDeadline, "Deadline for live-backup peer gates and pin fan-out") + backupSnapshotHeadroomEntries = flag.Uint64( + "backupSnapshotHeadroomEntries", defaultBackupHeadroom, + "Minimum Raft entries remaining before the next snapshot when a live backup begins", + ) + backupScanPageSize = flag.Int("backupScanPageSize", defaultBackupPageSize, "Count-only scan page size used by live-backup baselines") + backupMaxActivePins = flag.Int("backupMaxActivePins", defaultBackupPinLimit, "Maximum concurrent logical live-backup pins") // Admin HTTP listener flags (PR #545's parallel work merged into // main; serves the cookie/SigV4-authenticated admin dashboard). @@ -398,7 +415,7 @@ func startFSMCompactorIfEnabled(ctx context.Context, eg *errgroup.Group, runtime } func run() error { - cfg, engineType, bootstrapCfg, bootstrap, err := resolveRuntimeInputs() + cfg, engineType, bootstrapCfg, bootstrap, err := resolveValidatedRuntimeInputs() if err != nil { return err } @@ -446,7 +463,7 @@ func run() error { } keystore := encryption.NewKeystore() redisApplyObserver := adapter.NewRedisApplyObserver() - readTracker := kv.NewActiveTimestampTracker() + readTracker := kv.NewActiveTimestampTracker(kv.WithActiveTimestampTrackerMaxBackupPins(*backupMaxActivePins)) // Stage 6D-6c: buildShardGroupsWithEncryptionWiring assembles the // storage-envelope write-path wiring (cipher + deterministic nonce @@ -619,6 +636,47 @@ func run() error { return nil } +func resolveValidatedRuntimeInputs() (runtimeConfig, raftEngineType, raftBootstrapConfig, bool, error) { + cfg, engineType, bootstrapCfg, bootstrap, err := resolveRuntimeInputs() + if err != nil { + return runtimeConfig{}, "", raftBootstrapConfig{}, false, err + } + err = validateLiveBackupConfig( + *backupDefaultTTL, *backupMaxTTL, *backupBeginDeadline, + *backupSnapshotHeadroomEntries, *backupScanPageSize, *backupMaxActivePins, + ) + return cfg, engineType, bootstrapCfg, bootstrap, err +} + +func validateLiveBackupConfig( + defaultTTL time.Duration, + maxTTL time.Duration, + beginDeadline time.Duration, + snapshotHeadroom uint64, + scanPageSize int, + maxActivePins int, +) error { + const minTTL = time.Minute + switch { + case defaultTTL < minTTL: + return errors.Wrapf(errInvalidLiveBackupConfig, "backupDefaultTTL must be at least %s", minTTL) + case maxTTL < minTTL: + return errors.Wrapf(errInvalidLiveBackupConfig, "backupMaxTTL must be at least %s", minTTL) + case defaultTTL > maxTTL: + return errors.Wrap(errInvalidLiveBackupConfig, "backupDefaultTTL must not exceed backupMaxTTL") + case beginDeadline <= 0: + return errors.Wrap(errInvalidLiveBackupConfig, "backupBeginDeadline must be positive") + case snapshotHeadroom == 0: + return errors.Wrap(errInvalidLiveBackupConfig, "backupSnapshotHeadroomEntries must be positive") + case scanPageSize <= 0: + return errors.Wrap(errInvalidLiveBackupConfig, "backupScanPageSize must be positive") + case maxActivePins <= 0: + return errors.Wrap(errInvalidLiveBackupConfig, "backupMaxActivePins must be positive") + default: + return nil + } +} + func startRaftEngineLifecycleWatchers(ctx context.Context, eg *errgroup.Group, runtimes []*raftGroupRuntime) { for _, rt := range runtimes { if rt == nil { @@ -1813,7 +1871,10 @@ func startServersAfterStartupRotation(waitRotateOnStartup startupRotationWaiter, // connCache is also used by the gRPC Admin GetNodeVersion probe, so create // it when either admin HTTP or gRPC Admin is enabled. connCache = prepareAdminConnCache(in.ctx, in.eg, *adminEnabled || adminGRPCEnabled) - adminServer, adminGRPCOpts, err := setupAdminService(*raftId, *myAddr, in.runtimes, in.bootstrapServers, in.keyvizSampler, connCache) + adminServer, adminGRPCOpts, err := setupAdminService( + *raftId, *myAddr, in.runtimes, in.shardGroups, in.bootstrapServers, + in.shardStore, in.coordinate, in.readTracker, in.keyvizSampler, connCache, + ) if err != nil { return err } @@ -2051,7 +2112,11 @@ func startHLCLeaseRenewal(ctx context.Context, eg *errgroup.Group, coordinate kv func setupAdminService( nodeID, grpcAddress string, runtimes []*raftGroupRuntime, + shardGroups map[uint64]*kv.ShardGroup, bootstrapServers []raftengine.Server, + shardStore *kv.ShardStore, + coordinate kv.Coordinator, + readTracker *kv.ActiveTimestampTracker, keyvizSampler *keyviz.MemSampler, connCache *kv.GRPCConnCache, ) (*adapter.AdminServer, adminGRPCInterceptors, error) { @@ -2068,6 +2133,9 @@ func setupAdminService( adapter.NodeIdentity{NodeID: nodeID, GRPCAddress: selfAddr}, members, connCache, + adminBackupDependencies{ + store: shardStore, coordinate: coordinate, tracker: readTracker, + }, ) if err != nil { return nil, adminGRPCInterceptors{}, err @@ -2077,6 +2145,9 @@ func setupAdminService( } for _, rt := range runtimes { srv.RegisterGroup(rt.spec.id, rt.engine) + if group := shardGroups[rt.spec.id]; group != nil { + srv.RegisterBackupProposer(rt.spec.id, kv.NewLeaderAdminProposer(rt.engine, group.Proposer(), connCache)) + } } // Only register a real sampler. Passing a typed-nil *MemSampler // would store a non-nil interface and make GetKeyVizMatrix @@ -2284,31 +2355,28 @@ func startupRotationGatedMethod(fullMethod string) bool { // service is intentionally disabled. It is mutually exclusive with // --adminInsecureNoAuth so operators have to opt into the unauthenticated // mode explicitly. +type adminBackupDependencies struct { + store *kv.ShardStore + coordinate kv.Coordinator + tracker *kv.ActiveTimestampTracker +} + func configureAdminService( tokenPath string, insecureNoAuth bool, self adapter.NodeIdentity, members []adapter.NodeIdentity, connCache *kv.GRPCConnCache, + backupDeps ...adminBackupDependencies, ) (*adapter.AdminServer, adminGRPCInterceptors, error) { - if tokenPath == "" && !insecureNoAuth { - return nil, adminGRPCInterceptors{}, nil - } - if tokenPath != "" && insecureNoAuth { - return nil, adminGRPCInterceptors{}, errors.New("--adminInsecureNoAuth and --adminTokenFile are mutually exclusive") - } - token := "" - if tokenPath != "" { - loaded, err := loadAdminTokenFile(tokenPath) - if err != nil { - return nil, adminGRPCInterceptors{}, err - } - token = loaded + token, enabled, err := resolveAdminGRPCToken(tokenPath, insecureNoAuth) + if err != nil { + return nil, adminGRPCInterceptors{}, err } - opts := []adapter.AdminOption{adapter.WithAdminNodeVersion(buildVersion())} - if probe := adminLeaderVersionProbe(connCache); probe != nil { - opts = append(opts, adapter.WithAdminLeaderVersionProbe(probe)) + if !enabled { + return nil, adminGRPCInterceptors{}, nil } + opts := adminServerOptions(token, insecureNoAuth, connCache, backupDeps) srv := adapter.NewAdminServer(self, members, opts...) srv.SetCapability(adapter.S3BlobOffloadCapabilityName, adapter.S3BlobOffloadLocalCapability()) unary, stream := adapter.AdminTokenAuth(token) @@ -2322,6 +2390,94 @@ func configureAdminService( return srv, icept, nil } +func resolveAdminGRPCToken(tokenPath string, insecureNoAuth bool) (string, bool, error) { + switch { + case tokenPath == "" && !insecureNoAuth: + return "", false, nil + case tokenPath != "" && insecureNoAuth: + return "", false, errors.New("--adminInsecureNoAuth and --adminTokenFile are mutually exclusive") + case insecureNoAuth: + return "", true, nil + default: + token, err := loadAdminTokenFile(tokenPath) + return token, err == nil, err + } +} + +func adminServerOptions( + token string, + insecureNoAuth bool, + connCache *kv.GRPCConnCache, + backupDeps []adminBackupDependencies, +) []adapter.AdminOption { + opts := []adapter.AdminOption{adapter.WithAdminNodeVersion(buildVersion())} + if probe := adminLeaderVersionProbe(connCache); probe != nil { + opts = append(opts, adapter.WithAdminLeaderVersionProbe(probe)) + } + if len(backupDeps) > 0 { + deps := backupDeps[0] + tokenKey := []byte(token) + if insecureNoAuth { + tokenKey = []byte("elastickv-insecure-live-backup-token-v1") + } + opts = append(opts, + adapter.WithAdminBackupControl( + deps.store, + adminBackupReadFence(deps.coordinate, deps.store), + adminBackupPeerProbe(connCache), + deps.tracker, + tokenKey, + ), + adapter.WithAdminBackupConfig(adapter.AdminBackupConfig{ + DefaultTTL: *backupDefaultTTL, + MinTTL: time.Minute, + MaxTTL: *backupMaxTTL, + BeginDeadline: *backupBeginDeadline, + SnapshotHeadroomEntries: *backupSnapshotHeadroomEntries, + ScanPageSize: *backupScanPageSize, + }), + ) + } + return opts +} + +func adminBackupReadFence(coordinate kv.Coordinator, shardStore *kv.ShardStore) adapter.BackupReadFence { + if coordinate == nil || shardStore == nil || coordinate.Clock() == nil { + return nil + } + return func(ctx context.Context) (uint64, error) { + clock := coordinate.Clock() + if err := kv.LeaseReadAllGroupsThrough(coordinate, ctx); err != nil { + return 0, errors.Wrap(err, "backup: fence raft groups") + } + clock.Observe(shardStore.LastCommitTS()) + readTS, err := clock.NextFenced() + if err != nil { + return 0, errors.Wrap(err, "backup: issue read timestamp") + } + return readTS, nil + } +} + +func adminBackupPeerProbe(connCache *kv.GRPCConnCache) adapter.BackupPeerProbe { + if connCache == nil { + return nil + } + return func(ctx context.Context, address string) (adapter.BackupPeerVersion, error) { + conn, err := connCache.ConnFor(address) + if err != nil { + return adapter.BackupPeerVersion{}, errors.Wrap(err, "backup peer probe: dial peer") + } + resp, err := pb.NewAdminClient(conn).GetNodeVersion(ctx, &pb.GetNodeVersionRequest{}) + if err != nil { + return adapter.BackupPeerVersion{}, errors.Wrap(err, "backup peer probe: get node version") + } + return adapter.BackupPeerVersion{ + NodeVersion: resp.GetNodeVersion(), BackupProtocolVersion: resp.GetBackupProtocolVersion(), + }, nil + } +} + func adminLeaderVersionProbe(connCache *kv.GRPCConnCache) adapter.LeaderVersionProbe { if connCache == nil { return nil @@ -2547,7 +2703,7 @@ func startRaftServers( rt.engine, coordinate.Clock(), relay, - internalTimestampOptions(coordinate)..., + append(internalTimestampOptions(coordinate), adapter.WithInternalAdminProposer(proposerForGroup(rt, shardGroups)))..., )) pb.RegisterDistributionServer(gs, distServer) if adminServer != nil { diff --git a/main_admin_test.go b/main_admin_test.go index 98d52f857..5c597f628 100644 --- a/main_admin_test.go +++ b/main_admin_test.go @@ -23,10 +23,12 @@ import ( "time" "github.com/bootjp/elastickv/adapter" + "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/admin" "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" "google.golang.org/grpc" @@ -43,6 +45,68 @@ func TestConfigureAdminServiceDisabledByDefault(t *testing.T) { } } +func TestValidateLiveBackupConfig(t *testing.T) { + t.Parallel() + require.NoError(t, validateLiveBackupConfig(30*time.Minute, time.Hour, 5*time.Second, 1000, 1024, 4)) + tests := []struct { + name string + defaultTTL, maxTTL, beginDeadline time.Duration + headroom uint64 + pageSize, maxPins int + }{ + {name: "short default ttl", defaultTTL: time.Second, maxTTL: time.Hour, beginDeadline: time.Second, headroom: 1, pageSize: 1, maxPins: 1}, + {name: "default exceeds max", defaultTTL: 2 * time.Hour, maxTTL: time.Hour, beginDeadline: time.Second, headroom: 1, pageSize: 1, maxPins: 1}, + {name: "zero begin deadline", defaultTTL: time.Minute, maxTTL: time.Hour, headroom: 1, pageSize: 1, maxPins: 1}, + {name: "zero headroom", defaultTTL: time.Minute, maxTTL: time.Hour, beginDeadline: time.Second, pageSize: 1, maxPins: 1}, + {name: "zero page", defaultTTL: time.Minute, maxTTL: time.Hour, beginDeadline: time.Second, headroom: 1, maxPins: 1}, + {name: "zero pins", defaultTTL: time.Minute, maxTTL: time.Hour, beginDeadline: time.Second, headroom: 1, pageSize: 1}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + require.Error(t, validateLiveBackupConfig( + tc.defaultTTL, tc.maxTTL, tc.beginDeadline, tc.headroom, tc.pageSize, tc.maxPins, + )) + }) + } +} + +type backupFenceOrderCoordinator struct { + stubStartupCoordinator + timestampDuringBarrier uint64 + onBarrier func(uint64) +} + +func (c *backupFenceOrderCoordinator) LeaseReadAllGroups(context.Context) error { + ts, err := c.Clock().NextFenced() + if err != nil { + return err + } + c.timestampDuringBarrier = ts + if c.onBarrier != nil { + c.onBarrier(ts) + } + return nil +} + +func TestAdminBackupReadFenceAllocatesTimestampAfterBarrier(t *testing.T) { + t.Parallel() + coordinate := &backupFenceOrderCoordinator{} + groupStore := store.NewMVCCStore() + shardStore := kv.NewShardStore(distribution.NewEngineWithDefaultRoute(), map[uint64]*kv.ShardGroup{ + 1: {Store: groupStore}, + }) + coordinate.onBarrier = func(ts uint64) { + require.NoError(t, groupStore.PutAt(context.Background(), []byte("committed"), []byte("value"), ts, 0)) + } + fence := adminBackupReadFence(coordinate, shardStore) + require.NotNil(t, fence) + + readTS, err := fence(context.Background()) + require.NoError(t, err) + require.Equal(t, coordinate.timestampDuringBarrier, shardStore.LastCommitTS()) + require.Greater(t, readTS, coordinate.timestampDuringBarrier) +} + func TestConfigureAdminServiceRejectsMutualExclusion(t *testing.T) { t.Parallel() dir := t.TempDir() diff --git a/proto/admin.pb.go b/proto/admin.pb.go index 58165ee74..f6da3a84b 100644 --- a/proto/admin.pb.go +++ b/proto/admin.pb.go @@ -1178,12 +1178,13 @@ func (x *BackupShardApplied) GetAppliedIndex() uint64 { } type BackupExpectedKeys struct { - state protoimpl.MessageState `protogen:"open.v1"` - Adapter string `protobuf:"bytes,1,opt,name=adapter,proto3" json:"adapter,omitempty"` - Scope string `protobuf:"bytes,2,opt,name=scope,proto3" json:"scope,omitempty"` - KeyCount uint64 `protobuf:"varint,3,opt,name=key_count,json=keyCount,proto3" json:"key_count,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Adapter string `protobuf:"bytes,1,opt,name=adapter,proto3" json:"adapter,omitempty"` + Scope string `protobuf:"bytes,2,opt,name=scope,proto3" json:"scope,omitempty"` + KeyCount uint64 `protobuf:"varint,3,opt,name=key_count,json=keyCount,proto3" json:"key_count,omitempty"` + AppliedIndexAtCount uint64 `protobuf:"varint,4,opt,name=applied_index_at_count,json=appliedIndexAtCount,proto3" json:"applied_index_at_count,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *BackupExpectedKeys) Reset() { @@ -1237,6 +1238,13 @@ func (x *BackupExpectedKeys) GetKeyCount() uint64 { return 0 } +func (x *BackupExpectedKeys) GetAppliedIndexAtCount() uint64 { + if x != nil { + return x.AppliedIndexAtCount + } + return 0 +} + type BeginBackupRequest struct { state protoimpl.MessageState `protogen:"open.v1"` TtlMs uint64 `protobuf:"varint,1,opt,name=ttl_ms,json=ttlMs,proto3" json:"ttl_ms,omitempty"` @@ -1412,8 +1420,10 @@ func (x *RenewBackupRequest) GetTtlMs() uint64 { type RenewBackupResponse struct { state protoimpl.MessageState `protogen:"open.v1"` TtlMsEffective uint64 `protobuf:"varint,1,opt,name=ttl_ms_effective,json=ttlMsEffective,proto3" json:"ttl_ms_effective,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Replaces the request token and carries the newly committed hard deadline. + PinToken []byte `protobuf:"bytes,2,opt,name=pin_token,json=pinToken,proto3" json:"pin_token,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RenewBackupResponse) Reset() { @@ -1453,6 +1463,13 @@ func (x *RenewBackupResponse) GetTtlMsEffective() uint64 { return 0 } +func (x *RenewBackupResponse) GetPinToken() []byte { + if x != nil { + return x.PinToken + } + return nil +} + type EndBackupRequest struct { state protoimpl.MessageState `protogen:"open.v1"` PinToken []byte `protobuf:"bytes,1,opt,name=pin_token,json=pinToken,proto3" json:"pin_token,omitempty"` @@ -1673,6 +1690,112 @@ func (x *ListAdaptersAndScopesResponse) GetScopes() []*BackupScope { return nil } +type StreamBackupRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + PinToken []byte `protobuf:"bytes,1,opt,name=pin_token,json=pinToken,proto3" json:"pin_token,omitempty"` + // Empty means every scope present at read_ts. Otherwise only the exact + // adapter/scope pairs are emitted. + Scopes []*BackupScope `protobuf:"bytes,2,rep,name=scopes,proto3" json:"scopes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StreamBackupRequest) Reset() { + *x = StreamBackupRequest{} + mi := &file_admin_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StreamBackupRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamBackupRequest) ProtoMessage() {} + +func (x *StreamBackupRequest) ProtoReflect() protoreflect.Message { + mi := &file_admin_proto_msgTypes[26] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamBackupRequest.ProtoReflect.Descriptor instead. +func (*StreamBackupRequest) Descriptor() ([]byte, []int) { + return file_admin_proto_rawDescGZIP(), []int{26} +} + +func (x *StreamBackupRequest) GetPinToken() []byte { + if x != nil { + return x.PinToken + } + return nil +} + +func (x *StreamBackupRequest) GetScopes() []*BackupScope { + if x != nil { + return x.Scopes + } + return nil +} + +type BackupKV struct { + state protoimpl.MessageState `protogen:"open.v1"` + Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BackupKV) Reset() { + *x = BackupKV{} + mi := &file_admin_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BackupKV) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BackupKV) ProtoMessage() {} + +func (x *BackupKV) ProtoReflect() protoreflect.Message { + mi := &file_admin_proto_msgTypes[27] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BackupKV.ProtoReflect.Descriptor instead. +func (*BackupKV) Descriptor() ([]byte, []int) { + return file_admin_proto_rawDescGZIP(), []int{27} +} + +func (x *BackupKV) GetKey() []byte { + if x != nil { + return x.Key + } + return nil +} + +func (x *BackupKV) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + type GetNodeVersionRequest struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -1681,7 +1804,7 @@ type GetNodeVersionRequest struct { func (x *GetNodeVersionRequest) Reset() { *x = GetNodeVersionRequest{} - mi := &file_admin_proto_msgTypes[26] + mi := &file_admin_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1693,7 +1816,7 @@ func (x *GetNodeVersionRequest) String() string { func (*GetNodeVersionRequest) ProtoMessage() {} func (x *GetNodeVersionRequest) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[26] + mi := &file_admin_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1706,19 +1829,23 @@ func (x *GetNodeVersionRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetNodeVersionRequest.ProtoReflect.Descriptor instead. func (*GetNodeVersionRequest) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{26} + return file_admin_proto_rawDescGZIP(), []int{28} } type GetNodeVersionResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` - NodeVersion string `protobuf:"bytes,1,opt,name=node_version,json=nodeVersion,proto3" json:"node_version,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + NodeVersion string `protobuf:"bytes,1,opt,name=node_version,json=nodeVersion,proto3" json:"node_version,omitempty"` + // backup_protocol_version is a capability gate, not a release-version + // comparison. Zero means the node cannot safely apply live-backup FSM + // entries; version 1 supports reservation, pin, renew, release and stream. + BackupProtocolVersion uint32 `protobuf:"varint,2,opt,name=backup_protocol_version,json=backupProtocolVersion,proto3" json:"backup_protocol_version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GetNodeVersionResponse) Reset() { *x = GetNodeVersionResponse{} - mi := &file_admin_proto_msgTypes[27] + mi := &file_admin_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1730,7 +1857,7 @@ func (x *GetNodeVersionResponse) String() string { func (*GetNodeVersionResponse) ProtoMessage() {} func (x *GetNodeVersionResponse) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[27] + mi := &file_admin_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1743,7 +1870,7 @@ func (x *GetNodeVersionResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetNodeVersionResponse.ProtoReflect.Descriptor instead. func (*GetNodeVersionResponse) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{27} + return file_admin_proto_rawDescGZIP(), []int{29} } func (x *GetNodeVersionResponse) GetNodeVersion() string { @@ -1753,6 +1880,13 @@ func (x *GetNodeVersionResponse) GetNodeVersion() string { return "" } +func (x *GetNodeVersionResponse) GetBackupProtocolVersion() uint32 { + if x != nil { + return x.BackupProtocolVersion + } + return 0 +} + type StreamEventsRequest struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -1761,7 +1895,7 @@ type StreamEventsRequest struct { func (x *StreamEventsRequest) Reset() { *x = StreamEventsRequest{} - mi := &file_admin_proto_msgTypes[28] + mi := &file_admin_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1773,7 +1907,7 @@ func (x *StreamEventsRequest) String() string { func (*StreamEventsRequest) ProtoMessage() {} func (x *StreamEventsRequest) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[28] + mi := &file_admin_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1786,7 +1920,7 @@ func (x *StreamEventsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use StreamEventsRequest.ProtoReflect.Descriptor instead. func (*StreamEventsRequest) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{28} + return file_admin_proto_rawDescGZIP(), []int{30} } type StreamEventsEvent struct { @@ -1802,7 +1936,7 @@ type StreamEventsEvent struct { func (x *StreamEventsEvent) Reset() { *x = StreamEventsEvent{} - mi := &file_admin_proto_msgTypes[29] + mi := &file_admin_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1814,7 +1948,7 @@ func (x *StreamEventsEvent) String() string { func (*StreamEventsEvent) ProtoMessage() {} func (x *StreamEventsEvent) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[29] + mi := &file_admin_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1827,7 +1961,7 @@ func (x *StreamEventsEvent) ProtoReflect() protoreflect.Message { // Deprecated: Use StreamEventsEvent.ProtoReflect.Descriptor instead. func (*StreamEventsEvent) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{29} + return file_admin_proto_rawDescGZIP(), []int{31} } func (x *StreamEventsEvent) GetEvent() isStreamEventsEvent_Event { @@ -1883,7 +2017,7 @@ type RouteTransition struct { func (x *RouteTransition) Reset() { *x = RouteTransition{} - mi := &file_admin_proto_msgTypes[30] + mi := &file_admin_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1895,7 +2029,7 @@ func (x *RouteTransition) String() string { func (*RouteTransition) ProtoMessage() {} func (x *RouteTransition) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[30] + mi := &file_admin_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1908,7 +2042,7 @@ func (x *RouteTransition) ProtoReflect() protoreflect.Message { // Deprecated: Use RouteTransition.ProtoReflect.Descriptor instead. func (*RouteTransition) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{30} + return file_admin_proto_rawDescGZIP(), []int{32} } func (x *RouteTransition) GetParentRouteId() uint64 { @@ -1950,7 +2084,7 @@ type KeyVizColumn struct { func (x *KeyVizColumn) Reset() { *x = KeyVizColumn{} - mi := &file_admin_proto_msgTypes[31] + mi := &file_admin_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1962,7 +2096,7 @@ func (x *KeyVizColumn) String() string { func (*KeyVizColumn) ProtoMessage() {} func (x *KeyVizColumn) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[31] + mi := &file_admin_proto_msgTypes[33] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1975,7 +2109,7 @@ func (x *KeyVizColumn) ProtoReflect() protoreflect.Message { // Deprecated: Use KeyVizColumn.ProtoReflect.Descriptor instead. func (*KeyVizColumn) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{31} + return file_admin_proto_rawDescGZIP(), []int{33} } func (x *KeyVizColumn) GetColumnUnixMs() int64 { @@ -2089,11 +2223,12 @@ const file_admin_proto_rawDesc = "" + "perAdapter\"]\n" + "\x12BackupShardApplied\x12\"\n" + "\rraft_group_id\x18\x01 \x01(\x04R\vraftGroupId\x12#\n" + - "\rapplied_index\x18\x02 \x01(\x04R\fappliedIndex\"a\n" + + "\rapplied_index\x18\x02 \x01(\x04R\fappliedIndex\"\x96\x01\n" + "\x12BackupExpectedKeys\x12\x18\n" + "\aadapter\x18\x01 \x01(\tR\aadapter\x12\x14\n" + "\x05scope\x18\x02 \x01(\tR\x05scope\x12\x1b\n" + - "\tkey_count\x18\x03 \x01(\x04R\bkeyCount\"+\n" + + "\tkey_count\x18\x03 \x01(\x04R\bkeyCount\x123\n" + + "\x16applied_index_at_count\x18\x04 \x01(\x04R\x13appliedIndexAtCount\"+\n" + "\x12BeginBackupRequest\x12\x15\n" + "\x06ttl_ms\x18\x01 \x01(\x04R\x05ttlMs\"\xdc\x01\n" + "\x13BeginBackupResponse\x12\x17\n" + @@ -2104,9 +2239,10 @@ const file_admin_proto_rawDesc = "" + "\rexpected_keys\x18\x05 \x03(\v2\x13.BackupExpectedKeysR\fexpectedKeys\"H\n" + "\x12RenewBackupRequest\x12\x1b\n" + "\tpin_token\x18\x01 \x01(\fR\bpinToken\x12\x15\n" + - "\x06ttl_ms\x18\x02 \x01(\x04R\x05ttlMs\"?\n" + + "\x06ttl_ms\x18\x02 \x01(\x04R\x05ttlMs\"\\\n" + "\x13RenewBackupResponse\x12(\n" + - "\x10ttl_ms_effective\x18\x01 \x01(\x04R\x0ettlMsEffective\"/\n" + + "\x10ttl_ms_effective\x18\x01 \x01(\x04R\x0ettlMsEffective\x12\x1b\n" + + "\tpin_token\x18\x02 \x01(\fR\bpinToken\"/\n" + "\x10EndBackupRequest\x12\x1b\n" + "\tpin_token\x18\x01 \x01(\fR\bpinToken\"\x13\n" + "\x11EndBackupResponse\";\n" + @@ -2116,10 +2252,17 @@ const file_admin_proto_rawDesc = "" + "\aadapter\x18\x01 \x01(\tR\aadapter\x12\x14\n" + "\x05scope\x18\x02 \x01(\tR\x05scope\"E\n" + "\x1dListAdaptersAndScopesResponse\x12$\n" + - "\x06scopes\x18\x01 \x03(\v2\f.BackupScopeR\x06scopes\"\x17\n" + - "\x15GetNodeVersionRequest\";\n" + + "\x06scopes\x18\x01 \x03(\v2\f.BackupScopeR\x06scopes\"X\n" + + "\x13StreamBackupRequest\x12\x1b\n" + + "\tpin_token\x18\x01 \x01(\fR\bpinToken\x12$\n" + + "\x06scopes\x18\x02 \x03(\v2\f.BackupScopeR\x06scopes\"2\n" + + "\bBackupKV\x12\x10\n" + + "\x03key\x18\x01 \x01(\fR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\fR\x05value\"\x17\n" + + "\x15GetNodeVersionRequest\"s\n" + "\x16GetNodeVersionResponse\x12!\n" + - "\fnode_version\x18\x01 \x01(\tR\vnodeVersion\"\x15\n" + + "\fnode_version\x18\x01 \x01(\tR\vnodeVersion\x126\n" + + "\x17backup_protocol_version\x18\x02 \x01(\rR\x15backupProtocolVersion\"\x15\n" + "\x13StreamEventsRequest\"\x91\x01\n" + "\x11StreamEventsEvent\x12=\n" + "\x10route_transition\x18\x01 \x01(\v2\x10.RouteTransitionH\x00R\x0frouteTransition\x124\n" + @@ -2147,7 +2290,7 @@ const file_admin_proto_rawDesc = "" + "\x17SAMPLE_ROLE_UNSPECIFIED\x10\x00\x12\x1c\n" + "\x18SAMPLE_ROLE_LEADER_WRITE\x10\x01\x12\x1b\n" + "\x17SAMPLE_ROLE_LEADER_READ\x10\x02\x12\x1d\n" + - "\x19SAMPLE_ROLE_FOLLOWER_READ\x10\x032\x80\x06\n" + + "\x19SAMPLE_ROLE_FOLLOWER_READ\x10\x032\xb5\x06\n" + "\x05Admin\x12O\n" + "\x12GetClusterOverview\x12\x1a.GetClusterOverviewRequest\x1a\x1b.GetClusterOverviewResponse\"\x00\x12@\n" + "\rGetRaftGroups\x12\x15.GetRaftGroupsRequest\x1a\x16.GetRaftGroupsResponse\"\x00\x12L\n" + @@ -2157,7 +2300,8 @@ const file_admin_proto_rawDesc = "" + "\vBeginBackup\x12\x13.BeginBackupRequest\x1a\x14.BeginBackupResponse\"\x00\x12:\n" + "\vRenewBackup\x12\x13.RenewBackupRequest\x1a\x14.RenewBackupResponse\"\x00\x124\n" + "\tEndBackup\x12\x11.EndBackupRequest\x1a\x12.EndBackupResponse\"\x00\x12X\n" + - "\x15ListAdaptersAndScopes\x12\x1d.ListAdaptersAndScopesRequest\x1a\x1e.ListAdaptersAndScopesResponse\"\x00\x12C\n" + + "\x15ListAdaptersAndScopes\x12\x1d.ListAdaptersAndScopesRequest\x1a\x1e.ListAdaptersAndScopesResponse\"\x00\x123\n" + + "\fStreamBackup\x12\x14.StreamBackupRequest\x1a\t.BackupKV\"\x000\x01\x12C\n" + "\x0eGetNodeVersion\x12\x16.GetNodeVersionRequest\x1a\x17.GetNodeVersionResponse\"\x00\x12<\n" + "\fStreamEvents\x12\x14.StreamEventsRequest\x1a\x12.StreamEventsEvent\"\x000\x01B#Z!github.com/bootjp/elastickv/protob\x06proto3" @@ -2174,7 +2318,7 @@ func file_admin_proto_rawDescGZIP() []byte { } var file_admin_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_admin_proto_msgTypes = make([]protoimpl.MessageInfo, 33) +var file_admin_proto_msgTypes = make([]protoimpl.MessageInfo, 35) var file_admin_proto_goTypes = []any{ (KeyVizSeries)(0), // 0: KeyVizSeries (SampleRole)(0), // 1: SampleRole @@ -2204,19 +2348,21 @@ var file_admin_proto_goTypes = []any{ (*ListAdaptersAndScopesRequest)(nil), // 25: ListAdaptersAndScopesRequest (*BackupScope)(nil), // 26: BackupScope (*ListAdaptersAndScopesResponse)(nil), // 27: ListAdaptersAndScopesResponse - (*GetNodeVersionRequest)(nil), // 28: GetNodeVersionRequest - (*GetNodeVersionResponse)(nil), // 29: GetNodeVersionResponse - (*StreamEventsRequest)(nil), // 30: StreamEventsRequest - (*StreamEventsEvent)(nil), // 31: StreamEventsEvent - (*RouteTransition)(nil), // 32: RouteTransition - (*KeyVizColumn)(nil), // 33: KeyVizColumn - nil, // 34: GetClusterOverviewResponse.CapabilitiesEntry + (*StreamBackupRequest)(nil), // 28: StreamBackupRequest + (*BackupKV)(nil), // 29: BackupKV + (*GetNodeVersionRequest)(nil), // 30: GetNodeVersionRequest + (*GetNodeVersionResponse)(nil), // 31: GetNodeVersionResponse + (*StreamEventsRequest)(nil), // 32: StreamEventsRequest + (*StreamEventsEvent)(nil), // 33: StreamEventsEvent + (*RouteTransition)(nil), // 34: RouteTransition + (*KeyVizColumn)(nil), // 35: KeyVizColumn + nil, // 36: GetClusterOverviewResponse.CapabilitiesEntry } var file_admin_proto_depIdxs = []int32{ 2, // 0: GetClusterOverviewResponse.self:type_name -> NodeIdentity 2, // 1: GetClusterOverviewResponse.members:type_name -> NodeIdentity 3, // 2: GetClusterOverviewResponse.group_leaders:type_name -> GroupLeader - 34, // 3: GetClusterOverviewResponse.capabilities:type_name -> GetClusterOverviewResponse.CapabilitiesEntry + 36, // 3: GetClusterOverviewResponse.capabilities:type_name -> GetClusterOverviewResponse.CapabilitiesEntry 6, // 4: GetRaftGroupsResponse.groups:type_name -> RaftGroupState 9, // 5: GetAdapterSummaryResponse.summaries:type_name -> AdapterSummary 1, // 6: KeyVizRow.sample_roles:type_name -> SampleRole @@ -2227,37 +2373,40 @@ var file_admin_proto_depIdxs = []int32{ 17, // 11: BeginBackupResponse.shards:type_name -> BackupShardApplied 18, // 12: BeginBackupResponse.expected_keys:type_name -> BackupExpectedKeys 26, // 13: ListAdaptersAndScopesResponse.scopes:type_name -> BackupScope - 32, // 14: StreamEventsEvent.route_transition:type_name -> RouteTransition - 33, // 15: StreamEventsEvent.keyviz_column:type_name -> KeyVizColumn - 0, // 16: KeyVizColumn.series:type_name -> KeyVizSeries - 12, // 17: KeyVizColumn.rows:type_name -> KeyVizRow - 4, // 18: Admin.GetClusterOverview:input_type -> GetClusterOverviewRequest - 7, // 19: Admin.GetRaftGroups:input_type -> GetRaftGroupsRequest - 10, // 20: Admin.GetAdapterSummary:input_type -> GetAdapterSummaryRequest - 13, // 21: Admin.GetKeyVizMatrix:input_type -> GetKeyVizMatrixRequest - 15, // 22: Admin.GetRouteDetail:input_type -> GetRouteDetailRequest - 19, // 23: Admin.BeginBackup:input_type -> BeginBackupRequest - 21, // 24: Admin.RenewBackup:input_type -> RenewBackupRequest - 23, // 25: Admin.EndBackup:input_type -> EndBackupRequest - 25, // 26: Admin.ListAdaptersAndScopes:input_type -> ListAdaptersAndScopesRequest - 28, // 27: Admin.GetNodeVersion:input_type -> GetNodeVersionRequest - 30, // 28: Admin.StreamEvents:input_type -> StreamEventsRequest - 5, // 29: Admin.GetClusterOverview:output_type -> GetClusterOverviewResponse - 8, // 30: Admin.GetRaftGroups:output_type -> GetRaftGroupsResponse - 11, // 31: Admin.GetAdapterSummary:output_type -> GetAdapterSummaryResponse - 14, // 32: Admin.GetKeyVizMatrix:output_type -> GetKeyVizMatrixResponse - 16, // 33: Admin.GetRouteDetail:output_type -> GetRouteDetailResponse - 20, // 34: Admin.BeginBackup:output_type -> BeginBackupResponse - 22, // 35: Admin.RenewBackup:output_type -> RenewBackupResponse - 24, // 36: Admin.EndBackup:output_type -> EndBackupResponse - 27, // 37: Admin.ListAdaptersAndScopes:output_type -> ListAdaptersAndScopesResponse - 29, // 38: Admin.GetNodeVersion:output_type -> GetNodeVersionResponse - 31, // 39: Admin.StreamEvents:output_type -> StreamEventsEvent - 29, // [29:40] is the sub-list for method output_type - 18, // [18:29] is the sub-list for method input_type - 18, // [18:18] is the sub-list for extension type_name - 18, // [18:18] is the sub-list for extension extendee - 0, // [0:18] is the sub-list for field type_name + 26, // 14: StreamBackupRequest.scopes:type_name -> BackupScope + 34, // 15: StreamEventsEvent.route_transition:type_name -> RouteTransition + 35, // 16: StreamEventsEvent.keyviz_column:type_name -> KeyVizColumn + 0, // 17: KeyVizColumn.series:type_name -> KeyVizSeries + 12, // 18: KeyVizColumn.rows:type_name -> KeyVizRow + 4, // 19: Admin.GetClusterOverview:input_type -> GetClusterOverviewRequest + 7, // 20: Admin.GetRaftGroups:input_type -> GetRaftGroupsRequest + 10, // 21: Admin.GetAdapterSummary:input_type -> GetAdapterSummaryRequest + 13, // 22: Admin.GetKeyVizMatrix:input_type -> GetKeyVizMatrixRequest + 15, // 23: Admin.GetRouteDetail:input_type -> GetRouteDetailRequest + 19, // 24: Admin.BeginBackup:input_type -> BeginBackupRequest + 21, // 25: Admin.RenewBackup:input_type -> RenewBackupRequest + 23, // 26: Admin.EndBackup:input_type -> EndBackupRequest + 25, // 27: Admin.ListAdaptersAndScopes:input_type -> ListAdaptersAndScopesRequest + 28, // 28: Admin.StreamBackup:input_type -> StreamBackupRequest + 30, // 29: Admin.GetNodeVersion:input_type -> GetNodeVersionRequest + 32, // 30: Admin.StreamEvents:input_type -> StreamEventsRequest + 5, // 31: Admin.GetClusterOverview:output_type -> GetClusterOverviewResponse + 8, // 32: Admin.GetRaftGroups:output_type -> GetRaftGroupsResponse + 11, // 33: Admin.GetAdapterSummary:output_type -> GetAdapterSummaryResponse + 14, // 34: Admin.GetKeyVizMatrix:output_type -> GetKeyVizMatrixResponse + 16, // 35: Admin.GetRouteDetail:output_type -> GetRouteDetailResponse + 20, // 36: Admin.BeginBackup:output_type -> BeginBackupResponse + 22, // 37: Admin.RenewBackup:output_type -> RenewBackupResponse + 24, // 38: Admin.EndBackup:output_type -> EndBackupResponse + 27, // 39: Admin.ListAdaptersAndScopes:output_type -> ListAdaptersAndScopesResponse + 29, // 40: Admin.StreamBackup:output_type -> BackupKV + 31, // 41: Admin.GetNodeVersion:output_type -> GetNodeVersionResponse + 33, // 42: Admin.StreamEvents:output_type -> StreamEventsEvent + 31, // [31:43] is the sub-list for method output_type + 19, // [19:31] is the sub-list for method input_type + 19, // [19:19] is the sub-list for extension type_name + 19, // [19:19] is the sub-list for extension extendee + 0, // [0:19] is the sub-list for field type_name } func init() { file_admin_proto_init() } @@ -2265,7 +2414,7 @@ func file_admin_proto_init() { if File_admin_proto != nil { return } - file_admin_proto_msgTypes[29].OneofWrappers = []any{ + file_admin_proto_msgTypes[31].OneofWrappers = []any{ (*StreamEventsEvent_RouteTransition)(nil), (*StreamEventsEvent_KeyvizColumn)(nil), } @@ -2275,7 +2424,7 @@ func file_admin_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_admin_proto_rawDesc), len(file_admin_proto_rawDesc)), NumEnums: 2, - NumMessages: 33, + NumMessages: 35, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/admin.proto b/proto/admin.proto index 5d9fefa77..bcd8b55a7 100644 --- a/proto/admin.proto +++ b/proto/admin.proto @@ -2,7 +2,7 @@ syntax = "proto3"; option go_package = "github.com/bootjp/elastickv/proto"; -// Admin is the node-side read-only admin gRPC service consumed by +// Admin is the node-side admin gRPC service consumed by // cmd/elastickv-admin. Every method requires "authorization: Bearer " // metadata unless the node was started with --adminInsecureNoAuth. // See docs/admin_ui_key_visualizer_design.md §4 (Layer A). @@ -16,6 +16,7 @@ service Admin { rpc RenewBackup (RenewBackupRequest) returns (RenewBackupResponse) {} rpc EndBackup (EndBackupRequest) returns (EndBackupResponse) {} rpc ListAdaptersAndScopes (ListAdaptersAndScopesRequest) returns (ListAdaptersAndScopesResponse) {} + rpc StreamBackup (StreamBackupRequest) returns (stream BackupKV) {} rpc GetNodeVersion (GetNodeVersionRequest) returns (GetNodeVersionResponse) {} rpc StreamEvents (StreamEventsRequest) returns (stream StreamEventsEvent) {} } @@ -172,6 +173,7 @@ message BackupExpectedKeys { string adapter = 1; string scope = 2; uint64 key_count = 3; + uint64 applied_index_at_count = 4; } message BeginBackupRequest { @@ -193,6 +195,8 @@ message RenewBackupRequest { message RenewBackupResponse { uint64 ttl_ms_effective = 1; + // Replaces the request token and carries the newly committed hard deadline. + bytes pin_token = 2; } message EndBackupRequest { @@ -214,10 +218,26 @@ message ListAdaptersAndScopesResponse { repeated BackupScope scopes = 1; } +message StreamBackupRequest { + bytes pin_token = 1; + // Empty means every scope present at read_ts. Otherwise only the exact + // adapter/scope pairs are emitted. + repeated BackupScope scopes = 2; +} + +message BackupKV { + bytes key = 1; + bytes value = 2; +} + message GetNodeVersionRequest {} message GetNodeVersionResponse { string node_version = 1; + // backup_protocol_version is a capability gate, not a release-version + // comparison. Zero means the node cannot safely apply live-backup FSM + // entries; version 1 supports reservation, pin, renew, release and stream. + uint32 backup_protocol_version = 2; } message StreamEventsRequest {} diff --git a/proto/admin_grpc.pb.go b/proto/admin_grpc.pb.go index 06829837d..407e95369 100644 --- a/proto/admin_grpc.pb.go +++ b/proto/admin_grpc.pb.go @@ -28,6 +28,7 @@ const ( Admin_RenewBackup_FullMethodName = "/Admin/RenewBackup" Admin_EndBackup_FullMethodName = "/Admin/EndBackup" Admin_ListAdaptersAndScopes_FullMethodName = "/Admin/ListAdaptersAndScopes" + Admin_StreamBackup_FullMethodName = "/Admin/StreamBackup" Admin_GetNodeVersion_FullMethodName = "/Admin/GetNodeVersion" Admin_StreamEvents_FullMethodName = "/Admin/StreamEvents" ) @@ -36,7 +37,7 @@ const ( // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. // -// Admin is the node-side read-only admin gRPC service consumed by +// Admin is the node-side admin gRPC service consumed by // cmd/elastickv-admin. Every method requires "authorization: Bearer " // metadata unless the node was started with --adminInsecureNoAuth. // See docs/admin_ui_key_visualizer_design.md §4 (Layer A). @@ -50,6 +51,7 @@ type AdminClient interface { RenewBackup(ctx context.Context, in *RenewBackupRequest, opts ...grpc.CallOption) (*RenewBackupResponse, error) EndBackup(ctx context.Context, in *EndBackupRequest, opts ...grpc.CallOption) (*EndBackupResponse, error) ListAdaptersAndScopes(ctx context.Context, in *ListAdaptersAndScopesRequest, opts ...grpc.CallOption) (*ListAdaptersAndScopesResponse, error) + StreamBackup(ctx context.Context, in *StreamBackupRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[BackupKV], error) GetNodeVersion(ctx context.Context, in *GetNodeVersionRequest, opts ...grpc.CallOption) (*GetNodeVersionResponse, error) StreamEvents(ctx context.Context, in *StreamEventsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[StreamEventsEvent], error) } @@ -152,6 +154,25 @@ func (c *adminClient) ListAdaptersAndScopes(ctx context.Context, in *ListAdapter return out, nil } +func (c *adminClient) StreamBackup(ctx context.Context, in *StreamBackupRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[BackupKV], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Admin_ServiceDesc.Streams[0], Admin_StreamBackup_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[StreamBackupRequest, BackupKV]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Admin_StreamBackupClient = grpc.ServerStreamingClient[BackupKV] + func (c *adminClient) GetNodeVersion(ctx context.Context, in *GetNodeVersionRequest, opts ...grpc.CallOption) (*GetNodeVersionResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GetNodeVersionResponse) @@ -164,7 +185,7 @@ func (c *adminClient) GetNodeVersion(ctx context.Context, in *GetNodeVersionRequ func (c *adminClient) StreamEvents(ctx context.Context, in *StreamEventsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[StreamEventsEvent], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - stream, err := c.cc.NewStream(ctx, &Admin_ServiceDesc.Streams[0], Admin_StreamEvents_FullMethodName, cOpts...) + stream, err := c.cc.NewStream(ctx, &Admin_ServiceDesc.Streams[1], Admin_StreamEvents_FullMethodName, cOpts...) if err != nil { return nil, err } @@ -185,7 +206,7 @@ type Admin_StreamEventsClient = grpc.ServerStreamingClient[StreamEventsEvent] // All implementations must embed UnimplementedAdminServer // for forward compatibility. // -// Admin is the node-side read-only admin gRPC service consumed by +// Admin is the node-side admin gRPC service consumed by // cmd/elastickv-admin. Every method requires "authorization: Bearer " // metadata unless the node was started with --adminInsecureNoAuth. // See docs/admin_ui_key_visualizer_design.md §4 (Layer A). @@ -199,6 +220,7 @@ type AdminServer interface { RenewBackup(context.Context, *RenewBackupRequest) (*RenewBackupResponse, error) EndBackup(context.Context, *EndBackupRequest) (*EndBackupResponse, error) ListAdaptersAndScopes(context.Context, *ListAdaptersAndScopesRequest) (*ListAdaptersAndScopesResponse, error) + StreamBackup(*StreamBackupRequest, grpc.ServerStreamingServer[BackupKV]) error GetNodeVersion(context.Context, *GetNodeVersionRequest) (*GetNodeVersionResponse, error) StreamEvents(*StreamEventsRequest, grpc.ServerStreamingServer[StreamEventsEvent]) error mustEmbedUnimplementedAdminServer() @@ -238,6 +260,9 @@ func (UnimplementedAdminServer) EndBackup(context.Context, *EndBackupRequest) (* func (UnimplementedAdminServer) ListAdaptersAndScopes(context.Context, *ListAdaptersAndScopesRequest) (*ListAdaptersAndScopesResponse, error) { return nil, status.Error(codes.Unimplemented, "method ListAdaptersAndScopes not implemented") } +func (UnimplementedAdminServer) StreamBackup(*StreamBackupRequest, grpc.ServerStreamingServer[BackupKV]) error { + return status.Error(codes.Unimplemented, "method StreamBackup not implemented") +} func (UnimplementedAdminServer) GetNodeVersion(context.Context, *GetNodeVersionRequest) (*GetNodeVersionResponse, error) { return nil, status.Error(codes.Unimplemented, "method GetNodeVersion not implemented") } @@ -427,6 +452,17 @@ func _Admin_ListAdaptersAndScopes_Handler(srv interface{}, ctx context.Context, return interceptor(ctx, in, info, handler) } +func _Admin_StreamBackup_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(StreamBackupRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(AdminServer).StreamBackup(m, &grpc.GenericServerStream[StreamBackupRequest, BackupKV]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Admin_StreamBackupServer = grpc.ServerStreamingServer[BackupKV] + func _Admin_GetNodeVersion_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetNodeVersionRequest) if err := dec(in); err != nil { @@ -505,6 +541,11 @@ var Admin_ServiceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{ + { + StreamName: "StreamBackup", + Handler: _Admin_StreamBackup_Handler, + ServerStreams: true, + }, { StreamName: "StreamEvents", Handler: _Admin_StreamEvents_Handler, diff --git a/proto/internal.pb.go b/proto/internal.pb.go index 720eee66a..7513776a8 100644 --- a/proto/internal.pb.go +++ b/proto/internal.pb.go @@ -454,6 +454,94 @@ func (x *ForwardResponse) GetCommitTs() uint64 { return 0 } +type ForwardAdminProposalRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Payload []byte `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ForwardAdminProposalRequest) Reset() { + *x = ForwardAdminProposalRequest{} + mi := &file_internal_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ForwardAdminProposalRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ForwardAdminProposalRequest) ProtoMessage() {} + +func (x *ForwardAdminProposalRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ForwardAdminProposalRequest.ProtoReflect.Descriptor instead. +func (*ForwardAdminProposalRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{5} +} + +func (x *ForwardAdminProposalRequest) GetPayload() []byte { + if x != nil { + return x.Payload + } + return nil +} + +type ForwardAdminProposalResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + CommitIndex uint64 `protobuf:"varint,1,opt,name=commit_index,json=commitIndex,proto3" json:"commit_index,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ForwardAdminProposalResponse) Reset() { + *x = ForwardAdminProposalResponse{} + mi := &file_internal_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ForwardAdminProposalResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ForwardAdminProposalResponse) ProtoMessage() {} + +func (x *ForwardAdminProposalResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ForwardAdminProposalResponse.ProtoReflect.Descriptor instead. +func (*ForwardAdminProposalResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{6} +} + +func (x *ForwardAdminProposalResponse) GetCommitIndex() uint64 { + if x != nil { + return x.CommitIndex + } + return 0 +} + type RelayPublishRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Channel []byte `protobuf:"bytes,1,opt,name=channel,proto3" json:"channel,omitempty"` @@ -464,7 +552,7 @@ type RelayPublishRequest struct { func (x *RelayPublishRequest) Reset() { *x = RelayPublishRequest{} - mi := &file_internal_proto_msgTypes[5] + mi := &file_internal_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -476,7 +564,7 @@ func (x *RelayPublishRequest) String() string { func (*RelayPublishRequest) ProtoMessage() {} func (x *RelayPublishRequest) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_msgTypes[5] + mi := &file_internal_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -489,7 +577,7 @@ func (x *RelayPublishRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RelayPublishRequest.ProtoReflect.Descriptor instead. func (*RelayPublishRequest) Descriptor() ([]byte, []int) { - return file_internal_proto_rawDescGZIP(), []int{5} + return file_internal_proto_rawDescGZIP(), []int{7} } func (x *RelayPublishRequest) GetChannel() []byte { @@ -515,7 +603,7 @@ type RelayPublishResponse struct { func (x *RelayPublishResponse) Reset() { *x = RelayPublishResponse{} - mi := &file_internal_proto_msgTypes[6] + mi := &file_internal_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -527,7 +615,7 @@ func (x *RelayPublishResponse) String() string { func (*RelayPublishResponse) ProtoMessage() {} func (x *RelayPublishResponse) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_msgTypes[6] + mi := &file_internal_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -540,7 +628,7 @@ func (x *RelayPublishResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RelayPublishResponse.ProtoReflect.Descriptor instead. func (*RelayPublishResponse) Descriptor() ([]byte, []int) { - return file_internal_proto_rawDescGZIP(), []int{6} + return file_internal_proto_rawDescGZIP(), []int{8} } func (x *RelayPublishResponse) GetSubscribers() int64 { @@ -575,7 +663,11 @@ const file_internal_proto_rawDesc = "" + "\x0fForwardResponse\x12\x18\n" + "\asuccess\x18\x01 \x01(\bR\asuccess\x12!\n" + "\fcommit_index\x18\x02 \x01(\x04R\vcommitIndex\x12\x1b\n" + - "\tcommit_ts\x18\x03 \x01(\x04R\bcommitTs\"I\n" + + "\tcommit_ts\x18\x03 \x01(\x04R\bcommitTs\"7\n" + + "\x1bForwardAdminProposalRequest\x12\x18\n" + + "\apayload\x18\x01 \x01(\fR\apayload\"A\n" + + "\x1cForwardAdminProposalResponse\x12!\n" + + "\fcommit_index\x18\x01 \x01(\x04R\vcommitIndex\"I\n" + "\x13RelayPublishRequest\x12\x18\n" + "\achannel\x18\x01 \x01(\fR\achannel\x12\x18\n" + "\amessage\x18\x02 \x01(\fR\amessage\"8\n" + @@ -591,9 +683,10 @@ const file_internal_proto_rawDesc = "" + "\aPREPARE\x10\x01\x12\n" + "\n" + "\x06COMMIT\x10\x02\x12\t\n" + - "\x05ABORT\x10\x032y\n" + + "\x05ABORT\x10\x032\xd0\x01\n" + "\bInternal\x12.\n" + - "\aForward\x12\x0f.ForwardRequest\x1a\x10.ForwardResponse\"\x00\x12=\n" + + "\aForward\x12\x0f.ForwardRequest\x1a\x10.ForwardResponse\"\x00\x12U\n" + + "\x14ForwardAdminProposal\x12\x1c.ForwardAdminProposalRequest\x1a\x1d.ForwardAdminProposalResponse\"\x00\x12=\n" + "\fRelayPublish\x12\x14.RelayPublishRequest\x1a\x15.RelayPublishResponse\"\x00B#Z!github.com/bootjp/elastickv/protob\x06proto3" var ( @@ -609,33 +702,37 @@ func file_internal_proto_rawDescGZIP() []byte { } var file_internal_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 9) var file_internal_proto_goTypes = []any{ - (Op)(0), // 0: Op - (Phase)(0), // 1: Phase - (*Mutation)(nil), // 2: Mutation - (*Request)(nil), // 3: Request - (*RaftCommand)(nil), // 4: RaftCommand - (*ForwardRequest)(nil), // 5: ForwardRequest - (*ForwardResponse)(nil), // 6: ForwardResponse - (*RelayPublishRequest)(nil), // 7: RelayPublishRequest - (*RelayPublishResponse)(nil), // 8: RelayPublishResponse + (Op)(0), // 0: Op + (Phase)(0), // 1: Phase + (*Mutation)(nil), // 2: Mutation + (*Request)(nil), // 3: Request + (*RaftCommand)(nil), // 4: RaftCommand + (*ForwardRequest)(nil), // 5: ForwardRequest + (*ForwardResponse)(nil), // 6: ForwardResponse + (*ForwardAdminProposalRequest)(nil), // 7: ForwardAdminProposalRequest + (*ForwardAdminProposalResponse)(nil), // 8: ForwardAdminProposalResponse + (*RelayPublishRequest)(nil), // 9: RelayPublishRequest + (*RelayPublishResponse)(nil), // 10: RelayPublishResponse } var file_internal_proto_depIdxs = []int32{ - 0, // 0: Mutation.op:type_name -> Op - 1, // 1: Request.phase:type_name -> Phase - 2, // 2: Request.mutations:type_name -> Mutation - 3, // 3: RaftCommand.requests:type_name -> Request - 3, // 4: ForwardRequest.requests:type_name -> Request - 5, // 5: Internal.Forward:input_type -> ForwardRequest - 7, // 6: Internal.RelayPublish:input_type -> RelayPublishRequest - 6, // 7: Internal.Forward:output_type -> ForwardResponse - 8, // 8: Internal.RelayPublish:output_type -> RelayPublishResponse - 7, // [7:9] is the sub-list for method output_type - 5, // [5:7] is the sub-list for method input_type - 5, // [5:5] is the sub-list for extension type_name - 5, // [5:5] is the sub-list for extension extendee - 0, // [0:5] is the sub-list for field type_name + 0, // 0: Mutation.op:type_name -> Op + 1, // 1: Request.phase:type_name -> Phase + 2, // 2: Request.mutations:type_name -> Mutation + 3, // 3: RaftCommand.requests:type_name -> Request + 3, // 4: ForwardRequest.requests:type_name -> Request + 5, // 5: Internal.Forward:input_type -> ForwardRequest + 7, // 6: Internal.ForwardAdminProposal:input_type -> ForwardAdminProposalRequest + 9, // 7: Internal.RelayPublish:input_type -> RelayPublishRequest + 6, // 8: Internal.Forward:output_type -> ForwardResponse + 8, // 9: Internal.ForwardAdminProposal:output_type -> ForwardAdminProposalResponse + 10, // 10: Internal.RelayPublish:output_type -> RelayPublishResponse + 8, // [8:11] is the sub-list for method output_type + 5, // [5:8] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name } func init() { file_internal_proto_init() } @@ -649,7 +746,7 @@ func file_internal_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_internal_proto_rawDesc), len(file_internal_proto_rawDesc)), NumEnums: 2, - NumMessages: 7, + NumMessages: 9, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/internal.proto b/proto/internal.proto index fb8f70e55..24396b703 100644 --- a/proto/internal.proto +++ b/proto/internal.proto @@ -6,6 +6,7 @@ option go_package = "github.com/bootjp/elastickv/proto"; service Internal { // for internal leader redirect only rpc Forward(ForwardRequest) returns (ForwardResponse) {} + rpc ForwardAdminProposal(ForwardAdminProposalRequest) returns (ForwardAdminProposalResponse) {} rpc RelayPublish(RelayPublishRequest) returns (RelayPublishResponse) {} } @@ -77,6 +78,14 @@ message ForwardResponse { uint64 commit_ts = 3; } +message ForwardAdminProposalRequest { + bytes payload = 1; +} + +message ForwardAdminProposalResponse { + uint64 commit_index = 1; +} + message RelayPublishRequest { bytes channel = 1; bytes message = 2; diff --git a/proto/internal_grpc.pb.go b/proto/internal_grpc.pb.go index 3828db02a..c528a33b7 100644 --- a/proto/internal_grpc.pb.go +++ b/proto/internal_grpc.pb.go @@ -19,8 +19,9 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - Internal_Forward_FullMethodName = "/Internal/Forward" - Internal_RelayPublish_FullMethodName = "/Internal/RelayPublish" + Internal_Forward_FullMethodName = "/Internal/Forward" + Internal_ForwardAdminProposal_FullMethodName = "/Internal/ForwardAdminProposal" + Internal_RelayPublish_FullMethodName = "/Internal/RelayPublish" ) // InternalClient is the client API for Internal service. @@ -29,6 +30,7 @@ const ( type InternalClient interface { // for internal leader redirect only Forward(ctx context.Context, in *ForwardRequest, opts ...grpc.CallOption) (*ForwardResponse, error) + ForwardAdminProposal(ctx context.Context, in *ForwardAdminProposalRequest, opts ...grpc.CallOption) (*ForwardAdminProposalResponse, error) RelayPublish(ctx context.Context, in *RelayPublishRequest, opts ...grpc.CallOption) (*RelayPublishResponse, error) } @@ -50,6 +52,16 @@ func (c *internalClient) Forward(ctx context.Context, in *ForwardRequest, opts . return out, nil } +func (c *internalClient) ForwardAdminProposal(ctx context.Context, in *ForwardAdminProposalRequest, opts ...grpc.CallOption) (*ForwardAdminProposalResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ForwardAdminProposalResponse) + err := c.cc.Invoke(ctx, Internal_ForwardAdminProposal_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *internalClient) RelayPublish(ctx context.Context, in *RelayPublishRequest, opts ...grpc.CallOption) (*RelayPublishResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(RelayPublishResponse) @@ -66,6 +78,7 @@ func (c *internalClient) RelayPublish(ctx context.Context, in *RelayPublishReque type InternalServer interface { // for internal leader redirect only Forward(context.Context, *ForwardRequest) (*ForwardResponse, error) + ForwardAdminProposal(context.Context, *ForwardAdminProposalRequest) (*ForwardAdminProposalResponse, error) RelayPublish(context.Context, *RelayPublishRequest) (*RelayPublishResponse, error) mustEmbedUnimplementedInternalServer() } @@ -80,6 +93,9 @@ type UnimplementedInternalServer struct{} func (UnimplementedInternalServer) Forward(context.Context, *ForwardRequest) (*ForwardResponse, error) { return nil, status.Error(codes.Unimplemented, "method Forward not implemented") } +func (UnimplementedInternalServer) ForwardAdminProposal(context.Context, *ForwardAdminProposalRequest) (*ForwardAdminProposalResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ForwardAdminProposal not implemented") +} func (UnimplementedInternalServer) RelayPublish(context.Context, *RelayPublishRequest) (*RelayPublishResponse, error) { return nil, status.Error(codes.Unimplemented, "method RelayPublish not implemented") } @@ -122,6 +138,24 @@ func _Internal_Forward_Handler(srv interface{}, ctx context.Context, dec func(in return interceptor(ctx, in, info, handler) } +func _Internal_ForwardAdminProposal_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ForwardAdminProposalRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InternalServer).ForwardAdminProposal(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Internal_ForwardAdminProposal_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InternalServer).ForwardAdminProposal(ctx, req.(*ForwardAdminProposalRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Internal_RelayPublish_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(RelayPublishRequest) if err := dec(in); err != nil { @@ -151,6 +185,10 @@ var Internal_ServiceDesc = grpc.ServiceDesc{ MethodName: "Forward", Handler: _Internal_Forward_Handler, }, + { + MethodName: "ForwardAdminProposal", + Handler: _Internal_ForwardAdminProposal_Handler, + }, { MethodName: "RelayPublish", Handler: _Internal_RelayPublish_Handler, From f30bd74b5e64064d6a846ac640cf35d639a9d03b Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 22:25:19 +0900 Subject: [PATCH 3/9] backup: harden control-plane fencing --- adapter/admin_backup.go | 7 +- adapter/admin_backup_test.go | 7 +- adapter/admin_grpc.go | 9 +- adapter/admin_grpc_test.go | 14 +++ adapter/internal.go | 14 +++ adapter/internal_admin_proposal_test.go | 24 ++++- distribution/engine.go | 7 ++ kv/backup_scan.go | 19 ++++ kv/leader_admin_proposer.go | 28 +++++- kv/leader_admin_proposer_test.go | 40 +++++++- kv/leader_proxy.go | 40 ++++++++ kv/shard_store_test.go | 30 ++++++ kv/sharded_coordinator.go | 30 +++++- kv/sharded_coordinator_leader_test.go | 44 +++++++++ main.go | 70 ++++++++++++- main_admin_test.go | 42 ++++++++ proto/internal.pb.go | 126 ++++++++++++++++++++---- proto/internal.proto | 7 ++ proto/internal_grpc.pb.go | 38 +++++++ 19 files changed, 551 insertions(+), 45 deletions(-) diff --git a/adapter/admin_backup.go b/adapter/admin_backup.go index 51b24d72e..4055f1d01 100644 --- a/adapter/admin_backup.go +++ b/adapter/admin_backup.go @@ -54,7 +54,7 @@ type BackupPeerVersion struct { type BackupPeerProbe func(context.Context, string) (BackupPeerVersion, error) type BackupStore interface { - CaptureBackupRouteSnapshot(start, end []byte) kv.BackupRouteSnapshot + CaptureBackupRouteSnapshotAt(context.Context, uint64) (kv.BackupRouteSnapshot, error) NewBackupKeyScannerAtSnapshot(snapshot kv.BackupRouteSnapshot, ts uint64, pageSize int) kv.BackupKeyScanner NewBackupScannerAtSnapshot(snapshot kv.BackupRouteSnapshot, ts uint64, pageSize int) kv.BackupScanner } @@ -259,7 +259,10 @@ func (s *AdminServer) prepareBackup(ctx context.Context, ttl time.Duration) (pre if readTS == 0 || readTS == ^uint64(0) { return preparedBackup{}, status.Errorf(codes.FailedPrecondition, "%s", "backup read fence returned an invalid timestamp") } - routes := s.backupStore.CaptureBackupRouteSnapshot(nil, nil) + routes, err := s.backupStore.CaptureBackupRouteSnapshotAt(ctx, readTS) + if err != nil { + return preparedBackup{}, status.Errorf(codes.FailedPrecondition, "capture backup routes at read timestamp: %v", err) + } pinID, err := newBackupPinID() if err != nil { return preparedBackup{}, status.Errorf(codes.Internal, "generate backup pin id: %v", err) diff --git a/adapter/admin_backup_test.go b/adapter/admin_backup_test.go index 2df70a6e3..6ca3e4cfb 100644 --- a/adapter/admin_backup_test.go +++ b/adapter/admin_backup_test.go @@ -114,6 +114,7 @@ type backupTestStore struct { mu sync.Mutex keys [][]byte readTS []uint64 + capturedTS []uint64 onCapture func() onExhaust func() scanDelay time.Duration @@ -121,14 +122,15 @@ type backupTestStore struct { pairCloseErr error } -func (s *backupTestStore) CaptureBackupRouteSnapshot(_, _ []byte) kv.BackupRouteSnapshot { +func (s *backupTestStore) CaptureBackupRouteSnapshotAt(_ context.Context, ts uint64) (kv.BackupRouteSnapshot, error) { s.mu.Lock() + s.capturedTS = append(s.capturedTS, ts) onCapture := s.onCapture s.mu.Unlock() if onCapture != nil { onCapture() } - return kv.BackupRouteSnapshot{} + return kv.BackupRouteSnapshot{}, nil } func (s *backupTestStore) NewBackupKeyScannerAtSnapshot(_ kv.BackupRouteSnapshot, ts uint64, _ int) kv.BackupKeyScanner { @@ -337,6 +339,7 @@ func TestBeginBackupCapturesRouteSnapshotAfterReadFence(t *testing.T) { _, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) require.NoError(t, err) require.Equal(t, int32(2), stage.Load()) + require.Equal(t, []uint64{42}, store.capturedTS) } func TestBeginBackupRenewsWhileBaselineRuns(t *testing.T) { diff --git a/adapter/admin_grpc.go b/adapter/admin_grpc.go index 58d2e5601..b905cd281 100644 --- a/adapter/admin_grpc.go +++ b/adapter/admin_grpc.go @@ -722,6 +722,11 @@ func sortedGroupIDs(m map[uint64]AdminGroup) []uint64 { // package-qualify the service name) does not silently bypass the auth gate. var adminMethodPrefix = "/" + pb.Admin_ServiceDesc.ServiceName + "/" +func adminAuthenticatedMethod(fullMethod string) bool { + return strings.HasPrefix(fullMethod, adminMethodPrefix) || + fullMethod == pb.Internal_ForwardAdminProposal_FullMethodName +} + // AdminTokenAuth builds a gRPC unary+stream interceptor pair enforcing // "authorization: Bearer " metadata against the supplied token. An // empty token disables enforcement; callers should pair that mode with a @@ -755,7 +760,7 @@ func AdminTokenAuth(token string) (grpc.UnaryServerInterceptor, grpc.StreamServe info *grpc.UnaryServerInfo, handler grpc.UnaryHandler, ) (any, error) { - if !strings.HasPrefix(info.FullMethod, adminMethodPrefix) { + if !adminAuthenticatedMethod(info.FullMethod) { return handler(ctx, req) } if err := check(ctx); err != nil { @@ -769,7 +774,7 @@ func AdminTokenAuth(token string) (grpc.UnaryServerInterceptor, grpc.StreamServe info *grpc.StreamServerInfo, handler grpc.StreamHandler, ) error { - if !strings.HasPrefix(info.FullMethod, adminMethodPrefix) { + if !adminAuthenticatedMethod(info.FullMethod) { return handler(srv, ss) } if err := check(ss.Context()); err != nil { diff --git a/adapter/admin_grpc_test.go b/adapter/admin_grpc_test.go index 8671fb137..629d18321 100644 --- a/adapter/admin_grpc_test.go +++ b/adapter/admin_grpc_test.go @@ -1021,6 +1021,20 @@ func TestAdminTokenAuthSkipsOtherServices(t *testing.T) { } } +func TestAdminTokenAuthProtectsForwardAdminProposal(t *testing.T) { + t.Parallel() + unary, _ := AdminTokenAuth("s3cret") + info := &grpc.UnaryServerInfo{FullMethod: pb.Internal_ForwardAdminProposal_FullMethodName} + handler := func(_ context.Context, _ any) (any, error) { return "ok", nil } + + _, err := unary(context.Background(), nil, info, handler) + require.Equal(t, codes.Unauthenticated, status.Code(err)) + ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer s3cret")) + resp, err := unary(ctx, nil, info, handler) + require.NoError(t, err) + require.Equal(t, "ok", resp) +} + func TestAdminTokenAuthEmptyTokenDisabled(t *testing.T) { t.Parallel() unary, stream := AdminTokenAuth("") diff --git a/adapter/internal.go b/adapter/internal.go index b3e828d93..824abc85e 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -108,6 +108,20 @@ func (i *Internal) ForwardAdminProposal( return &pb.ForwardAdminProposalResponse{CommitIndex: result.CommitIndex}, nil } +func (i *Internal) ForwardLeaseRead( + ctx context.Context, + _ *pb.ForwardLeaseReadRequest, +) (*pb.ForwardLeaseReadResponse, error) { + if i.leader == nil || i.leader.State() != raftengine.StateLeader { + return nil, errors.WithStack(ErrNotLeader) + } + index, err := i.leader.LinearizableRead(ctx) + if err != nil { + return nil, errors.WithStack(err) + } + return &pb.ForwardLeaseReadResponse{AppliedIndex: index}, nil +} + func forwardedAdminProposalResponseError(result *raftengine.ProposalResult) error { if result == nil { return errors.New("admin proposal returned nil result") diff --git a/adapter/internal_admin_proposal_test.go b/adapter/internal_admin_proposal_test.go index ef2f87e97..0f295905b 100644 --- a/adapter/internal_admin_proposal_test.go +++ b/adapter/internal_admin_proposal_test.go @@ -11,7 +11,8 @@ import ( ) type internalAdminLeaderView struct { - state raftengine.State + state raftengine.State + readIndex uint64 } func (v internalAdminLeaderView) State() raftengine.State { return v.state } @@ -19,8 +20,25 @@ func (internalAdminLeaderView) Leader() raftengine.LeaderInfo { return raftengine.LeaderInfo{ID: "leader", Address: "leader:50051"} } func (internalAdminLeaderView) VerifyLeader(context.Context) error { return nil } -func (internalAdminLeaderView) LinearizableRead(context.Context) (uint64, error) { - return 0, nil +func (v internalAdminLeaderView) LinearizableRead(context.Context) (uint64, error) { + return v.readIndex, nil +} + +func TestInternalForwardLeaseReadUsesLeaderBarrier(t *testing.T) { + t.Parallel() + internal := NewInternalWithEngine( + nil, + internalAdminLeaderView{state: raftengine.StateLeader, readIndex: 23}, + nil, + nil, + ) + resp, err := internal.ForwardLeaseRead(context.Background(), &pb.ForwardLeaseReadRequest{}) + require.NoError(t, err) + require.Equal(t, uint64(23), resp.GetAppliedIndex()) + + internal.leader = internalAdminLeaderView{state: raftengine.StateFollower} + _, err = internal.ForwardLeaseRead(context.Background(), &pb.ForwardLeaseReadRequest{}) + require.ErrorIs(t, err, ErrNotLeader) } type internalAdminProposer struct { diff --git a/distribution/engine.go b/distribution/engine.go index bc6613894..4e0debc16 100644 --- a/distribution/engine.go +++ b/distribution/engine.go @@ -440,6 +440,13 @@ func routesFromCatalog(routes []RouteDescriptor) ([]Route, error) { return out, nil } +// RoutesFromCatalogSnapshot validates and materializes the immutable route +// view contained in a durable catalog snapshot. Callers that need ownership +// as of an MVCC timestamp must use this view instead of the live Engine. +func RoutesFromCatalogSnapshot(snapshot CatalogSnapshot) ([]Route, error) { + return routesFromCatalog(snapshot.Routes) +} + func validateRouteOrder(routes []Route) error { if len(routes) < minRouteCountForOrderValidation { return nil diff --git a/kv/backup_scan.go b/kv/backup_scan.go index ea1d46b2a..b30f21ac8 100644 --- a/kv/backup_scan.go +++ b/kv/backup_scan.go @@ -89,6 +89,25 @@ func (s *ShardStore) CaptureBackupRouteSnapshot(start []byte, end []byte) Backup } } +// CaptureBackupRouteSnapshotAt reads the durable distribution catalog at ts. +// This keeps backup ownership aligned with the same MVCC timestamp used to +// materialize values even if the live route watcher advances immediately +// after the backup read fence. +func (s *ShardStore) CaptureBackupRouteSnapshotAt(ctx context.Context, ts uint64) (BackupRouteSnapshot, error) { + if s == nil { + return BackupRouteSnapshot{}, errors.New("backup route store is unavailable") + } + snapshot, err := distribution.NewCatalogStore(s).SnapshotAt(ctx, ts) + if err != nil { + return BackupRouteSnapshot{}, errors.Wrap(err, "read distribution catalog at backup timestamp") + } + routes, err := distribution.RoutesFromCatalogSnapshot(snapshot) + if err != nil { + return BackupRouteSnapshot{}, errors.Wrap(err, "materialize backup route snapshot") + } + return BackupRouteSnapshot{routes: cloneBackupRoutes(routes)}, nil +} + // NewBackupScannerAtSnapshot creates a value scanner from a captured route view. func NewBackupScannerAtSnapshot(st *ShardStore, snapshot BackupRouteSnapshot, ts uint64, pageSize int) BackupScanner { if pageSize <= 0 { diff --git a/kv/leader_admin_proposer.go b/kv/leader_admin_proposer.go index 8b6d41334..c22d05314 100644 --- a/kv/leader_admin_proposer.go +++ b/kv/leader_admin_proposer.go @@ -7,23 +7,40 @@ import ( "github.com/bootjp/elastickv/internal/raftengine" pb "github.com/bootjp/elastickv/proto" "github.com/cockroachdb/errors" + "google.golang.org/grpc/metadata" ) +type LeaderAdminProposerOption func(*leaderAdminProposer) + +func WithLeaderAdminToken(token string) LeaderAdminProposerOption { + return func(p *leaderAdminProposer) { + p.adminToken = token + } +} + // leaderAdminProposer forwards ProposeAdmin calls to the current group leader. // Propose remains local because only barrier-exempt, idempotent admin entries // are safe to retry across a leadership change. type leaderAdminProposer struct { - leader raftengine.LeaderView - local raftengine.Proposer - connCache *GRPCConnCache + leader raftengine.LeaderView + local raftengine.Proposer + connCache *GRPCConnCache + adminToken string } func NewLeaderAdminProposer( leader raftengine.LeaderView, local raftengine.Proposer, connCache *GRPCConnCache, + opts ...LeaderAdminProposerOption, ) raftengine.Proposer { - return &leaderAdminProposer{leader: leader, local: local, connCache: connCache} + p := &leaderAdminProposer{leader: leader, local: local, connCache: connCache} + for _, opt := range opts { + if opt != nil { + opt(p) + } + } + return p } func (p *leaderAdminProposer) Propose( @@ -125,6 +142,9 @@ func (p *leaderAdminProposer) forwardAdmin( } ctx, cancel := context.WithTimeout(parentCtx, leaderForwardTimeout) defer cancel() + if p.adminToken != "" { + ctx = metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+p.adminToken) + } resp, err := pb.NewInternalClient(conn).ForwardAdminProposal(ctx, &pb.ForwardAdminProposalRequest{Payload: data}) if err != nil { return nil, errors.WithStack(err) diff --git a/kv/leader_admin_proposer_test.go b/kv/leader_admin_proposer_test.go index 13c7d1c77..e7b7911aa 100644 --- a/kv/leader_admin_proposer_test.go +++ b/kv/leader_admin_proposer_test.go @@ -10,6 +10,7 @@ import ( pb "github.com/bootjp/elastickv/proto" "github.com/stretchr/testify/require" "google.golang.org/grpc" + "google.golang.org/grpc/metadata" ) type adminProposalLeaderView struct { @@ -47,16 +48,20 @@ func (p *recordingAdminProposer) ProposeAdmin(_ context.Context, data []byte) (* type forwardingAdminServer struct { pb.UnimplementedInternalServer - mu sync.Mutex - payload []byte + mu sync.Mutex + payload []byte + authorization []string } func (s *forwardingAdminServer) ForwardAdminProposal( - _ context.Context, + ctx context.Context, req *pb.ForwardAdminProposalRequest, ) (*pb.ForwardAdminProposalResponse, error) { s.mu.Lock() s.payload = append([]byte(nil), req.GetPayload()...) + if md, ok := metadata.FromIncomingContext(ctx); ok { + s.authorization = append([]string(nil), md.Get("authorization")...) + } s.mu.Unlock() return &pb.ForwardAdminProposalResponse{CommitIndex: 123}, nil } @@ -107,3 +112,32 @@ func TestLeaderAdminProposerForwardsFollowerProposal(t *testing.T) { require.Equal(t, []byte("pin"), service.payload) service.mu.Unlock() } + +func TestLeaderAdminProposerForwardsBearerToken(t *testing.T) { + t.Parallel() + var lc net.ListenConfig + lis, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + service := &forwardingAdminServer{} + server := grpc.NewServer() + pb.RegisterInternalServer(server, service) + go func() { _ = server.Serve(lis) }() + t.Cleanup(func() { + server.Stop() + _ = lis.Close() + }) + + cache := &GRPCConnCache{} + t.Cleanup(func() { require.NoError(t, cache.Close()) }) + proposer := NewLeaderAdminProposer( + &adminProposalLeaderView{state: raftengine.StateFollower, addr: lis.Addr().String()}, + &recordingAdminProposer{}, + cache, + WithLeaderAdminToken("s3cret"), + ) + _, err = proposer.ProposeAdmin(context.Background(), []byte("pin")) + require.NoError(t, err) + service.mu.Lock() + require.Equal(t, []string{"Bearer s3cret"}, service.authorization) + service.mu.Unlock() +} diff --git a/kv/leader_proxy.go b/kv/leader_proxy.go index 2428c1bf9..9be953e08 100644 --- a/kv/leader_proxy.go +++ b/kv/leader_proxy.go @@ -223,6 +223,46 @@ func (p *LeaderProxy) forward(parentCtx context.Context, reqs []*pb.Request) (*T return &TransactionResponse{CommitIndex: resp.CommitIndex}, nil } +func (p *LeaderProxy) forwardLeaseRead(callerCtx context.Context) (uint64, error) { + deadline := time.Now().Add(leaderProxyRetryBudget) + ctx, cancel := context.WithDeadline(callerCtx, deadline) + defer cancel() + var lastErr error + for time.Now().Before(deadline) { + index, err := p.forwardLeaseReadOnce(ctx) + if err == nil { + return index, nil + } + lastErr = err + if !isTransientLeaderError(err) { + return 0, err + } + waitLeaderProxyBackoff(ctx, leaderProxyRetryInterval, deadline) + } + if lastErr == nil { + lastErr = ErrLeaderNotFound + } + return 0, errors.WithStack(lastErr) +} + +func (p *LeaderProxy) forwardLeaseReadOnce(parentCtx context.Context) (uint64, error) { + addr := leaderAddrFromEngine(p.engine) + if addr == "" { + return 0, errors.WithStack(ErrLeaderNotFound) + } + conn, err := p.connCache.ConnFor(addr) + if err != nil { + return 0, err + } + ctx, cancel := context.WithTimeout(parentCtx, leaderForwardTimeout) + defer cancel() + resp, err := pb.NewInternalClient(conn).ForwardLeaseRead(ctx, &pb.ForwardLeaseReadRequest{}) + if err != nil { + return 0, errors.WithStack(err) + } + return resp.GetAppliedIndex(), nil +} + var _ Transactional = (*LeaderProxy)(nil) var _ io.Closer = (*LeaderProxy)(nil) diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index ae0302f36..ed67a0c3c 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -669,6 +669,36 @@ func TestBackupScannerReusesCapturedRouteSnapshot(t *testing.T) { require.Equal(t, []byte("pinned-owner"), pair.Value) } +func TestCaptureBackupRouteSnapshotAtUsesDurableCatalogTimestamp(t *testing.T) { + t.Parallel() + ctx := context.Background() + engine := distribution.NewEngine() + groups := map[uint64]*ShardGroup{ + 1: {Store: store.NewMVCCStore()}, + 2: {Store: store.NewMVCCStore()}, + } + shards := NewShardStore(engine, groups) + catalog := distribution.NewCatalogStore(groups[1].Store) + old, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), GroupID: 2, State: distribution.RouteStateActive}, + }) + require.NoError(t, err) + require.NoError(t, engine.ApplySnapshot(old)) + oldReadTS := groups[1].Store.LastCommitTS() + current, err := catalog.Save(ctx, old.Version, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), GroupID: 1, State: distribution.RouteStateActive}, + }) + require.NoError(t, err) + require.NoError(t, engine.ApplySnapshot(current)) + + snapshot, err := shards.CaptureBackupRouteSnapshotAt(ctx, oldReadTS) + require.NoError(t, err) + require.Len(t, snapshot.routes, 2) + require.Equal(t, uint64(1), snapshot.routes[0].GroupID) + require.Equal(t, uint64(2), snapshot.routes[1].GroupID) +} + func TestBackupScannerMaterializesFromCapturedRoute(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index 37b963c90..e8a70cc8f 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -60,6 +60,9 @@ type ShardGroup struct { // NewLeaderProxyForShardGroup), so the no-wrap default keeps // working. proposer raftengine.Proposer + // leaderRead forwards a read barrier to this group's current leader when + // the local replica is a follower. It is installed with the LeaderProxy. + leaderRead func(context.Context) (uint64, error) } // Proposer returns the wrap-aware proposer chain installed by @@ -207,10 +210,12 @@ func NewLeaderProxyForShardGroup(g *ShardGroup, opts ...TransactionOption) *Lead // (codex P2 round-1); routing both paths through the same // proposer closes that hole. g.proposer = newDynamicWrappedProposer(g.Engine, &g.raftPayloadWrap) - return &LeaderProxy{ + p := &LeaderProxy{ engine: g.Engine, tm: NewTransactionWithProposer(g.proposer, opts...), } + g.leaderRead = p.forwardLeaseRead + return p } // leaseRefreshingTxn wraps a Transactional so every Commit / Abort that @@ -1820,8 +1825,26 @@ func observeLeaseRead(observer LeaseReadObserver, hit bool) { } } +func shouldForwardGroupLeaseRead(g *ShardGroup, engine raftengine.Engine) bool { + return g != nil && g.leaderRead != nil && engine != nil && engine.State() != raftengine.StateLeader +} + +func handleGroupLeaseReadError(ctx context.Context, g *ShardGroup, err error) (uint64, error) { + if g != nil && isLeadershipLossError(err) { + g.lease.invalidate() + } + if g != nil && g.leaderRead != nil && isTransientLeaderError(err) { + return g.leaderRead(ctx) + } + return 0, err +} + func groupLeaseRead(ctx context.Context, g *ShardGroup, observer LeaseReadObserver) (uint64, error) { engine := engineForGroup(g) + if shouldForwardGroupLeaseRead(g, engine) { + observeLeaseRead(observer, false) + return g.leaderRead(ctx) + } // g.lp caches the LeaseProvider assertion done once at construction // (NewShardedCoordinator); a nil group or an engine without the // capability falls through to the linearizable slow path. The nil-g @@ -1852,10 +1875,7 @@ func groupLeaseRead(ctx context.Context, g *ShardGroup, observer LeaseReadObserv observeLeaseRead(observer, false) idx, err := linearizableReadEngineCtx(ctx, engine) if err != nil { - if isLeadershipLossError(err) { - g.lease.invalidate() - } - return 0, err + return handleGroupLeaseReadError(ctx, g, err) } g.lease.extend(now.Add(leaseDur), expectedGen) return idx, nil diff --git a/kv/sharded_coordinator_leader_test.go b/kv/sharded_coordinator_leader_test.go index 0fda49e77..631fe2204 100644 --- a/kv/sharded_coordinator_leader_test.go +++ b/kv/sharded_coordinator_leader_test.go @@ -2,13 +2,30 @@ package kv import ( "context" + "net" + "sync/atomic" "testing" "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" + "google.golang.org/grpc" ) +type leaseReadForwardServer struct { + pb.UnimplementedInternalServer + calls atomic.Int32 +} + +func (s *leaseReadForwardServer) ForwardLeaseRead( + context.Context, + *pb.ForwardLeaseReadRequest, +) (*pb.ForwardLeaseReadResponse, error) { + s.calls.Add(1) + return &pb.ForwardLeaseReadResponse{AppliedIndex: 44}, nil +} + func TestShardedCoordinatorVerifyLeader_LeaderReturnsNil(t *testing.T) { t.Parallel() @@ -66,6 +83,33 @@ func TestShardedCoordinatorLeaseReadAllGroups_FencesEveryLeader(t *testing.T) { require.NoError(t, coord.LeaseReadAllGroups(context.Background())) } +func TestShardedCoordinatorLeaseReadAllGroups_ForwardsFollowerToGroupLeader(t *testing.T) { + t.Parallel() + var lc net.ListenConfig + lis, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + service := &leaseReadForwardServer{} + server := grpc.NewServer() + pb.RegisterInternalServer(server, service) + go func() { _ = server.Serve(lis) }() + t.Cleanup(func() { + server.Stop() + _ = lis.Close() + }) + + engine := distribution.NewEngine() + engine.UpdateRoute([]byte(""), nil, 1) + follower := &stubFollowerEngine{leaderAddr: lis.Addr().String()} + group := &ShardGroup{Engine: follower, Store: store.NewMVCCStore()} + proxy := NewLeaderProxyForShardGroup(group) + t.Cleanup(func() { require.NoError(t, proxy.Close()) }) + group.Txn = proxy + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{1: group}, 1, NewHLC(), nil) + + require.NoError(t, coord.LeaseReadAllGroups(context.Background())) + require.Equal(t, int32(1), service.calls.Load()) +} + func TestShardedCoordinatorLeaseReadAllGroups_UsesConfiguredAllShardGroups(t *testing.T) { t.Parallel() diff --git a/main.go b/main.go index 3eeddc69a..4c168b59f 100644 --- a/main.go +++ b/main.go @@ -2146,7 +2146,12 @@ func setupAdminService( for _, rt := range runtimes { srv.RegisterGroup(rt.spec.id, rt.engine) if group := shardGroups[rt.spec.id]; group != nil { - srv.RegisterBackupProposer(rt.spec.id, kv.NewLeaderAdminProposer(rt.engine, group.Proposer(), connCache)) + srv.RegisterBackupProposer(rt.spec.id, kv.NewLeaderAdminProposer( + rt.engine, + group.Proposer(), + connCache, + kv.WithLeaderAdminToken(icept.token), + )) } } // Only register a real sampler. Passing a typed-nil *MemSampler @@ -2218,6 +2223,7 @@ func adminMembersFromBootstrap(selfID string, servers []raftengine.Server) []ada type adminGRPCInterceptors struct { unary []grpc.UnaryServerInterceptor stream []grpc.StreamServerInterceptor + token string } func (a adminGRPCInterceptors) empty() bool { @@ -2233,6 +2239,8 @@ var _ kv.Coordinator = (*startupGatedCoordinator)(nil) var _ kv.LeaseReadableCoordinator = (*startupGatedCoordinator)(nil) var _ kv.AllGroupsLeaseReadableCoordinator = (*startupGatedCoordinator)(nil) var _ kv.GroupRoutableCoordinator = (*startupGatedCoordinator)(nil) +var _ kv.TimestampAllocator = (*startupGatedCoordinator)(nil) +var _ kv.TimestampAfterAllocator = (*startupGatedCoordinator)(nil) func (c startupGatedCoordinator) Dispatch(ctx context.Context, reqs *kv.OperationGroup[kv.OP]) (*kv.CoordinateResponse, error) { if c.gate != nil && c.gate.blocked() { @@ -2273,6 +2281,22 @@ func (c startupGatedCoordinator) Clock() *kv.HLC { return c.inner.Clock() } +func (c startupGatedCoordinator) Next(ctx context.Context) (uint64, error) { + alloc, ok := c.inner.(kv.TimestampAllocator) + if !ok { + return 0, errors.New("startup coordinator timestamp allocator is unavailable") + } + return alloc.Next(ctx) //nolint:wrapcheck // Preserve allocator errors for callers. +} + +func (c startupGatedCoordinator) NextAfter(ctx context.Context, min uint64) (uint64, error) { + alloc, ok := c.inner.(kv.TimestampAfterAllocator) + if !ok { + return 0, errors.New("startup coordinator timestamp-after allocator is unavailable") + } + return alloc.NextAfter(ctx, min) //nolint:wrapcheck // Preserve allocator errors for callers. +} + func (c startupGatedCoordinator) LeaseRead(ctx context.Context) (uint64, error) { return kv.LeaseReadThrough(c.inner, ctx) //nolint:wrapcheck // Pass through coordinator errors unchanged. } @@ -2380,7 +2404,7 @@ func configureAdminService( srv := adapter.NewAdminServer(self, members, opts...) srv.SetCapability(adapter.S3BlobOffloadCapabilityName, adapter.S3BlobOffloadLocalCapability()) unary, stream := adapter.AdminTokenAuth(token) - var icept adminGRPCInterceptors + icept := adminGRPCInterceptors{token: token} if unary != nil { icept.unary = append(icept.unary, unary) } @@ -2450,8 +2474,9 @@ func adminBackupReadFence(coordinate kv.Coordinator, shardStore *kv.ShardStore) if err := kv.LeaseReadAllGroupsThrough(coordinate, ctx); err != nil { return 0, errors.Wrap(err, "backup: fence raft groups") } - clock.Observe(shardStore.LastCommitTS()) - readTS, err := clock.NextFenced() + lastCommitTS := shardStore.LastCommitTS() + clock.Observe(lastCommitTS) + readTS, err := allocateBackupReadTimestamp(ctx, coordinate, lastCommitTS) if err != nil { return 0, errors.Wrap(err, "backup: issue read timestamp") } @@ -2459,6 +2484,28 @@ func adminBackupReadFence(coordinate kv.Coordinator, shardStore *kv.ShardStore) } } +func allocateBackupReadTimestamp(ctx context.Context, coordinate kv.Coordinator, min uint64) (uint64, error) { + if alloc, ok := coordinate.(kv.TimestampAfterAllocator); ok { + return alloc.NextAfter(ctx, min) //nolint:wrapcheck // Canonical coordinator/TSO error. + } + if alloc, ok := coordinate.(kv.TimestampAllocator); ok { + ts, err := alloc.Next(ctx) + if err != nil { + return 0, err //nolint:wrapcheck // Wrapped at the backup call site. + } + if ts <= min { + return 0, errors.Errorf("timestamp allocator returned %d, not after %d", ts, min) + } + return ts, nil + } + clock := coordinate.Clock() + if clock == nil { + return 0, errors.New("backup timestamp allocator is unavailable") + } + clock.Observe(min) + return clock.NextFenced() //nolint:wrapcheck // Wrapped at the backup call site. +} + func adminBackupPeerProbe(connCache *kv.GRPCConnCache) adapter.BackupPeerProbe { if connCache == nil { return nil @@ -2698,12 +2745,13 @@ func startRaftServers( grpcSvc := adapter.NewGRPCServer(shardStore, coordinate) pb.RegisterRawKVServer(gs, grpcSvc) pb.RegisterTransactionalKVServer(gs, grpcSvc) + internalOpts := internalServerOptions(coordinate, adminServer, proposerForGroup(rt, shardGroups)) pb.RegisterInternalServer(gs, adapter.NewInternalWithEngine( trx, rt.engine, coordinate.Clock(), relay, - append(internalTimestampOptions(coordinate), adapter.WithInternalAdminProposer(proposerForGroup(rt, shardGroups)))..., + internalOpts..., )) pb.RegisterDistributionServer(gs, distServer) if adminServer != nil { @@ -2778,6 +2826,18 @@ func startRaftServers( return nil } +func internalServerOptions( + coordinate kv.Coordinator, + adminServer *adapter.AdminServer, + proposer raftengine.Proposer, +) []adapter.InternalOption { + opts := internalTimestampOptions(coordinate) + if adminServer != nil { + return append(opts, adapter.WithInternalAdminProposer(proposer)) + } + return opts +} + func internalTimestampOptions(coordinate kv.Coordinator) []adapter.InternalOption { if alloc, ok := coordinate.(kv.TimestampAllocator); ok { return []adapter.InternalOption{adapter.WithInternalTimestampAllocator(alloc)} diff --git a/main_admin_test.go b/main_admin_test.go index 5c597f628..0baefbe36 100644 --- a/main_admin_test.go +++ b/main_admin_test.go @@ -19,6 +19,7 @@ import ( "os" "path/filepath" "strings" + "sync/atomic" "testing" "time" @@ -76,6 +77,30 @@ type backupFenceOrderCoordinator struct { onBarrier func(uint64) } +type backupFenceTSOCoordinator struct { + stubStartupCoordinator + min uint64 + nextAfter uint64 + barriers atomic.Int32 + barrierErr error + nextErr error + afterErr error +} + +func (c *backupFenceTSOCoordinator) LeaseReadAllGroups(context.Context) error { + c.barriers.Add(1) + return c.barrierErr +} + +func (c *backupFenceTSOCoordinator) Next(context.Context) (uint64, error) { + return c.nextAfter, c.nextErr +} + +func (c *backupFenceTSOCoordinator) NextAfter(_ context.Context, min uint64) (uint64, error) { + c.min = min + return c.nextAfter, c.afterErr +} + func (c *backupFenceOrderCoordinator) LeaseReadAllGroups(context.Context) error { ts, err := c.Clock().NextFenced() if err != nil { @@ -107,6 +132,23 @@ func TestAdminBackupReadFenceAllocatesTimestampAfterBarrier(t *testing.T) { require.Greater(t, readTS, coordinate.timestampDuringBarrier) } +func TestAdminBackupReadFenceUsesCoordinatorTimestampAllocator(t *testing.T) { + t.Parallel() + coordinate := &backupFenceTSOCoordinator{nextAfter: 9_000} + groupStore := store.NewMVCCStore() + require.NoError(t, groupStore.PutAt(context.Background(), []byte("committed"), []byte("value"), 7_000, 0)) + shardStore := kv.NewShardStore(distribution.NewEngineWithDefaultRoute(), map[uint64]*kv.ShardGroup{ + 1: {Store: groupStore}, + }) + fence := adminBackupReadFence(coordinate, shardStore) + + readTS, err := fence(context.Background()) + require.NoError(t, err) + require.Equal(t, uint64(9_000), readTS) + require.Equal(t, uint64(7_000), coordinate.min) + require.Equal(t, int32(1), coordinate.barriers.Load()) +} + func TestConfigureAdminServiceRejectsMutualExclusion(t *testing.T) { t.Parallel() dir := t.TempDir() diff --git a/proto/internal.pb.go b/proto/internal.pb.go index 7513776a8..36e9607d2 100644 --- a/proto/internal.pb.go +++ b/proto/internal.pb.go @@ -542,6 +542,86 @@ func (x *ForwardAdminProposalResponse) GetCommitIndex() uint64 { return 0 } +type ForwardLeaseReadRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ForwardLeaseReadRequest) Reset() { + *x = ForwardLeaseReadRequest{} + mi := &file_internal_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ForwardLeaseReadRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ForwardLeaseReadRequest) ProtoMessage() {} + +func (x *ForwardLeaseReadRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ForwardLeaseReadRequest.ProtoReflect.Descriptor instead. +func (*ForwardLeaseReadRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{7} +} + +type ForwardLeaseReadResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + AppliedIndex uint64 `protobuf:"varint,1,opt,name=applied_index,json=appliedIndex,proto3" json:"applied_index,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ForwardLeaseReadResponse) Reset() { + *x = ForwardLeaseReadResponse{} + mi := &file_internal_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ForwardLeaseReadResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ForwardLeaseReadResponse) ProtoMessage() {} + +func (x *ForwardLeaseReadResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ForwardLeaseReadResponse.ProtoReflect.Descriptor instead. +func (*ForwardLeaseReadResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{8} +} + +func (x *ForwardLeaseReadResponse) GetAppliedIndex() uint64 { + if x != nil { + return x.AppliedIndex + } + return 0 +} + type RelayPublishRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Channel []byte `protobuf:"bytes,1,opt,name=channel,proto3" json:"channel,omitempty"` @@ -552,7 +632,7 @@ type RelayPublishRequest struct { func (x *RelayPublishRequest) Reset() { *x = RelayPublishRequest{} - mi := &file_internal_proto_msgTypes[7] + mi := &file_internal_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -564,7 +644,7 @@ func (x *RelayPublishRequest) String() string { func (*RelayPublishRequest) ProtoMessage() {} func (x *RelayPublishRequest) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_msgTypes[7] + mi := &file_internal_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -577,7 +657,7 @@ func (x *RelayPublishRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RelayPublishRequest.ProtoReflect.Descriptor instead. func (*RelayPublishRequest) Descriptor() ([]byte, []int) { - return file_internal_proto_rawDescGZIP(), []int{7} + return file_internal_proto_rawDescGZIP(), []int{9} } func (x *RelayPublishRequest) GetChannel() []byte { @@ -603,7 +683,7 @@ type RelayPublishResponse struct { func (x *RelayPublishResponse) Reset() { *x = RelayPublishResponse{} - mi := &file_internal_proto_msgTypes[8] + mi := &file_internal_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -615,7 +695,7 @@ func (x *RelayPublishResponse) String() string { func (*RelayPublishResponse) ProtoMessage() {} func (x *RelayPublishResponse) ProtoReflect() protoreflect.Message { - mi := &file_internal_proto_msgTypes[8] + mi := &file_internal_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -628,7 +708,7 @@ func (x *RelayPublishResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RelayPublishResponse.ProtoReflect.Descriptor instead. func (*RelayPublishResponse) Descriptor() ([]byte, []int) { - return file_internal_proto_rawDescGZIP(), []int{8} + return file_internal_proto_rawDescGZIP(), []int{10} } func (x *RelayPublishResponse) GetSubscribers() int64 { @@ -667,7 +747,10 @@ const file_internal_proto_rawDesc = "" + "\x1bForwardAdminProposalRequest\x12\x18\n" + "\apayload\x18\x01 \x01(\fR\apayload\"A\n" + "\x1cForwardAdminProposalResponse\x12!\n" + - "\fcommit_index\x18\x01 \x01(\x04R\vcommitIndex\"I\n" + + "\fcommit_index\x18\x01 \x01(\x04R\vcommitIndex\"\x19\n" + + "\x17ForwardLeaseReadRequest\"?\n" + + "\x18ForwardLeaseReadResponse\x12#\n" + + "\rapplied_index\x18\x01 \x01(\x04R\fappliedIndex\"I\n" + "\x13RelayPublishRequest\x12\x18\n" + "\achannel\x18\x01 \x01(\fR\achannel\x12\x18\n" + "\amessage\x18\x02 \x01(\fR\amessage\"8\n" + @@ -683,10 +766,11 @@ const file_internal_proto_rawDesc = "" + "\aPREPARE\x10\x01\x12\n" + "\n" + "\x06COMMIT\x10\x02\x12\t\n" + - "\x05ABORT\x10\x032\xd0\x01\n" + + "\x05ABORT\x10\x032\x9b\x02\n" + "\bInternal\x12.\n" + "\aForward\x12\x0f.ForwardRequest\x1a\x10.ForwardResponse\"\x00\x12U\n" + - "\x14ForwardAdminProposal\x12\x1c.ForwardAdminProposalRequest\x1a\x1d.ForwardAdminProposalResponse\"\x00\x12=\n" + + "\x14ForwardAdminProposal\x12\x1c.ForwardAdminProposalRequest\x1a\x1d.ForwardAdminProposalResponse\"\x00\x12I\n" + + "\x10ForwardLeaseRead\x12\x18.ForwardLeaseReadRequest\x1a\x19.ForwardLeaseReadResponse\"\x00\x12=\n" + "\fRelayPublish\x12\x14.RelayPublishRequest\x1a\x15.RelayPublishResponse\"\x00B#Z!github.com/bootjp/elastickv/protob\x06proto3" var ( @@ -702,7 +786,7 @@ func file_internal_proto_rawDescGZIP() []byte { } var file_internal_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 9) +var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 11) var file_internal_proto_goTypes = []any{ (Op)(0), // 0: Op (Phase)(0), // 1: Phase @@ -713,8 +797,10 @@ var file_internal_proto_goTypes = []any{ (*ForwardResponse)(nil), // 6: ForwardResponse (*ForwardAdminProposalRequest)(nil), // 7: ForwardAdminProposalRequest (*ForwardAdminProposalResponse)(nil), // 8: ForwardAdminProposalResponse - (*RelayPublishRequest)(nil), // 9: RelayPublishRequest - (*RelayPublishResponse)(nil), // 10: RelayPublishResponse + (*ForwardLeaseReadRequest)(nil), // 9: ForwardLeaseReadRequest + (*ForwardLeaseReadResponse)(nil), // 10: ForwardLeaseReadResponse + (*RelayPublishRequest)(nil), // 11: RelayPublishRequest + (*RelayPublishResponse)(nil), // 12: RelayPublishResponse } var file_internal_proto_depIdxs = []int32{ 0, // 0: Mutation.op:type_name -> Op @@ -724,12 +810,14 @@ var file_internal_proto_depIdxs = []int32{ 3, // 4: ForwardRequest.requests:type_name -> Request 5, // 5: Internal.Forward:input_type -> ForwardRequest 7, // 6: Internal.ForwardAdminProposal:input_type -> ForwardAdminProposalRequest - 9, // 7: Internal.RelayPublish:input_type -> RelayPublishRequest - 6, // 8: Internal.Forward:output_type -> ForwardResponse - 8, // 9: Internal.ForwardAdminProposal:output_type -> ForwardAdminProposalResponse - 10, // 10: Internal.RelayPublish:output_type -> RelayPublishResponse - 8, // [8:11] is the sub-list for method output_type - 5, // [5:8] is the sub-list for method input_type + 9, // 7: Internal.ForwardLeaseRead:input_type -> ForwardLeaseReadRequest + 11, // 8: Internal.RelayPublish:input_type -> RelayPublishRequest + 6, // 9: Internal.Forward:output_type -> ForwardResponse + 8, // 10: Internal.ForwardAdminProposal:output_type -> ForwardAdminProposalResponse + 10, // 11: Internal.ForwardLeaseRead:output_type -> ForwardLeaseReadResponse + 12, // 12: Internal.RelayPublish:output_type -> RelayPublishResponse + 9, // [9:13] is the sub-list for method output_type + 5, // [5:9] is the sub-list for method input_type 5, // [5:5] is the sub-list for extension type_name 5, // [5:5] is the sub-list for extension extendee 0, // [0:5] is the sub-list for field type_name @@ -746,7 +834,7 @@ func file_internal_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_internal_proto_rawDesc), len(file_internal_proto_rawDesc)), NumEnums: 2, - NumMessages: 9, + NumMessages: 11, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/internal.proto b/proto/internal.proto index 24396b703..536f7b593 100644 --- a/proto/internal.proto +++ b/proto/internal.proto @@ -7,6 +7,7 @@ service Internal { // for internal leader redirect only rpc Forward(ForwardRequest) returns (ForwardResponse) {} rpc ForwardAdminProposal(ForwardAdminProposalRequest) returns (ForwardAdminProposalResponse) {} + rpc ForwardLeaseRead(ForwardLeaseReadRequest) returns (ForwardLeaseReadResponse) {} rpc RelayPublish(RelayPublishRequest) returns (RelayPublishResponse) {} } @@ -86,6 +87,12 @@ message ForwardAdminProposalResponse { uint64 commit_index = 1; } +message ForwardLeaseReadRequest {} + +message ForwardLeaseReadResponse { + uint64 applied_index = 1; +} + message RelayPublishRequest { bytes channel = 1; bytes message = 2; diff --git a/proto/internal_grpc.pb.go b/proto/internal_grpc.pb.go index c528a33b7..e059b67c7 100644 --- a/proto/internal_grpc.pb.go +++ b/proto/internal_grpc.pb.go @@ -21,6 +21,7 @@ const _ = grpc.SupportPackageIsVersion9 const ( Internal_Forward_FullMethodName = "/Internal/Forward" Internal_ForwardAdminProposal_FullMethodName = "/Internal/ForwardAdminProposal" + Internal_ForwardLeaseRead_FullMethodName = "/Internal/ForwardLeaseRead" Internal_RelayPublish_FullMethodName = "/Internal/RelayPublish" ) @@ -31,6 +32,7 @@ type InternalClient interface { // for internal leader redirect only Forward(ctx context.Context, in *ForwardRequest, opts ...grpc.CallOption) (*ForwardResponse, error) ForwardAdminProposal(ctx context.Context, in *ForwardAdminProposalRequest, opts ...grpc.CallOption) (*ForwardAdminProposalResponse, error) + ForwardLeaseRead(ctx context.Context, in *ForwardLeaseReadRequest, opts ...grpc.CallOption) (*ForwardLeaseReadResponse, error) RelayPublish(ctx context.Context, in *RelayPublishRequest, opts ...grpc.CallOption) (*RelayPublishResponse, error) } @@ -62,6 +64,16 @@ func (c *internalClient) ForwardAdminProposal(ctx context.Context, in *ForwardAd return out, nil } +func (c *internalClient) ForwardLeaseRead(ctx context.Context, in *ForwardLeaseReadRequest, opts ...grpc.CallOption) (*ForwardLeaseReadResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ForwardLeaseReadResponse) + err := c.cc.Invoke(ctx, Internal_ForwardLeaseRead_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *internalClient) RelayPublish(ctx context.Context, in *RelayPublishRequest, opts ...grpc.CallOption) (*RelayPublishResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(RelayPublishResponse) @@ -79,6 +91,7 @@ type InternalServer interface { // for internal leader redirect only Forward(context.Context, *ForwardRequest) (*ForwardResponse, error) ForwardAdminProposal(context.Context, *ForwardAdminProposalRequest) (*ForwardAdminProposalResponse, error) + ForwardLeaseRead(context.Context, *ForwardLeaseReadRequest) (*ForwardLeaseReadResponse, error) RelayPublish(context.Context, *RelayPublishRequest) (*RelayPublishResponse, error) mustEmbedUnimplementedInternalServer() } @@ -96,6 +109,9 @@ func (UnimplementedInternalServer) Forward(context.Context, *ForwardRequest) (*F func (UnimplementedInternalServer) ForwardAdminProposal(context.Context, *ForwardAdminProposalRequest) (*ForwardAdminProposalResponse, error) { return nil, status.Error(codes.Unimplemented, "method ForwardAdminProposal not implemented") } +func (UnimplementedInternalServer) ForwardLeaseRead(context.Context, *ForwardLeaseReadRequest) (*ForwardLeaseReadResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ForwardLeaseRead not implemented") +} func (UnimplementedInternalServer) RelayPublish(context.Context, *RelayPublishRequest) (*RelayPublishResponse, error) { return nil, status.Error(codes.Unimplemented, "method RelayPublish not implemented") } @@ -156,6 +172,24 @@ func _Internal_ForwardAdminProposal_Handler(srv interface{}, ctx context.Context return interceptor(ctx, in, info, handler) } +func _Internal_ForwardLeaseRead_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ForwardLeaseReadRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InternalServer).ForwardLeaseRead(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Internal_ForwardLeaseRead_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InternalServer).ForwardLeaseRead(ctx, req.(*ForwardLeaseReadRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Internal_RelayPublish_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(RelayPublishRequest) if err := dec(in); err != nil { @@ -189,6 +223,10 @@ var Internal_ServiceDesc = grpc.ServiceDesc{ MethodName: "ForwardAdminProposal", Handler: _Internal_ForwardAdminProposal_Handler, }, + { + MethodName: "ForwardLeaseRead", + Handler: _Internal_ForwardLeaseRead_Handler, + }, { MethodName: "RelayPublish", Handler: _Internal_RelayPublish_Handler, From 85e98d736480612bc71dba11ae4ca84685f5e676 Mon Sep 17 00:00:00 2001 From: bootjp Date: Mon, 20 Jul 2026 00:04:04 +0900 Subject: [PATCH 4/9] backup: fence live snapshot timestamp races --- adapter/admin_backup.go | 18 ++- adapter/admin_backup_test.go | 46 +++++- adapter/admin_grpc.go | 3 +- adapter/admin_grpc_test.go | 26 ++-- .../2026_04_29_proposed_logical_backup.md | 53 +++---- kv/active_timestamp_tracker.go | 36 +++-- kv/active_timestamp_tracker_test.go | 13 ++ kv/backup_scan.go | 12 +- kv/fsm.go | 33 ++++- kv/fsm_backup.go | 77 +++++++++- kv/fsm_backup_test.go | 140 ++++++++++++++++++ kv/leader_proxy.go | 5 + kv/shard_store_test.go | 31 +++- kv/sharded_coordinator.go | 52 +++++++ kv/sharded_coordinator_leader_test.go | 18 ++- kv/tso_test.go | 32 ++++ main.go | 29 +++- 17 files changed, 545 insertions(+), 79 deletions(-) diff --git a/adapter/admin_backup.go b/adapter/admin_backup.go index 4055f1d01..ff4d0649a 100644 --- a/adapter/admin_backup.go +++ b/adapter/admin_backup.go @@ -259,10 +259,6 @@ func (s *AdminServer) prepareBackup(ctx context.Context, ttl time.Duration) (pre if readTS == 0 || readTS == ^uint64(0) { return preparedBackup{}, status.Errorf(codes.FailedPrecondition, "%s", "backup read fence returned an invalid timestamp") } - routes, err := s.backupStore.CaptureBackupRouteSnapshotAt(ctx, readTS) - if err != nil { - return preparedBackup{}, status.Errorf(codes.FailedPrecondition, "capture backup routes at read timestamp: %v", err) - } pinID, err := newBackupPinID() if err != nil { return preparedBackup{}, status.Errorf(codes.Internal, "generate backup pin id: %v", err) @@ -273,6 +269,15 @@ func (s *AdminServer) prepareBackup(ctx context.Context, ttl time.Duration) (pre if err != nil { return preparedBackup{}, err } + // Capture catalog ownership only after every data group has applied the + // pin's timestamp floor. A preallocated catalog write at or below readTS + // either lands before this point and is visible here, or is rejected by the + // floor; it cannot race the captured route view after this call returns. + routes, err := s.backupStore.CaptureBackupRouteSnapshotAt(ctx, readTS) + if err != nil { + s.compensateBackupRelease(controlGroup, groups, pinID) + return preparedBackup{}, status.Errorf(codes.FailedPrecondition, "capture backup routes at read timestamp: %v", err) + } return preparedBackup{ groups: groups, commits: commits, controlGroup: controlGroup, pinID: pinID, readTS: readTS, ttl: ttl, routes: routes, @@ -573,6 +578,11 @@ func (s *AdminServer) snapshotBackupGroups() ([]backupGroup, error) { s.groupsMu.RLock() groups := make([]backupGroup, 0, len(s.groups)) for id, group := range s.groups { + // Group zero is reserved for TSO state and has no user keyspace. + // Backup tokens also reserve zero as an invalid group ID. + if id == 0 { + continue + } proposer := s.backupProposers[id] if group == nil || proposer == nil { s.groupsMu.RUnlock() diff --git a/adapter/admin_backup_test.go b/adapter/admin_backup_test.go index 6ca3e4cfb..0341752a3 100644 --- a/adapter/admin_backup_test.go +++ b/adapter/admin_backup_test.go @@ -120,6 +120,7 @@ type backupTestStore struct { scanDelay time.Duration keyCloseErr error pairCloseErr error + captureErr error } func (s *backupTestStore) CaptureBackupRouteSnapshotAt(_ context.Context, ts uint64) (kv.BackupRouteSnapshot, error) { @@ -130,7 +131,7 @@ func (s *backupTestStore) CaptureBackupRouteSnapshotAt(_ context.Context, ts uin if onCapture != nil { onCapture() } - return kv.BackupRouteSnapshot{}, nil + return kv.BackupRouteSnapshot{}, s.captureErr } func (s *backupTestStore) NewBackupKeyScannerAtSnapshot(_ kv.BackupRouteSnapshot, ts uint64, _ int) kv.BackupKeyScanner { @@ -308,16 +309,35 @@ func TestBeginBackupLifecycleAndBaselineAtPinnedTimestamp(t *testing.T) { } } +func TestSnapshotBackupGroupsExcludesReservedTSOGroup(t *testing.T) { + t.Parallel() + groups := map[uint64]*backupTestGroup{ + 0: {status: raftengine.Status{AppliedIndex: 100}, every: 10_000}, + 1: {status: raftengine.Status{AppliedIndex: 100}, every: 10_000}, + } + proposers := map[uint64]*backupTestProposer{ + 0: newBackupTestProposer(), + 1: newBackupTestProposer(), + } + srv := newBackupControlTestServer(t, &backupTestStore{}, groups, proposers, nil) + + got, err := srv.snapshotBackupGroups() + require.NoError(t, err) + require.Len(t, got, 1) + require.Equal(t, uint64(1), got[0].id) +} + func TestBeginBackupCapturesRouteSnapshotAfterReadFence(t *testing.T) { t.Parallel() stage := atomic.Int32{} + proposer := newBackupTestProposer() store := &backupTestStore{} store.onCapture = func() { require.Equal(t, int32(1), stage.Load()) + require.Equal(t, []byte{backupSubtypeReserve, backupSubtypePin}, proposer.subtypes()) stage.Store(2) } group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} - proposer := newBackupTestProposer() probe := func(context.Context, string) (BackupPeerVersion, error) { return BackupPeerVersion{NodeVersion: "test", BackupProtocolVersion: backupProtocolVersionV1}, nil } @@ -342,6 +362,28 @@ func TestBeginBackupCapturesRouteSnapshotAfterReadFence(t *testing.T) { require.Equal(t, []uint64{42}, store.capturedTS) } +func TestBeginBackupCompensatesWhenPinnedRouteCaptureFails(t *testing.T) { + t.Parallel() + store := &backupTestStore{captureErr: stderrors.New("catalog unavailable")} + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + proposer := newBackupTestProposer() + srv := newBackupControlTestServer( + t, store, + map[uint64]*backupTestGroup{1: group}, + map[uint64]*backupTestProposer{1: proposer}, + nil, + ) + + _, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Equal(t, []byte{ + backupSubtypeReserve, + backupSubtypePin, + backupSubtypeRelease, + backupSubtypeUnreserve, + }, proposer.subtypes()) +} + func TestBeginBackupRenewsWhileBaselineRuns(t *testing.T) { t.Parallel() store := &backupTestStore{ diff --git a/adapter/admin_grpc.go b/adapter/admin_grpc.go index b905cd281..cb5a1bde4 100644 --- a/adapter/admin_grpc.go +++ b/adapter/admin_grpc.go @@ -724,7 +724,8 @@ var adminMethodPrefix = "/" + pb.Admin_ServiceDesc.ServiceName + "/" func adminAuthenticatedMethod(fullMethod string) bool { return strings.HasPrefix(fullMethod, adminMethodPrefix) || - fullMethod == pb.Internal_ForwardAdminProposal_FullMethodName + fullMethod == pb.Internal_ForwardAdminProposal_FullMethodName || + fullMethod == pb.Internal_ForwardLeaseRead_FullMethodName } // AdminTokenAuth builds a gRPC unary+stream interceptor pair enforcing diff --git a/adapter/admin_grpc_test.go b/adapter/admin_grpc_test.go index 629d18321..06436e347 100644 --- a/adapter/admin_grpc_test.go +++ b/adapter/admin_grpc_test.go @@ -1021,18 +1021,26 @@ func TestAdminTokenAuthSkipsOtherServices(t *testing.T) { } } -func TestAdminTokenAuthProtectsForwardAdminProposal(t *testing.T) { +func TestAdminTokenAuthProtectsInternalAdminMethods(t *testing.T) { t.Parallel() unary, _ := AdminTokenAuth("s3cret") - info := &grpc.UnaryServerInfo{FullMethod: pb.Internal_ForwardAdminProposal_FullMethodName} handler := func(_ context.Context, _ any) (any, error) { return "ok", nil } - - _, err := unary(context.Background(), nil, info, handler) - require.Equal(t, codes.Unauthenticated, status.Code(err)) - ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer s3cret")) - resp, err := unary(ctx, nil, info, handler) - require.NoError(t, err) - require.Equal(t, "ok", resp) + methods := []string{ + pb.Internal_ForwardAdminProposal_FullMethodName, + pb.Internal_ForwardLeaseRead_FullMethodName, + } + for _, method := range methods { + t.Run(method, func(t *testing.T) { + t.Parallel() + info := &grpc.UnaryServerInfo{FullMethod: method} + _, err := unary(context.Background(), nil, info, handler) + require.Equal(t, codes.Unauthenticated, status.Code(err)) + ctx := metadata.NewIncomingContext(context.Background(), metadata.Pairs("authorization", "Bearer s3cret")) + resp, err := unary(ctx, nil, info, handler) + require.NoError(t, err) + require.Equal(t, "ok", resp) + }) + } } func TestAdminTokenAuthEmptyTokenDisabled(t *testing.T) { diff --git a/docs/design/2026_04_29_proposed_logical_backup.md b/docs/design/2026_04_29_proposed_logical_backup.md index 39350d188..c5e97c6b7 100644 --- a/docs/design/2026_04_29_proposed_logical_backup.md +++ b/docs/design/2026_04_29_proposed_logical_backup.md @@ -995,25 +995,26 @@ dumps with retention pressure. ### BeginBackup → EndBackup flow -1. **Pick `read_ts`**: `BeginBackup` reads the lease-read timestamp - pipeline (`kv/lease_state.go`, see - `2026_04_20_implemented_lease_read.md`) and snapshots - `applied_index` per Raft group. -2. **Wait for shards to catch up**: every group is required to report - `applied_index ≥ commit_index_at_pin` for the default group's HLC - ceiling proposal that produced `read_ts`. `BeginBackup` polls each - group's `Status.AppliedIndex` (already exposed via the existing - raftengine status interface used by `AdminServer.GetRaftGroups`) - with a 500 ms tick and a configurable deadline (default 5 s; surfaced - as `--begin-backup-deadline` on the CLI). **This is the binding wait - in practice** — a healthy group commits a Raft entry in <100 ms, - while a lagging shard recovering from a leader change or restart - can take seconds. Operators tuning `--begin-backup-deadline` are - adjusting tolerance for shard lag; it does not need to scale with - pin-fan-out latency. If any group fails to reach the threshold - within the deadline, `BeginBackup` returns `FailedPrecondition` - and the producer aborts — the dump is not started until every - group can serve `read_ts` consistently. +1. **Pick `read_ts` after an all-group applied barrier**: + `BeginBackup` executes `LeaseReadAllGroups`, reads the maximum + applied `LastCommitTS`, and allocates `read_ts` strictly above it + through the configured HLC/TSO allocator. The barrier proves the + observed store watermark is applied; the replicated floor in step 2 + closes the separate case where a writer already obtained a lower + timestamp but has not reached Raft apply yet. +2. **Close the timestamp cut on every data group**: every `BackupPin` + persists a monotone `backup timestamp floor = read_ts` in the group + FSM before baseline or stream scanning. Raw, one-phase, and PREPARE + entries that arrive later with `commit_ts <= read_ts` fail closed. + COMMIT/ABORT entries for transactions prepared before the pin remain + resolvable: the snapshot scanner resolves committed locks and fails + `BeginBackup` on a still-pending primary. Applying the pin also + observes `read_ts` into the shared HLC. This covers cached TSO batch + values as well as HLC values and survives snapshots/restarts. + The durable catalog route snapshot is read from the catalog owner + group only after every data group has committed this floor, so a + delayed catalog write cannot change ownership at or below `read_ts` + after route capture. 3. **Pin `read_ts` cluster-wide**, not just on the node that received the RPC. **Per-group `BackupPin` proposals are issued concurrently** — one goroutine per group — so a 100-shard cluster does not pay @@ -1077,13 +1078,13 @@ keyspace, not a streaming tail. ### Cross-shard consistency -Step 2 above is the mechanism. Without it, picking `read_ts` from -`max(group_commit_ts) + 50 ms` is only a *liveness* assertion: a -lagging shard might not yet have applied through `read_ts`, and -`ScanAt(at_ts=read_ts)` on that shard would either block forever or -return a partial view. The `applied_index` poll-and-wait in -`BeginBackup` makes the constraint explicit and bounded — every shard -provably has the data at `read_ts` before any scan begins. +Steps 1 and 2 are the mechanism. The all-group lease barrier establishes an +applied starting point, while the replicated timestamp floor prevents an +already allocated low timestamp from arriving after that point and changing +the snapshot. Either the write applies before its group pin and is visible to +the baseline, or it reaches apply after the pin and is rejected. A prepared +transaction is the deliberate exception: its resolution remains legal, and a +pending primary makes the baseline fail instead of emitting a partial view. ## Internal-State Handling diff --git a/kv/active_timestamp_tracker.go b/kv/active_timestamp_tracker.go index 26d68be3a..73ab5d978 100644 --- a/kv/active_timestamp_tracker.go +++ b/kv/active_timestamp_tracker.go @@ -70,16 +70,17 @@ func WithActiveTimestampTrackerLogger(logger *slog.Logger) ActiveTimestampTracke // ActiveTimestampTracker tracks in-flight read or transaction timestamps that // must remain readable while background compaction is running. type ActiveTimestampTracker struct { - mu sync.Mutex - nextID uint64 - active map[uint64]uint64 - backupPins map[backupPinKey]backupDeadlinePin - maxBackupPins int - sweepEvery time.Duration - sweepOnce sync.Once - stopCh chan struct{} - closeOnce sync.Once - logger *slog.Logger + mu sync.Mutex + nextID uint64 + active map[uint64]uint64 + backupPins map[backupPinKey]backupDeadlinePin + maxBackupPins int + sweepEvery time.Duration + sweepOnce sync.Once + stopCh chan struct{} + closeOnce sync.Once + logger *slog.Logger + backupFloorObserver func(uint64) } // ActiveTimestampToken releases one tracked timestamp when the owning @@ -107,6 +108,17 @@ func NewActiveTimestampTracker(opts ...ActiveTimestampTrackerOption) *ActiveTime return t } +// SetBackupTimestampFloorObserver installs the process-local timestamp-cache +// invalidation callback invoked after a replicated backup pin is accepted. +func (t *ActiveTimestampTracker) SetBackupTimestampFloorObserver(observer func(uint64)) { + if t == nil { + return + } + t.mu.Lock() + t.backupFloorObserver = observer + t.mu.Unlock() +} + func (t *ActiveTimestampTracker) Pin(ts uint64) *ActiveTimestampToken { if t == nil || ts == 0 || ts == ^uint64(0) { return &ActiveTimestampToken{} @@ -206,8 +218,12 @@ func (t *ActiveTimestampTracker) pinWithDeadlineForGroup(pinID BackupPinID, grou t.backupPins[key] = mergeBackupDeadlinePin( t.backupPins[key], backupDeadlinePin{readTS: readTS, deadline: deadline}, ) + observer := t.backupFloorObserver t.startBackupPinSweeperLocked() t.mu.Unlock() + if observer != nil { + observer(readTS) + } t.logExpiredBackupPins(expired) return nil } diff --git a/kv/active_timestamp_tracker_test.go b/kv/active_timestamp_tracker_test.go index aa4d5701d..ee35c42f7 100644 --- a/kv/active_timestamp_tracker_test.go +++ b/kv/active_timestamp_tracker_test.go @@ -1,12 +1,25 @@ package kv import ( + "sync/atomic" "testing" "time" "github.com/stretchr/testify/require" ) +func TestActiveTimestampTrackerNotifiesBackupTimestampFloor(t *testing.T) { + t.Parallel() + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + var observed atomic.Uint64 + tracker.SetBackupTimestampFloorObserver(observed.Store) + + require.NoError(t, tracker.ApplyPinWithDeadlineForGroup( + backupTrackerTestPinID(1), 7, 42, time.Now().Add(time.Hour), + )) + require.Equal(t, uint64(42), observed.Load()) +} + func TestActiveTimestampTrackerOldest(t *testing.T) { tracker := NewActiveTimestampTracker() diff --git a/kv/backup_scan.go b/kv/backup_scan.go index b30f21ac8..f892f6b53 100644 --- a/kv/backup_scan.go +++ b/kv/backup_scan.go @@ -90,14 +90,14 @@ func (s *ShardStore) CaptureBackupRouteSnapshot(start []byte, end []byte) Backup } // CaptureBackupRouteSnapshotAt reads the durable distribution catalog at ts. -// This keeps backup ownership aligned with the same MVCC timestamp used to -// materialize values even if the live route watcher advances immediately -// after the backup read fence. -func (s *ShardStore) CaptureBackupRouteSnapshotAt(ctx context.Context, ts uint64) (BackupRouteSnapshot, error) { - if s == nil { +// The caller must pass the CatalogStore bound to the catalog owner group; +// using a normally routed ShardStore can split the version read from the +// route-row scan when those reserved prefixes route to different groups. +func CaptureBackupRouteSnapshotAt(ctx context.Context, catalog *distribution.CatalogStore, ts uint64) (BackupRouteSnapshot, error) { + if catalog == nil { return BackupRouteSnapshot{}, errors.New("backup route store is unavailable") } - snapshot, err := distribution.NewCatalogStore(s).SnapshotAt(ctx, ts) + snapshot, err := catalog.SnapshotAt(ctx, ts) if err != nil { return BackupRouteSnapshot{}, errors.Wrap(err, "read distribution catalog at backup timestamp") } diff --git a/kv/fsm.go b/kv/fsm.go index 8d0f52cfb..7d72499c3 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -8,6 +8,7 @@ import ( "io" "log/slog" "os" + "sync/atomic" "github.com/bootjp/elastickv/internal/encryption/fsmwire" "github.com/bootjp/elastickv/internal/raftengine" @@ -98,6 +99,11 @@ type kvFSM struct { // applyObservers are called after successful logical mutations. // They are never mutated after NewKvFSMWithHLC returns. applyObservers []ApplyObserver + // backupTimestampFloor is the highest replicated BackupPin read timestamp. + // New raw, one-phase, and PREPARE entries at or below this floor were + // timestamped before the backup cut and reached apply too late. + backupTimestampFloor atomic.Uint64 + backupFloorLoadErr error } // RouteHistory is the kv-side interface to the route catalog's @@ -270,6 +276,7 @@ func NewKvFSMWithHLC(store store.MVCCStore, hlc *HLC, opts ...FSMOption) FSM { for _, opt := range opts { opt(f) } + f.backupFloorLoadErr = f.reloadBackupTimestampFloor(context.Background()) f.snapLatch.log = f.log observeStoreLastCommitTS(hlc, store) return f @@ -324,6 +331,9 @@ type fsmApplyResponse struct { } func (f *kvFSM) Apply(data []byte) any { + if f.backupFloorLoadErr != nil { + return haltErr(errors.Wrap(errors.Mark(f.backupFloorLoadErr, ErrBackupApply), "kv/fsm: load backup timestamp floor")) + } if resp, handled := f.applyReservedOpcode(data); handled { return resp } @@ -481,6 +491,9 @@ func (f *kvFSM) applyRequestErr(ctx context.Context, r *pb.Request) error { if err != nil { return err } + if err := f.verifyBackupTimestampFloor(r, commitTS); err != nil { + return err + } if err := f.handleRequest(ctx, r, commitTS); err != nil { return errors.WithStack(err) } @@ -601,6 +614,10 @@ func (f *kvFSM) Restore(r io.Reader) error { if err := f.store.Restore(io.NopCloser(br)); err != nil { return errors.WithStack(err) } + f.backupFloorLoadErr = f.reloadBackupTimestampFloor(context.Background()) + if f.backupFloorLoadErr != nil { + return errors.Wrap(f.backupFloorLoadErr, "restore backup timestamp floor") + } observeStoreLastCommitTS(f.hlc, f.store) return nil } @@ -657,14 +674,14 @@ func (f *kvFSM) ApplySnapshotHeader(ceiling, cutover uint64) { } // IsVolatileOnlyPayload satisfies raftengine.VolatileEntryClassifier. -// Returns true iff payload is an HLC lease entry (raftEncodeHLCLease -// tag, 0x02) — those entries only call HLC.SetPhysicalCeiling, which -// is monotonic and lives purely in memory. After the cold-start skip -// gate fires, the engine still delivers WAL committed-tail entries -// past snapshot.Metadata.Index; without this classifier those -// volatile entries get dropped along with KV/MVCC duplicates. HLC -// would lose the post-snapshot ceiling raise; backup pins would lose -// a post-snapshot retention fence. +// Returns true for HLC lease and backup-pin payloads. HLC leases only call +// HLC.SetPhysicalCeiling, which is monotonic and lives purely in memory. +// Backup pins restore volatile retention state and also persist a monotonic +// timestamp floor; replaying the same floor is idempotent. After the cold-start +// skip gate fires, the engine still delivers WAL committed-tail entries past +// snapshot.Metadata.Index; without this classifier those entries get dropped +// along with KV/MVCC duplicates. HLC would lose the post-snapshot ceiling +// raise; backup pins would lose a post-snapshot retention fence. // // Re-applying KV/MVCC entries would re-execute OCC validation against // store state that has already moved past commit_ts, surfacing diff --git a/kv/fsm_backup.go b/kv/fsm_backup.go index aee16aaed..0cf249821 100644 --- a/kv/fsm_backup.go +++ b/kv/fsm_backup.go @@ -1,8 +1,20 @@ package kv -import "github.com/cockroachdb/errors" +import ( + "context" + "encoding/binary" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) var ErrBackupApply = errors.New("backup fsm apply failed") +var ErrBackupTimestampFenced = errors.New("backup timestamp fence rejects stale write") + +var backupTimestampFloorKey = []byte(TxnKeyPrefix + "backup|timestamp_floor") + +const backupTimestampFloorValueSize = 8 func (f *kvFSM) applyBackup(data []byte) any { if f.readTracker == nil { @@ -26,7 +38,7 @@ func (f *kvFSM) applyDecodedBackup(entry backupEntry) error { switch entry.subtype { case backupSubtypePin: err := f.readTracker.ApplyPinWithDeadlineForGroup(entry.pin.PinID, f.shardGroupID, entry.pin.ReadTS, entry.pin.Deadline) - return f.observeBackupReadTimestamp(entry.pin.ReadTS, err) + return f.applyBackupTimestampFence(entry.pin.ReadTS, err) case backupSubtypeExtend: return f.readTracker.ApplyExtendForGroup(entry.extend.PinID, f.shardGroupID, entry.extend.Deadline) case backupSubtypeRelease: @@ -34,7 +46,7 @@ func (f *kvFSM) applyDecodedBackup(entry backupEntry) error { return nil case backupSubtypeReserve: err := f.readTracker.PinWithDeadline(entry.pin.PinID, entry.pin.ReadTS, entry.pin.Deadline) - return f.observeBackupReadTimestamp(entry.pin.ReadTS, err) + return f.applyBackupTimestampFence(entry.pin.ReadTS, err) case backupSubtypeUnreserve: f.readTracker.ReleaseBackupPin(entry.release.PinID) return nil @@ -43,6 +55,65 @@ func (f *kvFSM) applyDecodedBackup(entry backupEntry) error { } } +func (f *kvFSM) applyBackupTimestampFence(readTS uint64, applyErr error) error { + if applyErr != nil { + return applyErr + } + if err := f.persistBackupTimestampFloor(context.Background(), readTS); err != nil { + return err + } + return f.observeBackupReadTimestamp(readTS, nil) +} + +func (f *kvFSM) persistBackupTimestampFloor(ctx context.Context, readTS uint64) error { + if readTS <= f.backupTimestampFloor.Load() { + return nil + } + value := make([]byte, backupTimestampFloorValueSize) + binary.BigEndian.PutUint64(value, readTS) + err := f.store.ApplyMutationsRaftAt(ctx, []*store.KVPairMutation{{ + Op: store.OpTypePut, Key: backupTimestampFloorKey, Value: value, + }}, nil, readTS, readTS, f.pendingApplyIdx) + if err != nil { + return errors.Wrap(err, "persist backup timestamp floor") + } + f.backupTimestampFloor.Store(readTS) + return nil +} + +func (f *kvFSM) reloadBackupTimestampFloor(ctx context.Context) error { + if f == nil || f.store == nil { + return nil + } + value, err := f.store.GetAt(ctx, backupTimestampFloorKey, ^uint64(0)) + if err != nil { + if errors.Is(err, store.ErrKeyNotFound) { + f.backupTimestampFloor.Store(0) + return nil + } + return errors.WithStack(err) + } + if len(value) != backupTimestampFloorValueSize { + return errors.Wrapf(ErrBackupApply, "backup timestamp floor has %d bytes, want %d", len(value), backupTimestampFloorValueSize) + } + f.backupTimestampFloor.Store(binary.BigEndian.Uint64(value)) + return nil +} + +func (f *kvFSM) verifyBackupTimestampFloor(r *pb.Request, commitTS uint64) error { + floor := f.backupTimestampFloor.Load() + if floor == 0 || commitTS > floor || r == nil { + return nil + } + if r.IsTxn && (r.Phase == pb.Phase_COMMIT || r.Phase == pb.Phase_ABORT) { + // A transaction prepared before the pin must remain resolvable. The + // backup scanner resolves its existing lock or fails BeginBackup while + // its primary remains pending; new PREPARE entries at this ts are fenced. + return nil + } + return errors.Wrapf(ErrBackupTimestampFenced, "commit_ts %d is not above backup read_ts %d", commitTS, floor) +} + func (f *kvFSM) observeBackupReadTimestamp(readTS uint64, applyErr error) error { if applyErr == nil && f.hlc != nil { // The pin entry is also the per-group timestamp barrier. Once it diff --git a/kv/fsm_backup_test.go b/kv/fsm_backup_test.go index d6b584ff9..4501e5367 100644 --- a/kv/fsm_backup_test.go +++ b/kv/fsm_backup_test.go @@ -1,9 +1,12 @@ package kv import ( + "bytes" + "context" "testing" "time" + pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" @@ -142,6 +145,143 @@ func TestApplyBackupObservesPinnedReadTimestamp(t *testing.T) { require.GreaterOrEqual(t, hlc.Current(), readTS) } +func TestApplyBackupFencesPreallocatedWrites(t *testing.T) { + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + fsm := newBackupTestFSM(t, tracker) + readTS := uint64(100) + require.NoError(t, haltApplyOf(fsm.Apply(EncodeBackupPinEntry(BackupPinEntry{ + PinID: backupTrackerTestPinID(1), ReadTS: readTS, Deadline: time.Now().Add(time.Hour), + })))) + + tests := []struct { + name string + req *pb.Request + }{ + { + name: "raw", + req: &pb.Request{Ts: readTS - 1, Mutations: []*pb.Mutation{{ + Op: pb.Op_PUT, Key: []byte("raw"), Value: []byte("stale"), + }}}, + }, + { + name: "one phase", + req: &pb.Request{IsTxn: true, Phase: pb.Phase_NONE, Ts: readTS - 2, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: []byte("one-phase"), CommitTS: readTS - 1, + })}, + {Op: pb.Op_PUT, Key: []byte("one-phase"), Value: []byte("stale")}, + }}, + }, + { + name: "prepare", + req: &pb.Request{IsTxn: true, Phase: pb.Phase_PREPARE, Ts: readTS - 1, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: []byte("prepare"), LockTTLms: defaultTxnLockTTLms, + })}, + {Op: pb.Op_PUT, Key: []byte("prepare"), Value: []byte("stale")}, + }}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + requireBackupTimestampFenced(t, applyBackupTestRequest(t, fsm, tc.req)) + }) + } +} + +func TestApplyBackupAllowsResolutionOfPrePinTransaction(t *testing.T) { + fsm := newBackupTestFSM(t, NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0))) + primary := []byte("primary") + startTS := uint64(30) + commitTS := uint64(40) + prepare := &pb.Request{IsTxn: true, Phase: pb.Phase_PREPARE, Ts: startTS, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: primary, LockTTLms: defaultTxnLockTTLms, + })}, + {Op: pb.Op_PUT, Key: primary, Value: []byte("committed")}, + }} + require.Nil(t, applyBackupTestRequest(t, fsm, prepare)) + require.NoError(t, haltApplyOf(fsm.Apply(EncodeBackupPinEntry(BackupPinEntry{ + PinID: backupTrackerTestPinID(1), ReadTS: 50, Deadline: time.Now().Add(time.Hour), + })))) + commit := &pb.Request{IsTxn: true, Phase: pb.Phase_COMMIT, Ts: startTS, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: primary, CommitTS: commitTS, + })}, + {Op: pb.Op_PUT, Key: primary}, + }} + require.Nil(t, applyBackupTestRequest(t, fsm, commit)) + value, err := fsm.store.GetAt(context.Background(), primary, 50) + require.NoError(t, err) + require.Equal(t, []byte("committed"), value) +} + +func TestBackupTimestampFloorSurvivesSnapshotRestore(t *testing.T) { + srcStore := store.NewMVCCStore() + src, ok := NewKvFSMWithHLCAndTracker( + srcStore, NewHLC(), NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)), + ).(*kvFSM) + require.True(t, ok) + require.NoError(t, haltApplyOf(src.Apply(EncodeBackupPinEntry(BackupPinEntry{ + PinID: backupTrackerTestPinID(1), ReadTS: 75, Deadline: time.Now().Add(time.Hour), + })))) + snapshot, err := srcStore.Snapshot() + require.NoError(t, err) + defer snapshot.Close() + var raw bytes.Buffer + _, err = snapshot.WriteTo(&raw) + require.NoError(t, err) + + dstStore := store.NewMVCCStore() + dst, ok := NewKvFSMWithHLCAndTracker( + dstStore, NewHLC(), NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)), + ).(*kvFSM) + require.True(t, ok) + require.NoError(t, dst.Restore(bytes.NewReader(raw.Bytes()))) + requireBackupTimestampFenced(t, applyBackupTestRequest(t, dst, &pb.Request{Ts: 74, Mutations: []*pb.Mutation{{ + Op: pb.Op_PUT, Key: []byte("late"), Value: []byte("stale"), + }}})) +} + +func TestBackupTimestampFloorRejectsDelayedRaftProposal(t *testing.T) { + st := store.NewMVCCStore() + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + r, stop := newSingleRaft(t, "backup-floor-delayed", NewKvFSMWithHLCAndTracker(st, NewHLC(), tracker)) + t.Cleanup(stop) + readTS := uint64(500) + result, err := r.ProposeAdmin(context.Background(), EncodeBackupPinEntry(BackupPinEntry{ + PinID: backupTrackerTestPinID(1), ReadTS: readTS, Deadline: time.Now().Add(time.Hour), + })) + require.NoError(t, err) + require.NotNil(t, result) + require.Nil(t, result.Response) + + txn := NewTransactionWithProposer(r) + _, err = txn.Commit(context.Background(), []*pb.Request{{ + Ts: readTS - 1, + Mutations: []*pb.Mutation{{ + Op: pb.Op_PUT, Key: []byte("delayed"), Value: []byte("stale"), + }}, + }}) + require.ErrorIs(t, err, ErrBackupTimestampFenced) + _, err = st.GetAt(context.Background(), []byte("delayed"), readTS) + require.ErrorIs(t, err, store.ErrKeyNotFound) +} + +func applyBackupTestRequest(t *testing.T, fsm *kvFSM, req *pb.Request) any { + t.Helper() + payload, err := marshalRaftCommand([]*pb.Request{req}) + require.NoError(t, err) + return fsm.Apply(payload) +} + +func requireBackupTimestampFenced(t *testing.T, resp any) { + t.Helper() + err, ok := resp.(error) + require.True(t, ok) + require.ErrorIs(t, err, ErrBackupTimestampFenced) +} + func TestApplyBackupReserveEnforcesCapacityAndUnreserveReleases(t *testing.T) { tracker := NewActiveTimestampTracker( WithActiveTimestampTrackerSweepInterval(0), diff --git a/kv/leader_proxy.go b/kv/leader_proxy.go index 9be953e08..fc3f89d4a 100644 --- a/kv/leader_proxy.go +++ b/kv/leader_proxy.go @@ -8,6 +8,7 @@ import ( "github.com/bootjp/elastickv/internal/raftengine" pb "github.com/bootjp/elastickv/proto" "github.com/cockroachdb/errors" + "google.golang.org/grpc/metadata" ) const leaderForwardTimeout = 5 * time.Second @@ -28,6 +29,7 @@ const leaderProxyRetryInterval = 25 * time.Millisecond type LeaderProxy struct { engine raftengine.Engine tm *TransactionManager + group *ShardGroup connCache GRPCConnCache } @@ -256,6 +258,9 @@ func (p *LeaderProxy) forwardLeaseReadOnce(parentCtx context.Context) (uint64, e } ctx, cancel := context.WithTimeout(parentCtx, leaderForwardTimeout) defer cancel() + if token := p.group.forwardedLeaderReadToken(); token != "" { + ctx = metadata.AppendToOutgoingContext(ctx, "authorization", "Bearer "+token) + } resp, err := pb.NewInternalClient(conn).ForwardLeaseRead(ctx, &pb.ForwardLeaseReadRequest{}) if err != nil { return 0, errors.WithStack(err) diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index ed67a0c3c..64398e21c 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -677,7 +677,6 @@ func TestCaptureBackupRouteSnapshotAtUsesDurableCatalogTimestamp(t *testing.T) { 1: {Store: store.NewMVCCStore()}, 2: {Store: store.NewMVCCStore()}, } - shards := NewShardStore(engine, groups) catalog := distribution.NewCatalogStore(groups[1].Store) old, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{ {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, @@ -692,13 +691,41 @@ func TestCaptureBackupRouteSnapshotAtUsesDurableCatalogTimestamp(t *testing.T) { require.NoError(t, err) require.NoError(t, engine.ApplySnapshot(current)) - snapshot, err := shards.CaptureBackupRouteSnapshotAt(ctx, oldReadTS) + snapshot, err := CaptureBackupRouteSnapshotAt(ctx, catalog, oldReadTS) require.NoError(t, err) require.Len(t, snapshot.routes, 2) require.Equal(t, uint64(1), snapshot.routes[0].GroupID) require.Equal(t, uint64(2), snapshot.routes[1].GroupID) } +func TestCaptureBackupRouteSnapshotAtReadsRowsFromCatalogOwner(t *testing.T) { + t.Parallel() + ctx := context.Background() + owner := store.NewMVCCStore() + wronglyRouted := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(owner) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), GroupID: 2, State: distribution.RouteStateActive}, + }) + require.NoError(t, err) + + // A live shard router could send the reserved route-row prefix to group 2. + // The backup reader must stay on the durable catalog owner instead. + engine := distribution.NewEngine() + engine.UpdateRoute([]byte(""), nil, 2) + _ = NewShardStore(engine, map[uint64]*ShardGroup{ + 1: {Store: owner}, + 2: {Store: wronglyRouted}, + }) + + snapshot, err := CaptureBackupRouteSnapshotAt(ctx, catalog, owner.LastCommitTS()) + require.NoError(t, err) + require.Len(t, snapshot.routes, len(saved.Routes)) + require.Equal(t, uint64(1), snapshot.routes[0].GroupID) + require.Equal(t, uint64(2), snapshot.routes[1].GroupID) +} + func TestBackupScannerMaterializesFromCapturedRoute(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index e8a70cc8f..b80167c33 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -63,6 +63,31 @@ type ShardGroup struct { // leaderRead forwards a read barrier to this group's current leader when // the local replica is a follower. It is installed with the LeaderProxy. leaderRead func(context.Context) (uint64, error) + // leaderReadToken authenticates follower-to-leader lease-read RPCs. Admin + // service setup publishes it once before the public listeners are opened. + leaderReadToken atomic.Pointer[string] +} + +// SetLeaderReadToken configures the bearer token used by forwarded lease-read +// RPCs. An empty token preserves explicitly configured insecure admin mode. +func (g *ShardGroup) SetLeaderReadToken(token string) { + if token == "" { + g.leaderReadToken.Store(nil) + return + } + t := token + g.leaderReadToken.Store(&t) +} + +func (g *ShardGroup) forwardedLeaderReadToken() string { + if g == nil { + return "" + } + token := g.leaderReadToken.Load() + if token == nil { + return "" + } + return *token } // Proposer returns the wrap-aware proposer chain installed by @@ -213,6 +238,7 @@ func NewLeaderProxyForShardGroup(g *ShardGroup, opts ...TransactionOption) *Lead p := &LeaderProxy{ engine: g.Engine, tm: NewTransactionWithProposer(g.proposer, opts...), + group: g, } g.leaderRead = p.forwardLeaseRead return p @@ -380,6 +406,9 @@ type ShardedCoordinator struct { // coordinator-owned persistence timestamp. Nil preserves the legacy shared // HLC path. tsAllocator TimestampAllocator + // timestampFloor is the highest replicated backup cut observed by this + // process. It deduplicates the same cut as each local shard applies its pin. + timestampFloor atomic.Uint64 // timestampGroup pins IsTimestampLeader to a dedicated Raft group when // timestampGroupConfigured is true. Nil/false preserves the M3 bridge // behavior where any locally-led shard group can issue TSO timestamps. @@ -468,6 +497,29 @@ func (c *ShardedCoordinator) WithTSOAllocator(alloc TimestampAllocator) *Sharded return c } +// ObserveTimestampFloor advances the process clock past a replicated backup +// cut and invalidates any cached TSO batch that could still contain values at +// or below it. Claims returned before invalidation remain protected by the +// FSM's durable backup timestamp floor. +func (c *ShardedCoordinator) ObserveTimestampFloor(ts uint64) { + if c == nil || ts == 0 { + return + } + for { + floor := c.timestampFloor.Load() + if ts <= floor { + return + } + if c.timestampFloor.CompareAndSwap(floor, ts) { + break + } + } + if c.clock != nil { + c.clock.Observe(ts) + } + invalidateTimestampWindow(c.tsAllocator) +} + // WithTimestampGroup pins timestamp issuance leadership to one Raft group. // Callers should only enable this once a data-shard leader can redirect // timestamp allocation to that group; otherwise data leaders would stop being diff --git a/kv/sharded_coordinator_leader_test.go b/kv/sharded_coordinator_leader_test.go index 631fe2204..64989adee 100644 --- a/kv/sharded_coordinator_leader_test.go +++ b/kv/sharded_coordinator_leader_test.go @@ -3,6 +3,7 @@ package kv import ( "context" "net" + "sync" "sync/atomic" "testing" @@ -11,18 +12,25 @@ import ( "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" "google.golang.org/grpc" + "google.golang.org/grpc/metadata" ) type leaseReadForwardServer struct { pb.UnimplementedInternalServer - calls atomic.Int32 + calls atomic.Int32 + mu sync.Mutex + authorization []string } func (s *leaseReadForwardServer) ForwardLeaseRead( - context.Context, - *pb.ForwardLeaseReadRequest, + ctx context.Context, + _ *pb.ForwardLeaseReadRequest, ) (*pb.ForwardLeaseReadResponse, error) { s.calls.Add(1) + md, _ := metadata.FromIncomingContext(ctx) + s.mu.Lock() + s.authorization = append([]string(nil), md.Get("authorization")...) + s.mu.Unlock() return &pb.ForwardLeaseReadResponse{AppliedIndex: 44}, nil } @@ -102,12 +110,16 @@ func TestShardedCoordinatorLeaseReadAllGroups_ForwardsFollowerToGroupLeader(t *t follower := &stubFollowerEngine{leaderAddr: lis.Addr().String()} group := &ShardGroup{Engine: follower, Store: store.NewMVCCStore()} proxy := NewLeaderProxyForShardGroup(group) + group.SetLeaderReadToken("admin-secret") t.Cleanup(func() { require.NoError(t, proxy.Close()) }) group.Txn = proxy coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{1: group}, 1, NewHLC(), nil) require.NoError(t, coord.LeaseReadAllGroups(context.Background())) require.Equal(t, int32(1), service.calls.Load()) + service.mu.Lock() + require.Equal(t, []string{"Bearer admin-secret"}, service.authorization) + service.mu.Unlock() } func TestShardedCoordinatorLeaseReadAllGroups_UsesConfiguredAllShardGroups(t *testing.T) { diff --git a/kv/tso_test.go b/kv/tso_test.go index ef4e58510..9295d019c 100644 --- a/kv/tso_test.go +++ b/kv/tso_test.go @@ -81,6 +81,38 @@ func TestBatchAllocatorInvalidateDropsCachedWindow(t *testing.T) { require.EqualValues(t, 2, tso.calls.Load()) } +func TestShardedCoordinatorBackupFloorDropsCachedTSOWindow(t *testing.T) { + t.Parallel() + clock := NewHLC() + clock.SetPhysicalCeiling(time.Now().Add(testTSOFutureCeiling).UnixMilli()) + coord := NewShardedCoordinator( + distribution.NewEngineWithDefaultRoute(), + map[uint64]*ShardGroup{1: {Engine: stubLeaderEngine{}}}, + 1, + clock, + nil, + ) + local, err := NewLocalTSOAllocator(coord) + require.NoError(t, err) + batch, err := NewBatchAllocator(local, testTSOBatchSize) + require.NoError(t, err) + coord.WithTSOAllocator(batch) + + first, err := coord.Next(context.Background()) + require.NoError(t, err) + floor := first + 100 + coord.ObserveTimestampFloor(floor) + + next, err := coord.Next(context.Background()) + require.NoError(t, err) + require.Greater(t, next, floor) + + coord.ObserveTimestampFloor(floor) + afterDuplicateFloor, err := coord.Next(context.Background()) + require.NoError(t, err) + require.Equal(t, next+1, afterDuplicateFloor) +} + func TestBatchAllocatorNextAfterSkipsCachedSlotsBelowFloor(t *testing.T) { tso := &fakeTSOAllocator{nextBase: testTSOInitialBase, leader: true} alloc, err := NewBatchAllocator(tso, testTSOBatchSize) diff --git a/main.go b/main.go index 4c168b59f..26e2de2b3 100644 --- a/main.go +++ b/main.go @@ -535,6 +535,7 @@ func run() error { if err := configureCoordinatorTSO(coordinate); err != nil { return err } + readTracker.SetBackupTimestampFloorObserver(coordinate.ObserveTimestampFloor) // SQS HT-FIFO §8 leadership-refusal: install per-group // observers that step the local node down via @@ -613,7 +614,7 @@ func run() error { ctx: runCtx, eg: eg, cancel: cancel, lc: &lc, runtimes: runtimes, shardGroups: shardGroups, bootstrapServers: bootstrapCfg.adminSeed(cfg.defaultGroup), shardStore: shardStore, coordinate: coordinate, - distServer: distServer, readTracker: readTracker, + distServer: distServer, distCatalog: distCatalog, readTracker: readTracker, metricsRegistry: metricsRegistry, cfg: cfg, redisApplyObserver: redisApplyObserver, cleanup: &cleanup, @@ -1830,6 +1831,7 @@ type serversInput struct { shardStore *kv.ShardStore coordinate kv.Coordinator distServer *adapter.DistributionServer + distCatalog *distribution.CatalogStore readTracker *kv.ActiveTimestampTracker metricsRegistry *monitoring.Registry cfg runtimeConfig @@ -1873,7 +1875,7 @@ func startServersAfterStartupRotation(waitRotateOnStartup startupRotationWaiter, connCache = prepareAdminConnCache(in.ctx, in.eg, *adminEnabled || adminGRPCEnabled) adminServer, adminGRPCOpts, err := setupAdminService( *raftId, *myAddr, in.runtimes, in.shardGroups, in.bootstrapServers, - in.shardStore, in.coordinate, in.readTracker, in.keyvizSampler, connCache, + in.shardStore, in.distCatalog, in.coordinate, in.readTracker, in.keyvizSampler, connCache, ) if err != nil { return err @@ -2115,6 +2117,7 @@ func setupAdminService( shardGroups map[uint64]*kv.ShardGroup, bootstrapServers []raftengine.Server, shardStore *kv.ShardStore, + distCatalog *distribution.CatalogStore, coordinate kv.Coordinator, readTracker *kv.ActiveTimestampTracker, keyvizSampler *keyviz.MemSampler, @@ -2134,7 +2137,8 @@ func setupAdminService( members, connCache, adminBackupDependencies{ - store: shardStore, coordinate: coordinate, tracker: readTracker, + store: &adminBackupStore{ShardStore: shardStore, catalog: distCatalog}, + shardStore: shardStore, coordinate: coordinate, tracker: readTracker, }, ) if err != nil { @@ -2146,6 +2150,7 @@ func setupAdminService( for _, rt := range runtimes { srv.RegisterGroup(rt.spec.id, rt.engine) if group := shardGroups[rt.spec.id]; group != nil { + group.SetLeaderReadToken(icept.token) srv.RegisterBackupProposer(rt.spec.id, kv.NewLeaderAdminProposer( rt.engine, group.Proposer(), @@ -2380,11 +2385,25 @@ func startupRotationGatedMethod(fullMethod string) bool { // --adminInsecureNoAuth so operators have to opt into the unauthenticated // mode explicitly. type adminBackupDependencies struct { - store *kv.ShardStore + store adapter.BackupStore + shardStore *kv.ShardStore coordinate kv.Coordinator tracker *kv.ActiveTimestampTracker } +type adminBackupStore struct { + *kv.ShardStore + catalog *distribution.CatalogStore +} + +func (s *adminBackupStore) CaptureBackupRouteSnapshotAt(ctx context.Context, ts uint64) (kv.BackupRouteSnapshot, error) { + if s == nil { + return kv.BackupRouteSnapshot{}, errors.New("backup store is unavailable") + } + snapshot, err := kv.CaptureBackupRouteSnapshotAt(ctx, s.catalog, ts) + return snapshot, errors.Wrap(err, "capture backup route snapshot") +} + func configureAdminService( tokenPath string, insecureNoAuth bool, @@ -2447,7 +2466,7 @@ func adminServerOptions( opts = append(opts, adapter.WithAdminBackupControl( deps.store, - adminBackupReadFence(deps.coordinate, deps.store), + adminBackupReadFence(deps.coordinate, deps.shardStore), adminBackupPeerProbe(connCache), deps.tracker, tokenKey, From 2f927a610f85c2e4c74f9cd5fb5ba65320841a26 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 16:12:44 +0900 Subject: [PATCH 5/9] backup: harden live backup fences --- adapter/admin_backup.go | 87 ++++++++++++++-- adapter/admin_backup_test.go | 33 ++++++ adapter/internal.go | 25 +++-- adapter/internal_admin_proposal_test.go | 2 + internal/backup/live.go | 10 +- internal/backup/live_test.go | 2 + kv/backup_scan.go | 130 ++++++++++++++++++++---- kv/coordinator.go | 20 ++++ kv/fsm_backup_test.go | 10 ++ kv/keyviz_label.go | 4 + kv/leader_proxy.go | 20 ++-- kv/shard_store.go | 16 ++- kv/shard_store_test.go | 99 ++++++++++++++++++ kv/sharded_coordinator.go | 56 +++++++--- kv/sharded_coordinator_leader_test.go | 12 ++- kv/txn_keys.go | 9 +- main.go | 23 ++++- main_admin_test.go | 22 ++-- proto/internal.pb.go | 13 ++- proto/internal.proto | 1 + proto/service.pb.go | 4 +- proto/service.proto | 3 + 22 files changed, 504 insertions(+), 97 deletions(-) diff --git a/adapter/admin_backup.go b/adapter/admin_backup.go index ff4d0649a..a66c815ef 100644 --- a/adapter/admin_backup.go +++ b/adapter/admin_backup.go @@ -37,6 +37,7 @@ const ( defaultLiveBackupScanPageSize = 1024 defaultLiveBackupRenewAttempts = 3 defaultLiveBackupRenewBackoff = 500 * time.Millisecond + backupAppliedPollInterval = 10 * time.Millisecond ) var ( @@ -55,6 +56,7 @@ type BackupPeerProbe func(context.Context, string) (BackupPeerVersion, error) type BackupStore interface { CaptureBackupRouteSnapshotAt(context.Context, uint64) (kv.BackupRouteSnapshot, error) + ValidateBackupSnapshotAt(context.Context, kv.BackupRouteSnapshot, uint64, int) error NewBackupKeyScannerAtSnapshot(snapshot kv.BackupRouteSnapshot, ts uint64, pageSize int) kv.BackupKeyScanner NewBackupScannerAtSnapshot(snapshot kv.BackupRouteSnapshot, ts uint64, pageSize int) kv.BackupScanner } @@ -252,12 +254,9 @@ func (s *AdminServer) prepareBackup(ctx context.Context, ttl time.Duration) (pre if err := s.checkBackupSnapshotHeadroom(groups); err != nil { return preparedBackup{}, err } - readTS, err := s.backupReadFence(ctx) + readTS, err := s.prepareBackupReadTimestamp(ctx) if err != nil { - return preparedBackup{}, status.Errorf(codes.FailedPrecondition, "backup read fence failed: %v", err) - } - if readTS == 0 || readTS == ^uint64(0) { - return preparedBackup{}, status.Errorf(codes.FailedPrecondition, "%s", "backup read fence returned an invalid timestamp") + return preparedBackup{}, err } pinID, err := newBackupPinID() if err != nil { @@ -269,6 +268,39 @@ func (s *AdminServer) prepareBackup(ctx context.Context, ttl time.Duration) (pre if err != nil { return preparedBackup{}, err } + routes, err := s.captureBackupRoutesAfterPin(ctx, groups, controlGroup, pinID, commits, readTS) + if err != nil { + return preparedBackup{}, err + } + return preparedBackup{ + groups: groups, commits: commits, controlGroup: controlGroup, + pinID: pinID, readTS: readTS, ttl: ttl, routes: routes, + }, nil +} + +func (s *AdminServer) prepareBackupReadTimestamp(ctx context.Context) (uint64, error) { + readTS, err := s.backupReadFence(ctx) + if err != nil { + return 0, status.Errorf(codes.FailedPrecondition, "backup read fence failed: %v", err) + } + if readTS == 0 || readTS == ^uint64(0) { + return 0, status.Errorf(codes.FailedPrecondition, "%s", "backup read fence returned an invalid timestamp") + } + return readTS, nil +} + +func (s *AdminServer) captureBackupRoutesAfterPin( + ctx context.Context, + groups []backupGroup, + controlGroup backupGroup, + pinID kv.BackupPinID, + commits map[uint64]uint64, + readTS uint64, +) (kv.BackupRouteSnapshot, error) { + if err := waitBackupGroupsApplied(ctx, groups, commits); err != nil { + s.compensateBackupRelease(controlGroup, groups, pinID) + return kv.BackupRouteSnapshot{}, status.Errorf(codes.FailedPrecondition, "wait for local backup pin apply: %v", err) + } // Capture catalog ownership only after every data group has applied the // pin's timestamp floor. A preallocated catalog write at or below readTS // either lands before this point and is visible here, or is rejected by the @@ -276,12 +308,9 @@ func (s *AdminServer) prepareBackup(ctx context.Context, ttl time.Duration) (pre routes, err := s.backupStore.CaptureBackupRouteSnapshotAt(ctx, readTS) if err != nil { s.compensateBackupRelease(controlGroup, groups, pinID) - return preparedBackup{}, status.Errorf(codes.FailedPrecondition, "capture backup routes at read timestamp: %v", err) + return kv.BackupRouteSnapshot{}, status.Errorf(codes.FailedPrecondition, "capture backup routes at read timestamp: %v", err) } - return preparedBackup{ - groups: groups, commits: commits, controlGroup: controlGroup, - pinID: pinID, readTS: readTS, ttl: ttl, routes: routes, - }, nil + return routes, nil } func (s *AdminServer) pinBackupGroups( @@ -320,7 +349,17 @@ func (s *AdminServer) buildExpectedBackupBaseline( ctx, stopRenew, prepared.groups, prepared.pinID, prepared.readTS, prepared.ttl, ) }() - counts, appliedAtCount, scanErr := s.scanBackupScopeCounts(ctx, prepared.routes, prepared.readTS, prepared.groups) + validateErr := s.backupStore.ValidateBackupSnapshotAt( + ctx, prepared.routes, prepared.readTS, s.backupConfig.scanPageSize, + ) + var counts map[logicalbackup.Scope]uint64 + var appliedAtCount uint64 + var scanErr error + if validateErr != nil { + scanErr = errors.Wrap(validateErr, "validate backup transaction locks") + } else { + counts, appliedAtCount, scanErr = s.scanBackupScopeCounts(ctx, prepared.routes, prepared.readTS, prepared.groups) + } close(stopRenew) renewErr := <-renewDone if scanErr != nil { @@ -332,6 +371,32 @@ func (s *AdminServer) buildExpectedBackupBaseline( return counts, appliedAtCount, nil } +func waitBackupGroupsApplied(ctx context.Context, groups []backupGroup, commits map[uint64]uint64) error { + for { + pending := false + for _, group := range groups { + target := commits[group.id] + if group.reader == nil || target == 0 { + return errors.Wrapf(ErrBackupUnavailable, "raft group %d has no local apply target", group.id) + } + if group.reader.Status().AppliedIndex < target { + pending = true + break + } + } + if !pending { + return nil + } + timer := time.NewTimer(backupAppliedPollInterval) + select { + case <-ctx.Done(): + timer.Stop() + return errors.WithStack(ctx.Err()) + case <-timer.C: + } + } +} + func (s *AdminServer) RenewBackup(ctx context.Context, req *pb.RenewBackupRequest) (*pb.RenewBackupResponse, error) { if err := s.requireBackupControl(); err != nil { return nil, err diff --git a/adapter/admin_backup_test.go b/adapter/admin_backup_test.go index 0341752a3..bff60375c 100644 --- a/adapter/admin_backup_test.go +++ b/adapter/admin_backup_test.go @@ -121,6 +121,11 @@ type backupTestStore struct { keyCloseErr error pairCloseErr error captureErr error + validateErr error +} + +func (s *backupTestStore) ValidateBackupSnapshotAt(context.Context, kv.BackupRouteSnapshot, uint64, int) error { + return s.validateErr } func (s *backupTestStore) CaptureBackupRouteSnapshotAt(_ context.Context, ts uint64) (kv.BackupRouteSnapshot, error) { @@ -362,6 +367,34 @@ func TestBeginBackupCapturesRouteSnapshotAfterReadFence(t *testing.T) { require.Equal(t, []uint64{42}, store.capturedTS) } +func TestBeginBackupWaitsForLocalPinApplyBeforeCatalogCapture(t *testing.T) { + store := &backupTestStore{} + group := &backupTestGroup{status: raftengine.Status{}, every: 10_000} + proposer := newBackupTestProposer() + proposer.onPropose = func(subtype byte) { + if subtype != backupSubtypePin { + return + } + go func() { + time.Sleep(25 * time.Millisecond) + group.setApplied(2) + }() + } + store.onCapture = func() { + require.GreaterOrEqual(t, group.Status().AppliedIndex, uint64(2)) + } + srv := newBackupControlTestServer( + t, store, + map[uint64]*backupTestGroup{1: group}, + map[uint64]*backupTestProposer{1: proposer}, + nil, + WithAdminBackupConfig(AdminBackupConfig{BeginDeadline: time.Second}), + ) + + _, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.NoError(t, err) +} + func TestBeginBackupCompensatesWhenPinnedRouteCaptureFails(t *testing.T) { t.Parallel() store := &backupTestStore{captureErr: stderrors.New("catalog unavailable")} diff --git a/adapter/internal.go b/adapter/internal.go index 824abc85e..1d479af50 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -24,6 +24,12 @@ func WithInternalAdminProposer(proposer raftengine.Proposer) InternalOption { } } +func WithInternalLastCommitTimestamp(reader func() uint64) InternalOption { + return func(i *Internal) { + i.lastCommitTimestamp = reader + } +} + func NewInternalWithEngine(txm kv.Transactional, leader raftengine.LeaderView, clock *kv.HLC, relay *RedisPubSubRelay, opts ...InternalOption) *Internal { i := &Internal{ leader: leader, @@ -38,12 +44,13 @@ func NewInternalWithEngine(txm kv.Transactional, leader raftengine.LeaderView, c } type Internal struct { - leader raftengine.LeaderView - transactionManager kv.Transactional - clock *kv.HLC - tsAllocator kv.TimestampAllocator - adminProposer raftengine.Proposer - relay *RedisPubSubRelay + leader raftengine.LeaderView + transactionManager kv.Transactional + clock *kv.HLC + tsAllocator kv.TimestampAllocator + adminProposer raftengine.Proposer + lastCommitTimestamp func() uint64 + relay *RedisPubSubRelay pb.UnimplementedInternalServer } @@ -119,7 +126,11 @@ func (i *Internal) ForwardLeaseRead( if err != nil { return nil, errors.WithStack(err) } - return &pb.ForwardLeaseReadResponse{AppliedIndex: index}, nil + lastCommitTS := uint64(0) + if i.lastCommitTimestamp != nil { + lastCommitTS = i.lastCommitTimestamp() + } + return &pb.ForwardLeaseReadResponse{AppliedIndex: index, LastCommitTs: lastCommitTS}, nil } func forwardedAdminProposalResponseError(result *raftengine.ProposalResult) error { diff --git a/adapter/internal_admin_proposal_test.go b/adapter/internal_admin_proposal_test.go index 0f295905b..24439777e 100644 --- a/adapter/internal_admin_proposal_test.go +++ b/adapter/internal_admin_proposal_test.go @@ -31,10 +31,12 @@ func TestInternalForwardLeaseReadUsesLeaderBarrier(t *testing.T) { internalAdminLeaderView{state: raftengine.StateLeader, readIndex: 23}, nil, nil, + WithInternalLastCommitTimestamp(func() uint64 { return 37 }), ) resp, err := internal.ForwardLeaseRead(context.Background(), &pb.ForwardLeaseReadRequest{}) require.NoError(t, err) require.Equal(t, uint64(23), resp.GetAppliedIndex()) + require.Equal(t, uint64(37), resp.GetLastCommitTs()) internal.leader = internalAdminLeaderView{state: raftengine.StateFollower} _, err = internal.ForwardLeaseRead(context.Background(), &pb.ForwardLeaseReadRequest{}) diff --git a/internal/backup/live.go b/internal/backup/live.go index f033e7028..7f6256aa6 100644 --- a/internal/backup/live.go +++ b/internal/backup/live.go @@ -33,10 +33,10 @@ var ErrScopeKeyMalformed = errors.New("backup: malformed scoped key") // Internal control-plane keys and derivable indexes return (_, false, nil). func ScopeForKey(key []byte) (Scope, bool, error) { switch { - case hasAnyBackupPrefix(key, DDBTableMetaPrefix, DDBTableGenPrefix, DDBItemPrefix, DDBGSIPrefix): + case hasAnyBackupPrefix(key, DDBTableMetaPrefix, DDBItemPrefix, DDBGSIPrefix): return scopeForDDBKey(key) case hasAnyBackupPrefix(key, - S3BucketMetaPrefix, S3BucketGenPrefix, S3ObjectManifestPrefix, + S3BucketMetaPrefix, S3ObjectManifestPrefix, S3UploadMetaPrefix, S3UploadPartPrefix, S3BlobPrefix, S3GCUploadPrefix, S3RoutePrefix, ): return scopeForS3Key(key) @@ -56,8 +56,6 @@ func scopeForDDBKey(key []byte) (Scope, bool, error) { switch { case bytes.HasPrefix(key, []byte(DDBTableMetaPrefix)): return ddbScopeFromDirectSegment(key, DDBTableMetaPrefix) - case bytes.HasPrefix(key, []byte(DDBTableGenPrefix)): - return ddbScopeFromDirectSegment(key, DDBTableGenPrefix) case bytes.HasPrefix(key, []byte(DDBItemPrefix)): encoded, _, err := parseDDBItemKey(key) if err != nil { @@ -76,10 +74,6 @@ func scopeForS3Key(key []byte) (Scope, bool, error) { case bytes.HasPrefix(key, []byte(S3BucketMetaPrefix)): bucket, ok := s3keys.ParseBucketMetaKey(key) return parsedS3Scope(bucket, ok, key) - case bytes.HasPrefix(key, []byte(S3BucketGenPrefix)): - metaKey := append([]byte(S3BucketMetaPrefix), key[len(S3BucketGenPrefix):]...) - bucket, ok := s3keys.ParseBucketMetaKey(metaKey) - return parsedS3Scope(bucket, ok, key) case bytes.HasPrefix(key, []byte(S3ObjectManifestPrefix)): bucket, _, _, ok := s3keys.ParseObjectManifestKey(key) return parsedS3Scope(bucket, ok, key) diff --git a/internal/backup/live_test.go b/internal/backup/live_test.go index cc4b2ed11..ace0d9933 100644 --- a/internal/backup/live_test.go +++ b/internal/backup/live_test.go @@ -24,6 +24,8 @@ func TestScopeForKey(t *testing.T) { {name: "sqs sequence", key: []byte(SQSQueueSeqPrefix + enc("jobs")), want: Scope{Adapter: "sqs", Name: "jobs"}, scoped: true}, {name: "redis", key: []byte(RedisStringPrefix + "key"), want: Scope{Adapter: "redis", Name: "db_0"}, scoped: true}, {name: "ddb derived gsi", key: []byte(DDBGSIPrefix + "ignored"), scoped: false}, + {name: "dynamodb generation counter", key: []byte(DDBTableGenPrefix + enc("deleted")), scoped: false}, + {name: "s3 generation counter", key: []byte(S3BucketGenPrefix + "deleted"), scoped: false}, {name: "transaction", key: []byte("!txn|lock|ignored"), scoped: false}, } for _, tc := range cases { diff --git a/kv/backup_scan.go b/kv/backup_scan.go index f892f6b53..1a81bbb50 100644 --- a/kv/backup_scan.go +++ b/kv/backup_scan.go @@ -108,6 +108,65 @@ func CaptureBackupRouteSnapshotAt(ctx context.Context, catalog *distribution.Cat return BackupRouteSnapshot{routes: cloneBackupRoutes(routes)}, nil } +// ValidateBackupSnapshotAt resolves committed or rolled-back transaction +// locks and fails closed while any prepared transaction remains pending at the +// backup cut. The scan covers lock-only inserts that have no visible user key. +func (s *ShardStore) ValidateBackupSnapshotAt(ctx context.Context, snapshot BackupRouteSnapshot, ts uint64, pageSize int) error { + if s == nil { + return errors.New("backup store is unavailable") + } + if pageSize <= 0 { + pageSize = defaultBackupScanPageSize + } + seenGroups := make(map[uint64]struct{}, len(snapshot.routes)) + for _, route := range snapshot.routes { + if _, seen := seenGroups[route.GroupID]; seen { + continue + } + seenGroups[route.GroupID] = struct{}{} + group, ok := s.groupForID(route.GroupID) + if !ok || group == nil || group.Store == nil { + return errors.Wrapf(ErrLeaderNotFound, "backup lock validation group %d is unavailable", route.GroupID) + } + if err := s.validateBackupGroupLocksAt(ctx, group, ts, pageSize); err != nil { + return errors.Wrapf(err, "validate backup locks for group %d", route.GroupID) + } + } + return nil +} + +func (s *ShardStore) validateBackupGroupLocksAt(ctx context.Context, group *ShardGroup, ts uint64, pageSize int) error { + cursor := txnLockKey(nil) + end := prefixScanEnd([]byte(txnLockPrefix)) + for { + locks, err := group.Store.ScanAt(ctx, cursor, end, pageSize, ts) + if err != nil { + return errors.WithStack(err) + } + if len(locks) == 0 { + return nil + } + plan, err := s.planScanLockResolutions(ctx, group, nil, locks, ts) + if err != nil { + return err + } + if err := applyScanLockResolutions(ctx, group, plan); err != nil { + return err + } + last := locks[len(locks)-1] + if len(locks) < pageSize { + return nil + } + if last == nil || len(last.Key) == 0 { + return errors.New("backup lock scan returned an invalid cursor") + } + cursor = nextScanCursor(last.Key) + if bytes.Compare(cursor, end) >= 0 { + return nil + } + } +} + // NewBackupScannerAtSnapshot creates a value scanner from a captured route view. func NewBackupScannerAtSnapshot(st *ShardStore, snapshot BackupRouteSnapshot, ts uint64, pageSize int) BackupScanner { if pageSize <= 0 { @@ -232,7 +291,9 @@ func (s *backupKeyScanner) loadNextPage(ctx context.Context) error { } s.page = s.page[:0] for _, item := range keys { - if _, ok := routeForRoutedKey(item, s.routes); ok { + if _, ok, err := s.store.routeForRoutedKey(item, s.routes); err != nil { + return err + } else if ok { s.page = append(s.page, item) } } @@ -262,7 +323,10 @@ func (s *backupScanner) loadNextPage(ctx context.Context) error { } s.page = s.page[:0] for _, item := range keys { - route, ok := s.materializeRouteForKey(item) + route, ok, err := s.materializeRouteForKey(item) + if err != nil { + return err + } if !ok { continue } @@ -289,24 +353,37 @@ func (s *backupScanner) loadNextPage(ctx context.Context) error { return nil } -func (s *backupScanner) materializeRouteForKey(item routedScanKey) (distribution.Route, bool) { - return routeForRoutedKey(item, s.routes) +func (s *backupScanner) materializeRouteForKey(item routedScanKey) (distribution.Route, bool, error) { + return s.store.routeForRoutedKey(item, s.routes) } -func routeForRoutedKey(item routedScanKey, routes []distribution.Route) (distribution.Route, bool) { +func (s *ShardStore) routeForRoutedKey(item routedScanKey, routes []distribution.Route) (distribution.Route, bool, error) { + if s != nil && s.partitionResolver != nil { + groupID, ok := s.partitionResolver.ResolveGroup(item.key) + if ok { + if groupID == item.route.GroupID { + return distribution.Route{GroupID: groupID}, true, nil + } + return distribution.Route{}, false, nil + } + if s.partitionResolver.RecognisesPartitionedKey(item.key) { + return distribution.Route{}, false, errors.Wrapf(ErrInvalidRequest, "no partition route for backup key %q", item.key) + } + } + key := routeKey(item.key) if routeContainsKey(item.route, key) { - return item.route, true + return item.route, true, nil } for _, route := range routes { if route.GroupID != item.route.GroupID { continue } if routeContainsKey(route, key) { - return route, true + return route, true, nil } } - return distribution.Route{}, false + return distribution.Route{}, false, nil } func cloneBackupRouteSnapshot(snapshot BackupRouteSnapshot) BackupRouteSnapshot { @@ -356,7 +433,10 @@ func (s *ShardStore) scanKeyRoutesWithSourceAt( if err != nil { return nil, err } - out = mergeAndTrimRoutedScanKeys(out, routedScanKeys(route, keys), routes, limit) + out, err = s.mergeAndTrimRoutedScanKeys(out, routedScanKeys(route, keys), routes, limit) + if err != nil { + return nil, err + } if clampToRoutes && len(out) >= limit { break } @@ -375,14 +455,14 @@ func routedScanKeys(route distribution.Route, keys [][]byte) []routedScanKey { return items } -func mergeAndTrimRoutedScanKeys( +func (s *ShardStore) mergeAndTrimRoutedScanKeys( out []routedScanKey, keys []routedScanKey, routes []distribution.Route, limit int, -) []routedScanKey { +) ([]routedScanKey, error) { if len(keys) == 0 { - return out + return out, nil } out = append(out, keys...) sort.SliceStable(out, func(i, j int) bool { @@ -394,7 +474,11 @@ func mergeAndTrimRoutedScanKeys( continue } if write > 0 && bytes.Equal(out[write-1].key, item.key) { - out[write-1] = preferredRoutedScanKey(out[write-1], item, routes) + preferred, err := s.preferredRoutedScanKey(out[write-1], item, routes) + if err != nil { + return nil, err + } + out[write-1] = preferred continue } out[write] = item @@ -403,19 +487,25 @@ func mergeAndTrimRoutedScanKeys( clear(out[write:]) out = out[:write] if len(out) <= limit { - return out + return out, nil } clear(out[limit:]) - return out[:limit] + return out[:limit], nil } -func preferredRoutedScanKey(current, candidate routedScanKey, routes []distribution.Route) routedScanKey { - _, currentOwned := routeForRoutedKey(current, routes) - _, candidateOwned := routeForRoutedKey(candidate, routes) +func (s *ShardStore) preferredRoutedScanKey(current, candidate routedScanKey, routes []distribution.Route) (routedScanKey, error) { + _, currentOwned, err := s.routeForRoutedKey(current, routes) + if err != nil { + return routedScanKey{}, err + } + _, candidateOwned, err := s.routeForRoutedKey(candidate, routes) + if err != nil { + return routedScanKey{}, err + } if candidateOwned && !currentOwned { - return candidate + return candidate, nil } - return current + return current, nil } func lastRoutedScanKey(keys []routedScanKey) []byte { diff --git a/kv/coordinator.go b/kv/coordinator.go index 5ee5c4f46..558b9ac20 100644 --- a/kv/coordinator.go +++ b/kv/coordinator.go @@ -296,6 +296,12 @@ type AllGroupsLeaseReadableCoordinator interface { LeaseReadAllGroups(ctx context.Context) error } +// AllGroupsLeaseTimestampCoordinator extends the all-group barrier with the +// greatest commit timestamp applied by any fenced group leader. +type AllGroupsLeaseTimestampCoordinator interface { + LeaseReadAllGroupsTimestamp(ctx context.Context) (uint64, error) +} + // LeaseReadAllGroupsThrough establishes the lease freshness bound across // every shard group a multi-shard read can touch. When the coordinator owns // multiple groups (AllGroupsLeaseReadableCoordinator) it fences all of them; @@ -311,6 +317,20 @@ func LeaseReadAllGroupsThrough(c Coordinator, ctx context.Context) error { return errors.WithStack(err) } +// LeaseReadAllGroupsTimestampThrough returns the greatest leader-side commit +// timestamp when the coordinator exposes it. Legacy coordinators still execute +// their all-group barrier and return a zero watermark to their caller. +func LeaseReadAllGroupsTimestampThrough(c Coordinator, ctx context.Context) (uint64, error) { + if ag, ok := c.(AllGroupsLeaseTimestampCoordinator); ok { + ts, err := ag.LeaseReadAllGroupsTimestamp(ctx) + return ts, errors.WithStack(err) + } + if err := LeaseReadAllGroupsThrough(c, ctx); err != nil { + return 0, err + } + return 0, nil +} + // GroupRoutableCoordinator is the optional capability implemented by // coordinators that can resolve the owning Raft group of a key without // any I/O. Callers that need to lease-check a set of keys use it to diff --git a/kv/fsm_backup_test.go b/kv/fsm_backup_test.go index 4501e5367..71e7233a2 100644 --- a/kv/fsm_backup_test.go +++ b/kv/fsm_backup_test.go @@ -189,6 +189,16 @@ func TestApplyBackupFencesPreallocatedWrites(t *testing.T) { } } +func TestBackupTimestampFloorKeyRejectsRawMutation(t *testing.T) { + fsm := newBackupTestFSM(t, NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0))) + resp := applyBackupTestRequest(t, fsm, &pb.Request{Ts: 10, Mutations: []*pb.Mutation{{ + Op: pb.Op_PUT, Key: bytes.Clone(backupTimestampFloorKey), Value: make([]byte, backupTimestampFloorValueSize), + }}}) + err, ok := resp.(error) + require.True(t, ok) + require.ErrorIs(t, err, ErrInvalidRequest) +} + func TestApplyBackupAllowsResolutionOfPrePinTransaction(t *testing.T) { fsm := newBackupTestFSM(t, NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0))) primary := []byte("primary") diff --git a/kv/keyviz_label.go b/kv/keyviz_label.go index eb1949076..b7cf20957 100644 --- a/kv/keyviz_label.go +++ b/kv/keyviz_label.go @@ -117,6 +117,10 @@ func (c keyVizLabeledCoordinator) LeaseReadAllGroups(ctx context.Context) error return errors.WithStack(err) } +func (c keyVizLabeledCoordinator) LeaseReadAllGroupsTimestamp(ctx context.Context) (uint64, error) { + return LeaseReadAllGroupsTimestampThrough(c.inner, ctx) +} + func (c keyVizLabeledCoordinator) EngineGroupIDForKey(key []byte) uint64 { if gr, ok := c.inner.(GroupRoutableCoordinator); ok { return gr.EngineGroupIDForKey(key) diff --git a/kv/leader_proxy.go b/kv/leader_proxy.go index fc3f89d4a..4b42f6224 100644 --- a/kv/leader_proxy.go +++ b/kv/leader_proxy.go @@ -225,36 +225,36 @@ func (p *LeaderProxy) forward(parentCtx context.Context, reqs []*pb.Request) (*T return &TransactionResponse{CommitIndex: resp.CommitIndex}, nil } -func (p *LeaderProxy) forwardLeaseRead(callerCtx context.Context) (uint64, error) { +func (p *LeaderProxy) forwardLeaseRead(callerCtx context.Context) (leaseReadResult, error) { deadline := time.Now().Add(leaderProxyRetryBudget) ctx, cancel := context.WithDeadline(callerCtx, deadline) defer cancel() var lastErr error for time.Now().Before(deadline) { - index, err := p.forwardLeaseReadOnce(ctx) + result, err := p.forwardLeaseReadOnce(ctx) if err == nil { - return index, nil + return result, nil } lastErr = err if !isTransientLeaderError(err) { - return 0, err + return leaseReadResult{}, err } waitLeaderProxyBackoff(ctx, leaderProxyRetryInterval, deadline) } if lastErr == nil { lastErr = ErrLeaderNotFound } - return 0, errors.WithStack(lastErr) + return leaseReadResult{}, errors.WithStack(lastErr) } -func (p *LeaderProxy) forwardLeaseReadOnce(parentCtx context.Context) (uint64, error) { +func (p *LeaderProxy) forwardLeaseReadOnce(parentCtx context.Context) (leaseReadResult, error) { addr := leaderAddrFromEngine(p.engine) if addr == "" { - return 0, errors.WithStack(ErrLeaderNotFound) + return leaseReadResult{}, errors.WithStack(ErrLeaderNotFound) } conn, err := p.connCache.ConnFor(addr) if err != nil { - return 0, err + return leaseReadResult{}, err } ctx, cancel := context.WithTimeout(parentCtx, leaderForwardTimeout) defer cancel() @@ -263,9 +263,9 @@ func (p *LeaderProxy) forwardLeaseReadOnce(parentCtx context.Context) (uint64, e } resp, err := pb.NewInternalClient(conn).ForwardLeaseRead(ctx, &pb.ForwardLeaseReadRequest{}) if err != nil { - return 0, errors.WithStack(err) + return leaseReadResult{}, errors.WithStack(err) } - return resp.GetAppliedIndex(), nil + return leaseReadResult{appliedIndex: resp.GetAppliedIndex(), lastCommitTS: resp.GetLastCommitTs()}, nil } var _ Transactional = (*LeaderProxy)(nil) diff --git a/kv/shard_store.go b/kv/shard_store.go index 14c72332f..d06086182 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -23,8 +23,9 @@ const proxyForwardTimeout = 5 * time.Second // ShardStore routes MVCC reads to shard-specific stores and proxies to leaders when needed. type ShardStore struct { - engine *distribution.Engine - groups map[uint64]*ShardGroup + engine *distribution.Engine + groups map[uint64]*ShardGroup + partitionResolver PartitionResolver connCache GRPCConnCache } @@ -42,6 +43,17 @@ func NewShardStore(engine *distribution.Engine, groups map[uint64]*ShardGroup) * } } +// WithPartitionResolver installs the same partition-keyspace resolver used by +// ShardedCoordinator. ShardStore keeps normal byte-range routing for ordinary +// calls, but backup scanners need the resolver to decide which physical group +// owns partition-routed keys discovered while scanning every group. +func (s *ShardStore) WithPartitionResolver(r PartitionResolver) *ShardStore { + if s != nil { + s.partitionResolver = r + } + return s +} + // FilesystemGroupForHome resolves the group that owns one file-home route. func (s *ShardStore) FilesystemGroupForHome(homeSlot uint64, inode uint64) (uint64, bool) { if s == nil || s.engine == nil { diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 64398e21c..9767d65c3 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -726,6 +726,29 @@ func TestCaptureBackupRouteSnapshotAtReadsRowsFromCatalogOwner(t *testing.T) { require.Equal(t, uint64(2), snapshot.routes[1].GroupID) } +func TestValidateBackupSnapshotAtRejectsPreparedInsert(t *testing.T) { + ctx := context.Background() + st := store.NewMVCCStore() + fsm, ok := NewKvFSMWithHLCAndTracker( + st, NewHLC(), NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)), + ).(*kvFSM) + require.True(t, ok) + primary := []byte("insert-only") + prepare := &pb.Request{IsTxn: true, Phase: pb.Phase_PREPARE, Ts: 30, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: primary, LockTTLms: defaultTxnLockTTLms, + })}, + {Op: pb.Op_PUT, Key: primary, Value: []byte("pending")}, + }} + require.Nil(t, applyBackupTestRequest(t, fsm, prepare)) + + engine := distribution.NewEngineWithDefaultRoute() + shards := NewShardStore(engine, map[uint64]*ShardGroup{1: {Store: st}}) + snapshot := shards.CaptureBackupRouteSnapshot(nil, nil) + err := shards.ValidateBackupSnapshotAt(ctx, snapshot, 50, 16) + require.ErrorIs(t, err, ErrTxnLocked) +} + func TestBackupScannerMaterializesFromCapturedRoute(t *testing.T) { t.Parallel() @@ -833,6 +856,82 @@ func TestBackupScannersPreferCapturedOwnerForDuplicateKey(t *testing.T) { require.Equal(t, []byte("captured-owner"), pair.Value) } +func TestBackupScannerKeepsPartitionResolvedOwnerForSQSKey(t *testing.T) { + t.Parallel() + + ctx := context.Background() + key := []byte("!sqs|msg|data|p|orders|partition-2|msg-2") + groups := map[uint64]*ShardGroup{ + 1: {Store: store.NewMVCCStore()}, + 2: {Store: store.NewMVCCStore()}, + } + st := NewShardStore(distribution.NewEngine(), groups). + WithPartitionResolver(&fakePartitionResolver{ + routes: map[string]uint64{string(key): 2}, + recognisedPrefix: []byte("!sqs|msg|data|p|"), + }) + require.NoError(t, groups[1].Store.PutAt(ctx, key, []byte("stale-engine-owner"), 1, 0)) + require.NoError(t, groups[2].Store.PutAt(ctx, key, []byte("partition-owner"), 2, 0)) + routes := []distribution.Route{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1}, + {RouteID: 2, Start: []byte("m"), GroupID: 2}, + } + + keyScanner := &backupKeyScanner{ + store: st, routes: routes, clampToRoutes: false, + cursor: []byte(""), ts: ^uint64(0), pageSize: 1, + } + defer keyScanner.Close() + gotKey, ok, err := keyScanner.Next(ctx) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, key, gotKey) + _, ok, err = keyScanner.Next(ctx) + require.NoError(t, err) + require.False(t, ok) + + valueScanner := &backupScanner{ + store: st, routes: routes, clampToRoutes: false, + cursor: []byte(""), ts: ^uint64(0), pageSize: 1, + } + defer valueScanner.Close() + pair, ok, err := valueScanner.Next(ctx) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, key, pair.Key) + require.Equal(t, []byte("partition-owner"), pair.Value) +} + +func TestBackupScannerFailsClosedOnUnresolvedPartitionedSQSKey(t *testing.T) { + t.Parallel() + + ctx := context.Background() + key := []byte("!sqs|msg|data|p|orders|partition-9|msg-9") + groups := map[uint64]*ShardGroup{ + 1: {Store: store.NewMVCCStore()}, + } + st := NewShardStore(distribution.NewEngine(), groups). + WithPartitionResolver(&fakePartitionResolver{ + routes: map[string]uint64{}, + recognisedPrefix: []byte("!sqs|msg|data|p|"), + }) + require.NoError(t, groups[1].Store.PutAt(ctx, key, []byte("unresolved"), 1, 0)) + scanner := &backupScanner{ + store: st, + routes: []distribution.Route{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1}, + }, + clampToRoutes: false, + cursor: []byte(""), + ts: ^uint64(0), + pageSize: 1, + } + defer scanner.Close() + + _, _, err := scanner.Next(ctx) + require.ErrorIs(t, err, ErrInvalidRequest) +} + func TestBackupScannerPreservesFullRoutingAfterListKeyCursor(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index b80167c33..561c047f9 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -62,7 +62,7 @@ type ShardGroup struct { proposer raftengine.Proposer // leaderRead forwards a read barrier to this group's current leader when // the local replica is a follower. It is installed with the LeaderProxy. - leaderRead func(context.Context) (uint64, error) + leaderRead func(context.Context) (leaseReadResult, error) // leaderReadToken authenticates follower-to-leader lease-read RPCs. Admin // service setup publishes it once before the public listeners are opened. leaderReadToken atomic.Pointer[string] @@ -1818,16 +1818,28 @@ func (c *ShardedCoordinator) LeaseReadForKey(ctx context.Context, key []byte) (u // but correctness does not depend on it because every configured group must // succeed. func (c *ShardedCoordinator) LeaseReadAllGroups(ctx context.Context) error { + _, err := c.LeaseReadAllGroupsTimestamp(ctx) + return err +} + +// LeaseReadAllGroupsTimestamp fences every data-group leader and returns the +// maximum commit timestamp observed after those barriers. +func (c *ShardedCoordinator) LeaseReadAllGroupsTimestamp(ctx context.Context) (uint64, error) { groups, err := c.allShardGroups() if err != nil { - return err + return 0, err } + var maxCommitTS uint64 for _, g := range groups { - if _, err := groupLeaseRead(ctx, g, c.leaseObserver); err != nil { - return errors.WithStack(err) + result, err := groupLeaseReadResult(ctx, g, c.leaseObserver) + if err != nil { + return 0, errors.WithStack(err) + } + if result.lastCommitTS > maxCommitTS { + maxCommitTS = result.lastCommitTS } } - return nil + return maxCommitTS, nil } func (c *ShardedCoordinator) allShardGroups() ([]*ShardGroup, error) { @@ -1881,17 +1893,35 @@ func shouldForwardGroupLeaseRead(g *ShardGroup, engine raftengine.Engine) bool { return g != nil && g.leaderRead != nil && engine != nil && engine.State() != raftengine.StateLeader } -func handleGroupLeaseReadError(ctx context.Context, g *ShardGroup, err error) (uint64, error) { +type leaseReadResult struct { + appliedIndex uint64 + lastCommitTS uint64 +} + +func localLeaseReadResult(g *ShardGroup, appliedIndex uint64) leaseReadResult { + result := leaseReadResult{appliedIndex: appliedIndex} + if g != nil && g.Store != nil { + result.lastCommitTS = g.Store.LastCommitTS() + } + return result +} + +func handleGroupLeaseReadError(ctx context.Context, g *ShardGroup, err error) (leaseReadResult, error) { if g != nil && isLeadershipLossError(err) { g.lease.invalidate() } if g != nil && g.leaderRead != nil && isTransientLeaderError(err) { return g.leaderRead(ctx) } - return 0, err + return leaseReadResult{}, err } func groupLeaseRead(ctx context.Context, g *ShardGroup, observer LeaseReadObserver) (uint64, error) { + result, err := groupLeaseReadResult(ctx, g, observer) + return result.appliedIndex, err +} + +func groupLeaseReadResult(ctx context.Context, g *ShardGroup, observer LeaseReadObserver) (leaseReadResult, error) { engine := engineForGroup(g) if shouldForwardGroupLeaseRead(g, engine) { observeLeaseRead(observer, false) @@ -1903,12 +1933,14 @@ func groupLeaseRead(ctx context.Context, g *ShardGroup, observer LeaseReadObserv // guard preserves engineForGroup's nil-safety since g.lp would panic // on a nil receiver. if g == nil || g.lp == nil { - return linearizableReadEngineCtx(ctx, engine) + idx, err := linearizableReadEngineCtx(ctx, engine) + return localLeaseReadResult(g, idx), err } lp := g.lp leaseDur := lp.LeaseDuration() if leaseDur <= 0 { - return linearizableReadEngineCtx(ctx, engine) + idx, err := linearizableReadEngineCtx(ctx, engine) + return localLeaseReadResult(g, idx), err } // Single monoclock.Now() sample so primary/secondary/extension // all see the same monotonic-raw instant. Clock-skew safety @@ -1917,12 +1949,12 @@ func groupLeaseRead(ctx context.Context, g *ShardGroup, observer LeaseReadObserv state := engine.State() if engineLeaseAckValid(state, lp.LastQuorumAck(), now, leaseDur) { observeLeaseRead(observer, true) - return lp.AppliedIndex(), nil + return localLeaseReadResult(g, lp.AppliedIndex()), nil } expectedGen := g.lease.generation() if g.lease.valid(now) && state == raftengine.StateLeader { observeLeaseRead(observer, true) - return lp.AppliedIndex(), nil + return localLeaseReadResult(g, lp.AppliedIndex()), nil } observeLeaseRead(observer, false) idx, err := linearizableReadEngineCtx(ctx, engine) @@ -1930,7 +1962,7 @@ func groupLeaseRead(ctx context.Context, g *ShardGroup, observer LeaseReadObserv return handleGroupLeaseReadError(ctx, g, err) } g.lease.extend(now.Add(leaseDur), expectedGen) - return idx, nil + return localLeaseReadResult(g, idx), nil } func (c *ShardedCoordinator) Clock() *HLC { diff --git a/kv/sharded_coordinator_leader_test.go b/kv/sharded_coordinator_leader_test.go index 64989adee..5633fc186 100644 --- a/kv/sharded_coordinator_leader_test.go +++ b/kv/sharded_coordinator_leader_test.go @@ -31,7 +31,7 @@ func (s *leaseReadForwardServer) ForwardLeaseRead( s.mu.Lock() s.authorization = append([]string(nil), md.Get("authorization")...) s.mu.Unlock() - return &pb.ForwardLeaseReadResponse{AppliedIndex: 44}, nil + return &pb.ForwardLeaseReadResponse{AppliedIndex: 44, LastCommitTs: 91}, nil } func TestShardedCoordinatorVerifyLeader_LeaderReturnsNil(t *testing.T) { @@ -76,9 +76,11 @@ func TestShardedCoordinatorLeaseReadAllGroups_FencesEveryLeader(t *testing.T) { engine.UpdateRoute([]byte("m"), nil, 2) s1 := store.NewMVCCStore() + require.NoError(t, s1.PutAt(context.Background(), []byte("a"), []byte("one"), 70, 0)) r1, stop1 := newSingleRaft(t, "lrag-g1", NewKvFSMWithHLC(s1, NewHLC())) t.Cleanup(stop1) s2 := store.NewMVCCStore() + require.NoError(t, s2.PutAt(context.Background(), []byte("z"), []byte("two"), 90, 0)) r2, stop2 := newSingleRaft(t, "lrag-g2", NewKvFSMWithHLC(s2, NewHLC())) t.Cleanup(stop2) @@ -88,7 +90,9 @@ func TestShardedCoordinatorLeaseReadAllGroups_FencesEveryLeader(t *testing.T) { } coord := NewShardedCoordinator(engine, groups, 1, NewHLC(), NewShardStore(engine, groups)) - require.NoError(t, coord.LeaseReadAllGroups(context.Background())) + maxCommitTS, err := coord.LeaseReadAllGroupsTimestamp(context.Background()) + require.NoError(t, err) + require.Equal(t, uint64(90), maxCommitTS) } func TestShardedCoordinatorLeaseReadAllGroups_ForwardsFollowerToGroupLeader(t *testing.T) { @@ -115,7 +119,9 @@ func TestShardedCoordinatorLeaseReadAllGroups_ForwardsFollowerToGroupLeader(t *t group.Txn = proxy coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{1: group}, 1, NewHLC(), nil) - require.NoError(t, coord.LeaseReadAllGroups(context.Background())) + maxCommitTS, err := coord.LeaseReadAllGroupsTimestamp(context.Background()) + require.NoError(t, err) + require.Equal(t, uint64(91), maxCommitTS) require.Equal(t, int32(1), service.calls.Load()) service.mu.Lock() require.Equal(t, []string{"Bearer admin-secret"}, service.authorization) diff --git a/kv/txn_keys.go b/kv/txn_keys.go index e997722ea..a2818f7a3 100644 --- a/kv/txn_keys.go +++ b/kv/txn_keys.go @@ -68,14 +68,7 @@ func txnRollbackKey(primaryKey []byte, startTS uint64) []byte { } func isTxnInternalKey(key []byte) bool { - if !bytes.HasPrefix(key, txnCommonPrefix) { - return false - } - return bytes.HasPrefix(key, txnLockPrefixBytes) || - bytes.HasPrefix(key, txnIntentPrefixBytes) || - bytes.HasPrefix(key, txnCommitPrefixBytes) || - bytes.HasPrefix(key, txnRollbackPrefixBytes) || - bytes.HasPrefix(key, txnMetaPrefixBytes) + return bytes.HasPrefix(key, txnCommonPrefix) } func isTxnMetaKey(key []byte) bool { diff --git a/main.go b/main.go index 26e2de2b3..01966f0ab 100644 --- a/main.go +++ b/main.go @@ -526,12 +526,14 @@ func run() error { cleanup.Add(cancel) startLockResolverIfEnabled(shardStore, shardGroups, &cleanup) sampler := buildKeyVizSampler() + sqsPartitionResolver := buildSQSPartitionResolver(cfg.sqsFifoPartitionMap) + shardStore.WithPartitionResolver(sqsPartitionResolver) coordinate := kv.NewShardedCoordinator(cfg.engine, shardGroups, cfg.defaultGroup, clock, shardStore). WithLeaseReadObserver(metricsRegistry.LeaseReadObserver()). WithSampler(keyVizSamplerForCoordinator(sampler)). WithKeyVizLabelsEnabled(*keyvizLabelsEnabled). WithAllShardGroups(dataGroupIDs(cfg.groups)...). - WithPartitionResolver(buildSQSPartitionResolver(cfg.sqsFifoPartitionMap)) + WithPartitionResolver(sqsPartitionResolver) if err := configureCoordinatorTSO(coordinate); err != nil { return err } @@ -2243,6 +2245,7 @@ type startupGatedCoordinator struct { var _ kv.Coordinator = (*startupGatedCoordinator)(nil) var _ kv.LeaseReadableCoordinator = (*startupGatedCoordinator)(nil) var _ kv.AllGroupsLeaseReadableCoordinator = (*startupGatedCoordinator)(nil) +var _ kv.AllGroupsLeaseTimestampCoordinator = (*startupGatedCoordinator)(nil) var _ kv.GroupRoutableCoordinator = (*startupGatedCoordinator)(nil) var _ kv.TimestampAllocator = (*startupGatedCoordinator)(nil) var _ kv.TimestampAfterAllocator = (*startupGatedCoordinator)(nil) @@ -2314,6 +2317,10 @@ func (c startupGatedCoordinator) LeaseReadAllGroups(ctx context.Context) error { return kv.LeaseReadAllGroupsThrough(c.inner, ctx) //nolint:wrapcheck // Pass through coordinator errors unchanged. } +func (c startupGatedCoordinator) LeaseReadAllGroupsTimestamp(ctx context.Context) (uint64, error) { + return kv.LeaseReadAllGroupsTimestampThrough(c.inner, ctx) //nolint:wrapcheck // Pass through coordinator errors unchanged. +} + func (c startupGatedCoordinator) EngineGroupIDForKey(key []byte) uint64 { if router, ok := c.inner.(kv.GroupRoutableCoordinator); ok { return router.EngineGroupIDForKey(key) @@ -2490,10 +2497,14 @@ func adminBackupReadFence(coordinate kv.Coordinator, shardStore *kv.ShardStore) } return func(ctx context.Context) (uint64, error) { clock := coordinate.Clock() - if err := kv.LeaseReadAllGroupsThrough(coordinate, ctx); err != nil { + leaderCommitTS, err := kv.LeaseReadAllGroupsTimestampThrough(coordinate, ctx) + if err != nil { return 0, errors.Wrap(err, "backup: fence raft groups") } lastCommitTS := shardStore.LastCommitTS() + if leaderCommitTS > lastCommitTS { + lastCommitTS = leaderCommitTS + } clock.Observe(lastCommitTS) readTS, err := allocateBackupReadTimestamp(ctx, coordinate, lastCommitTS) if err != nil { @@ -2764,7 +2775,7 @@ func startRaftServers( grpcSvc := adapter.NewGRPCServer(shardStore, coordinate) pb.RegisterRawKVServer(gs, grpcSvc) pb.RegisterTransactionalKVServer(gs, grpcSvc) - internalOpts := internalServerOptions(coordinate, adminServer, proposerForGroup(rt, shardGroups)) + internalOpts := internalServerOptions(coordinate, adminServer, proposerForGroup(rt, shardGroups), shardGroups[rt.spec.id]) pb.RegisterInternalServer(gs, adapter.NewInternalWithEngine( trx, rt.engine, @@ -2849,10 +2860,14 @@ func internalServerOptions( coordinate kv.Coordinator, adminServer *adapter.AdminServer, proposer raftengine.Proposer, + group *kv.ShardGroup, ) []adapter.InternalOption { opts := internalTimestampOptions(coordinate) if adminServer != nil { - return append(opts, adapter.WithInternalAdminProposer(proposer)) + opts = append(opts, adapter.WithInternalAdminProposer(proposer)) + } + if group != nil && group.Store != nil { + opts = append(opts, adapter.WithInternalLastCommitTimestamp(group.Store.LastCommitTS)) } return opts } diff --git a/main_admin_test.go b/main_admin_test.go index 0baefbe36..2587e0993 100644 --- a/main_admin_test.go +++ b/main_admin_test.go @@ -79,12 +79,13 @@ type backupFenceOrderCoordinator struct { type backupFenceTSOCoordinator struct { stubStartupCoordinator - min uint64 - nextAfter uint64 - barriers atomic.Int32 - barrierErr error - nextErr error - afterErr error + min uint64 + nextAfter uint64 + barriers atomic.Int32 + barrierErr error + nextErr error + afterErr error + leaderCommitTS uint64 } func (c *backupFenceTSOCoordinator) LeaseReadAllGroups(context.Context) error { @@ -92,6 +93,11 @@ func (c *backupFenceTSOCoordinator) LeaseReadAllGroups(context.Context) error { return c.barrierErr } +func (c *backupFenceTSOCoordinator) LeaseReadAllGroupsTimestamp(context.Context) (uint64, error) { + c.barriers.Add(1) + return c.leaderCommitTS, c.barrierErr +} + func (c *backupFenceTSOCoordinator) Next(context.Context) (uint64, error) { return c.nextAfter, c.nextErr } @@ -134,7 +140,7 @@ func TestAdminBackupReadFenceAllocatesTimestampAfterBarrier(t *testing.T) { func TestAdminBackupReadFenceUsesCoordinatorTimestampAllocator(t *testing.T) { t.Parallel() - coordinate := &backupFenceTSOCoordinator{nextAfter: 9_000} + coordinate := &backupFenceTSOCoordinator{nextAfter: 9_000, leaderCommitTS: 8_000} groupStore := store.NewMVCCStore() require.NoError(t, groupStore.PutAt(context.Background(), []byte("committed"), []byte("value"), 7_000, 0)) shardStore := kv.NewShardStore(distribution.NewEngineWithDefaultRoute(), map[uint64]*kv.ShardGroup{ @@ -145,7 +151,7 @@ func TestAdminBackupReadFenceUsesCoordinatorTimestampAllocator(t *testing.T) { readTS, err := fence(context.Background()) require.NoError(t, err) require.Equal(t, uint64(9_000), readTS) - require.Equal(t, uint64(7_000), coordinate.min) + require.Equal(t, uint64(8_000), coordinate.min) require.Equal(t, int32(1), coordinate.barriers.Load()) } diff --git a/proto/internal.pb.go b/proto/internal.pb.go index 36e9607d2..f0be65369 100644 --- a/proto/internal.pb.go +++ b/proto/internal.pb.go @@ -581,6 +581,7 @@ func (*ForwardLeaseReadRequest) Descriptor() ([]byte, []int) { type ForwardLeaseReadResponse struct { state protoimpl.MessageState `protogen:"open.v1"` AppliedIndex uint64 `protobuf:"varint,1,opt,name=applied_index,json=appliedIndex,proto3" json:"applied_index,omitempty"` + LastCommitTs uint64 `protobuf:"varint,2,opt,name=last_commit_ts,json=lastCommitTs,proto3" json:"last_commit_ts,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -622,6 +623,13 @@ func (x *ForwardLeaseReadResponse) GetAppliedIndex() uint64 { return 0 } +func (x *ForwardLeaseReadResponse) GetLastCommitTs() uint64 { + if x != nil { + return x.LastCommitTs + } + return 0 +} + type RelayPublishRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Channel []byte `protobuf:"bytes,1,opt,name=channel,proto3" json:"channel,omitempty"` @@ -748,9 +756,10 @@ const file_internal_proto_rawDesc = "" + "\apayload\x18\x01 \x01(\fR\apayload\"A\n" + "\x1cForwardAdminProposalResponse\x12!\n" + "\fcommit_index\x18\x01 \x01(\x04R\vcommitIndex\"\x19\n" + - "\x17ForwardLeaseReadRequest\"?\n" + + "\x17ForwardLeaseReadRequest\"e\n" + "\x18ForwardLeaseReadResponse\x12#\n" + - "\rapplied_index\x18\x01 \x01(\x04R\fappliedIndex\"I\n" + + "\rapplied_index\x18\x01 \x01(\x04R\fappliedIndex\x12$\n" + + "\x0elast_commit_ts\x18\x02 \x01(\x04R\flastCommitTs\"I\n" + "\x13RelayPublishRequest\x12\x18\n" + "\achannel\x18\x01 \x01(\fR\achannel\x12\x18\n" + "\amessage\x18\x02 \x01(\fR\amessage\"8\n" + diff --git a/proto/internal.proto b/proto/internal.proto index 536f7b593..c847eb166 100644 --- a/proto/internal.proto +++ b/proto/internal.proto @@ -91,6 +91,7 @@ message ForwardLeaseReadRequest {} message ForwardLeaseReadResponse { uint64 applied_index = 1; + uint64 last_commit_ts = 2; } message RelayPublishRequest { diff --git a/proto/service.pb.go b/proto/service.pb.go index 00000ae2f..6484611fe 100644 --- a/proto/service.pb.go +++ b/proto/service.pb.go @@ -2350,7 +2350,7 @@ const file_service_proto_rawDesc = "" + "\bstart_ts\x18\x01 \x01(\x04R\astartTs\",\n" + "\x10RollbackResponse\x12\x18\n" + "\asuccess\x18\x01 \x01(\bR\asuccess\"\x18\n" + - "\x16RaftAdminStatusRequest\"\xa2\x03\n" + + "\x16RaftAdminStatusRequest\"\xd8\x03\n" + "\x17RaftAdminStatusResponse\x12%\n" + "\x05state\x18\x01 \x01(\x0e2\x0f.RaftAdminStateR\x05state\x12\x1b\n" + "\tleader_id\x18\x02 \x01(\tR\bleaderId\x12%\n" + @@ -2364,7 +2364,7 @@ const file_service_proto_rawDesc = "" + "fsmPending\x12\x1b\n" + "\tnum_peers\x18\n" + " \x01(\x04R\bnumPeers\x12,\n" + - "\x12last_contact_nanos\x18\v \x01(\x03R\x10lastContactNanos\"\x1f\n" + + "\x12last_contact_nanos\x18\v \x01(\x03R\x10lastContactNanosJ\x04\b\f\x10\rJ\x04\b\r\x10\x0eR\x13configuration_indexR\x13pending_conf_change\"\x1f\n" + "\x1dRaftAdminConfigurationRequest\"W\n" + "\x0fRaftAdminMember\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x18\n" + diff --git a/proto/service.proto b/proto/service.proto index 68d12d5f7..bb19c6ae7 100644 --- a/proto/service.proto +++ b/proto/service.proto @@ -184,6 +184,9 @@ enum RaftAdminState { message RaftAdminStatusRequest {} message RaftAdminStatusResponse { + reserved 12, 13; + reserved "configuration_index", "pending_conf_change"; + RaftAdminState state = 1; string leader_id = 2; string leader_address = 3; From dd9f499f9380933a0df6a3579619064d0259f846 Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 16:42:03 +0900 Subject: [PATCH 6/9] backup: stop unsafe live backup windows --- adapter/admin_backup.go | 32 ++++++++++++++++++------ adapter/admin_backup_test.go | 38 +++++++++++++++++++++++++++++ kv/active_timestamp_tracker.go | 12 +++++++++ kv/active_timestamp_tracker_test.go | 13 ++++++++++ kv/fsm.go | 3 +++ kv/fsm_backup.go | 11 +++++++++ kv/fsm_backup_test.go | 17 +++++++++++++ 7 files changed, 119 insertions(+), 7 deletions(-) diff --git a/adapter/admin_backup.go b/adapter/admin_backup.go index a66c815ef..da8c9919d 100644 --- a/adapter/admin_backup.go +++ b/adapter/admin_backup.go @@ -16,6 +16,7 @@ import ( "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -531,7 +532,9 @@ func (s *AdminServer) StreamBackup( if scanner == nil { return status.Errorf(codes.Unavailable, "%s", "backup scanner is nil") } - scanErr := streamBackupRecords(stream, scanner, selected) + scanErr := streamBackupRecords(stream, scanner, selected, func() error { + return s.requireUnexpiredBackupToken(tok) + }) if err := finishBackupScan(stream.Context(), scanner, scanErr); err != nil { if scanErr != nil { return err @@ -545,8 +548,12 @@ func streamBackupRecords( stream grpc.ServerStreamingServer[pb.BackupKV], scanner kv.BackupScanner, selected map[logicalbackup.Scope]bool, + requireLive func() error, ) error { for { + if err := requireLive(); err != nil { + return err + } pair, ok, err := scanner.Next(stream.Context()) if err != nil { return backupScanStreamError(err) @@ -554,22 +561,33 @@ func streamBackupRecords( if !ok { return nil } - if pair == nil { - return status.Errorf(codes.Internal, "%s", "backup scanner returned a nil record") - } - scope, scoped, err := logicalbackup.ScopeForKey(pair.Key) + selectedRecord, err := backupRecordSelected(pair, selected) if err != nil { - return status.Errorf(codes.FailedPrecondition, "classify backup key: %v", err) + return err } - if !scoped || (len(selected) > 0 && !selected[scope]) { + if !selectedRecord { continue } + if err := requireLive(); err != nil { + return err + } if err := stream.Send(&pb.BackupKV{Key: pair.Key, Value: pair.Value}); err != nil { return backupSendStreamError(err) } } } +func backupRecordSelected(pair *store.KVPair, selected map[logicalbackup.Scope]bool) (bool, error) { + if pair == nil { + return false, status.Errorf(codes.Internal, "%s", "backup scanner returned a nil record") + } + scope, scoped, err := logicalbackup.ScopeForKey(pair.Key) + if err != nil { + return false, status.Errorf(codes.FailedPrecondition, "classify backup key: %v", err) + } + return scoped && (len(selected) == 0 || selected[scope]), nil +} + func backupScanStreamError(err error) error { if contextErr := backupContextStreamError(err); contextErr != nil { return contextErr diff --git a/adapter/admin_backup_test.go b/adapter/admin_backup_test.go index bff60375c..c530c305b 100644 --- a/adapter/admin_backup_test.go +++ b/adapter/admin_backup_test.go @@ -227,6 +227,7 @@ type backupTestStream struct { ctx context.Context got []*pb.BackupKV sendErr error + onSend func() } func (s *backupTestStream) Context() context.Context { return s.ctx } @@ -236,6 +237,9 @@ func (s *backupTestStream) Send(pair *pb.BackupKV) error { return s.sendErr } s.got = append(s.got, pair) + if s.onSend != nil { + s.onSend() + } return nil } @@ -488,6 +492,40 @@ func TestStreamBackupPreservesContextStatusAndReportsCloseErrors(t *testing.T) { require.Equal(t, codes.Internal, status.Code(err)) } +func TestStreamBackupStopsWhenTokenExpiresMidStream(t *testing.T) { + t.Parallel() + const ttl = 30 * time.Millisecond + nowMS := atomic.Int64{} + base := time.Unix(1_000_000, 0) + nowMS.Store(base.UnixMilli()) + store := &backupTestStore{keys: [][]byte{ + []byte(logicalbackup.RedisStringPrefix + "a"), + []byte(logicalbackup.RedisStringPrefix + "b"), + }} + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + proposer := newBackupTestProposer() + srv := newBackupControlTestServer(t, store, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil, + WithAdminBackupConfig(AdminBackupConfig{DefaultTTL: ttl, MinTTL: time.Millisecond, MaxTTL: time.Second}), + ) + srv.SetClock(func() time.Time { return time.UnixMilli(nowMS.Load()) }) + begin, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.NoError(t, err) + + var expireOnce sync.Once + stream := &backupTestStream{ + ctx: context.Background(), + onSend: func() { + expireOnce.Do(func() { + nowMS.Add(ttl.Milliseconds()) + }) + }, + } + err = srv.StreamBackup(&pb.StreamBackupRequest{PinToken: begin.GetPinToken()}, stream) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Contains(t, err.Error(), "expired") + require.Len(t, stream.got, 1) +} + func TestStreamBackupFailsClosedWithoutPinnedRouteSnapshot(t *testing.T) { t.Parallel() store := &backupTestStore{keys: [][]byte{[]byte(logicalbackup.RedisStringPrefix + "key")}} diff --git a/kv/active_timestamp_tracker.go b/kv/active_timestamp_tracker.go index 73ab5d978..e41c59559 100644 --- a/kv/active_timestamp_tracker.go +++ b/kv/active_timestamp_tracker.go @@ -144,6 +144,18 @@ func (t *ActiveTimestampTracker) OldestForGroup(groupID uint64) uint64 { return t.oldestForGroup(groupID, true) } +// OldestBackupForGroup returns the oldest live backup pin for groupID, +// excluding ordinary read pins. FSM snapshot generation uses this to avoid +// emitting a snapshot that would drop active backup retention state. +func (t *ActiveTimestampTracker) OldestBackupForGroup(groupID uint64) uint64 { + if t == nil { + return 0 + } + t.mu.Lock() + defer t.mu.Unlock() + return oldestBackupTimestamp(t.backupPins, groupID, true, time.Now()) +} + func (t *ActiveTimestampTracker) oldestForGroup(groupID uint64, scoped bool) uint64 { if t == nil { return 0 diff --git a/kv/active_timestamp_tracker_test.go b/kv/active_timestamp_tracker_test.go index ee35c42f7..df0808f20 100644 --- a/kv/active_timestamp_tracker_test.go +++ b/kv/active_timestamp_tracker_test.go @@ -50,6 +50,19 @@ func TestActiveTimestampTrackerOldestIncludesBackupPins(t *testing.T) { require.Equal(t, uint64(30), tracker.Oldest()) } +func TestActiveTimestampTrackerOldestBackupForGroupIgnoresReadsAndOtherGroups(t *testing.T) { + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + read := tracker.Pin(5) + defer read.Release() + deadline := time.Now().Add(time.Hour) + require.NoError(t, tracker.PinWithDeadlineForGroup(backupTrackerTestPinID(1), 1, 30, deadline)) + require.NoError(t, tracker.PinWithDeadlineForGroup(backupTrackerTestPinID(2), 2, 20, deadline)) + + require.Equal(t, uint64(30), tracker.OldestBackupForGroup(1)) + require.Equal(t, uint64(20), tracker.OldestBackupForGroup(2)) + require.Equal(t, uint64(0), tracker.OldestBackupForGroup(3)) +} + func TestActiveTimestampTrackerBackupPinExpiry(t *testing.T) { tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) now := time.UnixMilli(3000) diff --git a/kv/fsm.go b/kv/fsm.go index 7d72499c3..765fbe1cb 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -568,6 +568,9 @@ func (f *kvFSM) handleDelPrefix(ctx context.Context, prefix []byte, commitTS uin var ErrNotImplemented = errors.New("not implemented") func (f *kvFSM) Snapshot() (raftengine.Snapshot, error) { + if err := f.rejectSnapshotWithActiveBackupPin(); err != nil { + return nil, err + } snapshot, err := f.store.Snapshot() if err != nil { return nil, errors.WithStack(err) diff --git a/kv/fsm_backup.go b/kv/fsm_backup.go index 0cf249821..9d0e12917 100644 --- a/kv/fsm_backup.go +++ b/kv/fsm_backup.go @@ -11,6 +11,7 @@ import ( var ErrBackupApply = errors.New("backup fsm apply failed") var ErrBackupTimestampFenced = errors.New("backup timestamp fence rejects stale write") +var ErrBackupSnapshotBlocked = errors.New("active backup pin blocks fsm snapshot") var backupTimestampFloorKey = []byte(TxnKeyPrefix + "backup|timestamp_floor") @@ -114,6 +115,16 @@ func (f *kvFSM) verifyBackupTimestampFloor(r *pb.Request, commitTS uint64) error return errors.Wrapf(ErrBackupTimestampFenced, "commit_ts %d is not above backup read_ts %d", commitTS, floor) } +func (f *kvFSM) rejectSnapshotWithActiveBackupPin() error { + if f == nil || f.readTracker == nil { + return nil + } + if readTS := f.readTracker.OldestBackupForGroup(f.shardGroupID); readTS != 0 { + return errors.Wrapf(ErrBackupSnapshotBlocked, "raft group %d has active backup read_ts %d", f.shardGroupID, readTS) + } + return nil +} + func (f *kvFSM) observeBackupReadTimestamp(readTS uint64, applyErr error) error { if applyErr == nil && f.hlc != nil { // The pin entry is also the per-group timestamp barrier. Once it diff --git a/kv/fsm_backup_test.go b/kv/fsm_backup_test.go index 71e7233a2..116974e0a 100644 --- a/kv/fsm_backup_test.go +++ b/kv/fsm_backup_test.go @@ -253,6 +253,23 @@ func TestBackupTimestampFloorSurvivesSnapshotRestore(t *testing.T) { }}})) } +func TestFSMSnapshotRejectsActiveBackupPin(t *testing.T) { + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + fsm := newBackupTestFSMWithGroup(t, tracker, 7) + pinID := backupTrackerTestPinID(1) + require.NoError(t, haltApplyOf(fsm.Apply(EncodeBackupPinEntry(BackupPinEntry{ + PinID: pinID, ReadTS: 75, Deadline: time.Now().Add(time.Hour), + })))) + + _, err := fsm.Snapshot() + require.ErrorIs(t, err, ErrBackupSnapshotBlocked) + + require.NoError(t, haltApplyOf(fsm.Apply(EncodeBackupReleaseEntry(BackupReleaseEntry{PinID: pinID})))) + snapshot, err := fsm.Snapshot() + require.NoError(t, err) + require.NoError(t, snapshot.Close()) +} + func TestBackupTimestampFloorRejectsDelayedRaftProposal(t *testing.T) { st := store.NewMVCCStore() tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) From 6608a1cd8f4e25c10b869c6d131ac6d17bf020ce Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 18:32:46 +0900 Subject: [PATCH 7/9] backup: harden renewal deadline publication --- adapter/admin_backup.go | 55 +++++++++--- adapter/admin_backup_test.go | 112 ++++++++++++++++++++++-- adapter/internal.go | 5 ++ adapter/internal_admin_proposal_test.go | 13 +++ kv/leader_admin_proposer.go | 7 +- 5 files changed, 176 insertions(+), 16 deletions(-) diff --git a/adapter/admin_backup.go b/adapter/admin_backup.go index da8c9919d..8a3dbea50 100644 --- a/adapter/admin_backup.go +++ b/adapter/admin_backup.go @@ -324,7 +324,7 @@ func (s *AdminServer) pinBackupGroups( ) (map[uint64]uint64, error) { reserveEntry := kv.EncodeBackupReserveEntry(kv.BackupReserveEntry{PinID: pinID, ReadTS: readTS, Deadline: deadline}) if _, _, err := proposeBackupAll(ctx, []backupGroup{controlGroup}, reserveEntry); err != nil { - if errors.Is(err, kv.ErrTooManyActiveBackups) { + if backupCapacityReservationFull(err) { return nil, status.Errorf(codes.ResourceExhausted, "%s", kv.ErrTooManyActiveBackups) } return nil, status.Errorf(codes.Unavailable, "reserve backup capacity: %v", err) @@ -533,7 +533,7 @@ func (s *AdminServer) StreamBackup( return status.Errorf(codes.Unavailable, "%s", "backup scanner is nil") } scanErr := streamBackupRecords(stream, scanner, selected, func() error { - return s.requireUnexpiredBackupToken(tok) + return s.requireLiveBackupSession(tok) }) if err := finishBackupScan(stream.Context(), scanner, scanErr); err != nil { if scanErr != nil { @@ -847,7 +847,7 @@ func proposeBackupAll(ctx context.Context, groups []backupGroup, entry []byte) ( result := <-results if result.err != nil { if firstErr == nil { - firstErr = errors.Wrapf(result.err, "raft group %d", result.group.id) + firstErr = backupProposalGroupError(result.group.id, result.err) } continue } @@ -870,6 +870,17 @@ func backupProposalResponseError(result *raftengine.ProposalResult) error { return errors.Wrapf(ErrBackupUnavailable, "unexpected backup apply response %T", result.Response) } +func backupProposalGroupError(groupID uint64, err error) error { + if status.Code(err) == codes.ResourceExhausted { + return status.Errorf(codes.ResourceExhausted, "raft group %d: %v", groupID, err) + } + return errors.Wrapf(err, "raft group %d", groupID) +} + +func backupCapacityReservationFull(err error) bool { + return errors.Is(err, kv.ErrTooManyActiveBackups) || status.Code(err) == codes.ResourceExhausted +} + func (s *AdminServer) compensateBackupRelease(control backupGroup, groups []backupGroup, pinID kv.BackupPinID) { ctx, cancel := context.WithTimeout(context.Background(), s.backupConfig.beginDeadline) defer cancel() @@ -881,23 +892,31 @@ func (s *AdminServer) compensateBackupRelease(control backupGroup, groups []back _, _, _ = proposeBackupAll(ctx, []backupGroup{control}, unreserve) } -func (s *AdminServer) proposeBackupWithRetry(ctx context.Context, groups []backupGroup, entry []byte) error { +func (s *AdminServer) proposeBackupWithRetry( + ctx context.Context, + groups []backupGroup, + entry []byte, +) (map[uint64]uint64, error) { pending := append([]backupGroup(nil), groups...) + commits := make(map[uint64]uint64, len(groups)) var firstErr error for attempt := 0; attempt < s.backupConfig.renewAttempts && len(pending) > 0; attempt++ { - _, committed, err := proposeBackupAll(ctx, pending, entry) + attemptCommits, committed, err := proposeBackupAll(ctx, pending, entry) + for groupID, index := range attemptCommits { + commits[groupID] = index + } if err == nil { - return nil + return commits, nil } firstErr = err pending = remainingBackupGroups(pending, committed) if len(pending) > 0 && attempt+1 < s.backupConfig.renewAttempts { if err := waitBackupRetry(ctx, s.backupConfig.renewBackoff); err != nil { - return err + return commits, err } } } - return firstErr + return commits, firstErr } func remainingBackupGroups(pending, committed []backupGroup) []backupGroup { @@ -937,16 +956,20 @@ func (s *AdminServer) renewBackupGroups( } deadline := s.nowSnapshot().Add(ttl) reserveEntry := kv.EncodeBackupReserveEntry(kv.BackupReserveEntry{PinID: pinID, ReadTS: readTS, Deadline: deadline}) - if err := s.proposeBackupWithRetry(ctx, groups[:1], reserveEntry); err != nil { + if _, err := s.proposeBackupWithRetry(ctx, groups[:1], reserveEntry); err != nil { return time.Time{}, errors.Wrap(err, "capacity reservation") } // Reapply the complete pin rather than only its deadline. This restores a // missing replica-local fence after partial delivery while duplicate Pin // apply preserves the earliest read timestamp and latest deadline. pinEntry := kv.EncodeBackupPinEntry(kv.BackupPinEntry{PinID: pinID, ReadTS: readTS, Deadline: deadline}) - if err := s.proposeBackupWithRetry(ctx, groups, pinEntry); err != nil { + commits, err := s.proposeBackupWithRetry(ctx, groups, pinEntry) + if err != nil { return time.Time{}, errors.Wrap(err, "group pins") } + if err := waitBackupGroupsApplied(ctx, groups, commits); err != nil { + return time.Time{}, errors.Wrap(err, "wait for renewed group pins") + } return deadline, nil } @@ -1039,6 +1062,18 @@ func (s *AdminServer) backupRouteSnapshotForToken(tok backupToken) (kv.BackupRou return session.routes, nil } +func (s *AdminServer) requireLiveBackupSession(tok backupToken) error { + now := s.nowSnapshot() + s.backupStateMu.Lock() + defer s.backupStateMu.Unlock() + s.reapBackupSessionsLocked(now) + session, ok := s.backupSessions[tok.pinID] + if !ok || session.readTS != tok.readTS { + return status.Errorf(codes.FailedPrecondition, "%s", "backup pin token has expired") + } + return nil +} + func (s *AdminServer) extendBackupSession(tok backupToken) { s.backupStateMu.Lock() defer s.backupStateMu.Unlock() diff --git a/adapter/admin_backup_test.go b/adapter/admin_backup_test.go index c530c305b..145e58207 100644 --- a/adapter/admin_backup_test.go +++ b/adapter/admin_backup_test.go @@ -67,7 +67,7 @@ type backupTestProposer struct { failures map[byte]int transportError map[byte]error responseError map[byte]error - onPropose func(byte) + onPropose func(byte, uint64) } func newBackupTestProposer() *backupTestProposer { @@ -95,7 +95,7 @@ func (p *backupTestProposer) ProposeAdmin(ctx context.Context, data []byte) (*ra return nil, p.transportError[subtype] } if p.onPropose != nil { - p.onPropose(subtype) + p.onPropose(subtype, p.commit) } return &raftengine.ProposalResult{CommitIndex: p.commit, Response: p.responseError[subtype]}, nil } @@ -375,13 +375,13 @@ func TestBeginBackupWaitsForLocalPinApplyBeforeCatalogCapture(t *testing.T) { store := &backupTestStore{} group := &backupTestGroup{status: raftengine.Status{}, every: 10_000} proposer := newBackupTestProposer() - proposer.onPropose = func(subtype byte) { + proposer.onPropose = func(subtype byte, commit uint64) { if subtype != backupSubtypePin { return } go func() { time.Sleep(25 * time.Millisecond) - group.setApplied(2) + group.setApplied(commit) }() } store.onCapture = func() { @@ -526,6 +526,42 @@ func TestStreamBackupStopsWhenTokenExpiresMidStream(t *testing.T) { require.Len(t, stream.got, 1) } +func TestStreamBackupHonorsRenewedSessionDeadline(t *testing.T) { + t.Parallel() + const ttl = 30 * time.Millisecond + nowMS := atomic.Int64{} + base := time.Unix(1_000_000, 0) + nowMS.Store(base.UnixMilli()) + store := &backupTestStore{keys: [][]byte{ + []byte(logicalbackup.RedisStringPrefix + "a"), + []byte(logicalbackup.RedisStringPrefix + "b"), + }} + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + proposer := newBackupTestProposer() + srv := newBackupControlTestServer(t, store, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil, + WithAdminBackupConfig(AdminBackupConfig{DefaultTTL: ttl, MinTTL: time.Millisecond, MaxTTL: time.Second}), + ) + srv.SetClock(func() time.Time { return time.UnixMilli(nowMS.Load()) }) + begin, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.NoError(t, err) + + var renewOnce sync.Once + stream := &backupTestStream{ + ctx: context.Background(), + onSend: func() { + renewOnce.Do(func() { + nowMS.Store(base.Add(10 * time.Millisecond).UnixMilli()) + _, err := srv.RenewBackup(context.Background(), &pb.RenewBackupRequest{PinToken: begin.GetPinToken()}) + require.NoError(t, err) + nowMS.Store(base.Add(ttl + time.Millisecond).UnixMilli()) + }) + }, + } + err = srv.StreamBackup(&pb.StreamBackupRequest{PinToken: begin.GetPinToken()}, stream) + require.NoError(t, err) + require.Len(t, stream.got, 2) +} + func TestStreamBackupFailsClosedWithoutPinnedRouteSnapshot(t *testing.T) { t.Parallel() store := &backupTestStore{keys: [][]byte{[]byte(logicalbackup.RedisStringPrefix + "key")}} @@ -688,6 +724,60 @@ func TestRenewBackupRetriesAndRejectsTamperedToken(t *testing.T) { require.Equal(t, codes.InvalidArgument, status.Code(err)) } +func TestRenewBackupWaitsForLocalPinApplyBeforePublishing(t *testing.T) { + t.Parallel() + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + proposer := newBackupTestProposer() + srv := newBackupControlTestServer( + t, + &backupTestStore{}, + map[uint64]*backupTestGroup{1: group}, + map[uint64]*backupTestProposer{1: proposer}, + nil, + WithAdminBackupConfig(AdminBackupConfig{BeginDeadline: time.Second}), + ) + begin, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.NoError(t, err) + group.setApplied(0) + + pinCommit := make(chan uint64, 1) + releaseApply := make(chan struct{}) + proposer.mu.Lock() + proposer.onPropose = func(subtype byte, commit uint64) { + if subtype != backupSubtypePin { + return + } + pinCommit <- commit + go func() { + <-releaseApply + group.setApplied(commit) + }() + } + proposer.mu.Unlock() + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := srv.RenewBackup(ctx, &pb.RenewBackupRequest{PinToken: begin.GetPinToken()}) + done <- err + }() + var commit uint64 + select { + case commit = <-pinCommit: + case <-ctx.Done(): + require.FailNow(t, "RenewBackup did not propose renewed pin") + } + select { + case err := <-done: + require.Failf(t, "RenewBackup returned before local apply", "err=%v", err) + default: + } + close(releaseApply) + require.NoError(t, <-done) + require.GreaterOrEqual(t, group.Status().AppliedIndex, commit) +} + func TestBackupTokenDeadlineRotatesAndFailsClosed(t *testing.T) { t.Parallel() const ttl = 30 * time.Millisecond @@ -744,7 +834,7 @@ func TestRenewBackupReleasesResourcesWhenTokenExpiresDuringRenewal(t *testing.T) decoded, err := srv.decodeBackupToken(begin.GetPinToken()) require.NoError(t, err) proposer.mu.Lock() - proposer.onPropose = func(subtype byte) { + proposer.onPropose = func(subtype byte, _ uint64) { if subtype == backupSubtypePin { nowMS.Store(base.Add(ttl).UnixMilli()) } @@ -802,3 +892,15 @@ func TestBeginBackupMapsCapacityReservationToResourceExhausted(t *testing.T) { _, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) require.Equal(t, codes.ResourceExhausted, status.Code(err)) } + +func TestBeginBackupMapsForwardedCapacityReservationStatusToResourceExhausted(t *testing.T) { + t.Parallel() + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + proposer := newBackupTestProposer() + proposer.failures[backupSubtypeReserve] = 1 + proposer.transportError[backupSubtypeReserve] = status.Errorf(codes.ResourceExhausted, "%s", kv.ErrTooManyActiveBackups) + srv := newBackupControlTestServer(t, &backupTestStore{}, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil) + + _, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) +} diff --git a/adapter/internal.go b/adapter/internal.go index 1d479af50..2f38bf1eb 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -8,6 +8,8 @@ import ( "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/cockroachdb/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) type InternalOption func(*Internal) @@ -141,6 +143,9 @@ func forwardedAdminProposalResponseError(result *raftengine.ProposalResult) erro return nil } if err, ok := result.Response.(error); ok { + if errors.Is(err, kv.ErrTooManyActiveBackups) { + return status.Errorf(codes.ResourceExhausted, "%s", kv.ErrTooManyActiveBackups) + } return errors.WithStack(err) } return errors.Errorf("unexpected admin proposal response %T", result.Response) diff --git a/adapter/internal_admin_proposal_test.go b/adapter/internal_admin_proposal_test.go index 24439777e..1294d2d57 100644 --- a/adapter/internal_admin_proposal_test.go +++ b/adapter/internal_admin_proposal_test.go @@ -6,8 +6,11 @@ import ( "testing" "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) type internalAdminLeaderView struct { @@ -87,6 +90,7 @@ func TestInternalForwardAdminProposalFailsClosed(t *testing.T) { state raftengine.State proposer raftengine.Proposer errTarget error + errCode codes.Code errText string }{ {name: "follower", state: raftengine.StateFollower, errTarget: ErrNotLeader}, @@ -95,6 +99,12 @@ func TestInternalForwardAdminProposalFailsClosed(t *testing.T) { proposer: &internalAdminProposer{response: stderrors.New("apply failed")}, errText: "apply failed", }, + { + name: "capacity response", state: raftengine.StateLeader, + proposer: &internalAdminProposer{response: kv.ErrTooManyActiveBackups}, + errCode: codes.ResourceExhausted, + errText: kv.ErrTooManyActiveBackups.Error(), + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { @@ -109,6 +119,9 @@ func TestInternalForwardAdminProposalFailsClosed(t *testing.T) { if tc.errTarget != nil { require.ErrorIs(t, err, tc.errTarget) } + if tc.errCode != codes.OK { + require.Equal(t, tc.errCode, status.Code(err)) + } if tc.errText != "" { require.ErrorContains(t, err, tc.errText) } diff --git a/kv/leader_admin_proposer.go b/kv/leader_admin_proposer.go index c22d05314..662608d5c 100644 --- a/kv/leader_admin_proposer.go +++ b/kv/leader_admin_proposer.go @@ -7,7 +7,9 @@ import ( "github.com/bootjp/elastickv/internal/raftengine" pb "github.com/bootjp/elastickv/proto" "github.com/cockroachdb/errors" + "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" ) type LeaderAdminProposerOption func(*leaderAdminProposer) @@ -147,7 +149,10 @@ func (p *leaderAdminProposer) forwardAdmin( } resp, err := pb.NewInternalClient(conn).ForwardAdminProposal(ctx, &pb.ForwardAdminProposalRequest{Payload: data}) if err != nil { - return nil, errors.WithStack(err) + if code := status.Code(err); code != codes.Unknown { + return nil, status.Errorf(code, "forward admin proposal: %v", err) + } + return nil, errors.Wrap(err, "forward admin proposal") } return &raftengine.ProposalResult{CommitIndex: resp.GetCommitIndex()}, nil } From b3b8ab9f7fb2d4c161d4226bcafe75e45c977b1d Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 19:13:59 +0900 Subject: [PATCH 8/9] backup: gate renewals on live sessions --- adapter/admin_backup.go | 51 +++++++++++++++++++++++++--- adapter/admin_backup_test.go | 59 ++++++++++++++++++++++++++++++-- kv/backup_scan.go | 65 +++++++++++++++++++++++++++++++----- 3 files changed, 159 insertions(+), 16 deletions(-) diff --git a/adapter/admin_backup.go b/adapter/admin_backup.go index 8a3dbea50..8414cf397 100644 --- a/adapter/admin_backup.go +++ b/adapter/admin_backup.go @@ -62,6 +62,15 @@ type BackupStore interface { NewBackupScannerAtSnapshot(snapshot kv.BackupRouteSnapshot, ts uint64, pageSize int) kv.BackupScanner } +type filteredBackupScanStore interface { + NewFilteredBackupScannerAtSnapshot( + snapshot kv.BackupRouteSnapshot, + ts uint64, + pageSize int, + keyFilter kv.BackupKeyFilter, + ) kv.BackupScanner +} + type BackupPinLimiter interface { PinWithDeadline(pinID kv.BackupPinID, readTS uint64, deadline time.Time) error ReleaseBackupPin(pinID kv.BackupPinID) @@ -431,12 +440,24 @@ func (s *AdminServer) RenewBackup(ctx context.Context, req *pb.RenewBackupReques s.forgetBackupSession(tok.pinID) return nil, err } + return s.finishRenewBackup(groups, tok, ttl, deadline) +} + +func (s *AdminServer) finishRenewBackup( + groups []backupGroup, + tok backupToken, + ttl time.Duration, + deadline time.Time, +) (*pb.RenewBackupResponse, error) { tok.deadline = deadline + if !s.extendBackupSession(tok) { + s.compensateBackupRelease(groups[0], groups, tok.pinID) + return nil, status.Errorf(codes.FailedPrecondition, "%s", "backup pin token has expired") + } encodedToken, err := s.encodeBackupToken(tok) if err != nil { return nil, status.Errorf(codes.Internal, "encode renewed backup token: %v", err) } - s.extendBackupSession(tok) return &pb.RenewBackupResponse{ TtlMsEffective: uint64(ttl / time.Millisecond), //nolint:gosec // validated positive. PinToken: encodedToken, @@ -528,7 +549,7 @@ func (s *AdminServer) StreamBackup( if err != nil { return err } - scanner := s.backupStore.NewBackupScannerAtSnapshot(routes, tok.readTS, s.backupConfig.scanPageSize) + scanner := s.newBackupStreamScanner(routes, tok.readTS, selected) if scanner == nil { return status.Errorf(codes.Unavailable, "%s", "backup scanner is nil") } @@ -544,6 +565,21 @@ func (s *AdminServer) StreamBackup( return nil } +func (s *AdminServer) newBackupStreamScanner( + routes kv.BackupRouteSnapshot, + readTS uint64, + selected map[logicalbackup.Scope]bool, +) kv.BackupScanner { + if len(selected) > 0 { + if filtered, ok := s.backupStore.(filteredBackupScanStore); ok { + return filtered.NewFilteredBackupScannerAtSnapshot(routes, readTS, s.backupConfig.scanPageSize, func(key []byte) (bool, error) { + return backupKeySelected(key, selected) + }) + } + } + return s.backupStore.NewBackupScannerAtSnapshot(routes, readTS, s.backupConfig.scanPageSize) +} + func streamBackupRecords( stream grpc.ServerStreamingServer[pb.BackupKV], scanner kv.BackupScanner, @@ -581,7 +617,11 @@ func backupRecordSelected(pair *store.KVPair, selected map[logicalbackup.Scope]b if pair == nil { return false, status.Errorf(codes.Internal, "%s", "backup scanner returned a nil record") } - scope, scoped, err := logicalbackup.ScopeForKey(pair.Key) + return backupKeySelected(pair.Key, selected) +} + +func backupKeySelected(key []byte, selected map[logicalbackup.Scope]bool) (bool, error) { + scope, scoped, err := logicalbackup.ScopeForKey(key) if err != nil { return false, status.Errorf(codes.FailedPrecondition, "classify backup key: %v", err) } @@ -1074,17 +1114,18 @@ func (s *AdminServer) requireLiveBackupSession(tok backupToken) error { return nil } -func (s *AdminServer) extendBackupSession(tok backupToken) { +func (s *AdminServer) extendBackupSession(tok backupToken) bool { s.backupStateMu.Lock() defer s.backupStateMu.Unlock() session, ok := s.backupSessions[tok.pinID] if !ok || session.readTS != tok.readTS { - return + return false } if tok.deadline.After(session.deadline) { session.deadline = tok.deadline s.backupSessions[tok.pinID] = session } + return true } func (s *AdminServer) forgetBackupSession(pinID kv.BackupPinID) { diff --git a/adapter/admin_backup_test.go b/adapter/admin_backup_test.go index 145e58207..d07715807 100644 --- a/adapter/admin_backup_test.go +++ b/adapter/admin_backup_test.go @@ -122,6 +122,7 @@ type backupTestStore struct { pairCloseErr error captureErr error validateErr error + valueKeys [][]byte } func (s *backupTestStore) ValidateBackupSnapshotAt(context.Context, kv.BackupRouteSnapshot, uint64, int) error { @@ -154,15 +155,40 @@ func (s *backupTestStore) NewBackupKeyScannerAtSnapshot(_ kv.BackupRouteSnapshot } func (s *backupTestStore) NewBackupScannerAtSnapshot(_ kv.BackupRouteSnapshot, ts uint64, _ int) kv.BackupScanner { + return s.newBackupScannerAtSnapshot(ts, nil) +} + +func (s *backupTestStore) NewFilteredBackupScannerAtSnapshot( + _ kv.BackupRouteSnapshot, + ts uint64, + _ int, + keyFilter kv.BackupKeyFilter, +) kv.BackupScanner { + return s.newBackupScannerAtSnapshot(ts, keyFilter) +} + +func (s *backupTestStore) newBackupScannerAtSnapshot(ts uint64, keyFilter kv.BackupKeyFilter) kv.BackupScanner { s.mu.Lock() + defer s.mu.Unlock() s.readTS = append(s.readTS, ts) pairs := make([]*kvstore.KVPair, 0, len(s.keys)) + var filterErr error for _, key := range s.keys { + if keyFilter != nil { + selected, err := keyFilter(key) + if err != nil { + filterErr = err + break + } + if !selected { + continue + } + } + s.valueKeys = append(s.valueKeys, append([]byte(nil), key...)) pairs = append(pairs, &kvstore.KVPair{Key: append([]byte(nil), key...), Value: []byte("value")}) } closeErr := s.pairCloseErr - s.mu.Unlock() - return &backupPairScanner{pairs: pairs, closeErr: closeErr} + return &backupPairScanner{pairs: pairs, err: filterErr, closeErr: closeErr} } type backupSliceScanner struct { @@ -205,6 +231,7 @@ func (s *backupSliceScanner) Close() error { return s.closeErr } type backupPairScanner struct { pairs []*kvstore.KVPair index int + err error closeErr error } @@ -212,6 +239,11 @@ func (s *backupPairScanner) Next(ctx context.Context) (*kvstore.KVPair, bool, er if err := ctx.Err(); err != nil { return nil, false, err } + if s.err != nil { + err := s.err + s.err = nil + return nil, false, err + } if s.index >= len(s.pairs) { return nil, false, nil } @@ -464,6 +496,7 @@ func TestStreamBackupUsesPinTimestampAndScopeFilter(t *testing.T) { require.Len(t, stream.got, 1) require.Equal(t, redisKey, stream.got[0].GetKey()) require.Equal(t, uint64(42), store.readTS[len(store.readTS)-1]) + require.Equal(t, [][]byte{redisKey}, store.valueKeys) } func TestStreamBackupPreservesContextStatusAndReportsCloseErrors(t *testing.T) { @@ -562,6 +595,28 @@ func TestStreamBackupHonorsRenewedSessionDeadline(t *testing.T) { require.Len(t, stream.got, 2) } +func TestRenewBackupFailsWhenSessionEndsDuringRenew(t *testing.T) { + t.Parallel() + store := &backupTestStore{keys: [][]byte{[]byte(logicalbackup.RedisStringPrefix + "key")}} + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + proposer := newBackupTestProposer() + srv := newBackupControlTestServer(t, store, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil) + begin, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.NoError(t, err) + tok, err := srv.decodeBackupToken(begin.GetPinToken()) + require.NoError(t, err) + + proposer.onPropose = func(subtype byte, _ uint64) { + if subtype == backupSubtypePin { + srv.forgetBackupSession(tok.pinID) + } + } + _, err = srv.RenewBackup(context.Background(), &pb.RenewBackupRequest{PinToken: begin.GetPinToken()}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Contains(t, proposer.subtypes(), backupSubtypeRelease) + require.Contains(t, proposer.subtypes(), backupSubtypeUnreserve) +} + func TestStreamBackupFailsClosedWithoutPinnedRouteSnapshot(t *testing.T) { t.Parallel() store := &backupTestStore{keys: [][]byte{[]byte(logicalbackup.RedisStringPrefix + "key")}} diff --git a/kv/backup_scan.go b/kv/backup_scan.go index 1a81bbb50..4e1043047 100644 --- a/kv/backup_scan.go +++ b/kv/backup_scan.go @@ -19,6 +19,10 @@ type BackupScanner interface { Close() error } +// BackupKeyFilter decides whether a key should be materialized by a value +// scanner. It runs after route ownership filtering but before reading values. +type BackupKeyFilter func(key []byte) (bool, error) + // BackupKeyScanner is the count-only counterpart to BackupScanner. It pages // through the same captured route set without materializing values. type BackupKeyScanner interface { @@ -47,6 +51,7 @@ type backupScanner struct { cursor []byte page []*store.KVPair index int + keyFilter BackupKeyFilter closed bool exhausted bool } @@ -169,6 +174,18 @@ func (s *ShardStore) validateBackupGroupLocksAt(ctx context.Context, group *Shar // NewBackupScannerAtSnapshot creates a value scanner from a captured route view. func NewBackupScannerAtSnapshot(st *ShardStore, snapshot BackupRouteSnapshot, ts uint64, pageSize int) BackupScanner { + return NewFilteredBackupScannerAtSnapshot(st, snapshot, ts, pageSize, nil) +} + +// NewFilteredBackupScannerAtSnapshot creates a value scanner that skips +// filtered-out keys before materializing values. +func NewFilteredBackupScannerAtSnapshot( + st *ShardStore, + snapshot BackupRouteSnapshot, + ts uint64, + pageSize int, + keyFilter BackupKeyFilter, +) BackupScanner { if pageSize <= 0 { pageSize = defaultBackupScanPageSize } @@ -181,6 +198,7 @@ func NewBackupScannerAtSnapshot(st *ShardStore, snapshot BackupRouteSnapshot, ts end: snapshot.end, ts: ts, pageSize: pageSize, + keyFilter: keyFilter, } } @@ -192,6 +210,15 @@ func (s *ShardStore) NewBackupScannerAtSnapshot(snapshot BackupRouteSnapshot, ts return NewBackupScannerAtSnapshot(s, snapshot, ts, pageSize) } +func (s *ShardStore) NewFilteredBackupScannerAtSnapshot( + snapshot BackupRouteSnapshot, + ts uint64, + pageSize int, + keyFilter BackupKeyFilter, +) BackupScanner { + return NewFilteredBackupScannerAtSnapshot(s, snapshot, ts, pageSize, keyFilter) +} + func NewBackupKeyScanner(st *ShardStore, start []byte, end []byte, ts uint64, pageSize int) BackupKeyScanner { snapshot := st.CaptureBackupRouteSnapshot(start, end) return NewBackupKeyScannerAtSnapshot(st, snapshot, ts, pageSize) @@ -323,21 +350,14 @@ func (s *backupScanner) loadNextPage(ctx context.Context) error { } s.page = s.page[:0] for _, item := range keys { - route, ok, err := s.materializeRouteForKey(item) + kvp, ok, err := s.materializeBackupKey(ctx, item) if err != nil { return err } if !ok { continue } - val, err := s.store.getRouteAt(ctx, route, item.key, s.ts) - if errors.Is(err, store.ErrKeyNotFound) { - continue - } - if err != nil { - return err - } - s.page = append(s.page, &store.KVPair{Key: bytes.Clone(item.key), Value: bytes.Clone(val)}) + s.page = append(s.page, kvp) } s.index = 0 if len(keys) == 0 { @@ -353,6 +373,33 @@ func (s *backupScanner) loadNextPage(ctx context.Context) error { return nil } +func (s *backupScanner) materializeBackupKey(ctx context.Context, item routedScanKey) (*store.KVPair, bool, error) { + if s.keyFilter != nil { + selected, err := s.keyFilter(item.key) + if err != nil { + return nil, false, err + } + if !selected { + return nil, false, nil + } + } + route, ok, err := s.materializeRouteForKey(item) + if err != nil { + return nil, false, err + } + if !ok { + return nil, false, nil + } + val, err := s.store.getRouteAt(ctx, route, item.key, s.ts) + if errors.Is(err, store.ErrKeyNotFound) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + return &store.KVPair{Key: bytes.Clone(item.key), Value: bytes.Clone(val)}, true, nil +} + func (s *backupScanner) materializeRouteForKey(item routedScanKey) (distribution.Route, bool, error) { return s.store.routeForRoutedKey(item, s.routes) } From d91744b04be1ce78e16f49a0e488d31f0b340cbc Mon Sep 17 00:00:00 2001 From: bootjp Date: Thu, 23 Jul 2026 23:35:20 +0900 Subject: [PATCH 9/9] backup: harden live scan edge cases --- adapter/admin_backup.go | 60 +++++-- adapter/admin_backup_test.go | 21 +++ kv/backup_scan.go | 317 ++++++++++++++++++++++++++++++----- kv/fsm_backup.go | 3 +- kv/fsm_backup_test.go | 20 +++ kv/shard_store_test.go | 110 ++++++++++++ 6 files changed, 472 insertions(+), 59 deletions(-) diff --git a/adapter/admin_backup.go b/adapter/admin_backup.go index 8414cf397..f88db35f4 100644 --- a/adapter/admin_backup.go +++ b/adapter/admin_backup.go @@ -320,7 +320,7 @@ func (s *AdminServer) captureBackupRoutesAfterPin( s.compensateBackupRelease(controlGroup, groups, pinID) return kv.BackupRouteSnapshot{}, status.Errorf(codes.FailedPrecondition, "capture backup routes at read timestamp: %v", err) } - return routes, nil + return kv.BackupRouteSnapshotWithScanGroups(routes, backupGroupIDs(groups)), nil } func (s *AdminServer) pinBackupGroups( @@ -368,7 +368,7 @@ func (s *AdminServer) buildExpectedBackupBaseline( if validateErr != nil { scanErr = errors.Wrap(validateErr, "validate backup transaction locks") } else { - counts, appliedAtCount, scanErr = s.scanBackupScopeCounts(ctx, prepared.routes, prepared.readTS, prepared.groups) + counts, appliedAtCount, scanErr = s.scanBackupScopeCounts(ctx, prepared.routes, prepared.readTS, prepared.groups, nil) } close(stopRenew) renewErr := <-renewDone @@ -512,7 +512,9 @@ func (s *AdminServer) ListAdaptersAndScopes( if err != nil { return nil, err } - counts, _, err := s.scanBackupScopeCounts(ctx, routes, tok.readTS, groups) + counts, _, err := s.scanBackupScopeCounts(ctx, routes, tok.readTS, groups, func() error { + return s.requireLiveBackupSession(tok) + }) if err != nil { return nil, status.Errorf(codes.FailedPrecondition, "list backup scopes: %v", err) } @@ -1047,6 +1049,7 @@ func (s *AdminServer) scanBackupScopeCounts( routes kv.BackupRouteSnapshot, readTS uint64, groups []backupGroup, + requireLive func() error, ) (counts map[logicalbackup.Scope]uint64, applied uint64, retErr error) { scanner := s.backupStore.NewBackupKeyScannerAtSnapshot(routes, readTS, s.backupConfig.scanPageSize) if scanner == nil { @@ -1055,28 +1058,57 @@ func (s *AdminServer) scanBackupScopeCounts( defer func() { retErr = finishBackupScan(ctx, scanner, retErr) }() - counts = make(map[logicalbackup.Scope]uint64) + counts, err := collectBackupScopeCounts(ctx, scanner, requireLive) + if err != nil { + return nil, 0, err + } + applied, err = currentMinBackupAppliedIndex(groups) + if err != nil { + return nil, 0, err + } + return counts, applied, nil +} + +func collectBackupScopeCounts( + ctx context.Context, + scanner kv.BackupKeyScanner, + requireLive func() error, +) (map[logicalbackup.Scope]uint64, error) { + counts := make(map[logicalbackup.Scope]uint64) for { + if err := requireBackupScopeScanLive(requireLive); err != nil { + return nil, err + } key, ok, err := scanner.Next(ctx) if err != nil { - return nil, 0, errors.Wrap(err, "scan backup key baseline") + return nil, errors.Wrap(err, "scan backup key baseline") } if !ok { break } - scope, scoped, err := logicalbackup.ScopeForKey(key) - if err != nil { - return nil, 0, errors.Wrap(err, "classify backup key baseline") - } - if scoped { - counts[scope]++ + if err := countBackupScopeKey(counts, key); err != nil { + return nil, err } } - applied, err := currentMinBackupAppliedIndex(groups) + return counts, requireBackupScopeScanLive(requireLive) +} + +func countBackupScopeKey(counts map[logicalbackup.Scope]uint64, key []byte) error { + scope, scoped, err := logicalbackup.ScopeForKey(key) if err != nil { - return nil, 0, err + return errors.Wrap(err, "classify backup key baseline") } - return counts, applied, nil + if scoped { + counts[scope]++ + } + return nil +} + +func requireBackupScopeScanLive(requireLive func() error) error { + if requireLive == nil { + return nil + } + return requireLive() } func (s *AdminServer) rememberBackupSession(tok backupToken, routes kv.BackupRouteSnapshot) { diff --git a/adapter/admin_backup_test.go b/adapter/admin_backup_test.go index d07715807..56a5c037a 100644 --- a/adapter/admin_backup_test.go +++ b/adapter/admin_backup_test.go @@ -650,6 +650,27 @@ func TestListBackupScopesReportsScannerCloseError(t *testing.T) { require.Equal(t, codes.FailedPrecondition, status.Code(err)) } +func TestListBackupScopesFailsWhenSessionEndsDuringScan(t *testing.T) { + t.Parallel() + store := &backupTestStore{keys: [][]byte{[]byte(logicalbackup.RedisStringPrefix + "key")}} + group := &backupTestGroup{status: raftengine.Status{AppliedIndex: 100}, every: 10_000} + proposer := newBackupTestProposer() + srv := newBackupControlTestServer(t, store, map[uint64]*backupTestGroup{1: group}, map[uint64]*backupTestProposer{1: proposer}, nil) + begin, err := srv.BeginBackup(context.Background(), &pb.BeginBackupRequest{}) + require.NoError(t, err) + decoded, err := srv.decodeBackupToken(begin.GetPinToken()) + require.NoError(t, err) + store.mu.Lock() + store.onExhaust = func() { + srv.forgetBackupSession(decoded.pinID) + } + store.mu.Unlock() + + _, err = srv.ListAdaptersAndScopes(context.Background(), &pb.ListAdaptersAndScopesRequest{PinToken: begin.GetPinToken()}) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Contains(t, err.Error(), "expired") +} + func TestBackupProtocolVersionRequiresCompleteControlWiring(t *testing.T) { t.Parallel() srv := NewAdminServer(NodeIdentity{NodeID: "n1"}, nil, diff --git a/kv/backup_scan.go b/kv/backup_scan.go index 4e1043047..14fdf0f35 100644 --- a/kv/backup_scan.go +++ b/kv/backup_scan.go @@ -36,6 +36,7 @@ type BackupKeyScanner interface { // when the live route catalog changes between those RPCs. type BackupRouteSnapshot struct { routes []distribution.Route + scanGroups []uint64 clampToRoutes bool start []byte end []byte @@ -44,6 +45,7 @@ type BackupRouteSnapshot struct { type backupScanner struct { store *ShardStore routes []distribution.Route + scanGroups []uint64 clampToRoutes bool end []byte ts uint64 @@ -57,13 +59,15 @@ type backupScanner struct { } type routedScanKey struct { - key []byte - route distribution.Route + key []byte + route distribution.Route + partitionOnly bool } type backupKeyScanner struct { store *ShardStore routes []distribution.Route + scanGroups []uint64 clampToRoutes bool end []byte ts uint64 @@ -88,6 +92,7 @@ func (s *ShardStore) CaptureBackupRouteSnapshot(start []byte, end []byte) Backup routes, clampToRoutes := s.routesForForwardScan(start, end) return BackupRouteSnapshot{ routes: cloneBackupRoutes(routes), + scanGroups: backupRouteGroupIDs(routes), clampToRoutes: clampToRoutes, start: bytes.Clone(start), end: bytes.Clone(end), @@ -110,7 +115,16 @@ func CaptureBackupRouteSnapshotAt(ctx context.Context, catalog *distribution.Cat if err != nil { return BackupRouteSnapshot{}, errors.Wrap(err, "materialize backup route snapshot") } - return BackupRouteSnapshot{routes: cloneBackupRoutes(routes)}, nil + return BackupRouteSnapshot{routes: cloneBackupRoutes(routes), scanGroups: backupRouteGroupIDs(routes)}, nil +} + +// BackupRouteSnapshotWithScanGroups returns a snapshot that also scans the +// supplied Raft groups for resolver-owned partitioned keys even when the durable +// byte-range catalog has no route for those groups. +func BackupRouteSnapshotWithScanGroups(snapshot BackupRouteSnapshot, groupIDs []uint64) BackupRouteSnapshot { + out := cloneBackupRouteSnapshot(snapshot) + out.scanGroups = appendUniqueBackupGroups(out.scanGroups, groupIDs) + return out } // ValidateBackupSnapshotAt resolves committed or rolled-back transaction @@ -123,18 +137,13 @@ func (s *ShardStore) ValidateBackupSnapshotAt(ctx context.Context, snapshot Back if pageSize <= 0 { pageSize = defaultBackupScanPageSize } - seenGroups := make(map[uint64]struct{}, len(snapshot.routes)) - for _, route := range snapshot.routes { - if _, seen := seenGroups[route.GroupID]; seen { - continue - } - seenGroups[route.GroupID] = struct{}{} - group, ok := s.groupForID(route.GroupID) + for _, groupID := range backupSnapshotGroupIDs(snapshot) { + group, ok := s.groupForID(groupID) if !ok || group == nil || group.Store == nil { - return errors.Wrapf(ErrLeaderNotFound, "backup lock validation group %d is unavailable", route.GroupID) + return errors.Wrapf(ErrLeaderNotFound, "backup lock validation group %d is unavailable", groupID) } if err := s.validateBackupGroupLocksAt(ctx, group, ts, pageSize); err != nil { - return errors.Wrapf(err, "validate backup locks for group %d", route.GroupID) + return errors.Wrapf(err, "validate backup locks for group %d", groupID) } } return nil @@ -151,7 +160,7 @@ func (s *ShardStore) validateBackupGroupLocksAt(ctx context.Context, group *Shar if len(locks) == 0 { return nil } - plan, err := s.planScanLockResolutions(ctx, group, nil, locks, ts) + plan, err := s.planBackupLockResolutionsAt(ctx, locks, ts) if err != nil { return err } @@ -172,6 +181,107 @@ func (s *ShardStore) validateBackupGroupLocksAt(ctx context.Context, group *Shar } } +func (s *ShardStore) planBackupLockResolutionsAt(ctx context.Context, lockKVs []*store.KVPair, ts uint64) (*scanLockPlan, error) { + plan := newScanLockPlan(len(lockKVs)) + for _, kvp := range lockKVs { + if err := s.planBackupLockResolutionAt(ctx, plan, kvp, ts); err != nil { + return nil, err + } + } + return plan, nil +} + +func (s *ShardStore) planBackupLockResolutionAt(ctx context.Context, plan *scanLockPlan, kvp *store.KVPair, ts uint64) error { + if kvp == nil { + return nil + } + userKey, ok := txnUserKeyFromLockKey(kvp.Key) + if !ok { + return nil + } + lock, err := decodeTxnLock(kvp.Value) + if err != nil { + return errors.WithStack(err) + } + if len(lock.PrimaryKey) == 0 { + return errors.Wrapf(ErrTxnInvalidMeta, "missing txn primary key for key %s", string(userKey)) + } + txnKey := lockTxnKey{startTS: lock.StartTS, primary: string(lock.PrimaryKey)} + state, err := s.cachedBackupLockTxnStatusAt(ctx, plan, lock, txnKey, ts) + if err != nil { + return err + } + phase, resolveTS, err := lockResolutionForStatus(state, lock, userKey, plan.cleanupNow) + if err != nil { + return err + } + appendScanLockResolutionBatch(plan, txnKey, phase, resolveTS, lock, userKey) + return nil +} + +func (s *ShardStore) cachedBackupLockTxnStatusAt( + ctx context.Context, + plan *scanLockPlan, + lock txnLock, + txnKey lockTxnKey, + ts uint64, +) (lockTxnStatus, error) { + if state, ok := plan.statusCache[txnKey]; ok { + return state, nil + } + status, commitTS, err := s.primaryTxnRecordedStatusAt(ctx, lock.PrimaryKey, lock.StartTS, ts) + if err != nil { + return lockTxnStatus{}, err + } + state := lockTxnStatus{status: status, commitTS: commitTS} + plan.statusCache[txnKey] = state + return state, nil +} + +func (s *ShardStore) primaryTxnRecordedStatusAt(ctx context.Context, primaryKey []byte, startTS uint64, ts uint64) (txnStatus, uint64, error) { + commitTS, committed, err := s.txnCommitTSAt(ctx, primaryKey, startTS, ts) + if err != nil { + return txnStatusPending, 0, err + } + if committed && commitTS <= ts { + return txnStatusCommitted, commitTS, nil + } + rolledBack, err := s.hasTxnRollbackAt(ctx, primaryKey, startTS, ts) + if err != nil { + return txnStatusPending, 0, err + } + if rolledBack { + return txnStatusRolledBack, 0, nil + } + return txnStatusPending, 0, nil +} + +func (s *ShardStore) txnCommitTSAt(ctx context.Context, primaryKey []byte, startTS uint64, ts uint64) (uint64, bool, error) { + b, err := s.GetAt(ctx, txnCommitKey(primaryKey, startTS), ts) + if err != nil { + if errors.Is(err, store.ErrKeyNotFound) { + return 0, false, nil + } + return 0, false, err + } + cts, derr := decodeTxnCommitRecord(b) + if derr != nil { + return 0, false, errors.WithStack(derr) + } + return cts, true, nil +} + +func (s *ShardStore) hasTxnRollbackAt(ctx context.Context, primaryKey []byte, startTS uint64, ts uint64) (bool, error) { + _, err := s.GetAt(ctx, txnRollbackKey(primaryKey, startTS), ts) + if err != nil { + if errors.Is(err, store.ErrKeyNotFound) { + return false, nil + } + return false, err + } + return true, nil +} + // NewBackupScannerAtSnapshot creates a value scanner from a captured route view. func NewBackupScannerAtSnapshot(st *ShardStore, snapshot BackupRouteSnapshot, ts uint64, pageSize int) BackupScanner { return NewFilteredBackupScannerAtSnapshot(st, snapshot, ts, pageSize, nil) @@ -193,6 +303,7 @@ func NewFilteredBackupScannerAtSnapshot( return &backupScanner{ store: st, routes: snapshot.routes, + scanGroups: snapshot.scanGroups, clampToRoutes: snapshot.clampToRoutes, cursor: snapshot.start, end: snapshot.end, @@ -233,6 +344,7 @@ func NewBackupKeyScannerAtSnapshot(st *ShardStore, snapshot BackupRouteSnapshot, return &backupKeyScanner{ store: st, routes: snapshot.routes, + scanGroups: snapshot.scanGroups, clampToRoutes: snapshot.clampToRoutes, cursor: snapshot.start, end: snapshot.end, @@ -312,7 +424,9 @@ func (s *backupKeyScanner) loadNextPage(ctx context.Context) error { s.index = 0 return nil } - keys, err := s.store.scanKeyRoutesWithSourceAt(ctx, s.routes, s.cursor, s.end, s.pageSize, s.ts, s.clampToRoutes) + keys, err := s.store.scanKeyRoutesWithSourceAt( + ctx, s.routes, s.scanGroups, s.cursor, s.end, s.pageSize, s.ts, s.clampToRoutes, + ) if err != nil { return err } @@ -344,7 +458,9 @@ func (s *backupScanner) loadNextPage(ctx context.Context) error { s.index = 0 return nil } - keys, err := s.store.scanKeyRoutesWithSourceAt(ctx, s.routes, s.cursor, s.end, s.pageSize, s.ts, s.clampToRoutes) + keys, err := s.store.scanKeyRoutesWithSourceAt( + ctx, s.routes, s.scanGroups, s.cursor, s.end, s.pageSize, s.ts, s.clampToRoutes, + ) if err != nil { return err } @@ -374,6 +490,13 @@ func (s *backupScanner) loadNextPage(ctx context.Context) error { } func (s *backupScanner) materializeBackupKey(ctx context.Context, item routedScanKey) (*store.KVPair, bool, error) { + route, ok, err := s.materializeRouteForKey(item) + if err != nil { + return nil, false, err + } + if !ok { + return nil, false, nil + } if s.keyFilter != nil { selected, err := s.keyFilter(item.key) if err != nil { @@ -383,13 +506,6 @@ func (s *backupScanner) materializeBackupKey(ctx context.Context, item routedSca return nil, false, nil } } - route, ok, err := s.materializeRouteForKey(item) - if err != nil { - return nil, false, err - } - if !ok { - return nil, false, nil - } val, err := s.store.getRouteAt(ctx, route, item.key, s.ts) if errors.Is(err, store.ErrKeyNotFound) { return nil, false, nil @@ -405,19 +521,32 @@ func (s *backupScanner) materializeRouteForKey(item routedScanKey) (distribution } func (s *ShardStore) routeForRoutedKey(item routedScanKey, routes []distribution.Route) (distribution.Route, bool, error) { + if route, ok, handled, err := s.partitionRouteForRoutedKey(item); handled || err != nil { + return route, ok, err + } + if item.partitionOnly { + return distribution.Route{}, false, nil + } + return byteRangeRouteForRoutedKey(item, routes) +} + +func (s *ShardStore) partitionRouteForRoutedKey(item routedScanKey) (distribution.Route, bool, bool, error) { if s != nil && s.partitionResolver != nil { groupID, ok := s.partitionResolver.ResolveGroup(item.key) if ok { if groupID == item.route.GroupID { - return distribution.Route{GroupID: groupID}, true, nil + return distribution.Route{GroupID: groupID}, true, true, nil } - return distribution.Route{}, false, nil + return distribution.Route{}, false, true, nil } if s.partitionResolver.RecognisesPartitionedKey(item.key) { - return distribution.Route{}, false, errors.Wrapf(ErrInvalidRequest, "no partition route for backup key %q", item.key) + return distribution.Route{}, false, true, errors.Wrapf(ErrInvalidRequest, "no partition route for backup key %q", item.key) } } + return distribution.Route{}, false, false, nil +} +func byteRangeRouteForRoutedKey(item routedScanKey, routes []distribution.Route) (distribution.Route, bool, error) { key := routeKey(item.key) if routeContainsKey(item.route, key) { return item.route, true, nil @@ -436,6 +565,7 @@ func (s *ShardStore) routeForRoutedKey(item routedScanKey, routes []distribution func cloneBackupRouteSnapshot(snapshot BackupRouteSnapshot) BackupRouteSnapshot { return BackupRouteSnapshot{ routes: cloneBackupRoutes(snapshot.routes), + scanGroups: appendUniqueBackupGroups(nil, snapshot.scanGroups), clampToRoutes: snapshot.clampToRoutes, start: bytes.Clone(snapshot.start), end: bytes.Clone(snapshot.end), @@ -452,9 +582,41 @@ func cloneBackupRoutes(routes []distribution.Route) []distribution.Route { return out } +func backupRouteGroupIDs(routes []distribution.Route) []uint64 { + groups := make([]uint64, 0, len(routes)) + for _, route := range routes { + groups = appendUniqueBackupGroups(groups, []uint64{route.GroupID}) + } + return groups +} + +func backupSnapshotGroupIDs(snapshot BackupRouteSnapshot) []uint64 { + groups := backupRouteGroupIDs(snapshot.routes) + return appendUniqueBackupGroups(groups, snapshot.scanGroups) +} + +func appendUniqueBackupGroups(dst []uint64, src []uint64) []uint64 { + if len(src) == 0 { + return dst + } + seen := make(map[uint64]struct{}, len(dst)+len(src)) + for _, groupID := range dst { + seen[groupID] = struct{}{} + } + for _, groupID := range src { + if _, ok := seen[groupID]; ok { + continue + } + seen[groupID] = struct{}{} + dst = append(dst, groupID) + } + return dst +} + func (s *ShardStore) scanKeyRoutesWithSourceAt( ctx context.Context, routes []distribution.Route, + scanGroups []uint64, start []byte, end []byte, limit int, @@ -463,27 +625,34 @@ func (s *ShardStore) scanKeyRoutesWithSourceAt( ) ([]routedScanKey, error) { out := make([]routedScanKey, 0) seenGroups := make(map[uint64]struct{}) - for _, route := range routes { - scanStart := start - scanEnd := end - if clampToRoutes { - scanStart = clampScanStart(start, route.Start) - scanEnd = clampScanEnd(end, route.End) - } else { - if _, seen := seenGroups[route.GroupID]; seen { - continue - } - seenGroups[route.GroupID] = struct{}{} - } + out, err := s.scanSnapshotKeyRoutesWithSourceAt(ctx, out, routes, start, end, limit, ts, clampToRoutes, seenGroups) + if err != nil { + return nil, err + } + return s.scanPartitionOnlyKeyGroupsAt(ctx, out, routes, scanGroups, seenGroups, start, end, limit, ts) +} - keys, err := s.scanKeyRouteAt(ctx, route, scanStart, scanEnd, limit, ts) - if err != nil { - return nil, err +func (s *ShardStore) scanSnapshotKeyRoutesWithSourceAt( + ctx context.Context, + out []routedScanKey, + routes []distribution.Route, + start []byte, + end []byte, + limit int, + ts uint64, + clampToRoutes bool, + seenGroups map[uint64]struct{}, +) ([]routedScanKey, error) { + for _, route := range routes { + scanStart, scanEnd, skip := backupRouteScanBounds(route, start, end, clampToRoutes, seenGroups) + if skip { + continue } - out, err = s.mergeAndTrimRoutedScanKeys(out, routedScanKeys(route, keys), routes, limit) + next, err := s.appendBackupScanRouteKeysAt(ctx, out, routes, route, scanStart, scanEnd, limit, ts, false) if err != nil { return nil, err } + out = next if clampToRoutes && len(out) >= limit { break } @@ -491,13 +660,75 @@ func (s *ShardStore) scanKeyRoutesWithSourceAt( return out, nil } -func routedScanKeys(route distribution.Route, keys [][]byte) []routedScanKey { +func backupRouteScanBounds( + route distribution.Route, + start []byte, + end []byte, + clampToRoutes bool, + seenGroups map[uint64]struct{}, +) (scanStart []byte, scanEnd []byte, skip bool) { + if clampToRoutes { + seenGroups[route.GroupID] = struct{}{} + return clampScanStart(start, route.Start), clampScanEnd(end, route.End), false + } + if _, seen := seenGroups[route.GroupID]; seen { + return nil, nil, true + } + seenGroups[route.GroupID] = struct{}{} + return start, end, false +} + +func (s *ShardStore) scanPartitionOnlyKeyGroupsAt( + ctx context.Context, + out []routedScanKey, + routes []distribution.Route, + scanGroups []uint64, + seenGroups map[uint64]struct{}, + start []byte, + end []byte, + limit int, + ts uint64, +) ([]routedScanKey, error) { + for _, groupID := range scanGroups { + if _, seen := seenGroups[groupID]; seen { + continue + } + seenGroups[groupID] = struct{}{} + route := distribution.Route{GroupID: groupID} + next, err := s.appendBackupScanRouteKeysAt(ctx, out, routes, route, start, end, limit, ts, true) + if err != nil { + return nil, err + } + out = next + } + return out, nil +} + +func (s *ShardStore) appendBackupScanRouteKeysAt( + ctx context.Context, + out []routedScanKey, + routes []distribution.Route, + route distribution.Route, + start []byte, + end []byte, + limit int, + ts uint64, + partitionOnly bool, +) ([]routedScanKey, error) { + keys, err := s.scanKeyRouteAt(ctx, route, start, end, limit, ts) + if err != nil { + return nil, err + } + return s.mergeAndTrimRoutedScanKeys(out, routedScanKeys(route, keys, partitionOnly), routes, limit) +} + +func routedScanKeys(route distribution.Route, keys [][]byte, partitionOnly bool) []routedScanKey { items := make([]routedScanKey, 0, len(keys)) for _, key := range keys { if key == nil { continue } - items = append(items, routedScanKey{key: key, route: route}) + items = append(items, routedScanKey{key: key, route: route, partitionOnly: partitionOnly}) } return items } diff --git a/kv/fsm_backup.go b/kv/fsm_backup.go index 9d0e12917..11ed149e0 100644 --- a/kv/fsm_backup.go +++ b/kv/fsm_backup.go @@ -46,8 +46,7 @@ func (f *kvFSM) applyDecodedBackup(entry backupEntry) error { f.readTracker.ReleaseBackupPinForGroup(entry.release.PinID, f.shardGroupID) return nil case backupSubtypeReserve: - err := f.readTracker.PinWithDeadline(entry.pin.PinID, entry.pin.ReadTS, entry.pin.Deadline) - return f.applyBackupTimestampFence(entry.pin.ReadTS, err) + return f.readTracker.PinWithDeadline(entry.pin.PinID, entry.pin.ReadTS, entry.pin.Deadline) case backupSubtypeUnreserve: f.readTracker.ReleaseBackupPin(entry.release.PinID) return nil diff --git a/kv/fsm_backup_test.go b/kv/fsm_backup_test.go index 116974e0a..16b9d99c2 100644 --- a/kv/fsm_backup_test.go +++ b/kv/fsm_backup_test.go @@ -189,6 +189,26 @@ func TestApplyBackupFencesPreallocatedWrites(t *testing.T) { } } +func TestApplyBackupReserveDoesNotInstallTimestampFloor(t *testing.T) { + tracker := NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)) + fsm := newBackupTestFSM(t, tracker) + readTS := uint64(100) + require.NoError(t, haltApplyOf(fsm.Apply(EncodeBackupReserveEntry(BackupReserveEntry{ + PinID: backupTrackerTestPinID(1), ReadTS: readTS, Deadline: time.Now().Add(time.Hour), + })))) + + require.Nil(t, applyBackupTestRequest(t, fsm, &pb.Request{Ts: readTS - 1, Mutations: []*pb.Mutation{{ + Op: pb.Op_PUT, Key: []byte("reserved-only"), Value: []byte("allowed"), + }}})) + + require.NoError(t, haltApplyOf(fsm.Apply(EncodeBackupPinEntry(BackupPinEntry{ + PinID: backupTrackerTestPinID(1), ReadTS: readTS, Deadline: time.Now().Add(time.Hour), + })))) + requireBackupTimestampFenced(t, applyBackupTestRequest(t, fsm, &pb.Request{Ts: readTS - 1, Mutations: []*pb.Mutation{{ + Op: pb.Op_PUT, Key: []byte("after-pin"), Value: []byte("blocked"), + }}})) +} + func TestBackupTimestampFloorKeyRejectsRawMutation(t *testing.T) { fsm := newBackupTestFSM(t, NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0))) resp := applyBackupTestRequest(t, fsm, &pb.Request{Ts: 10, Mutations: []*pb.Mutation{{ diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 9767d65c3..7715a6105 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -749,6 +749,47 @@ func TestValidateBackupSnapshotAtRejectsPreparedInsert(t *testing.T) { require.ErrorIs(t, err, ErrTxnLocked) } +func TestValidateBackupSnapshotAtRejectsPostCutTxnDecision(t *testing.T) { + ctx := context.Background() + tests := []struct { + name string + phase pb.Phase + }{ + {name: "commit", phase: pb.Phase_COMMIT}, + {name: "rollback", phase: pb.Phase_ABORT}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + st := store.NewMVCCStore() + fsm, ok := NewKvFSMWithHLCAndTracker( + st, NewHLC(), NewActiveTimestampTracker(WithActiveTimestampTrackerSweepInterval(0)), + ).(*kvFSM) + require.True(t, ok) + primary := []byte("post-cut-" + tc.name) + prepare := &pb.Request{IsTxn: true, Phase: pb.Phase_PREPARE, Ts: 30, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: primary, LockTTLms: defaultTxnLockTTLms, + })}, + {Op: pb.Op_PUT, Key: primary, Value: []byte("pending")}, + }} + require.Nil(t, applyBackupTestRequest(t, fsm, prepare)) + decision := &pb.Request{IsTxn: true, Phase: tc.phase, Ts: 30, Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: primary, CommitTS: 60, + })}, + {Op: pb.Op_PUT, Key: primary}, + }} + require.Nil(t, applyBackupTestRequest(t, fsm, decision)) + + engine := distribution.NewEngineWithDefaultRoute() + shards := NewShardStore(engine, map[uint64]*ShardGroup{1: {Store: st}}) + snapshot := shards.CaptureBackupRouteSnapshot(nil, nil) + err := shards.ValidateBackupSnapshotAt(ctx, snapshot, 50, 16) + require.ErrorIs(t, err, ErrTxnLocked) + }) + } +} + func TestBackupScannerMaterializesFromCapturedRoute(t *testing.T) { t.Parallel() @@ -902,6 +943,75 @@ func TestBackupScannerKeepsPartitionResolvedOwnerForSQSKey(t *testing.T) { require.Equal(t, []byte("partition-owner"), pair.Value) } +func TestBackupScannersIncludePartitionOnlyGroups(t *testing.T) { + t.Parallel() + + ctx := context.Background() + key := []byte("!sqs|msg|data|p|orders|partition-2|msg-2") + groups := map[uint64]*ShardGroup{ + 1: {Store: store.NewMVCCStore()}, + 2: {Store: store.NewMVCCStore()}, + } + st := NewShardStore(distribution.NewEngine(), groups). + WithPartitionResolver(&fakePartitionResolver{ + routes: map[string]uint64{string(key): 2}, + recognisedPrefix: []byte("!sqs|msg|data|p|"), + }) + require.NoError(t, groups[2].Store.PutAt(ctx, key, []byte("partition-only"), 1, 0)) + snapshot := BackupRouteSnapshotWithScanGroups(BackupRouteSnapshot{ + routes: []distribution.Route{{RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1}}, + }, []uint64{1, 2}) + + keyScanner := st.NewBackupKeyScannerAtSnapshot(snapshot, ^uint64(0), 1) + defer keyScanner.Close() + gotKey, ok, err := keyScanner.Next(ctx) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, key, gotKey) + + valueScanner := st.NewBackupScannerAtSnapshot(snapshot, ^uint64(0), 1) + defer valueScanner.Close() + pair, ok, err := valueScanner.Next(ctx) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, key, pair.Key) + require.Equal(t, []byte("partition-only"), pair.Value) +} + +func TestFilteredBackupScannerAppliesFilterAfterOwnership(t *testing.T) { + t.Parallel() + + ctx := context.Background() + groups := map[uint64]*ShardGroup{ + 1: {Store: store.NewMVCCStore()}, + } + st := NewShardStore(distribution.NewEngine(), groups) + require.NoError(t, groups[1].Store.PutAt(ctx, []byte("bad"), []byte("stale-off-route"), 1, 0)) + require.NoError(t, groups[1].Store.PutAt(ctx, []byte("x"), []byte("owned"), 2, 0)) + scanner := &backupScanner{ + store: st, + routes: []distribution.Route{ + {RouteID: 1, Start: []byte("m"), GroupID: 1}, + }, + cursor: []byte(""), + ts: ^uint64(0), + pageSize: 1, + keyFilter: func(key []byte) (bool, error) { + if string(key) == "bad" { + return false, ErrInvalidRequest + } + return true, nil + }, + } + defer scanner.Close() + + pair, ok, err := scanner.Next(ctx) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, []byte("x"), pair.Key) + require.Equal(t, []byte("owned"), pair.Value) +} + func TestBackupScannerFailsClosedOnUnresolvedPartitionedSQSKey(t *testing.T) { t.Parallel()