diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index 65b448a10..22ea29731 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -3,32 +3,44 @@ package adapter import ( "bytes" "context" + "encoding/binary" "log/slog" "math" + "sort" "strings" "sync" "time" "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/fskeys" + "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/cockroachdb/errors" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) // DistributionServer serves distribution related gRPC APIs. type DistributionServer struct { - mu sync.Mutex - engine *distribution.Engine - catalog *distribution.CatalogStore - coordinator kv.Coordinator - readTracker *kv.ActiveTimestampTracker - fsObserver DistributionFilesystemObserver - readBlocked func() bool - reloadRetry struct { + mu sync.Mutex + engine *distribution.Engine + catalog *distribution.CatalogStore + coordinator kv.Coordinator + readTracker *kv.ActiveTimestampTracker + fsObserver DistributionFilesystemObserver + readBlocked func() bool + migrationCapabilityGate SplitMigrationCapabilityGate + splitJobRunnerReady bool + splitJobRunnerReadinessGate SplitMigrationCapabilityGate + splitPromotionClientFactory SplitPromotionClientFactory + splitMigrationClientFactory SplitMigrationClientFactory + splitMigrationVoterFactory SplitMigrationVoterFactory + knownRaftGroups map[uint64]struct{} + splitJobHistoryGCLast time.Time + reloadRetry struct { attempts int interval time.Duration } @@ -44,6 +56,46 @@ type DistributionFilesystemObserver interface { const DistributionFilePinnedHotspotSplitBoundary = "split_boundary" +// SplitMigrationCapabilityGate reports whether this node can safely create +// migration-only side effects. A nil gate keeps StartSplitMigration fail-closed. +type SplitMigrationCapabilityGate func(context.Context) error + +// SplitPromotionClient is the subset of the internal gRPC client the split +// job runner needs to promote target-local staged rows. +type SplitPromotionClient interface { + PromoteStagedVersions(context.Context, *pb.PromoteStagedVersionsRequest, ...grpc.CallOption) (*pb.PromoteStagedVersionsResponse, error) +} + +// SplitPromotionClientFactory dials the node currently leading a split job's +// target range and returns the internal client used by the runner. +type SplitPromotionClientFactory func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) + +// SplitMigrationClient is the internal data-plane client used for a split +// job's source export, target import, and durable source/target guards. +type SplitMigrationClient interface { + SplitPromotionClient + ExportRangeVersions(context.Context, *pb.ExportRangeVersionsRequest, ...grpc.CallOption) (grpc.ServerStreamingClient[pb.ExportRangeVersionsResponse], error) + ImportRangeVersions(context.Context, *pb.ImportRangeVersionsRequest, ...grpc.CallOption) (*pb.ImportRangeVersionsResponse, error) + ApplyTargetStagedReadiness(context.Context, *pb.TargetStagedReadinessRequest, ...grpc.CallOption) (*pb.TargetStagedReadinessResponse, error) + ProbeMigrationLocks(context.Context, *pb.ProbeMigrationLocksRequest, ...grpc.CallOption) (*pb.ProbeMigrationLocksResponse, error) + CleanupMigration(context.Context, *pb.CleanupMigrationRequest, ...grpc.CallOption) (*pb.CleanupMigrationResponse, error) + ProbeMigrationState(context.Context, *pb.ProbeMigrationStateRequest, ...grpc.CallOption) (*pb.ProbeMigrationStateResponse, error) + IssueMigrationTimestamp(context.Context, *pb.IssueMigrationTimestampRequest, ...grpc.CallOption) (*pb.IssueMigrationTimestampResponse, error) +} + +// SplitMigrationClientFactory resolves both participating group leaders. +type SplitMigrationClientFactory func(context.Context, distribution.SplitJob, uint64) (source SplitMigrationClient, target SplitMigrationClient, err error) + +type SplitMigrationVoter struct { + ID string + Address string + Client SplitMigrationClient +} + +// SplitMigrationVoterFactory resolves the current voter set and a direct +// client for each voter. The runner re-resolves it before every barrier. +type SplitMigrationVoterFactory func(context.Context, uint64) ([]SplitMigrationVoter, error) + // WithDistributionCoordinator configures the coordinator used for Raft-backed // catalog mutations in SplitRange. func WithDistributionCoordinator(coordinator kv.Coordinator) DistributionServerOption { @@ -70,6 +122,65 @@ func WithDistributionReadGate(blocked func() bool) DistributionServerOption { } } +func WithSplitMigrationCapabilityGate(gate SplitMigrationCapabilityGate) DistributionServerOption { + return func(s *DistributionServer) { + s.migrationCapabilityGate = gate + } +} + +// WithSplitJobRunnerReady opens the split migration capability probe once this +// node can advance planned split jobs. +func WithSplitJobRunnerReady() DistributionServerOption { + return func(s *DistributionServer) { + s.splitJobRunnerReady = true + } +} + +// WithSplitJobRunnerReadinessGate configures local runtime gates that must be +// open before this node advertises split migration capability. +func WithSplitJobRunnerReadinessGate(gate SplitMigrationCapabilityGate) DistributionServerOption { + return func(s *DistributionServer) { + s.splitJobRunnerReadinessGate = gate + } +} + +// WithSplitPromotionClientFactory configures the client factory used by the +// split job runner to promote target-local staged versions. +func WithSplitPromotionClientFactory(factory SplitPromotionClientFactory) DistributionServerOption { + return func(s *DistributionServer) { + s.splitPromotionClientFactory = factory + } +} + +// WithSplitMigrationClientFactory configures the source/target clients used by +// the production split job state machine. +func WithSplitMigrationClientFactory(factory SplitMigrationClientFactory) DistributionServerOption { + return func(s *DistributionServer) { + s.splitMigrationClientFactory = factory + } +} + +func WithSplitMigrationVoterFactory(factory SplitMigrationVoterFactory) DistributionServerOption { + return func(s *DistributionServer) { + s.splitMigrationVoterFactory = factory + } +} + +// WithDistributionKnownRaftGroups configures the Raft group IDs this node can +// route migration work to. +func WithDistributionKnownRaftGroups(groupIDs ...uint64) DistributionServerOption { + return func(s *DistributionServer) { + groups := make(map[uint64]struct{}, len(groupIDs)) + for _, groupID := range groupIDs { + if groupID == 0 { + continue + } + groups[groupID] = struct{}{} + } + s.knownRaftGroups = groups + } +} + // WithCatalogReloadRetryPolicy configures the retry policy used after split // commit when waiting for the local catalog snapshot to become visible. func WithCatalogReloadRetryPolicy(attempts int, interval time.Duration) DistributionServerOption { @@ -84,13 +195,32 @@ func WithCatalogReloadRetryPolicy(attempts int, interval time.Duration) Distribu } const ( - childRouteCount = 2 - splitMutationOpCount = childRouteCount + 3 + childRouteCount = 2 + splitMutationOpCount = childRouteCount + 3 + listSplitJobsDefaultPageSize = 200 + splitJobListCursorVersion = byte(1) + splitJobListCursorTerminalOff = 1 + splitJobListCursorJobIDOff = splitJobListCursorTerminalOff + 8 + splitJobListCursorEncodedBytes = splitJobListCursorJobIDOff + 8 + SplitMigrationCapabilityV2 = "cap_migration_v2" + splitMigrationCapabilityV2 = SplitMigrationCapabilityV2 + + splitPromotionDefaultMaxVersions = 1024 + splitPromotionBytesPerMiB = 1024 * 1024 + splitPromotionMaxBytesMiB = 4 + splitPromotionMaxScannedMiB = 16 + splitPromotionAttemptTimeoutSecs = 2 + promotionCompleteBaseOpCount = 2 ) var ( - defaultCatalogReloadRetryAttempts = 20 - defaultCatalogReloadRetryInterval = 10 * time.Millisecond + defaultCatalogReloadRetryAttempts = 20 + defaultCatalogReloadRetryInterval = 10 * time.Millisecond + defaultSplitJobRunnerInterval = time.Second + defaultSplitPromotionMaxVersions = uint32(splitPromotionDefaultMaxVersions) + defaultSplitPromotionMaxBytes = uint64(splitPromotionMaxBytesMiB * splitPromotionBytesPerMiB) + defaultSplitPromotionMaxScanned = uint64(splitPromotionMaxScannedMiB * splitPromotionBytesPerMiB) + defaultSplitPromotionAttemptTimeout = time.Duration(splitPromotionAttemptTimeoutSecs) * time.Second errDistributionCatalogNotConfigured = errors.New("route catalog is not configured") errDistributionUnknownRoute = errors.New("unknown route") @@ -102,6 +232,10 @@ var ( errDistributionCoordinatorRequired = errors.New("distribution coordinator is not configured") errDistributionEngineNotConfigured = errors.New("distribution engine is not configured") errDistributionCatalogVersionNotFound = errors.New("route catalog version not found") + errDistributionClusterNotReady = errors.New("cluster is not ready for split migration") + errDistributionRaftGroupsNotKnown = errors.New("distribution raft groups are not configured") + errDistributionUnknownTargetGroup = errors.New("unknown target group") + errDistributionSourceRouteNotActive = errors.New("source route is not active") ) // NewDistributionServer creates a new server. @@ -134,6 +268,230 @@ func (s *DistributionServer) requireReadReady() error { return nil } +func (s *DistributionServer) RunSplitJobRunner(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + ticker := time.NewTicker(defaultSplitJobRunnerInterval) + defer ticker.Stop() + for { + if err := s.RunSplitJobRunnerOnce(ctx); err != nil { + if !splitJobRunnerContextDone(err) { + slog.Warn("split job runner tick failed", "err", err) + } + } + select { + case <-ctx.Done(): + return nil + case <-ticker.C: + } + } +} + +func (s *DistributionServer) RunSplitJobRunnerOnce(ctx context.Context) error { + if !s.splitJobRunnerConfigured() { + return nil + } + leader, err := s.verifySplitJobRunnerLeader(ctx) + if err != nil || !leader { + return err + } + jobs, err := s.catalog.ListSplitJobs(ctx) + if err != nil { + return splitJobCatalogStatusError(err) + } + if job, ok := nextRunnableSplitJob(jobs); ok { + if job.Phase != distribution.SplitJobPhaseAbandoning { + ready, gateErr := s.splitJobCapabilityReady(ctx, job) + if gateErr != nil || !ready { + return gateErr + } + } + return s.runSplitJobPhase(ctx, job) + } + return s.gcSplitJobHistory(ctx, jobs, time.Now()) +} + +func (s *DistributionServer) splitJobCapabilityReady(ctx context.Context, job distribution.SplitJob) (bool, error) { + if s.migrationCapabilityGate == nil { + return true, nil + } + gateErr := s.migrationCapabilityGate(ctx) + if gateErr != nil { + return s.handleSplitJobCapabilityRegression(ctx, job, gateErr) + } + if !job.CapabilityRegressed { + return true, nil + } + return s.clearSplitJobCapabilityRegression(ctx, job) +} + +func (s *DistributionServer) handleSplitJobCapabilityRegression(ctx context.Context, job distribution.SplitJob, cause error) (bool, error) { + markedJob, marked, err := s.markSplitJobCapabilityRegressed(ctx, job, cause) + if err != nil || !marked { + return false, err + } + if splitJobCapabilityRegressionRequiresFailClosed(markedJob) { + closed, err := s.ensureSplitJobCapabilityRegressionFailClosed(ctx, markedJob) + if err != nil || !closed { + return false, err + } + } + return false, nil +} + +func (s *DistributionServer) markSplitJobCapabilityRegressed(ctx context.Context, job distribution.SplitJob, cause error) (distribution.SplitJob, bool, error) { + if job.CapabilityRegressed { + return job, true, nil + } + marked := false + err := s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase != job.Phase { + return current, nil + } + marked = true + if !current.CapabilityRegressed { + current.CapabilityRegressed = true + current.LastError = "cluster migration capability regressed: " + cause.Error() + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) + if err != nil || !marked { + return job, marked, err + } + job.CapabilityRegressed = true + return job, true, nil +} + +func (s *DistributionServer) clearSplitJobCapabilityRegression(ctx context.Context, job distribution.SplitJob) (bool, error) { + if splitJobCapabilityRegressionRequiresFailClosed(job) { + restored, err := s.restoreSplitJobCapabilityRegressionGuards(ctx, job) + if err != nil || !restored { + return false, err + } + } + err := s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == job.Phase && current.CapabilityRegressed { + current.CapabilityRegressed = false + current.LastError = "" + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) + return false, err +} + +func (s *DistributionServer) splitJobRunnerConfigured() bool { + return s != nil && s.catalog != nil && s.coordinator != nil && s.splitPromotionClientFactory != nil && s.splitMigrationClientFactory != nil +} + +func (s *DistributionServer) verifySplitJobRunnerLeader(ctx context.Context) (bool, error) { + if !s.coordinator.IsLeaderForKey(distribution.CatalogVersionKey()) { + return false, nil + } + if err := s.coordinator.VerifyLeaderForKey(ctx, distribution.CatalogVersionKey()); err != nil { + return false, errors.WithStack(err) + } + return true, nil +} + +func nextRunnableSplitJob(jobs []distribution.SplitJob) (distribution.SplitJob, bool) { + for _, job := range jobs { + switch job.Phase { + case distribution.SplitJobPhasePlanned, + distribution.SplitJobPhaseBackfill, + distribution.SplitJobPhaseFence, + distribution.SplitJobPhaseDeltaCopy, + distribution.SplitJobPhaseCutover, + distribution.SplitJobPhaseCleanup, + distribution.SplitJobPhaseAbandoning: + return job, true + case distribution.SplitJobPhaseNone, + distribution.SplitJobPhaseDone, + distribution.SplitJobPhaseFailed, + distribution.SplitJobPhaseAbandoned: + continue + } + } + return distribution.SplitJob{}, false +} + +func (s *DistributionServer) promoteSplitJobTargetAndComplete(ctx context.Context, job distribution.SplitJob) error { + if !job.TargetPromotionDone { + client, err := s.splitPromotionClientFactory(ctx, job) + if err != nil { + return errors.WithStack(err) + } + if client == nil { + return errors.New("split promotion client is nil") + } + if err := promoteSplitJobTarget(ctx, client, job); err != nil { + return errors.WithStack(err) + } + } + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return err + } + current, found, err := s.catalog.SplitJobAt(ctx, job.JobID, snapshot.ReadTS) + if err != nil { + return splitJobPromotionStatusError(err) + } + if !found { + return splitJobPromotionStatusError(distribution.ErrCatalogSplitJobConflict) + } + completed, _, err := s.completeSplitJobTargetPromotionViaCoordinator(ctx, snapshot.Version, current, time.Now().UnixMilli()) + if err != nil { + return err + } + return s.applyEngineSnapshot(completed) +} + +func promoteSplitJobTarget(ctx context.Context, client SplitPromotionClient, job distribution.SplitJob) error { + var cursor []byte + for { + attemptCtx, cancel := splitPromotionAttemptContext(ctx) + resp, err := client.PromoteStagedVersions(attemptCtx, &pb.PromoteStagedVersionsRequest{ + JobId: job.JobID, + Cursor: cursor, + MaxVersions: defaultSplitPromotionMaxVersions, + MaxBytes: defaultSplitPromotionMaxBytes, + MaxScannedBytes: defaultSplitPromotionMaxScanned, + }) + cancel() + if err != nil { + return errors.WithStack(err) + } + if resp.GetDone() { + return nil + } + next := resp.GetNextCursor() + if len(next) == 0 || bytes.Equal(next, cursor) { + return errors.New("split promotion made no cursor progress") + } + cursor = distribution.CloneBytes(next) + } +} + +func splitPromotionAttemptContext(ctx context.Context) (context.Context, context.CancelFunc) { + if ctx == nil { + ctx = context.Background() + } + if defaultSplitPromotionAttemptTimeout <= 0 { + return ctx, func() {} + } + return context.WithTimeout(ctx, defaultSplitPromotionAttemptTimeout) +} + +func splitJobRunnerContextDone(err error) bool { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return true + } + code := status.Code(errors.Cause(err)) + return code == codes.Canceled || code == codes.DeadlineExceeded +} + // UpdateRoute allows updating route information. func (s *DistributionServer) UpdateRoute(start, end []byte, group uint64) { s.engine.UpdateRoute(start, end, group) @@ -222,6 +580,215 @@ func (s *DistributionServer) GetIntersectingRoutes(ctx context.Context, req *pb. }, nil } +func (s *DistributionServer) StartSplitMigration(ctx context.Context, req *pb.StartSplitMigrationRequest) (*pb.StartSplitMigrationResponse, error) { + if err := s.verifyStartSplitMigrationPreflight(ctx, req); err != nil { + return nil, err + } + + s.mu.Lock() + defer s.mu.Unlock() + + if err := s.verifyStartSplitMigrationCatalogReady(ctx); err != nil { + return nil, err + } + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return nil, err + } + defer releaseReadPin(s.pinReadTS(snapshot.ReadTS)) + + parent, err := s.startSplitMigrationParent(ctx, snapshot, req) + if err != nil { + return nil, err + } + job, err := s.newSplitMigrationJob(ctx, snapshot, parent, req) + if err != nil { + return nil, err + } + if err := s.createSplitJobViaCoordinator(ctx, snapshot.ReadTS, job); err != nil { + return nil, err + } + return &pb.StartSplitMigrationResponse{ + CatalogVersion: snapshot.Version, + JobId: job.JobID, + }, nil +} + +func (s *DistributionServer) GetSplitMigrationCapability(ctx context.Context, _ *pb.GetSplitMigrationCapabilityRequest) (*pb.GetSplitMigrationCapabilityResponse, error) { + if !s.splitMigrationCapabilityReady(ctx) { + return &pb.GetSplitMigrationCapabilityResponse{}, nil + } + return &pb.GetSplitMigrationCapabilityResponse{ + MigrationCapable: true, + Capabilities: []string{splitMigrationCapabilityV2}, + }, nil +} + +func (s *DistributionServer) splitMigrationCapabilityReady(ctx context.Context) bool { + if !s.splitJobRunnerReady { + return false + } + if s.splitJobRunnerReadinessGate == nil { + return true + } + return s.splitJobRunnerReadinessGate(ctx) == nil +} + +func (s *DistributionServer) verifyStartSplitMigrationPreflight(ctx context.Context, req *pb.StartSplitMigrationRequest) error { + if req.GetTargetGroupId() == 0 { + return grpcStatusError(codes.InvalidArgument, distribution.ErrCatalogSplitJobTargetGroupRequired.Error()) + } + if err := s.verifyStartSplitMigrationCatalogReady(ctx); err != nil { + return err + } + return s.verifySplitMigrationCapability(ctx) +} + +func (s *DistributionServer) verifyStartSplitMigrationCatalogReady(ctx context.Context) error { + if err := s.verifyCatalogLeader(ctx); err != nil { + return err + } + if s.catalog == nil { + return grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + return nil +} + +func (s *DistributionServer) startSplitMigrationParent( + ctx context.Context, + snapshot distribution.CatalogSnapshot, + req *pb.StartSplitMigrationRequest, +) (distribution.RouteDescriptor, error) { + if err := validateExpectedCatalogVersion(snapshot.Version, req.GetExpectedCatalogVersion()); err != nil { + return distribution.RouteDescriptor{}, err + } + parent, found := findRouteByID(snapshot.Routes, req.GetRouteId()) + if !found { + return distribution.RouteDescriptor{}, grpcStatusError(codes.NotFound, errDistributionUnknownRoute.Error()) + } + if parent.State != distribution.RouteStateActive { + return distribution.RouteDescriptor{}, grpcStatusError(codes.FailedPrecondition, errDistributionSourceRouteNotActive.Error()) + } + splitKey := normalizeSplitMigrationSplitKey(req) + if err := validateSplitKey(parent, splitKey); err != nil { + return distribution.RouteDescriptor{}, err + } + if err := distribution.ValidateMigrationRouteRange(splitKey, parent.End); err != nil { + return distribution.RouteDescriptor{}, splitMigrationRangeStatusError(err) + } + if parent.GroupID == req.GetTargetGroupId() { + return distribution.RouteDescriptor{}, grpcStatusError(codes.InvalidArgument, "target group must differ from source route group") + } + if err := s.verifyKnownTargetGroup(req.GetTargetGroupId()); err != nil { + return distribution.RouteDescriptor{}, err + } + if err := s.rejectLiveSplitJob(ctx, snapshot, parent); err != nil { + return distribution.RouteDescriptor{}, err + } + return parent, nil +} + +func (s *DistributionServer) newSplitMigrationJob( + ctx context.Context, + snapshot distribution.CatalogSnapshot, + parent distribution.RouteDescriptor, + req *pb.StartSplitMigrationRequest, +) (distribution.SplitJob, error) { + jobID, err := s.catalog.NextSplitJobIDAt(ctx, snapshot.ReadTS) + if err != nil { + return distribution.SplitJob{}, splitJobCatalogStatusError(err) + } + job, err := distribution.InitializeSplitJobPlan(distribution.SplitJob{ + JobID: jobID, + SourceRouteID: parent.RouteID, + SplitKey: normalizeSplitMigrationSplitKey(req), + TargetGroupID: req.GetTargetGroupId(), + }, parent, time.Now().UnixMilli()) + if err != nil { + return distribution.SplitJob{}, splitJobCatalogStatusError(err) + } + return job, nil +} + +func normalizeSplitMigrationSplitKey(req *pb.StartSplitMigrationRequest) []byte { + return distribution.CloneBytes(fskeys.NormalizeSplitBoundary(kv.RouteKey(req.GetSplitKey()))) +} + +func (s *DistributionServer) GetSplitJob(ctx context.Context, req *pb.GetSplitJobRequest) (*pb.GetSplitJobResponse, error) { + if req.GetJobId() == 0 { + return nil, grpcStatusError(codes.InvalidArgument, distribution.ErrCatalogSplitJobIDRequired.Error()) + } + if s.catalog == nil { + return nil, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + job, found, err := s.catalog.SplitJob(ctx, req.GetJobId()) + if err != nil { + return nil, splitJobCatalogStatusError(err) + } + if !found { + return nil, grpcStatusError(codes.NotFound, distribution.ErrCatalogSplitJobNotFound.Error()) + } + return &pb.GetSplitJobResponse{Job: distribution.SplitJobToProto(job)}, nil +} + +func (s *DistributionServer) ListSplitJobs(ctx context.Context, req *pb.ListSplitJobsRequest) (*pb.ListSplitJobsResponse, error) { + if s.catalog == nil { + return nil, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + if req == nil { + req = &pb.ListSplitJobsRequest{} + } + phaseFilter := normalizeSplitJobPhaseFilter(req.GetPhase()) + if phaseFilter != "" && !validSplitJobPhaseFilter(phaseFilter) { + return nil, grpcStatusErrorf(codes.InvalidArgument, "unknown split job phase %q", req.GetPhase()) + } + jobs, err := s.catalog.ListSplitJobs(ctx) + if err != nil { + return nil, splitJobCatalogStatusError(err) + } + resp, err := splitJobListPage(jobs, req, phaseFilter) + if err != nil { + return nil, err + } + return resp, nil +} + +func (s *DistributionServer) AbandonSplitJob(ctx context.Context, req *pb.AbandonSplitJobRequest) (*pb.AbandonSplitJobResponse, error) { + if req.GetJobId() == 0 { + return nil, grpcStatusError(codes.InvalidArgument, distribution.ErrCatalogSplitJobIDRequired.Error()) + } + if err := s.verifyCatalogLeader(ctx); err != nil { + return nil, err + } + if s.catalog == nil { + return nil, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + if err := s.updateSplitJobViaCoordinator(ctx, req.GetJobId(), func(job distribution.SplitJob) (distribution.SplitJob, error) { + return distribution.BeginSplitJobAbandon(job, time.Now().UnixMilli()) + }); err != nil { + return nil, err + } + return &pb.AbandonSplitJobResponse{}, nil +} + +func (s *DistributionServer) RetrySplitJob(ctx context.Context, req *pb.RetrySplitJobRequest) (*pb.RetrySplitJobResponse, error) { + if req.GetJobId() == 0 { + return nil, grpcStatusError(codes.InvalidArgument, distribution.ErrCatalogSplitJobIDRequired.Error()) + } + if err := s.verifyCatalogLeader(ctx); err != nil { + return nil, err + } + if s.catalog == nil { + return nil, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + if err := s.updateSplitJobViaCoordinator(ctx, req.GetJobId(), func(job distribution.SplitJob) (distribution.SplitJob, error) { + return distribution.RetrySplitJobState(job, time.Now().UnixMilli()) + }); err != nil { + return nil, err + } + return &pb.RetrySplitJobResponse{}, nil +} + func (s *DistributionServer) routeSnapshotAt(version uint64) (distribution.RouteHistorySnapshot, error) { if s.engine == nil { return distribution.RouteHistorySnapshot{}, grpcStatusError(codes.FailedPrecondition, errDistributionEngineNotConfigured.Error()) @@ -248,8 +815,7 @@ func (s *DistributionServer) SplitRange(ctx context.Context, req *pb.SplitRangeR if err != nil { return nil, err } - readPin := s.pinReadTS(snapshot.ReadTS) - defer readPin.Release() + defer releaseReadPin(s.pinReadTS(snapshot.ReadTS)) if err := validateExpectedCatalogVersion(snapshot.Version, req.GetExpectedCatalogVersion()); err != nil { return nil, err } @@ -311,7 +877,7 @@ func (s *DistributionServer) planSplitRange(ctx context.Context, snapshot distri func (s *DistributionServer) pinReadTS(ts uint64) *kv.ActiveTimestampToken { if s == nil || s.readTracker == nil { - return nil + return &kv.ActiveTimestampToken{} } return s.readTracker.Pin(ts) } @@ -338,6 +904,13 @@ func isSplitBoundaryError(err error) bool { strings.Contains(err.Error(), errDistributionSplitKeyAtBoundary.Error()) } +func releaseReadPin(token *kv.ActiveTimestampToken) { + if token == nil { + return + } + token.Release() +} + func (s *DistributionServer) verifyCatalogLeader(ctx context.Context) error { if s.coordinator == nil { return grpcStatusError(codes.FailedPrecondition, errDistributionCoordinatorRequired.Error()) @@ -455,6 +1028,47 @@ func (s *DistributionServer) loadCatalogSnapshot(ctx context.Context) (distribut return snapshot, nil } +func (s *DistributionServer) loadCatalogSnapshotAtLeastVersion( + ctx context.Context, + minVersion uint64, +) (distribution.CatalogSnapshot, error) { + if s.catalog == nil { + return distribution.CatalogSnapshot{}, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + attempts := s.reloadRetry.attempts + if attempts <= 0 { + attempts = defaultCatalogReloadRetryAttempts + } + interval := s.reloadRetry.interval + if interval <= 0 { + interval = defaultCatalogReloadRetryInterval + } + + var last distribution.CatalogSnapshot + for attempt := 0; attempt < attempts; attempt++ { + snapshot, err := s.catalog.Snapshot(ctx) + if err != nil { + return distribution.CatalogSnapshot{}, grpcStatusErrorf(codes.Internal, "reload route catalog: %v", err) + } + if snapshot.Version >= minVersion { + return snapshot, nil + } + last = snapshot + if attempt == attempts-1 { + break + } + if err := waitWithContext(ctx, interval); err != nil { + return distribution.CatalogSnapshot{}, err + } + } + return distribution.CatalogSnapshot{}, grpcStatusErrorf( + codes.Internal, + "catalog split committed but local snapshot is stale: got %d, want at least %d", + last.Version, + minVersion, + ) +} + func (s *DistributionServer) loadCatalogSnapshotAtVersion( ctx context.Context, readTS uint64, @@ -519,6 +1133,313 @@ func (s *DistributionServer) applyEngineSnapshot(snapshot distribution.CatalogSn return nil } +func (s *DistributionServer) updateSplitJobViaCoordinator( + ctx context.Context, + jobID uint64, + transition func(distribution.SplitJob) (distribution.SplitJob, error), +) error { + expected, readTS, err := s.catalog.LiveSplitJobForUpdate(ctx, jobID) + if err != nil { + return splitJobCatalogStatusError(err) + } + next, err := transition(expected) + if err != nil { + return splitJobCatalogStatusError(err) + } + if distribution.SplitJobsEquivalent(expected, next) { + return nil + } + encoded, err := distribution.EncodeSplitJob(next) + if err != nil { + return splitJobCatalogStatusError(err) + } + key := distribution.CatalogSplitJobKey(jobID) + if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: []*kv.Elem[kv.OP]{{ + Op: kv.Put, + Key: key, + Value: encoded, + }}, + IsTxn: true, + StartTS: readTS, + ReadKeys: [][]byte{key}, + }); err != nil { + return splitJobCoordinatorStatusError(err) + } + return nil +} + +func (s *DistributionServer) completeSplitJobTargetPromotionViaCoordinator( + ctx context.Context, + expectedVersion uint64, + expected distribution.SplitJob, + nowMs int64, +) (distribution.CatalogSnapshot, distribution.SplitJob, error) { + if err := s.requirePromotionCompletionDeps(); err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, err + } + snapshot, current, alreadyDone, err := s.loadPromotionCompletionCandidate(ctx, expectedVersion, expected, nowMs) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, err + } + if alreadyDone { + return snapshot, current, nil + } + completion, err := distribution.CompleteTargetPromotionState(current, snapshot.Routes, nowMs) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, splitJobPromotionStatusError(err) + } + return s.dispatchPromotionCompletion(ctx, snapshot, expectedVersion, expected.JobID, completion) +} + +func (s *DistributionServer) requirePromotionCompletionDeps() error { + if s.catalog == nil { + return grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error()) + } + if s.coordinator == nil { + return grpcStatusError(codes.FailedPrecondition, errDistributionCoordinatorRequired.Error()) + } + return nil +} + +func (s *DistributionServer) loadPromotionCompletionCandidate( + ctx context.Context, + expectedVersion uint64, + expected distribution.SplitJob, + nowMs int64, +) (distribution.CatalogSnapshot, distribution.SplitJob, bool, error) { + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, false, err + } + current, found, err := s.catalog.SplitJobAt(ctx, expected.JobID, snapshot.ReadTS) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, false, splitJobPromotionStatusError(err) + } + if !found { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, false, splitJobPromotionStatusError(distribution.ErrCatalogSplitJobConflict) + } + if snapshot.Version == expectedVersion && distribution.SplitJobsEquivalent(current, expected) { + return snapshot, current, false, nil + } + completion, err := completedPromotionRetry(snapshot.Routes, expected, current, nowMs) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, false, splitJobPromotionStatusError(err) + } + if completion.Job.TargetPromotionDone { + return snapshot, current, !completion.Changed, nil + } + if snapshot.Version != expectedVersion { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, false, splitJobPromotionStatusError(distribution.ErrCatalogVersionMismatch) + } + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, false, splitJobPromotionStatusError(distribution.ErrCatalogSplitJobConflict) +} + +func completedPromotionRetry( + routes []distribution.RouteDescriptor, + expected distribution.SplitJob, + current distribution.SplitJob, + nowMs int64, +) (distribution.TargetPromotionCompletion, error) { + matches, err := splitJobPromotionMatchesExpected(expected, current) + if err != nil { + return distribution.TargetPromotionCompletion{}, errors.WithStack(err) + } + if !matches { + return distribution.TargetPromotionCompletion{}, nil + } + completion, err := distribution.CompleteTargetPromotionState(current, routes, nowMs) + if err != nil { + return distribution.TargetPromotionCompletion{}, errors.WithStack(err) + } + return completion, nil +} + +func (s *DistributionServer) dispatchPromotionCompletion( + ctx context.Context, + snapshot distribution.CatalogSnapshot, + expectedVersion uint64, + jobID uint64, + completion distribution.TargetPromotionCompletion, +) (distribution.CatalogSnapshot, distribution.SplitJob, error) { + if !completion.Changed { + return snapshot, completion.Job, nil + } + commitTS, err := s.nextPromotionCompleteCommitTS(ctx, snapshot.ReadTS) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, err + } + if completion.Job.PromotionCompletedTS == 0 { + completion.Job.PromotionCompletedTS = commitTS + } + ops, err := s.buildPromotionCompleteOps(expectedVersion, completion) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, splitJobPromotionStatusError(err) + } + jobKey := distribution.CatalogSplitJobKey(jobID) + if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: ops, + IsTxn: true, + StartTS: snapshot.ReadTS, + CommitTS: commitTS, + ReadKeys: [][]byte{ + distribution.CatalogVersionKey(), + jobKey, + }, + }); err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, splitJobCoordinatorStatusError(err) + } + loaded, err := s.loadCatalogSnapshotAtLeastVersion(ctx, expectedVersion+1) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.SplitJob{}, err + } + return loaded, completion.Job, nil +} + +func (s *DistributionServer) nextPromotionCompleteCommitTS(ctx context.Context, readTS uint64) (uint64, error) { + if readTS == math.MaxUint64 { + return 0, grpcStatusError(codes.Internal, distribution.ErrCatalogVersionOverflow.Error()) + } + commitTS, err := kv.NextTimestampAfterThrough(ctx, s.coordinator, readTS, "allocate promotion completion timestamp") + if err != nil { + return 0, grpcStatusErrorf(codes.FailedPrecondition, "allocate promotion completion timestamp: %v", err) + } + if commitTS <= readTS { + return 0, grpcStatusError(codes.Internal, "promotion completion timestamp did not advance") + } + return commitTS, nil +} + +func (s *DistributionServer) buildPromotionCompleteOps( + expectedVersion uint64, + completion distribution.TargetPromotionCompletion, +) ([]*kv.Elem[kv.OP], error) { + if expectedVersion == math.MaxUint64 { + return nil, errors.WithStack(distribution.ErrCatalogVersionOverflow) + } + ops := make([]*kv.Elem[kv.OP], 0, len(completion.ClearedRouteIDs)+promotionCompleteBaseOpCount) + for _, routeID := range completion.ClearedRouteIDs { + route, ok := promotionCompleteRouteByID(completion.Routes, routeID) + if !ok { + return nil, errors.WithStack(distribution.ErrMigrationPromotionTargetAbsent) + } + encoded, err := distribution.EncodeRouteDescriptorForCatalogWrite(route, s.catalog.AllowsRouteDescriptorV2Writes()) + if err != nil { + return nil, errors.WithStack(err) + } + ops = append(ops, &kv.Elem[kv.OP]{ + Op: kv.Put, + Key: distribution.CatalogRouteKey(route.RouteID), + Value: encoded, + }) + } + ops = append(ops, &kv.Elem[kv.OP]{ + Op: kv.Put, + Key: distribution.CatalogVersionKey(), + Value: distribution.EncodeCatalogVersion(expectedVersion + 1), + }) + encodedJob, err := distribution.EncodeSplitJob(completion.Job) + if err != nil { + return nil, errors.WithStack(err) + } + ops = append(ops, &kv.Elem[kv.OP]{ + Op: kv.Put, + Key: distribution.CatalogSplitJobKey(completion.Job.JobID), + Value: encodedJob, + }) + return ops, nil +} + +func promotionCompleteRouteByID(routes []distribution.RouteDescriptor, routeID uint64) (distribution.RouteDescriptor, bool) { + for _, route := range routes { + if route.RouteID == routeID { + return route, true + } + } + return distribution.RouteDescriptor{}, false +} + +func splitJobPromotionMatchesExpected(expected, current distribution.SplitJob) (bool, error) { + if expected.TargetPromotionDone || + !current.TargetPromotionDone || + current.PromotionCompletedTS == 0 || + (current.Phase != expected.Phase && current.Phase != distribution.SplitJobPhaseDone) { + return false, nil + } + normalized := distribution.CloneSplitJob(current) + normalized.Phase = expected.Phase + normalized.TargetPromotionDone = expected.TargetPromotionDone + normalized.PromotionCompletedTS = expected.PromotionCompletedTS + normalized.TerminalAtMs = expected.TerminalAtMs + normalized.UpdatedAtMs = expected.UpdatedAtMs + expectedRaw, err := distribution.EncodeSplitJob(expected) + if err != nil { + return false, errors.WithStack(err) + } + normalizedRaw, err := distribution.EncodeSplitJob(normalized) + if err != nil { + return false, errors.WithStack(err) + } + return bytes.Equal(expectedRaw, normalizedRaw), nil +} + +func (s *DistributionServer) verifyKnownTargetGroup(groupID uint64) error { + if len(s.knownRaftGroups) == 0 { + return grpcStatusError(codes.FailedPrecondition, errDistributionRaftGroupsNotKnown.Error()) + } + if _, ok := s.knownRaftGroups[groupID]; !ok { + return grpcStatusError(codes.InvalidArgument, errDistributionUnknownTargetGroup.Error()) + } + return nil +} + +func (s *DistributionServer) createSplitJobViaCoordinator(ctx context.Context, readTS uint64, job distribution.SplitJob) error { + if job.JobID == math.MaxUint64 { + return splitJobCatalogStatusError(distribution.ErrCatalogSplitJobIDOverflow) + } + encoded, err := distribution.EncodeSplitJob(job) + if err != nil { + return splitJobCatalogStatusError(err) + } + jobKey := distribution.CatalogSplitJobKey(job.JobID) + nextIDKey := distribution.CatalogNextSplitJobIDKey() + if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: []*kv.Elem[kv.OP]{ + { + Op: kv.Put, + Key: jobKey, + Value: encoded, + }, + { + Op: kv.Put, + Key: nextIDKey, + Value: distribution.EncodeCatalogNextSplitJobID(job.JobID + 1), + }, + }, + IsTxn: true, + StartTS: readTS, + ReadKeys: [][]byte{ + jobKey, + nextIDKey, + distribution.CatalogVersionKey(), + distribution.CatalogRouteKey(job.SourceRouteID), + }, + }); err != nil { + return splitJobCoordinatorStatusError(err) + } + return nil +} + +func (s *DistributionServer) verifySplitMigrationCapability(ctx context.Context) error { + if s.migrationCapabilityGate == nil { + return grpcStatusError(codes.FailedPrecondition, errDistributionClusterNotReady.Error()) + } + if err := s.migrationCapabilityGate(ctx); err != nil { + return err + } + return nil +} + func validateExpectedCatalogVersion(currentVersion, expectedVersion uint64) error { if currentVersion != expectedVersion { return grpcStatusError(codes.Aborted, errDistributionCatalogConflict.Error()) @@ -578,6 +1499,29 @@ func splitJobReadFenceKeys(jobs []distribution.SplitJob) [][]byte { return readKeys } +func (s *DistributionServer) rejectLiveSplitJob(ctx context.Context, snapshot distribution.CatalogSnapshot, parent distribution.RouteDescriptor) error { + jobs, err := s.catalog.ListSplitJobsAt(ctx, snapshot.ReadTS) + if err != nil { + return grpcStatusErrorf(codes.Internal, "load split jobs: %v", err) + } + liveJobs := 0 + for _, job := range jobs { + if !splitJobIsLive(job) { + continue + } + liveJobs++ + for _, interval := range liveSplitJobIntervals(job, snapshot.Routes) { + if routeRangeIntersects(parent.Start, parent.End, interval.start, interval.end) { + return grpcStatusError(codes.Aborted, distribution.ErrSplitJobOverlap.Error()) + } + } + } + if liveJobs > 0 { + return grpcStatusError(codes.ResourceExhausted, distribution.ErrTooManyInFlightSplitJobs.Error()) + } + return nil +} + func splitJobIsLive(job distribution.SplitJob) bool { return job.Phase != distribution.SplitJobPhaseDone && job.Phase != distribution.SplitJobPhaseAbandoned } @@ -623,6 +1567,218 @@ func routeRangeIntersects(aStart, aEnd, bStart, bEnd []byte) bool { return true } +func splitJobPassesListFilter(job distribution.SplitJob, sinceTerminalAtMs uint64, phaseFilter string) bool { + if sinceTerminalAtMs > 0 && job.TerminalAtMs > 0 && uint64(job.TerminalAtMs) < sinceTerminalAtMs { + return false + } + if phaseFilter == "" { + return true + } + return normalizeSplitJobPhaseFilter(pb.SplitJobPhase(job.Phase).String()) == phaseFilter || + normalizeSplitJobPhaseFilter(strings.TrimPrefix(pb.SplitJobPhase(job.Phase).String(), "SPLIT_JOB_PHASE_")) == phaseFilter +} + +func normalizeSplitJobPhaseFilter(phase string) string { + phase = strings.TrimSpace(phase) + if phase == "" { + return "" + } + phase = strings.ReplaceAll(phase, "-", "_") + phase = strings.ReplaceAll(phase, " ", "_") + return strings.ToUpper(phase) +} + +func validSplitJobPhaseFilter(phase string) bool { + for _, candidate := range pb.SplitJobPhase_name { + full := normalizeSplitJobPhaseFilter(candidate) + short := normalizeSplitJobPhaseFilter(strings.TrimPrefix(candidate, "SPLIT_JOB_PHASE_")) + if phase == full || phase == short { + return true + } + } + return false +} + +func splitJobCatalogStatusError(err error) error { + switch { + case errors.Is(err, distribution.ErrCatalogSplitJobIDRequired): + return grpcStatusError(codes.InvalidArgument, distribution.ErrCatalogSplitJobIDRequired.Error()) + case errors.Is(err, distribution.ErrCatalogSplitJobNotFound): + return grpcStatusError(codes.NotFound, distribution.ErrCatalogSplitJobNotFound.Error()) + case errors.Is(err, distribution.ErrCatalogSplitJobCannotRetry), + errors.Is(err, distribution.ErrCatalogSplitJobCannotAbandon): + return grpcStatusError(codes.FailedPrecondition, err.Error()) + case errors.Is(err, distribution.ErrCatalogSplitJobConflict): + return grpcStatusError(codes.Aborted, distribution.ErrCatalogSplitJobConflict.Error()) + default: + return grpcStatusErrorf(codes.Internal, "split job catalog: %v", err) + } +} + +func splitMigrationRangeStatusError(err error) error { + switch { + case errors.Is(err, distribution.ErrMigrationReservedRange), + errors.Is(err, distribution.ErrMigrationInvalidRoute): + return grpcStatusError(codes.InvalidArgument, err.Error()) + default: + return grpcStatusErrorf(codes.Internal, "validate split migration range: %v", err) + } +} + +func splitJobCoordinatorStatusError(err error) error { + switch { + case errors.Is(err, store.ErrWriteConflict): + return grpcStatusError(codes.Aborted, distribution.ErrCatalogSplitJobConflict.Error()) + case splitJobCoordinatorLeadershipError(err): + return grpcStatusError(codes.FailedPrecondition, errDistributionNotLeader.Error()) + default: + return grpcStatusErrorf(codes.Internal, "commit split job mutation: %v", err) + } +} + +func splitJobPromotionStatusError(err error) error { + switch { + case errors.Is(err, distribution.ErrCatalogVersionMismatch), + errors.Is(err, distribution.ErrCatalogSplitJobConflict), + errors.Is(err, store.ErrWriteConflict): + return grpcStatusError(codes.Aborted, err.Error()) + case errors.Is(err, distribution.ErrMigrationPromotionNotReady), + errors.Is(err, distribution.ErrMigrationPromotionTargetAbsent): + return grpcStatusError(codes.FailedPrecondition, err.Error()) + case errors.Is(err, distribution.ErrMigrationInvalidRoute), + errors.Is(err, distribution.ErrCatalogRouteV2WriteDisabled): + return grpcStatusError(codes.InvalidArgument, err.Error()) + default: + return splitJobCatalogStatusError(err) + } +} + +func splitJobListPage(jobs []distribution.SplitJob, req *pb.ListSplitJobsRequest, phaseFilter string) (*pb.ListSplitJobsResponse, error) { + filtered := make([]distribution.SplitJob, 0, len(jobs)) + for _, job := range jobs { + if splitJobPassesListFilter(job, req.GetSinceTerminalAtMs(), phaseFilter) { + filtered = append(filtered, job) + } + } + sortSplitJobsForList(filtered) + + start, err := splitJobListStartIndex(filtered, req.GetPageCursor()) + if err != nil { + return nil, err + } + end := start + listSplitJobsDefaultPageSize + if end > len(filtered) { + end = len(filtered) + } + + resp := &pb.ListSplitJobsResponse{ + Jobs: make([]*pb.SplitJob, 0, end-start), + } + for _, job := range filtered[start:end] { + resp.Jobs = append(resp.Jobs, distribution.SplitJobToProto(job)) + } + if end < len(filtered) { + resp.NextPageCursor = encodeSplitJobListCursor(filtered[end-1]) + } + return resp, nil +} + +func sortSplitJobsForList(jobs []distribution.SplitJob) { + sort.Slice(jobs, func(i, j int) bool { + left, right := jobs[i], jobs[j] + leftLive := left.TerminalAtMs <= 0 + rightLive := right.TerminalAtMs <= 0 + if leftLive != rightLive { + return leftLive + } + if leftLive { + if left.UpdatedAtMs != right.UpdatedAtMs { + return left.UpdatedAtMs > right.UpdatedAtMs + } + return left.JobID > right.JobID + } + if left.TerminalAtMs != right.TerminalAtMs { + return left.TerminalAtMs > right.TerminalAtMs + } + return left.JobID > right.JobID + }) +} + +func splitJobListStartIndex(jobs []distribution.SplitJob, cursor []byte) (int, error) { + if len(cursor) == 0 { + return 0, nil + } + terminalAtMs, jobID, err := decodeSplitJobListCursor(cursor) + if err != nil { + return 0, err + } + for i, job := range jobs { + if job.JobID == jobID && splitJobListCursorTerminalAtMs(job) == terminalAtMs { + return i + 1, nil + } + } + return 0, grpcStatusError(codes.InvalidArgument, "split job page cursor does not match the filtered result set") +} + +func encodeSplitJobListCursor(job distribution.SplitJob) []byte { + cursor := make([]byte, splitJobListCursorEncodedBytes) + cursor[0] = splitJobListCursorVersion + binary.BigEndian.PutUint64( + cursor[splitJobListCursorTerminalOff:splitJobListCursorJobIDOff], + splitJobListCursorTerminalAtMs(job), + ) + binary.BigEndian.PutUint64(cursor[splitJobListCursorJobIDOff:], job.JobID) + return cursor +} + +func decodeSplitJobListCursor(cursor []byte) (uint64, uint64, error) { + if len(cursor) != splitJobListCursorEncodedBytes || cursor[0] != splitJobListCursorVersion { + return 0, 0, grpcStatusError(codes.InvalidArgument, "invalid split job page cursor") + } + terminalAtMs := binary.BigEndian.Uint64(cursor[splitJobListCursorTerminalOff:splitJobListCursorJobIDOff]) + jobID := binary.BigEndian.Uint64(cursor[splitJobListCursorJobIDOff:]) + if jobID == 0 { + return 0, 0, grpcStatusError(codes.InvalidArgument, "invalid split job page cursor") + } + return terminalAtMs, jobID, nil +} + +func splitJobListCursorTerminalAtMs(job distribution.SplitJob) uint64 { + if job.TerminalAtMs <= 0 { + return 0 + } + return uint64(job.TerminalAtMs) +} + +func splitJobCoordinatorLeadershipError(err error) bool { + if err == nil { + return false + } + if errors.Is(err, kv.ErrLeaderNotFound) || + errors.Is(err, raftengine.ErrNotLeader) || + errors.Is(err, raftengine.ErrLeadershipLost) || + errors.Is(err, raftengine.ErrLeadershipTransferInProgress) { + return true + } + return hasSplitJobLeaderErrorSuffix(err.Error()) +} + +var splitJobLeaderErrorPhrases = []string{ + "not leader", + "leader not found", + "leadership lost", + "leadership transfer in progress", +} + +func hasSplitJobLeaderErrorSuffix(msg string) bool { + for _, phrase := range splitJobLeaderErrorPhrases { + if len(msg) >= len(phrase) && strings.EqualFold(msg[len(msg)-len(phrase):], phrase) { + return true + } + } + return false +} + func splitCatalogRoutes( parent distribution.RouteDescriptor, splitKey []byte, diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index 46a6502e0..8ad36bf96 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -4,15 +4,20 @@ import ( "bytes" "context" "encoding/binary" + "errors" + "io" "testing" "time" "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/fskeys" + "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" + crdberrors "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" + "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -154,63 +159,1375 @@ func TestWithCatalogReloadRetryPolicy_OverridesDefaults(t *testing.T) { require.Equal(t, 5*time.Millisecond, s.reloadRetry.interval) } +func TestDistributionServerPinReadTSWithoutTrackerReturnsReleasableToken(t *testing.T) { + t.Parallel() + + s := NewDistributionServer(distribution.NewEngine(), nil) + token := s.pinReadTS(10) + require.NotNil(t, token) + require.NotPanics(t, func() { + token.Release() + }) + + var nilServer *DistributionServer + nilToken := nilServer.pinReadTS(10) + require.NotNil(t, nilToken) + require.NotPanics(t, func() { + nilToken.Release() + }) +} + func TestDistributionServerListRoutes_ReadsDurableCatalog(t *testing.T) { t.Parallel() ctx := context.Background() - catalog := distribution.NewCatalogStore(store.NewMVCCStore(), distribution.WithCatalogRouteDescriptorV2Writes(true)) - saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{ - { - RouteID: 2, - Start: []byte("m"), - End: nil, - GroupID: 2, - State: distribution.RouteStateWriteFenced, - ParentRouteID: 1, - StagedVisibilityActive: true, - MigrationJobID: 42, - MinWriteTSExclusive: 99, - SplitAtHLC: 100, - }, - { - RouteID: 1, - Start: []byte(""), - End: []byte("m"), - GroupID: 1, - State: distribution.RouteStateActive, - ParentRouteID: 0, + catalog := distribution.NewCatalogStore(store.NewMVCCStore(), distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{ + { + RouteID: 2, + Start: []byte("m"), + End: nil, + GroupID: 2, + State: distribution.RouteStateWriteFenced, + ParentRouteID: 1, + StagedVisibilityActive: true, + MigrationJobID: 42, + MinWriteTSExclusive: 99, + SplitAtHLC: 100, + }, + { + RouteID: 1, + Start: []byte(""), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + ParentRouteID: 0, + }, + }) + require.NoError(t, err) + + s := NewDistributionServer(distribution.NewEngine(), catalog) + resp, err := s.ListRoutes(ctx, &pb.ListRoutesRequest{}) + require.NoError(t, err) + + require.Equal(t, saved.Version, resp.CatalogVersion) + require.Len(t, resp.Routes, 2) + require.Equal(t, uint64(1), resp.Routes[0].RouteId) + require.Equal(t, []byte(""), resp.Routes[0].Start) + require.Equal(t, []byte("m"), resp.Routes[0].End) + require.Equal(t, uint64(1), resp.Routes[0].RaftGroupId) + require.Equal(t, pb.RouteState_ROUTE_STATE_ACTIVE, resp.Routes[0].State) + require.Equal(t, uint64(2), resp.Routes[1].RouteId) + require.Nil(t, resp.Routes[1].End) + require.Equal(t, pb.RouteState_ROUTE_STATE_WRITE_FENCED, resp.Routes[1].State) + require.True(t, resp.Routes[1].StagedVisibilityActive) + require.Equal(t, uint64(42), resp.Routes[1].MigrationJobId) + require.Equal(t, uint64(99), resp.Routes[1].MinWriteTsExclusive) + require.Equal(t, uint64(100), resp.Routes[1].SplitAtHlc) +} + +func TestDistributionServerListRoutes_RequiresCatalog(t *testing.T) { + t.Parallel() + + s := NewDistributionServer(distribution.NewEngine(), nil) + _, err := s.ListRoutes(context.Background(), &pb.ListRoutesRequest{}) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, errDistributionCatalogNotConfigured.Error()) +} + +func TestDistributionServerGetSplitMigrationCapabilityReportsNotReadyUntilRunnerReady(t *testing.T) { + t.Parallel() + + s := NewDistributionServer(distribution.NewEngine(), nil) + resp, err := s.GetSplitMigrationCapability(context.Background(), &pb.GetSplitMigrationCapabilityRequest{}) + require.NoError(t, err) + require.False(t, resp.GetMigrationCapable()) + require.NotContains(t, resp.GetCapabilities(), splitMigrationCapabilityV2) +} + +func TestDistributionServerGetSplitMigrationCapabilityReportsReadyWhenRunnerReady(t *testing.T) { + t.Parallel() + + s := NewDistributionServer(distribution.NewEngine(), nil, WithSplitJobRunnerReady()) + resp, err := s.GetSplitMigrationCapability(context.Background(), &pb.GetSplitMigrationCapabilityRequest{}) + require.NoError(t, err) + require.True(t, resp.GetMigrationCapable()) + require.Contains(t, resp.GetCapabilities(), splitMigrationCapabilityV2) +} + +func TestDistributionServerGetSplitMigrationCapabilityRespectsReadinessGate(t *testing.T) { + t.Parallel() + + s := NewDistributionServer( + distribution.NewEngine(), + nil, + WithSplitJobRunnerReady(), + WithSplitJobRunnerReadinessGate(func(context.Context) error { + return status.Error(codes.FailedPrecondition, "migration opcode disabled") + }), + ) + resp, err := s.GetSplitMigrationCapability(context.Background(), &pb.GetSplitMigrationCapabilityRequest{}) + require.NoError(t, err) + require.False(t, resp.GetMigrationCapable()) + require.NotContains(t, resp.GetCapabilities(), splitMigrationCapabilityV2) +} + +func TestDistributionServerStartSplitMigration_FailsClosedUntilCapabilityGate(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + + _, err := s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: 1, + RouteId: 1, + SplitKey: []byte("g"), + TargetGroupId: 2, + }) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, errDistributionClusterNotReady.Error()) + require.Zero(t, coordinator.dispatchCalls) + + jobs, listErr := catalog.ListSplitJobs(ctx) + require.NoError(t, listErr) + require.Empty(t, jobs) +} + +func TestDistributionServerStartSplitMigrationReturnsCapabilityGateError(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithSplitMigrationCapabilityGate(func(context.Context) error { + return status.Error(codes.Unavailable, "split migration capability not ready") + }), + ) + + _, err := s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: 1, + RouteId: 1, + SplitKey: []byte("g"), + TargetGroupId: 2, + }) + require.Error(t, err) + require.Equal(t, codes.Unavailable, status.Code(err)) + require.ErrorContains(t, err, "split migration capability not ready") + require.Zero(t, coordinator.dispatchCalls) +} + +func TestDistributionServerStartSplitMigrationCapabilityGateRunsOutsideCatalogLock(t *testing.T) { + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + ParentRouteID: 0, + }}) + require.NoError(t, err) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + gateEntered := make(chan struct{}) + releaseGate := make(chan struct{}) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithDistributionKnownRaftGroups(1, 2), + WithSplitMigrationCapabilityGate(func(context.Context) error { + close(gateEntered) + <-releaseGate + return status.Error(codes.Unavailable, "split migration capability not ready") + }), + ) + + startDone := make(chan error, 1) + go func() { + _, err := s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: []byte("g"), + TargetGroupId: 2, + }) + startDone <- err + }() + <-gateEntered + + splitDone := make(chan error, 1) + go func() { + _, err := s.SplitRange(ctx, &pb.SplitRangeRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: []byte("g"), + }) + splitDone <- err + }() + select { + case err := <-splitDone: + require.NoError(t, err) + case <-time.After(500 * time.Millisecond): + t.Fatal("SplitRange blocked behind StartSplitMigration capability gate") + } + + close(releaseGate) + err = <-startDone + require.Error(t, err) + require.Equal(t, codes.Unavailable, status.Code(err)) +} + +func TestDistributionServerStartSplitMigration_CreatesPlannedJobWhenGateOpen(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + ParentRouteID: 0, + }}) + require.NoError(t, err) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + gateCalls := 0 + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithDistributionKnownRaftGroups(1, 2), + WithSplitMigrationCapabilityGate(func(context.Context) error { + gateCalls++ + return nil + }), + ) + + resp, err := s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: []byte("g"), + TargetGroupId: 2, + }) + require.NoError(t, err) + require.Equal(t, saved.Version, resp.CatalogVersion) + require.Equal(t, uint64(1), resp.JobId) + require.Equal(t, 1, gateCalls) + require.Equal(t, 1, coordinator.dispatchCalls) + require.ElementsMatch(t, []string{ + string(distribution.CatalogSplitJobKey(1)), + string(distribution.CatalogNextSplitJobIDKey()), + string(distribution.CatalogVersionKey()), + string(distribution.CatalogRouteKey(1)), + }, byteSliceStrings(coordinator.lastReadKeys)) + + jobs, err := catalog.ListSplitJobs(ctx) + require.NoError(t, err) + require.Len(t, jobs, 1) + job := jobs[0] + require.Equal(t, uint64(1), job.JobID) + require.Equal(t, uint64(1), job.SourceRouteID) + require.Equal(t, []byte("g"), job.SplitKey) + require.Equal(t, uint64(2), job.TargetGroupID) + require.Equal(t, distribution.SplitJobPhasePlanned, job.Phase) + require.NotZero(t, job.StartedAtMs) + require.NotZero(t, job.UpdatedAtMs) + require.NotEmpty(t, job.BracketProgress) + next, err := catalog.NextSplitJobID(ctx) + require.NoError(t, err) + require.Equal(t, uint64(2), next) +} + +func TestDistributionServerStartSplitMigration_NormalizesFilesystemChunkSplitKey(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: fskeys.ChunkRouteKey(11, 21), + End: fskeys.ChunkRouteKey(11, 23), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + require.NoError(t, err) + + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(newDistributionCoordinatorStub(baseStore, true)), + WithDistributionKnownRaftGroups(1, 2), + WithSplitMigrationCapabilityGate(func(context.Context) error { return nil }), + ) + wantBoundary := fskeys.ChunkRouteKey(11, 22) + + resp, err := s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: fskeys.ChunkKey(11, 22, 99), + TargetGroupId: 2, + }) + require.NoError(t, err) + require.Equal(t, uint64(1), resp.JobId) + + jobs, err := catalog.ListSplitJobs(ctx) + require.NoError(t, err) + require.Len(t, jobs, 1) + require.Equal(t, wantBoundary, jobs[0].SplitKey) + require.NotEqual(t, fskeys.ChunkKey(11, 22, 99), jobs[0].SplitKey) +} + +func TestDistributionServerStartSplitMigration_RejectsUnknownTargetGroup(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + ParentRouteID: 0, + }}) + require.NoError(t, err) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithDistributionKnownRaftGroups(1, 2), + WithSplitMigrationCapabilityGate(func(context.Context) error { return nil }), + ) + + _, err = s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: []byte("g"), + TargetGroupId: 3, + }) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + require.ErrorContains(t, err, errDistributionUnknownTargetGroup.Error()) + require.Zero(t, coordinator.dispatchCalls) +} + +func TestDistributionServerStartSplitMigration_RejectsNonActiveSourceRoute(t *testing.T) { + t.Parallel() + + ctx := context.Background() + for _, tc := range []struct { + name string + state distribution.RouteState + }{ + {name: "write_fenced", state: distribution.RouteStateWriteFenced}, + {name: "migrating_source", state: distribution.RouteStateMigratingSource}, + {name: "migrating_target", state: distribution.RouteStateMigratingTarget}, + } { + t.Run(tc.name, func(t *testing.T) { + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: tc.state, + ParentRouteID: 0, + }}) + require.NoError(t, err) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithDistributionKnownRaftGroups(1, 2), + WithSplitMigrationCapabilityGate(func(context.Context) error { return nil }), + ) + + _, err = s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: []byte("g"), + TargetGroupId: 2, + }) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, errDistributionSourceRouteNotActive.Error()) + require.Zero(t, coordinator.dispatchCalls) + + jobs, listErr := catalog.ListSplitJobs(ctx) + require.NoError(t, listErr) + require.Empty(t, jobs) + }) + } +} + +func TestDistributionServerStartSplitMigration_RejectsSecondLiveJob(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + ParentRouteID: 0, + }, + { + RouteID: 2, + Start: []byte("m"), + End: nil, + GroupID: 2, + State: distribution.RouteStateActive, + ParentRouteID: 0, + }, + }) + require.NoError(t, err) + require.NoError(t, catalog.CreateSplitJob(ctx, distribution.SplitJob{ + JobID: 1, + SourceRouteID: 2, + SplitKey: []byte("t"), + TargetGroupID: 3, + Phase: distribution.SplitJobPhaseBackfill, + StartedAtMs: 1000, + UpdatedAtMs: 1000, + })) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithDistributionKnownRaftGroups(1, 2, 3, 4), + WithSplitMigrationCapabilityGate(func(context.Context) error { return nil }), + ) + + _, err = s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: []byte("g"), + TargetGroupId: 4, + }) + require.Error(t, err) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) + require.ErrorContains(t, err, distribution.ErrTooManyInFlightSplitJobs.Error()) + require.Zero(t, coordinator.dispatchCalls) +} + +func TestDistributionServerStartSplitMigration_RejectsReservedMigrationRange(t *testing.T) { + t.Parallel() + + ctx := context.Background() + for _, tc := range []struct { + name string + splitKey []byte + end []byte + }{ + {name: "dist catalog", splitKey: []byte("!dist|"), end: nil}, + {name: "dist staged catalog", splitKey: []byte("!dist|migstage|"), end: nil}, + {name: "migration staged", splitKey: []byte("!migstage|ready|1"), end: nil}, + {name: "migration tracker", splitKey: []byte("!migwrite|"), end: nil}, + {name: "migration fence", splitKey: []byte("!migfence|"), end: nil}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte(""), + End: tc.end, + GroupID: 1, + State: distribution.RouteStateActive, + ParentRouteID: 0, + }}) + require.NoError(t, err) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithDistributionKnownRaftGroups(1, 2), + WithSplitMigrationCapabilityGate(func(context.Context) error { return nil }), + ) + + _, err = s.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: saved.Version, + RouteId: 1, + SplitKey: tc.splitKey, + TargetGroupId: 2, + }) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + require.ErrorContains(t, err, distribution.ErrMigrationReservedRange.Error()) + require.Zero(t, coordinator.dispatchCalls) + + jobs, listErr := catalog.ListSplitJobs(ctx) + require.NoError(t, listErr) + require.Empty(t, jobs) + }) + } +} + +func TestDistributionServerSplitJobRPCs_ReadAndListCatalogJobs(t *testing.T) { + t.Parallel() + + ctx := context.Background() + catalog := distribution.NewCatalogStore(store.NewMVCCStore()) + live := sampleDistributionSplitJob(10) + live.Phase = distribution.SplitJobPhaseDeltaCopy + require.NoError(t, catalog.CreateSplitJob(ctx, live)) + history := sampleDistributionSplitJob(11) + history.Phase = distribution.SplitJobPhaseDone + history.TerminalAtMs = 2000 + require.NoError(t, catalog.CreateSplitJob(ctx, history)) + require.NoError(t, catalog.MoveSplitJobToHistory(ctx, history, history)) + + s := NewDistributionServer(distribution.NewEngine(), catalog) + got, err := s.GetSplitJob(ctx, &pb.GetSplitJobRequest{JobId: live.JobID}) + require.NoError(t, err) + require.Equal(t, live.JobID, got.Job.JobId) + require.Equal(t, pb.SplitJobPhase_SPLIT_JOB_PHASE_DELTA_COPY, got.Job.Phase) + + listAll, err := s.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{}) + require.NoError(t, err) + require.Len(t, listAll.Jobs, 2) + require.Equal(t, []uint64{live.JobID, history.JobID}, []uint64{listAll.Jobs[0].JobId, listAll.Jobs[1].JobId}) + + listDone, err := s.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{Phase: "done"}) + require.NoError(t, err) + require.Len(t, listDone.Jobs, 1) + require.Equal(t, history.JobID, listDone.Jobs[0].JobId) + + _, err = s.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{Phase: "not-a-phase"}) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) +} + +func TestDistributionServerListSplitJobs_PaginatesNewestHistory(t *testing.T) { + t.Parallel() + + ctx := context.Background() + catalog := distribution.NewCatalogStore(store.NewMVCCStore()) + for jobID, terminalAtMs := uint64(1), int64(1001); jobID <= 205; jobID, terminalAtMs = jobID+1, terminalAtMs+1 { + job := sampleDistributionSplitJob(jobID) + job.Phase = distribution.SplitJobPhaseDone + job.TerminalAtMs = terminalAtMs + job.UpdatedAtMs = job.TerminalAtMs + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + require.NoError(t, catalog.MoveSplitJobToHistory(ctx, job, job)) + } + + s := NewDistributionServer(distribution.NewEngine(), catalog) + first, err := s.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{}) + require.NoError(t, err) + require.Len(t, first.Jobs, listSplitJobsDefaultPageSize) + require.NotEmpty(t, first.NextPageCursor) + require.Equal(t, uint64(205), first.Jobs[0].JobId) + require.Equal(t, uint64(6), first.Jobs[len(first.Jobs)-1].JobId) + + second, err := s.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{PageCursor: first.NextPageCursor}) + require.NoError(t, err) + require.Empty(t, second.NextPageCursor) + require.Equal(t, []uint64{5, 4, 3, 2, 1}, splitJobIDs(second.Jobs)) +} + +func TestDistributionServerListSplitJobs_RejectsInvalidCursor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + catalog := distribution.NewCatalogStore(store.NewMVCCStore()) + job := sampleDistributionSplitJob(1) + job.Phase = distribution.SplitJobPhaseDone + job.TerminalAtMs = 1000 + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + require.NoError(t, catalog.MoveSplitJobToHistory(ctx, job, job)) + + s := NewDistributionServer(distribution.NewEngine(), catalog) + _, err := s.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{PageCursor: []byte("bad")}) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + + missing := sampleDistributionSplitJob(999) + missing.TerminalAtMs = 9999 + _, err = s.ListSplitJobs(ctx, &pb.ListSplitJobsRequest{PageCursor: encodeSplitJobListCursor(missing)}) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) +} + +func TestDistributionServerRetrySplitJob_UsesCoordinatorCAS(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + job := sampleDistributionSplitJob(12) + job.Phase = distribution.SplitJobPhaseFailed + job.RetryPhase = distribution.SplitJobPhaseFence + job.LastError = "retry me" + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + _, err := s.RetrySplitJob(ctx, &pb.RetrySplitJobRequest{JobId: job.JobID}) + require.NoError(t, err) + require.Equal(t, 1, coordinator.dispatchCalls) + + got, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseFence, got.Phase) + require.Equal(t, distribution.SplitJobPhaseNone, got.RetryPhase) + require.Empty(t, got.LastError) +} + +func TestDistributionServerRetrySplitJob_MapsDispatchLeadershipLoss(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + err error + }{ + {name: "leader not found", err: kv.ErrLeaderNotFound}, + {name: "raft not leader", err: raftengine.ErrNotLeader}, + {name: "leadership lost", err: raftengine.ErrLeadershipLost}, + {name: "transfer in progress", err: raftengine.ErrLeadershipTransferInProgress}, + {name: "wrapped grpc detail", err: errors.New("rpc error: code = Unknown desc = raft engine: leadership lost")}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + job := sampleDistributionSplitJob(15) + job.Phase = distribution.SplitJobPhaseFailed + job.RetryPhase = distribution.SplitJobPhaseFence + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + coordinator.dispatchErr = tc.err + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + _, err := s.RetrySplitJob(ctx, &pb.RetrySplitJobRequest{JobId: job.JobID}) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, errDistributionNotLeader.Error()) + require.Equal(t, 1, coordinator.dispatchCalls) + }) + } +} + +func TestDistributionServerAbandonSplitJob_RecordsAbandoningViaCoordinator(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + job := sampleDistributionSplitJob(13) + job.Phase = distribution.SplitJobPhaseFailed + job.RetryPhase = distribution.SplitJobPhaseBackfill + job.AbandonFromPhase = distribution.SplitJobPhaseNone + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + _, err := s.AbandonSplitJob(ctx, &pb.AbandonSplitJobRequest{JobId: job.JobID}) + require.NoError(t, err) + require.Equal(t, 1, coordinator.dispatchCalls) + + got, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseAbandoning, got.Phase) + require.Equal(t, distribution.SplitJobPhaseNone, got.RetryPhase) + require.Equal(t, distribution.SplitJobPhaseBackfill, got.AbandonFromPhase) +} + +func TestDistributionServerCompleteSplitJobTargetPromotion_UsesCoordinatorCAS(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, promotionCompleteDistributionRoutes()) + require.NoError(t, err) + job := promotionCompleteDistributionJob() + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + before, err := catalog.Snapshot(ctx) + require.NoError(t, err) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + coordinator.timestampNext = before.ReadTS + 10 + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + + snapshot, completed, err := s.completeSplitJobTargetPromotionViaCoordinator(ctx, saved.Version, job, 2000) + require.NoError(t, err) + require.Equal(t, saved.Version+1, snapshot.Version) + require.True(t, completed.TargetPromotionDone) + require.Equal(t, distribution.SplitJobPhaseCleanup, completed.Phase) + require.Equal(t, before.ReadTS+10, completed.PromotionCompletedTS) + require.Equal(t, int64(2000), completed.UpdatedAtMs) + require.Zero(t, completed.TerminalAtMs) + require.Equal(t, 1, coordinator.dispatchCalls) + require.Equal(t, 1, coordinator.timestampCalls) + require.Equal(t, before.ReadTS, coordinator.lastStartTS) + require.Equal(t, completed.PromotionCompletedTS, coordinator.lastCommitTS) + requireReadKeysContain(t, coordinator.lastReadKeys, distribution.CatalogVersionKey()) + requireReadKeysContain(t, coordinator.lastReadKeys, distribution.CatalogSplitJobKey(job.JobID)) + + loadedJob, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, completed, loadedJob) + + loaded, err := catalog.Snapshot(ctx) + require.NoError(t, err) + target := distributionRouteByID(t, loaded.Routes, 3) + require.False(t, target.StagedVisibilityActive) + require.Equal(t, uint64(0), target.MigrationJobID) + require.Equal(t, uint64(777), target.MinWriteTSExclusive) +} + +func TestDistributionServerCompleteSplitJobTargetPromotion_RetryAfterCommitIsIdempotent(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, promotionCompleteDistributionRoutes()) + require.NoError(t, err) + job := promotionCompleteDistributionJob() + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + + firstSnapshot, firstCompleted, err := s.completeSplitJobTargetPromotionViaCoordinator(ctx, saved.Version, job, 2000) + require.NoError(t, err) + retrySnapshot, retryCompleted, err := s.completeSplitJobTargetPromotionViaCoordinator(ctx, saved.Version, job, 3000) + require.NoError(t, err) + require.Equal(t, firstSnapshot.Version, retrySnapshot.Version) + require.Equal(t, firstCompleted, retryCompleted) + require.Equal(t, 1, coordinator.dispatchCalls) +} + +func TestDistributionServerCompleteSplitJobTargetPromotion_PersistsLegacyTerminalTime(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + routes := promotionCompleteDistributionRoutes() + routes[1].StagedVisibilityActive = false + routes[1].MigrationJobID = 0 + saved, err := catalog.Save(ctx, 0, routes) + require.NoError(t, err) + expected := promotionCompleteDistributionJob() + legacy := distribution.CloneSplitJob(expected) + legacy.Phase = distribution.SplitJobPhaseDone + legacy.TargetPromotionDone = true + legacy.PromotionCompletedTS = saved.ReadTS + 1 + require.NoError(t, catalog.CreateSplitJob(ctx, legacy)) + + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + + snapshot, completed, err := s.completeSplitJobTargetPromotionViaCoordinator(ctx, saved.Version, expected, 3000) + require.NoError(t, err) + require.Equal(t, saved.Version+1, snapshot.Version) + require.Equal(t, int64(3000), completed.TerminalAtMs) + require.Equal(t, int64(3000), completed.UpdatedAtMs) + require.Equal(t, 1, coordinator.dispatchCalls) + + loaded, found, err := catalog.SplitJob(ctx, legacy.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, completed, loaded) +} + +func TestDistributionServerRunSplitJobRunnerOnce_PromotesCleanupJob(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, promotionCompleteDistributionRoutes()) + require.NoError(t, err) + job := promotionCompleteDistributionJob() + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + client := &splitPromotionClientStub{ + responses: []*pb.PromoteStagedVersionsResponse{ + {NextCursor: []byte("cursor-1")}, + {Done: true}, + }, + } + sourceMigration := &splitMigrationClientStub{} + targetMigration := &splitMigrationClientStub{} + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithSplitPromotionClientFactory(func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) { + return client, nil + }), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return sourceMigration, targetMigration, nil + }), + ) + + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + require.Equal(t, 2, client.calls) + require.Equal(t, job.JobID, client.requests[0].GetJobId()) + require.Empty(t, client.requests[0].GetCursor()) + require.Equal(t, defaultSplitPromotionMaxVersions, client.requests[0].GetMaxVersions()) + require.Equal(t, defaultSplitPromotionMaxBytes, client.requests[0].GetMaxBytes()) + require.Equal(t, defaultSplitPromotionMaxScanned, client.requests[0].GetMaxScannedBytes()) + require.Equal(t, []byte("cursor-1"), client.requests[1].GetCursor()) + require.Equal(t, 1, coordinator.dispatchCalls) + + loadedJob, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.True(t, loadedJob.TargetPromotionDone) + require.Equal(t, distribution.SplitJobPhaseCleanup, loadedJob.Phase) + require.NotZero(t, loadedJob.PromotionCompletedTS) + require.Zero(t, loadedJob.TerminalAtMs) + + loaded, err := catalog.Snapshot(ctx) + require.NoError(t, err) + require.Equal(t, saved.Version+1, loaded.Version) + target := distributionRouteByID(t, loaded.Routes, 3) + require.False(t, target.StagedVisibilityActive) + require.Equal(t, uint64(0), target.MigrationJobID) + require.Equal(t, uint64(777), target.MinWriteTSExclusive) + + route, ok := s.engine.GetRoute([]byte("z")) + require.True(t, ok) + require.False(t, route.StagedVisibilityActive) + require.Equal(t, uint64(0), route.MigrationJobID) + require.Equal(t, saved.Version+1, s.engine.Version()) +} + +func TestDistributionServerRunSplitJobRunnerOnce_TerminalizesPromotedCleanupJob(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + routes := promotionCompleteDistributionRoutes() + routes[1].StagedVisibilityActive = false + routes[1].MigrationJobID = 0 + saved, err := catalog.Save(ctx, 0, routes) + require.NoError(t, err) + job := promotionCompleteDistributionJob() + job.TargetPromotionDone = true + job.PromotionCompletedTS = saved.ReadTS + 1 + job.UpdatedAtMs = 2000 + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + source := &splitMigrationClientStub{} + target := &splitMigrationClientStub{} + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithSplitPromotionClientFactory(func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) { + return target, nil + }), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return source, target, nil + }), + ) + + for range 200 { + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + current, found, loadErr := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, loadErr) + require.True(t, found) + if current.Phase == distribution.SplitJobPhaseDone { + break + } + } + require.Zero(t, target.calls) + + loadedJob, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseDone, loadedJob.Phase) + require.True(t, loadedJob.TargetPromotionDone) + require.Equal(t, job.PromotionCompletedTS, loadedJob.PromotionCompletedTS) + + loaded, err := catalog.Snapshot(ctx) + require.NoError(t, err) + require.Equal(t, saved.Version, loaded.Version) +} + +func TestDistributionServerRunSplitJobRunnerOnce_CompletesCrossGroupMigration(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + require.NoError(t, err) + job, err := distribution.InitializeSplitJobPlan(distribution.SplitJob{ + JobID: 1, + SourceRouteID: 1, + SplitKey: []byte("m"), + TargetGroupID: 2, + }, saved.Routes[0], time.Now().UnixMilli()) + require.NoError(t, err) + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(saved)) + source := &splitMigrationClientStub{} + target := &splitMigrationClientStub{} + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + engine, + catalog, + WithDistributionCoordinator(coordinator), + WithSplitPromotionClientFactory(func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) { + return target, nil + }), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return source, target, nil + }), + ) + + for range 200 { + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + current, found, loadErr := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, loadErr) + require.True(t, found) + if current.Phase == distribution.SplitJobPhaseDone { + break + } + } + + completed, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseDone, completed.Phase) + require.True(t, completed.WriteTrackerArmed) + require.True(t, completed.PostFenceDrainCompleted) + require.NotZero(t, completed.SnapshotTS) + require.NotZero(t, completed.FenceTS) + require.NotZero(t, completed.CutoverVersion) + require.True(t, completed.TargetPromotionDone) + require.NotEmpty(t, source.controls) + require.NotEmpty(t, target.controls) + require.NotEmpty(t, source.exports) + require.Len(t, target.imports, len(source.exports)) + require.GreaterOrEqual(t, len(source.events), 2) + require.Equal(t, []string{"control", "timestamp"}, source.events[:2], "the tracker barrier must precede snapshot timestamp selection") + + finalSnapshot, err := catalog.Snapshot(ctx) + require.NoError(t, err) + require.Len(t, finalSnapshot.Routes, 2) + right := distributionRouteByID(t, finalSnapshot.Routes, 3) + require.Equal(t, uint64(2), right.GroupID) + require.Equal(t, distribution.RouteStateActive, right.State) + require.False(t, right.StagedVisibilityActive) + require.Equal(t, completed.FenceTS, right.MinWriteTSExclusive) +} + +func TestDistributionServerRunSplitJobRunnerOnce_CompletesAbandonCleanup(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + require.NoError(t, err) + job, err := distribution.InitializeSplitJobPlan(distribution.SplitJob{ + JobID: 7, + SourceRouteID: 1, + SplitKey: []byte("m"), + TargetGroupID: 2, + }, saved.Routes[0], time.Now().UnixMilli()) + require.NoError(t, err) + job.Phase = distribution.SplitJobPhaseAbandoning + job.AbandonFromPhase = distribution.SplitJobPhaseBackfill + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + source := &splitMigrationClientStub{} + target := &splitMigrationClientStub{} + coordinator := newDistributionCoordinatorStub(baseStore, true) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithSplitPromotionClientFactory(func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) { + return target, nil + }), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return source, target, nil + }), + ) + + for range 10 { + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + current, found, loadErr := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, loadErr) + require.True(t, found) + if current.Phase == distribution.SplitJobPhaseAbandoned { + break + } + } + completed, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseAbandoned, completed.Phase) + require.Positive(t, completed.TerminalAtMs) + require.NotEmpty(t, source.cleanups) + require.NotEmpty(t, target.cleanups) + require.Equal(t, pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS, target.cleanups[0].GetMode()) +} + +func TestDistributionServerRunSplitJobRunnerOnce_PausesOnCapabilityRegression(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + require.NoError(t, err) + job, err := distribution.InitializeSplitJobPlan(distribution.SplitJob{ + JobID: 9, + SourceRouteID: 1, + SplitKey: []byte("m"), + TargetGroupID: 2, + }, saved.Routes[0], time.Now().UnixMilli()) + require.NoError(t, err) + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + source := &splitMigrationClientStub{} + target := &splitMigrationClientStub{} + coordinator := newDistributionCoordinatorStub(baseStore, true) + capabilityErr := errors.New("node capability missing") + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithSplitMigrationCapabilityGate(func(context.Context) error { return capabilityErr }), + WithSplitPromotionClientFactory(func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) { + return target, nil + }), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return source, target, nil + }), + ) + + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + paused, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.True(t, paused.CapabilityRegressed) + require.Equal(t, distribution.SplitJobPhasePlanned, paused.Phase) + require.Empty(t, source.controls) + + s.migrationCapabilityGate = func(context.Context) error { return nil } + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + resumed, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.False(t, resumed.CapabilityRegressed) + require.Equal(t, distribution.SplitJobPhasePlanned, resumed.Phase) + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + armed, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, distribution.SplitJobPhaseBackfill, armed.Phase) + require.NotZero(t, armed.SnapshotTS) +} + +func TestDistributionServerCleanupSplitJobSourceProofsWaitsForMetadataBarrier(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 7, + State: distribution.RouteStateActive, + }}) + require.NoError(t, err) + job := sampleDistributionSplitJob(17) + job.Phase = distribution.SplitJobPhaseCleanup + job.TargetPromotionDone = true + job.TargetClearedDescriptorAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) + job.SourceCutoverAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) + job.SourceReadDrainCursor = distribution.CloneBytes(splitCleanupDoneCursor) + job.Cursor = distribution.CloneBytes(splitCleanupDoneCursor) + job.SourceRetentionPinTS = 90 + job.CutoverVersion = saved.Version + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + source := &splitMigrationClientStub{} + target := &splitMigrationClientStub{} + readyVoter := &splitMigrationClientStub{} + blockedVoter := &splitMigrationClientStub{ + probeFn: func(req *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + require.Equal(t, pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, req.GetKind()) + return &pb.ProbeMigrationStateResponse{}, nil }, + } + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(newDistributionCoordinatorStub(baseStore, true)), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return source, target, nil + }), + WithSplitMigrationVoterFactory(func(_ context.Context, groupID uint64) ([]SplitMigrationVoter, error) { + require.Equal(t, uint64(7), groupID) + return []SplitMigrationVoter{ + {ID: "n1", Address: "n1:50051", Client: readyVoter}, + {ID: "n2", Address: "n2:50051", Client: blockedVoter}, + }, nil + }), + ) + + err = s.cleanupSplitJobSourceProofs(ctx, job) + require.ErrorIs(t, err, errSplitMigrationVoterBarrierIncomplete) + require.Len(t, source.cleanups, 1) + require.Equal(t, pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA, source.cleanups[0].GetMode()) + current, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, uint64(90), current.SourceRetentionPinTS) +} + +func TestDistributionServerRunSplitJobRunnerOnce_FailClosesCapabilityRegressionAfterSideEffects(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + saved, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + require.NoError(t, err) + job, err := distribution.InitializeSplitJobPlan(distribution.SplitJob{ + JobID: 10, + SourceRouteID: 1, + SplitKey: []byte("m"), + TargetGroupID: 2, + }, saved.Routes[0], time.Now().UnixMilli()) + require.NoError(t, err) + job.Phase = distribution.SplitJobPhaseBackfill + job.SnapshotTS = 50 + job.WriteTrackerArmed = true + job.SourceRetentionPinTS = initialMigrationRetentionPinTS + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + source := &splitMigrationClientStub{} + target := &splitMigrationClientStub{} + capabilityErr := errors.New("node capability missing") + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(newDistributionCoordinatorStub(baseStore, true)), + WithSplitMigrationCapabilityGate(func(context.Context) error { return capabilityErr }), + WithSplitPromotionClientFactory(func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) { + return target, nil + }), + WithSplitMigrationClientFactory(func(context.Context, distribution.SplitJob, uint64) (SplitMigrationClient, SplitMigrationClient, error) { + return source, target, nil + }), + ) + + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + regressed, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.True(t, regressed.CapabilityRegressed) + require.Len(t, source.controls, 1) + require.Len(t, target.controls, 1) + sourceFailClosed := source.controls[0] + require.Equal(t, job.JobID, sourceFailClosed.GetJobId()) + require.Equal(t, []byte("m"), sourceFailClosed.GetRouteStart()) + require.Equal(t, []byte("z"), sourceFailClosed.GetRouteEnd()) + require.True(t, sourceFailClosed.GetSourceWriteFence()) + require.True(t, sourceFailClosed.GetSourceReadFence()) + require.True(t, sourceFailClosed.GetTrackWrites()) + require.Equal(t, uint64(50), sourceFailClosed.GetMinWriteTsExclusive()) + targetFailClosed := target.controls[0] + require.False(t, targetFailClosed.GetSourceWriteFence()) + require.False(t, targetFailClosed.GetSourceReadFence()) + require.False(t, targetFailClosed.GetTrackWrites()) + require.Equal(t, uint64(50), targetFailClosed.GetMinWriteTsExclusive()) + + s.migrationCapabilityGate = func(context.Context) error { return nil } + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + resumed, found, err := catalog.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + require.False(t, resumed.CapabilityRegressed) + require.Len(t, source.controls, 2) + sourceRestored := source.controls[1] + require.False(t, sourceRestored.GetSourceWriteFence()) + require.False(t, sourceRestored.GetSourceReadFence()) + require.True(t, sourceRestored.GetTrackWrites()) + require.Equal(t, initialMigrationRetentionPinTS, sourceRestored.GetMinWriteTsExclusive()) + require.Len(t, target.controls, 1) +} + +func TestSplitJobHistoryGCKeysAppliesTTLAndCountBound(t *testing.T) { + t.Parallel() + + now := time.UnixMilli(10 * splitJobHistoryTTL.Milliseconds()) + jobs := make([]distribution.SplitJob, 0, splitJobHistoryLimit+3) + jobs = append(jobs, distribution.SplitJob{ + JobID: 1, + Phase: distribution.SplitJobPhaseDone, + TerminalAtMs: now.Add(-splitJobHistoryTTL - time.Second).UnixMilli(), }) + for i := 0; i < splitJobHistoryLimit+2; i++ { + jobs = append(jobs, distribution.SplitJob{ + JobID: uint64(i) + 2, //nolint:gosec // loop bound is splitJobHistoryLimit. + Phase: distribution.SplitJobPhaseDone, + TerminalAtMs: now.Add(-time.Hour).UnixMilli() + int64(i), + }) + } + + keys := splitJobHistoryGCKeys(jobs, now) + require.Len(t, keys, 3) + require.Equal(t, distribution.CatalogSplitJobHistoryKey(jobs[0].TerminalAtMs, jobs[0].JobID), keys[0]) +} + +func TestDistributionServerRunSplitJobRunnerOnce_SkipsFollower(t *testing.T) { + t.Parallel() + + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore, distribution.WithCatalogRouteDescriptorV2Writes(true)) + _, err := catalog.Save(ctx, 0, promotionCompleteDistributionRoutes()) require.NoError(t, err) + require.NoError(t, catalog.CreateSplitJob(ctx, promotionCompleteDistributionJob())) - s := NewDistributionServer(distribution.NewEngine(), catalog) - resp, err := s.ListRoutes(ctx, &pb.ListRoutesRequest{}) + client := &splitPromotionClientStub{ + responses: []*pb.PromoteStagedVersionsResponse{{Done: true}}, + } + coordinator := newDistributionCoordinatorStub(baseStore, false) + s := NewDistributionServer( + distribution.NewEngine(), + catalog, + WithDistributionCoordinator(coordinator), + WithSplitPromotionClientFactory(func(context.Context, distribution.SplitJob) (SplitPromotionClient, error) { + return client, nil + }), + ) + + require.NoError(t, s.RunSplitJobRunnerOnce(ctx)) + require.Zero(t, client.calls) + require.Zero(t, coordinator.dispatchCalls) +} + +func TestSplitJobRunnerContextDoneRecognizesWrappedGRPCStatus(t *testing.T) { + t.Parallel() + + require.True(t, splitJobRunnerContextDone(context.Canceled)) + require.True(t, splitJobRunnerContextDone(context.DeadlineExceeded)) + require.True(t, splitJobRunnerContextDone(crdberrors.WithStack(status.Error(codes.Canceled, "stopping")))) + require.True(t, splitJobRunnerContextDone(crdberrors.WithStack(status.Error(codes.DeadlineExceeded, "stopping")))) + require.False(t, splitJobRunnerContextDone(crdberrors.WithStack(status.Error(codes.Unavailable, "not leader")))) +} + +func TestPromoteSplitJobTargetRejectsNoCursorProgress(t *testing.T) { + t.Parallel() + + client := &splitPromotionClientStub{ + responses: []*pb.PromoteStagedVersionsResponse{{NextCursor: []byte("same")}}, + } + err := promoteSplitJobTarget(context.Background(), client, promotionCompleteDistributionJob()) require.NoError(t, err) - require.Equal(t, saved.Version, resp.CatalogVersion) - require.Len(t, resp.Routes, 2) - require.Equal(t, uint64(1), resp.Routes[0].RouteId) - require.Equal(t, []byte(""), resp.Routes[0].Start) - require.Equal(t, []byte("m"), resp.Routes[0].End) - require.Equal(t, uint64(1), resp.Routes[0].RaftGroupId) - require.Equal(t, pb.RouteState_ROUTE_STATE_ACTIVE, resp.Routes[0].State) - require.Equal(t, uint64(2), resp.Routes[1].RouteId) - require.Nil(t, resp.Routes[1].End) - require.Equal(t, pb.RouteState_ROUTE_STATE_WRITE_FENCED, resp.Routes[1].State) - require.True(t, resp.Routes[1].StagedVisibilityActive) - require.Equal(t, uint64(42), resp.Routes[1].MigrationJobId) - require.Equal(t, uint64(99), resp.Routes[1].MinWriteTsExclusive) - require.Equal(t, uint64(100), resp.Routes[1].SplitAtHlc) + client = &splitPromotionClientStub{ + responses: []*pb.PromoteStagedVersionsResponse{ + {NextCursor: []byte("same")}, + {NextCursor: []byte("same")}, + }, + } + err = promoteSplitJobTarget(context.Background(), client, promotionCompleteDistributionJob()) + require.Error(t, err) + require.ErrorContains(t, err, "split promotion made no cursor progress") } -func TestDistributionServerListRoutes_RequiresCatalog(t *testing.T) { +func TestDistributionServerRetrySplitJob_RequiresCatalogLeader(t *testing.T) { t.Parallel() - s := NewDistributionServer(distribution.NewEngine(), nil) - _, err := s.ListRoutes(context.Background(), &pb.ListRoutesRequest{}) + ctx := context.Background() + baseStore := store.NewMVCCStore() + catalog := distribution.NewCatalogStore(baseStore) + job := sampleDistributionSplitJob(14) + job.Phase = distribution.SplitJobPhaseFailed + job.RetryPhase = distribution.SplitJobPhaseBackfill + require.NoError(t, catalog.CreateSplitJob(ctx, job)) + + coordinator := newDistributionCoordinatorStub(baseStore, false) + s := NewDistributionServer(distribution.NewEngine(), catalog, WithDistributionCoordinator(coordinator)) + _, err := s.RetrySplitJob(ctx, &pb.RetrySplitJobRequest{JobId: job.JobID}) require.Error(t, err) require.Equal(t, codes.FailedPrecondition, status.Code(err)) - require.ErrorContains(t, err, errDistributionCatalogNotConfigured.Error()) + require.Zero(t, coordinator.dispatchCalls) } func TestDistributionServerGetRouteOwnership_UsesExactVersionSnapshot(t *testing.T) { @@ -1138,11 +2455,248 @@ func seededDistributionServerWithoutCoordinator(t *testing.T) (*DistributionServ return NewDistributionServer(distribution.NewEngine(), catalog), saved.Version } +func sampleDistributionSplitJob(jobID uint64) distribution.SplitJob { + return distribution.SplitJob{ + JobID: jobID, + SourceRouteID: 1, + SplitKey: []byte("g"), + TargetGroupID: 2, + Phase: distribution.SplitJobPhaseBackfill, + RetryPhase: distribution.SplitJobPhaseNone, + StartedAtMs: 1000, + UpdatedAtMs: 1000, + } +} + +func promotionCompleteDistributionJob() distribution.SplitJob { + return distribution.SplitJob{ + JobID: 99, + SourceRouteID: 42, + SplitKey: []byte("m"), + TargetGroupID: 2, + Phase: distribution.SplitJobPhaseCleanup, + } +} + +func promotionCompleteDistributionRoutes() []distribution.RouteDescriptor { + return []distribution.RouteDescriptor{ + { + RouteID: 2, + Start: []byte(""), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + ParentRouteID: 42, + }, + { + RouteID: 3, + Start: []byte("m"), + End: nil, + GroupID: 2, + State: distribution.RouteStateActive, + ParentRouteID: 42, + StagedVisibilityActive: true, + MigrationJobID: 99, + MinWriteTSExclusive: 777, + }, + } +} + +func distributionRouteByID(t *testing.T, routes []distribution.RouteDescriptor, routeID uint64) distribution.RouteDescriptor { + t.Helper() + for _, route := range routes { + if route.RouteID == routeID { + return route + } + } + t.Fatalf("route %d not found in %+v", routeID, routes) + return distribution.RouteDescriptor{} +} + +func splitJobIDs(jobs []*pb.SplitJob) []uint64 { + ids := make([]uint64, 0, len(jobs)) + for _, job := range jobs { + ids = append(ids, job.GetJobId()) + } + return ids +} + +func byteSliceStrings(in [][]byte) []string { + out := make([]string, 0, len(in)) + for _, item := range in { + out = append(out, string(item)) + } + return out +} + +type splitPromotionClientStub struct { + responses []*pb.PromoteStagedVersionsResponse + err error + requests []*pb.PromoteStagedVersionsRequest + calls int +} + +type splitMigrationClientStub struct { + splitPromotionClientStub + exports []*pb.ExportRangeVersionsRequest + imports []*pb.ImportRangeVersionsRequest + controls []*pb.TargetStagedReadinessRequest + cleanups []*pb.CleanupMigrationRequest + probes []*pb.ProbeMigrationStateRequest + probeFn func(*pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) + nextTS uint64 + events []string +} + +func (s *splitMigrationClientStub) ExportRangeVersions( + _ context.Context, + req *pb.ExportRangeVersionsRequest, + _ ...grpc.CallOption, +) (grpc.ServerStreamingClient[pb.ExportRangeVersionsResponse], error) { + cloned := &pb.ExportRangeVersionsRequest{ + RangeStart: distribution.CloneBytes(req.GetRangeStart()), + RangeEnd: distribution.CloneBytes(req.GetRangeEnd()), + MaxCommitTs: req.GetMaxCommitTs(), + MinCommitTs: req.GetMinCommitTs(), + Cursor: distribution.CloneBytes(req.GetCursor()), + ChunkBytes: req.GetChunkBytes(), + RouteStart: distribution.CloneBytes(req.GetRouteStart()), + RouteEnd: distribution.CloneBytes(req.GetRouteEnd()), + MaxScannedBytes: req.GetMaxScannedBytes(), + KeyFamily: req.GetKeyFamily(), + ExcludeKnownInternal: req.GetExcludeKnownInternal(), + ExcludePrefixes: req.GetExcludePrefixes(), + } + s.exports = append(s.exports, cloned) + phaseByte := byte(0) + if req.GetMinCommitTs() != 0 { + phaseByte = 1 + } + cursor := []byte{byte(req.GetKeyFamily()), phaseByte} + return &splitMigrationStreamStub{responses: []*pb.ExportRangeVersionsResponse{{ + Versions: []*pb.MVCCVersion{{ + Key: []byte("n"), + CommitTs: 10, + Value: []byte("v"), + KeyFamily: req.GetKeyFamily(), + }}, + NextCursor: cursor, + Done: true, + }}}, nil +} + +func (s *splitMigrationClientStub) ImportRangeVersions( + _ context.Context, + req *pb.ImportRangeVersionsRequest, + _ ...grpc.CallOption, +) (*pb.ImportRangeVersionsResponse, error) { + s.imports = append(s.imports, req) + return &pb.ImportRangeVersionsResponse{AckedCursor: distribution.CloneBytes(req.GetCursor())}, nil +} + +func (s *splitMigrationClientStub) ApplyTargetStagedReadiness( + _ context.Context, + req *pb.TargetStagedReadinessRequest, + _ ...grpc.CallOption, +) (*pb.TargetStagedReadinessResponse, error) { + s.controls = append(s.controls, req) + s.events = append(s.events, "control") + minAdmittedTS := uint64(0) + if req.GetTrackWrites() { + minAdmittedTS = 10 + } + return &pb.TargetStagedReadinessResponse{MinAdmittedTs: minAdmittedTS}, nil +} + +func (s *splitMigrationClientStub) ProbeMigrationLocks( + context.Context, + *pb.ProbeMigrationLocksRequest, + ...grpc.CallOption, +) (*pb.ProbeMigrationLocksResponse, error) { + return &pb.ProbeMigrationLocksResponse{}, nil +} + +func (s *splitMigrationClientStub) CleanupMigration( + _ context.Context, + req *pb.CleanupMigrationRequest, + _ ...grpc.CallOption, +) (*pb.CleanupMigrationResponse, error) { + s.cleanups = append(s.cleanups, req) + return &pb.CleanupMigrationResponse{Done: true, NextCursor: distribution.CloneBytes(req.GetCursor())}, nil +} + +func (s *splitMigrationClientStub) ProbeMigrationState( + _ context.Context, + req *pb.ProbeMigrationStateRequest, + _ ...grpc.CallOption, +) (*pb.ProbeMigrationStateResponse, error) { + s.probes = append(s.probes, req) + if s.probeFn != nil { + return s.probeFn(req) + } + return &pb.ProbeMigrationStateResponse{Ready: true, CatalogVersion: req.GetExpectedCatalogVersion()}, nil +} + +func (s *splitMigrationClientStub) IssueMigrationTimestamp( + context.Context, + *pb.IssueMigrationTimestampRequest, + ...grpc.CallOption, +) (*pb.IssueMigrationTimestampResponse, error) { + s.events = append(s.events, "timestamp") + if s.nextTS == 0 { + s.nextTS = 100 + } + s.nextTS++ + return &pb.IssueMigrationTimestampResponse{Timestamp: s.nextTS, LastCommitTs: s.nextTS - 1}, nil +} + +type splitMigrationStreamStub struct { + grpc.ClientStream + responses []*pb.ExportRangeVersionsResponse +} + +func (s *splitMigrationStreamStub) Recv() (*pb.ExportRangeVersionsResponse, error) { + if len(s.responses) == 0 { + return nil, io.EOF + } + resp := s.responses[0] + s.responses = s.responses[1:] + return resp, nil +} + +func (s *splitPromotionClientStub) PromoteStagedVersions( + _ context.Context, + req *pb.PromoteStagedVersionsRequest, + _ ...grpc.CallOption, +) (*pb.PromoteStagedVersionsResponse, error) { + s.calls++ + s.requests = append(s.requests, &pb.PromoteStagedVersionsRequest{ + JobId: req.GetJobId(), + Cursor: distribution.CloneBytes(req.GetCursor()), + MaxVersions: req.GetMaxVersions(), + MaxBytes: req.GetMaxBytes(), + MaxScannedBytes: req.GetMaxScannedBytes(), + }) + if s.err != nil { + return nil, s.err + } + if len(s.responses) == 0 { + return &pb.PromoteStagedVersionsResponse{Done: true}, nil + } + resp := s.responses[0] + s.responses = s.responses[1:] + return resp, nil +} + type distributionCoordinatorStub struct { store store.MVCCStore leader bool clock *kv.HLC + dispatchErr error nextTS uint64 + timestampNext uint64 + timestampErr error + timestampCalls int lastStartTS uint64 lastCommitTS uint64 lastRequestedCommitTS uint64 @@ -1168,6 +2722,9 @@ func (s *distributionCoordinatorStub) Dispatch(ctx context.Context, reqs *kv.Ope } s.dispatchCalls++ s.lastRequestedCommitTS = reqs.CommitTS + if s.dispatchErr != nil { + return nil, s.dispatchErr + } startTS, commitTS := s.nextTimestamps(reqs.StartTS, reqs.CommitTS) s.lastStartTS = startTS s.lastCommitTS = commitTS @@ -1214,6 +2771,9 @@ func (s *distributionCoordinatorStub) nextTimestamps(startTS uint64, requestedCo if s.clock != nil { s.clock.Observe(requestedCommitTS) } + if s.nextTS <= requestedCommitTS { + s.nextTS = requestedCommitTS + 1 + } return startTS, requestedCommitTS } if s.nextTS == 0 { @@ -1353,6 +2913,26 @@ func (s *distributionCoordinatorStub) Clock() *kv.HLC { return s.clock } +func (s *distributionCoordinatorStub) Next(ctx context.Context) (uint64, error) { + return s.NextAfter(ctx, 0) +} + +func (s *distributionCoordinatorStub) NextAfter(_ context.Context, min uint64) (uint64, error) { + s.timestampCalls++ + if s.timestampErr != nil { + return 0, s.timestampErr + } + next := s.timestampNext + if next == 0 { + next = s.store.LastCommitTS() + 1 + } + if next <= min { + next = min + 1 + } + s.timestampNext = next + 1 + return next, nil +} + func (s *distributionCoordinatorStub) LinearizableRead(_ context.Context) (uint64, error) { return 0, nil } diff --git a/adapter/internal.go b/adapter/internal.go index 4037f6b8d..fffeed445 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -5,6 +5,7 @@ import ( "context" "os" "strings" + "time" "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/raftengine" @@ -17,6 +18,18 @@ import ( "google.golang.org/grpc/status" ) +const ( + // MigrationImportOpcodeEnv enables the internal staged import opcode after + // every voter is running a compatible build. + MigrationImportOpcodeEnv = "ELASTICKV_ENABLE_MIGRATION_IMPORT_OPCODE" + // MigrationPromoteOpcodeEnv enables the internal staged promotion opcode + // after every voter is running a compatible build. + MigrationPromoteOpcodeEnv = "ELASTICKV_ENABLE_MIGRATION_PROMOTE_OPCODE" + // MigrationCleanupOpcodeEnv enables the internal migration cleanup opcode + // after every voter is running a compatible build. + MigrationCleanupOpcodeEnv = "ELASTICKV_ENABLE_MIGRATION_CLEANUP_OPCODE" +) + type InternalOption func(*Internal) func WithInternalTimestampAllocator(alloc kv.TimestampAllocator) InternalOption { @@ -49,12 +62,24 @@ func WithInternalMigrationPromoteGate(gate func(context.Context) error) Internal } } +func WithInternalMigrationCleanupGate(gate func(context.Context) error) InternalOption { + return func(i *Internal) { + i.migrationCleanupGate = gate + } +} + func WithInternalRouteEngine(engine *distribution.Engine) InternalOption { return func(i *Internal) { i.routeEngine = engine } } +func WithInternalActiveTimestampTracker(tracker *kv.ActiveTimestampTracker) InternalOption { + return func(i *Internal) { + i.readTracker = tracker + } +} + func WithInternalMigrationExportRouting(groupID uint64, resolver kv.PartitionResolver) InternalOption { return func(i *Internal) { i.migrationExportGroupID = groupID @@ -70,6 +95,7 @@ func NewInternalWithEngine(txm kv.Transactional, leader raftengine.LeaderView, c relay: relay, migrationImportGate: defaultMigrationImportGate, migrationPromoteGate: defaultMigrationPromoteGate, + migrationCleanupGate: defaultMigrationCleanupGate, } for _, opt := range opts { opt(i) @@ -87,7 +113,9 @@ type Internal struct { migrationProposer raftengine.Proposer migrationImportGate func(context.Context) error migrationPromoteGate func(context.Context) error + migrationCleanupGate func(context.Context) error routeEngine *distribution.Engine + readTracker *kv.ActiveTimestampTracker migrationExportGroupID uint64 migrationExportResolver kv.PartitionResolver @@ -137,6 +165,179 @@ func (i *Internal) Forward(ctx context.Context, req *pb.ForwardRequest) (*pb.For }, nil } +func (i *Internal) ProbeMigrationState(ctx context.Context, req *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + if req == nil || req.GetJobId() == 0 { + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "migration probe job_id is required")) + } + return i.probeMigrationStateKind(ctx, req) +} + +func (i *Internal) probeMigrationStateKind(ctx context.Context, req *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + switch req.GetKind() { + case pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED: + return i.probeMigrationControl(ctx, req) + case pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED: + return i.probeMigrationRoute(req, true), nil + case pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_ROUTE_REMOVED: + return i.probeMigrationRoute(req, false), nil + case pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED: + return &pb.ProbeMigrationStateResponse{ + Ready: time.Now().UnixMilli() >= req.GetReadDrainNotBeforeMs() && + (i.readTracker == nil || i.readTracker.Oldest() == 0), + CatalogVersion: i.migrationCatalogVersion(), + }, nil + case pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED: + return i.probeMigrationMetadataCleared(ctx, req.GetJobId()) + case pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_UNSPECIFIED: + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "migration probe kind is required")) + default: + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "unknown migration probe kind")) + } +} + +func (i *Internal) IssueMigrationTimestamp(ctx context.Context, _ *pb.IssueMigrationTimestampRequest) (*pb.IssueMigrationTimestampResponse, error) { + if err := i.verifyInternalLeaderApplied(ctx); err != nil { + return nil, err + } + if i.store == nil { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration timestamp store is not configured")) + } + lastCommitTS := i.store.LastCommitTS() + ts, err := i.nextTimestampAfter(ctx, lastCommitTS, "issue migration timestamp") + if err != nil { + return nil, errors.WithStack(err) + } + return &pb.IssueMigrationTimestampResponse{Timestamp: ts, LastCommitTs: lastCommitTS}, nil +} + +func (i *Internal) probeMigrationControl(ctx context.Context, req *pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + reader, ok := i.store.(store.MigrationTargetReadinessReader) + if !ok { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration readiness reader is not configured")) + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return nil, errors.WithStack(err) + } + for _, state := range states { + if state.JobID != req.GetJobId() { + continue + } + ready := migrationControlMatches(state, req) + return &pb.ProbeMigrationStateResponse{Ready: ready, CatalogVersion: i.migrationCatalogVersion(), MinAdmittedTs: state.MinAdmittedTS}, nil + } + return &pb.ProbeMigrationStateResponse{CatalogVersion: i.migrationCatalogVersion()}, nil +} + +func migrationControlMatches(state store.TargetStagedReadinessState, req *pb.ProbeMigrationStateRequest) bool { + return state.Armed && + bytes.Equal(state.RouteStart, req.GetRouteStart()) && + bytes.Equal(state.RouteEnd, normalizeOptionalEnd(req.GetRouteEnd())) && + state.ExpectedCutoverVersion == req.GetExpectedCatalogVersion() && + state.MigrationJobID == req.GetMigrationJobId() && + state.MinWriteTSExclusive == req.GetMinWriteTsExclusive() && + state.SourceWriteFence == req.GetSourceWriteFence() && + state.SourceReadFence == req.GetSourceReadFence() && + state.TrackWrites == req.GetTrackWrites() && + state.RetentionPinTS == req.GetRetentionPinTs() +} + +func (i *Internal) probeMigrationRoute(req *pb.ProbeMigrationStateRequest, requireCleared bool) *pb.ProbeMigrationStateResponse { + version := i.migrationCatalogVersion() + if i.routeEngine == nil || version < req.GetExpectedCatalogVersion() { + return &pb.ProbeMigrationStateResponse{CatalogVersion: version} + } + route, found := i.routeEngine.GetRoute(req.GetRouteStart()) + ready := found && route.GroupID == req.GetExpectedGroupId() && + bytes.Equal(route.Start, req.GetRouteStart()) && + bytes.Equal(route.End, normalizeOptionalEnd(req.GetRouteEnd())) + if requireCleared { + ready = ready && !route.StagedVisibilityActive && route.MigrationJobID == 0 && + route.MinWriteTSExclusive == req.GetMinWriteTsExclusive() + } + return &pb.ProbeMigrationStateResponse{Ready: ready, CatalogVersion: version} +} + +func (i *Internal) probeMigrationMetadataCleared(ctx context.Context, jobID uint64) (*pb.ProbeMigrationStateResponse, error) { + present, err := i.migrationReadinessMetadataPresent(ctx, jobID) + if err != nil { + return nil, err + } + if present { + return &pb.ProbeMigrationStateResponse{CatalogVersion: i.migrationCatalogVersion()}, nil + } + present, err = i.migrationPromotionMetadataPresent(ctx, jobID) + if err != nil { + return nil, err + } + if present { + return &pb.ProbeMigrationStateResponse{CatalogVersion: i.migrationCatalogVersion()}, nil + } + present, err = i.migrationImportMetadataPresent(ctx, jobID) + if err != nil { + return nil, err + } + if present { + return &pb.ProbeMigrationStateResponse{CatalogVersion: i.migrationCatalogVersion()}, nil + } + return &pb.ProbeMigrationStateResponse{Ready: true, CatalogVersion: i.migrationCatalogVersion()}, nil +} + +func (i *Internal) migrationReadinessMetadataPresent(ctx context.Context, jobID uint64) (bool, error) { + readiness, ok := i.store.(store.MigrationTargetReadinessReader) + if !ok { + return false, errors.WithStack(status.Error(codes.FailedPrecondition, "migration readiness reader is not configured")) + } + states, err := readiness.MigrationTargetReadinessStates(ctx) + if err != nil { + return false, errors.WithStack(err) + } + for _, state := range states { + if state.JobID == jobID { + return true, nil + } + } + return false, nil +} + +func (i *Internal) migrationPromotionMetadataPresent(ctx context.Context, jobID uint64) (bool, error) { + promotion, ok := i.store.(store.MigrationPromotionStateReader) + if !ok { + return false, nil + } + _, found, err := promotion.MigrationPromotionState(ctx, jobID) + if err != nil { + return false, errors.WithStack(err) + } + return found, nil +} + +func (i *Internal) migrationImportMetadataPresent(ctx context.Context, jobID uint64) (bool, error) { + importMetadata, ok := i.store.(store.MigrationImportMetadataReader) + if !ok { + return false, nil + } + present, err := importMetadata.MigrationImportMetadataPresent(ctx, jobID) + if err != nil { + return false, errors.WithStack(err) + } + return present, nil +} + +func (i *Internal) migrationCatalogVersion() uint64 { + if i.routeEngine == nil { + return 0 + } + return i.routeEngine.Version() +} + +func normalizeOptionalEnd(end []byte) []byte { + if len(end) == 0 { + return nil + } + return end +} + func (i *Internal) RelayPublish(_ context.Context, req *pb.RelayPublishRequest) (*pb.RelayPublishResponse, error) { if req == nil || i.relay == nil { return &pb.RelayPublishResponse{}, nil @@ -286,6 +487,121 @@ func (i *Internal) PromoteStagedVersions(ctx context.Context, req *pb.PromoteSta }, nil } +func (i *Internal) ApplyTargetStagedReadiness(ctx context.Context, req *pb.TargetStagedReadinessRequest) (*pb.TargetStagedReadinessResponse, error) { + if err := validateTargetStagedReadinessRequest(req); err != nil { + return nil, err + } + if i.migrationProposer == nil { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "target staged readiness proposer is not configured")) + } + if err := i.verifyInternalLeader(ctx); err != nil { + return nil, err + } + if err := i.proposeTargetStagedReadiness(ctx, req); err != nil { + return nil, errors.WithStack(err) + } + return &pb.TargetStagedReadinessResponse{MinAdmittedTs: i.migrationMinAdmittedTS(ctx, req.GetJobId())}, nil +} + +func (i *Internal) CleanupMigration(ctx context.Context, req *pb.CleanupMigrationRequest) (*pb.CleanupMigrationResponse, error) { + if err := validateCleanupMigrationRequest(req); err != nil { + return nil, err + } + if i.migrationProposer == nil { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration cleanup proposer is not configured")) + } + if err := i.verifyInternalLeader(ctx); err != nil { + return nil, err + } + if err := i.verifyMigrationCleanupEnabled(ctx); err != nil { + return nil, err + } + result, err := i.proposeMigrationCleanup(ctx, req) + if err != nil { + return nil, errors.WithStack(err) + } + return &pb.CleanupMigrationResponse{ + NextCursor: result.NextCursor, + Done: result.Done, + DeletedRows: result.DeletedRows, + DeletedBytes: result.DeletedBytes, + ScannedBytes: result.ScannedBytes, + }, nil +} + +func validateCleanupMigrationRequest(req *pb.CleanupMigrationRequest) error { + switch { + case req == nil: + return errors.WithStack(status.Error(codes.InvalidArgument, "migration cleanup request is nil")) + case req.GetJobId() == 0: + return errors.WithStack(status.Error(codes.InvalidArgument, "migration cleanup job_id is required")) + case req.GetMode() != pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS && + req.GetMode() != pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA: + return errors.WithStack(status.Error(codes.InvalidArgument, "migration cleanup mode is invalid")) + case req.GetMode() == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS && req.GetKeyFamily() == 0: + return errors.WithStack(status.Error(codes.InvalidArgument, "migration cleanup key_family is required")) + } + return nil +} + +func (i *Internal) migrationMinAdmittedTS(ctx context.Context, jobID uint64) uint64 { + reader, ok := i.store.(store.MigrationTargetReadinessReader) + if !ok { + return 0 + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return 0 + } + for _, state := range states { + if state.JobID == jobID { + return state.MinAdmittedTS + } + } + return 0 +} + +func validateTargetStagedReadinessRequest(req *pb.TargetStagedReadinessRequest) error { + switch { + case req == nil: + return errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness request is nil")) + case req.GetJobId() == 0: + return errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness job_id is required")) + case req.GetMigrationJobId() == 0: + return errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness migration_job_id is required")) + case req.GetArmed() && req.GetMinWriteTsExclusive() == 0: + return errors.WithStack(status.Error(codes.InvalidArgument, "target staged readiness min_write_ts_exclusive is required when armed")) + } + return validateSourceMigrationControlRequest(req) +} + +func validateSourceMigrationControlRequest(req *pb.TargetStagedReadinessRequest) error { + if req.GetSourceReadFence() && !req.GetSourceWriteFence() { + return errors.WithStack(status.Error(codes.InvalidArgument, "source read fence requires source write fence")) + } + if (req.GetSourceWriteFence() || req.GetSourceReadFence()) && req.GetRetentionPinTs() == 0 { + return errors.WithStack(status.Error(codes.InvalidArgument, "source migration control retention_pin_ts is required")) + } + return nil +} + +func (i *Internal) ProbeMigrationLocks(ctx context.Context, req *pb.ProbeMigrationLocksRequest) (*pb.ProbeMigrationLocksResponse, error) { + if req == nil || len(req.GetRouteStart()) == 0 { + return nil, errors.WithStack(status.Error(codes.InvalidArgument, "migration lock probe route_start is required")) + } + if i.store == nil { + return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "migration lock probe store is not configured")) + } + if err := i.verifyInternalLeaderApplied(ctx); err != nil { + return nil, err + } + locks, err := kv.PendingTxnLocksInRoute(ctx, i.store, req.GetRouteStart(), req.GetRouteEnd(), ^uint64(0), int(req.GetLimit())) + if err != nil { + return nil, errors.WithStack(err) + } + return &pb.ProbeMigrationLocksResponse{PendingCount: uint32(len(locks))}, nil //nolint:gosec // result is bounded by uint32 wire limit +} + func (i *Internal) verifyMigrationPromoteEnabled(ctx context.Context) error { if i.migrationPromoteGate == nil { return nil @@ -296,6 +612,16 @@ func (i *Internal) verifyMigrationPromoteEnabled(ctx context.Context) error { return nil } +func (i *Internal) verifyMigrationCleanupEnabled(ctx context.Context) error { + if i.migrationCleanupGate == nil { + return nil + } + if err := i.migrationCleanupGate(ctx); err != nil { + return errors.WithStack(err) + } + return nil +} + func validatePromoteStagedVersionsRequest(req *pb.PromoteStagedVersionsRequest) error { prefix := distribution.MigrationStagedDataKeyPrefix(req.GetJobId()) if err := store.ValidatePromotionCursorForRange(req.GetCursor(), prefix, store.PrefixScanEnd(prefix)); err != nil { @@ -315,7 +641,7 @@ func defaultMigrationImportGate(context.Context) error { } func migrationImportOpcodeEnabledFromEnv() bool { - return envFlagEnabled("ELASTICKV_ENABLE_MIGRATION_IMPORT_OPCODE") + return MigrationImportOpcodeEnabledFromEnv() } func defaultMigrationPromoteGate(context.Context) error { @@ -325,8 +651,33 @@ func defaultMigrationPromoteGate(context.Context) error { return errors.WithStack(status.Error(codes.FailedPrecondition, "migration promote opcode is disabled; enable after every voter is running a build that supports migration promotion")) } +func defaultMigrationCleanupGate(context.Context) error { + if MigrationCleanupOpcodeEnabledFromEnv() { + return nil + } + return errors.WithStack(status.Error(codes.FailedPrecondition, "migration cleanup opcode is disabled; enable after every voter is running a build that supports migration cleanup")) +} + func migrationPromoteOpcodeEnabledFromEnv() bool { - return envFlagEnabled("ELASTICKV_ENABLE_MIGRATION_PROMOTE_OPCODE") + return MigrationPromoteOpcodeEnabledFromEnv() +} + +// MigrationImportOpcodeEnabledFromEnv reports whether the staged import opcode +// is enabled for this process. +func MigrationImportOpcodeEnabledFromEnv() bool { + return envFlagEnabled(MigrationImportOpcodeEnv) +} + +// MigrationPromoteOpcodeEnabledFromEnv reports whether the staged promotion +// opcode is enabled for this process. +func MigrationPromoteOpcodeEnabledFromEnv() bool { + return envFlagEnabled(MigrationPromoteOpcodeEnv) +} + +// MigrationCleanupOpcodeEnabledFromEnv reports whether the migration cleanup +// opcode is enabled for this process. +func MigrationCleanupOpcodeEnabledFromEnv() bool { + return envFlagEnabled(MigrationCleanupOpcodeEnv) } func envFlagEnabled(name string) bool { @@ -410,6 +761,51 @@ func (i *Internal) proposeMigrationPromote(ctx context.Context, req *pb.PromoteS } } +func (i *Internal) proposeMigrationCleanup(ctx context.Context, req *pb.CleanupMigrationRequest) (store.CleanupVersionsResult, error) { + cmd, err := kv.MarshalMigrationCleanupCommand(req) + if err != nil { + return store.CleanupVersionsResult{}, errors.WithStack(err) + } + resp, err := i.proposeMigrationCommand(ctx, cmd, "migration cleanup") + if err != nil { + return store.CleanupVersionsResult{}, errors.WithStack(err) + } + if req.GetMode() == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA && resp == nil { + return store.CleanupVersionsResult{Done: true}, nil + } + switch resp := resp.(type) { + case store.CleanupVersionsResult: + return resp, nil + case *store.CleanupVersionsResult: + if resp == nil { + return store.CleanupVersionsResult{}, errors.New("migration cleanup apply returned nil result") + } + return *resp, nil + case error: + return store.CleanupVersionsResult{}, errors.WithStack(resp) + default: + return store.CleanupVersionsResult{}, errors.WithStack(errors.Newf("unexpected migration cleanup apply response type %T", resp)) + } +} + +func (i *Internal) proposeTargetStagedReadiness(ctx context.Context, req *pb.TargetStagedReadinessRequest) error { + cmd, err := kv.MarshalTargetStagedReadinessCommand(req) + if err != nil { + return errors.WithStack(err) + } + resp, err := i.proposeMigrationCommand(ctx, cmd, "target staged readiness") + if err != nil { + return errors.WithStack(err) + } + if resp == nil { + return nil + } + if err, ok := resp.(error); ok { + return errors.WithStack(err) + } + return errors.WithStack(errors.Newf("unexpected target readiness apply response type %T", resp)) +} + func (i *Internal) proposeMigrationCommand(ctx context.Context, cmd []byte, label string) (any, error) { result, err := i.migrationProposer.Propose(ctx, cmd) if err != nil { diff --git a/adapter/internal_migration_probe_test.go b/adapter/internal_migration_probe_test.go new file mode 100644 index 000000000..53d309036 --- /dev/null +++ b/adapter/internal_migration_probe_test.go @@ -0,0 +1,154 @@ +package adapter + +import ( + "context" + "testing" + "time" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestInternalProbeMigrationStateUsesLocalFSMAndCatalog(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 9, + Routes: []distribution.RouteDescriptor{{ + RouteID: 2, + Start: []byte("m"), + GroupID: 2, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 88, + }}, + })) + tracker := kv.NewActiveTimestampTracker() + internal := NewInternalWithEngine(nil, nil, nil, nil, + WithInternalStore(st), + WithInternalRouteEngine(engine), + WithInternalActiveTimestampTracker(tracker), + ) + + state := store.TargetStagedReadinessState{ + JobID: 7, + RouteStart: []byte("m"), + ExpectedCutoverVersion: 9, + MigrationJobID: 7, + MinWriteTSExclusive: 88, + Armed: true, + } + readinessWriter, ok := st.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, readinessWriter.ApplyTargetStagedReadiness(ctx, state)) + control, err := internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED, + RouteStart: []byte("m"), + ExpectedCatalogVersion: 9, + MigrationJobId: 7, + MinWriteTsExclusive: 88, + }) + require.NoError(t, err) + require.True(t, control.Ready) + + cleared, err := internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED, + RouteStart: []byte("m"), + ExpectedCatalogVersion: 9, + ExpectedGroupId: 2, + MinWriteTsExclusive: 88, + }) + require.NoError(t, err) + require.True(t, cleared.Ready) + + pin := tracker.Pin(100) + drained, err := internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED, + ReadDrainNotBeforeMs: time.Now().Add(-time.Second).UnixMilli(), + }) + require.NoError(t, err) + require.False(t, drained.Ready) + pin.Release() + drained, err = internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED, + ReadDrainNotBeforeMs: time.Now().Add(-time.Second).UnixMilli(), + }) + require.NoError(t, err) + require.True(t, drained.Ready) + + metadata, err := internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, + }) + require.NoError(t, err) + require.False(t, metadata.Ready) + cleaner, ok := st.(store.MigrationCleaner) + require.True(t, ok) + require.NoError(t, cleaner.ClearMigrationState(ctx, 7, 0)) + metadata, err = internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, + }) + require.NoError(t, err) + require.True(t, metadata.Ready) +} + +func TestInternalProbeMigrationMetadataClearedWaitsForImportMetadata(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + internal := NewInternalWithEngine(nil, nil, nil, nil, WithInternalStore(st)) + _, err := st.ImportVersions(ctx, store.ImportVersionsOptions{ + JobID: 7, + BracketID: 1, + BatchSeq: 1, + Cursor: []byte("cursor-1"), + Versions: []store.MVCCVersion{{ + Key: []byte("m/key"), + Value: []byte("value"), + CommitTS: 42, + }}, + }) + require.NoError(t, err) + + metadata, err := internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, + }) + require.NoError(t, err) + require.False(t, metadata.Ready) + + cleaner, ok := st.(store.MigrationCleaner) + require.True(t, ok) + require.NoError(t, cleaner.ClearMigrationState(ctx, 7, 0)) + metadata, err = internal.ProbeMigrationState(ctx, &pb.ProbeMigrationStateRequest{ + JobId: 7, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, + }) + require.NoError(t, err) + require.True(t, metadata.Ready) +} + +func TestInternalIssueMigrationTimestampFollowsSourceLastCommit(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + require.NoError(t, st.ApplyMutations(ctx, []*store.KVPairMutation{{Key: []byte("m"), Value: []byte("value")}}, nil, 50, 50)) + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, WithInternalStore(st)) + + resp, err := internal.IssueMigrationTimestamp(ctx, &pb.IssueMigrationTimestampRequest{}) + require.NoError(t, err) + require.Equal(t, uint64(50), resp.GetLastCommitTs()) + require.Greater(t, resp.GetTimestamp(), resp.GetLastCommitTs()) +} diff --git a/adapter/internal_migration_test.go b/adapter/internal_migration_test.go index e2c162c96..e7ebce4a1 100644 --- a/adapter/internal_migration_test.go +++ b/adapter/internal_migration_test.go @@ -659,3 +659,137 @@ func TestInternalPromoteStagedVersionsAppliesStoreBatch(t *testing.T) { require.ErrorIs(t, err, store.ErrKeyNotFound) require.GreaterOrEqual(t, clock.Current(), uint64(30)) } + +func TestInternalApplyTargetStagedReadinessBypassesOpcodeGateForSafetyGuard(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + proposer := &applyingMigrationProposer{ + fsm: kv.NewKvFSMWithHLC(st, nil), + } + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, + WithInternalStore(st), + WithInternalMigrationProposer(proposer), + WithInternalMigrationPromoteGate(func(context.Context) error { + return status.Error(codes.FailedPrecondition, "migration opcode disabled for test") + }), + ) + + resp, err := internal.ApplyTargetStagedReadiness(ctx, &pb.TargetStagedReadinessRequest{ + JobId: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobId: 7, + MinWriteTsExclusive: 100, + Armed: true, + }) + require.NoError(t, err) + require.NotNil(t, resp) + require.Equal(t, uint64(1), proposer.calls) + + reader, ok := st.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []store.TargetStagedReadinessState{{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + }}, states) +} + +func TestInternalApplyTargetStagedReadinessProposesThroughRaft(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + proposer := &applyingMigrationProposer{ + fsm: kv.NewKvFSMWithHLC(st, nil), + } + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, + WithInternalStore(st), + WithInternalMigrationProposer(proposer), + WithInternalMigrationPromoteGate(func(context.Context) error { return nil }), + ) + + _, err := internal.ApplyTargetStagedReadiness(ctx, &pb.TargetStagedReadinessRequest{ + JobId: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobId: 7, + MinWriteTsExclusive: 100, + Armed: true, + }) + require.NoError(t, err) + require.Equal(t, uint64(1), proposer.calls) + + reader, ok := st.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []store.TargetStagedReadinessState{{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + }}, states) +} + +func TestInternalApplyTargetStagedReadinessRejectsArmedZeroMinWriteTS(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + proposer := &applyingMigrationProposer{ + fsm: kv.NewKvFSMWithHLC(st, nil), + } + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, + WithInternalStore(st), + WithInternalMigrationProposer(proposer), + WithInternalMigrationPromoteGate(func(context.Context) error { return nil }), + ) + + resp, err := internal.ApplyTargetStagedReadiness(ctx, &pb.TargetStagedReadinessRequest{ + JobId: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobId: 7, + Armed: true, + }) + require.Nil(t, resp) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + require.Equal(t, uint64(0), proposer.calls) +} + +func TestInternalCleanupMigrationRejectsWhenOpcodeGateClosed(t *testing.T) { + t.Parallel() + + proposer := &applyingMigrationProposer{fsm: kv.NewKvFSMWithHLC(store.NewMVCCStore(), nil)} + internal := NewInternalWithEngine(nil, mockInternalLeader{}, nil, nil, + WithInternalMigrationProposer(proposer), + WithInternalMigrationCleanupGate(func(context.Context) error { + return status.Error(codes.FailedPrecondition, "migration cleanup disabled for test") + }), + ) + + resp, err := internal.CleanupMigration(context.Background(), &pb.CleanupMigrationRequest{ + JobId: 9, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS, + KeyFamily: distribution.MigrationFamilyUser, + }) + require.Nil(t, resp) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.Zero(t, proposer.calls) +} diff --git a/adapter/redis_compat_commands_stream_test.go b/adapter/redis_compat_commands_stream_test.go index cbab8fa66..b862c5e16 100644 --- a/adapter/redis_compat_commands_stream_test.go +++ b/adapter/redis_compat_commands_stream_test.go @@ -279,19 +279,27 @@ func TestRedis_StreamXReadLatencyIsConstant(t *testing.T) { rdb := redis.NewClient(&redis.Options{Addr: nodes[0].redisAddress}) defer func() { _ = rdb.Close() }() - ctx := context.Background() + ctx := t.Context() const probes = 100 lastID := seedStreamEntriesForXReadLatency(t, nodes[0].redisServer, ctx, "stream-lat") measure := func() time.Duration { - start := time.Now() - streams, err := rdb.XRead(ctx, &redis.XReadArgs{ - Streams: []string{"stream-lat", lastID}, - Count: 10, - Block: 10 * time.Millisecond, - }).Result() - elapsed := time.Since(start) + var ( + streams []redis.XStream + elapsed time.Duration + ) + err := retryNotLeader(ctx, func() error { + start := time.Now() + var xerr error + streams, xerr = rdb.XRead(ctx, &redis.XReadArgs{ + Streams: []string{"stream-lat", lastID}, + Count: 10, + Block: 10 * time.Millisecond, + }).Result() + elapsed = time.Since(start) + return xerr + }) require.True(t, errors.Is(err, redis.Nil) || err == nil) require.Empty(t, streams) return elapsed diff --git a/adapter/redis_delta_compactor_test.go b/adapter/redis_delta_compactor_test.go index 46a40d538..17600a4d7 100644 --- a/adapter/redis_delta_compactor_test.go +++ b/adapter/redis_delta_compactor_test.go @@ -1684,8 +1684,11 @@ func TestDeltaCompactor_UrgentCompactionPagination(t *testing.T) { require.NoError(t, st.PutAt(ctx, dKey, delta, i+1, 0)) } - // Queue and process the urgent compaction. - c.TriggerUrgentCompaction("hash", userKey) + // Exercise the urgent pagination loop directly. Running through c.Run would + // also start an initial background SyncOnce; the local test coordinator applies + // elems one-by-one instead of atomically, so a test read can observe the meta + // update before all delete elems have been applied under the race detector. + c.compactUrgentKey(ctx, urgentCompactionRequest{typeName: "hash", userKey: userKey}) runCtx, cancel := context.WithCancel(ctx) defer cancel() diff --git a/adapter/split_job_runner.go b/adapter/split_job_runner.go new file mode 100644 index 000000000..3a37caa63 --- /dev/null +++ b/adapter/split_job_runner.go @@ -0,0 +1,1738 @@ +package adapter + +import ( + "bytes" + "context" + "encoding/binary" + "io" + "math" + "sort" + "time" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" +) + +const ( + defaultSplitMigrationChunkBytes = 1 << 20 + defaultSplitMigrationMaxScannedBytes = 4 << 20 + defaultSplitMigrationLockProbeLimit = 1024 + defaultSplitMigrationReadFenceGrace = 30 * time.Second + splitCleanupCursorVersion = byte(1) + splitVoterAckCursorVersion = byte(1) + splitCleanupCursorHeaderBytes = 5 + splitJobHistoryLimit = 1000 + splitJobHistoryDeleteBatchLimit = 1000 + splitJobHistoryTTL = 7 * 24 * time.Hour + splitJobHistoryGCInterval = time.Minute +) + +var ( + splitCleanupDoneCursor = []byte{splitCleanupCursorVersion, 0xff, 0xff, 0xff, 0xff} + errSplitMigrationVoterBarrierIncomplete = errors.New("split migration voter barrier incomplete") +) + +type splitVoterAck struct { + id string + address string + acked bool +} + +func splitMigrationVoterBarrierIncompleteError(jobID uint64, kind pb.MigrationStateProbeKind) error { + return errors.Wrapf(errSplitMigrationVoterBarrierIncomplete, "job_id=%d kind=%s", jobID, kind.String()) +} + +func (s *DistributionServer) syncSplitMigrationVoterBarrier( + ctx context.Context, + groupID uint64, + existing []byte, + fallback SplitMigrationClient, + req *pb.ProbeMigrationStateRequest, +) ([]byte, bool, error) { + voters, err := s.splitMigrationVoters(ctx, groupID, fallback) + if err != nil { + return nil, false, err + } + prior, err := decodeSplitVoterAckCursor(existing) + if err != nil { + return nil, false, err + } + priorAcked := make(map[string]bool, len(prior)) + for _, voter := range prior { + priorAcked[splitVoterAckKey(voter.id, voter.address)] = voter.acked + } + acks := make([]splitVoterAck, 0, len(voters)) + complete := true + for _, voter := range voters { + key := splitVoterAckKey(voter.ID, voter.Address) + acked := priorAcked[key] + resp, probeErr := voter.Client.ProbeMigrationState(ctx, req) + if probeErr == nil { + acked = resp.GetReady() + } + complete = complete && acked + acks = append(acks, splitVoterAck{id: voter.ID, address: voter.Address, acked: acked}) + } + return encodeSplitVoterAckCursor(acks), complete, nil +} + +func (s *DistributionServer) splitMigrationVoters(ctx context.Context, groupID uint64, fallback SplitMigrationClient) ([]SplitMigrationVoter, error) { + if s.splitMigrationVoterFactory == nil { + if fallback == nil { + return nil, errors.New("split migration voter factory is not configured") + } + return []SplitMigrationVoter{{ID: "leader", Address: "leader", Client: fallback}}, nil + } + voters, err := s.splitMigrationVoterFactory(ctx, groupID) + if err != nil { + return nil, errors.WithStack(err) + } + if len(voters) == 0 { + return nil, errors.New("split migration voter set is empty") + } + if err := validateSplitMigrationVoters(voters); err != nil { + return nil, err + } + return voters, nil +} + +func validateSplitMigrationVoters(voters []SplitMigrationVoter) error { + sort.Slice(voters, func(i, j int) bool { + if voters[i].ID == voters[j].ID { + return voters[i].Address < voters[j].Address + } + return voters[i].ID < voters[j].ID + }) + for index, voter := range voters { + if voter.ID == "" || voter.Address == "" || voter.Client == nil { + return errors.New("split migration voter is incomplete") + } + if index > 0 && voter.ID == voters[index-1].ID { + return errors.New("split migration voter id is duplicated") + } + } + return nil +} + +func splitVoterAckKey(id, address string) string { + return id + "\x00" + address +} + +func encodeSplitVoterAckCursor(acks []splitVoterAck) []byte { + out := []byte{splitVoterAckCursorVersion} + out = binary.AppendUvarint(out, uint64(len(acks))) + for _, ack := range acks { + out = binary.AppendUvarint(out, uint64(len(ack.id))) + out = append(out, ack.id...) + out = binary.AppendUvarint(out, uint64(len(ack.address))) + out = append(out, ack.address...) + if ack.acked { + out = append(out, 1) + } else { + out = append(out, 0) + } + } + return out +} + +func decodeSplitVoterAckCursor(raw []byte) ([]splitVoterAck, error) { + if len(raw) == 0 { + return nil, nil + } + if raw[0] != splitVoterAckCursorVersion { + return nil, errors.New("invalid split voter ack cursor version") + } + raw = raw[1:] + count, rest, err := consumeSplitCursorUvarint(raw) + if err != nil { + return nil, err + } + raw = rest + acks := make([]splitVoterAck, 0, count) + for range count { + id, next, err := consumeSplitCursorString(raw) + if err != nil { + return nil, err + } + address, next, err := consumeSplitCursorString(next) + if err != nil || len(next) == 0 || next[0] > 1 { + return nil, errors.New("invalid split voter ack cursor entry") + } + acks = append(acks, splitVoterAck{id: id, address: address, acked: next[0] == 1}) + raw = next[1:] + } + if len(raw) != 0 { + return nil, errors.New("split voter ack cursor has trailing bytes") + } + return acks, nil +} + +func consumeSplitCursorString(raw []byte) (string, []byte, error) { + size, rest, err := consumeSplitCursorUvarint(raw) + if err != nil || size > uint64(len(rest)) { + return "", nil, errors.New("invalid split voter ack cursor string") + } + return string(rest[:size]), rest[size:], nil +} + +func consumeSplitCursorUvarint(raw []byte) (uint64, []byte, error) { + value, size := binary.Uvarint(raw) + if size <= 0 { + return 0, nil, errors.New("invalid split voter ack cursor integer") + } + return value, raw[size:], nil +} + +func (s *DistributionServer) runSplitJobPhase(ctx context.Context, job distribution.SplitJob) error { + if s.splitMigrationClientFactory == nil { + return errors.New("split migration client factory is not configured") + } + if job.Phase == distribution.SplitJobPhaseBackfill || job.Phase == distribution.SplitJobPhaseDeltaCopy { + return s.runSplitJobCopyPhase(ctx, job) + } + return s.runSplitJobControlPhase(ctx, job) +} + +func (s *DistributionServer) runSplitJobCopyPhase(ctx context.Context, job distribution.SplitJob) error { + if job.Phase == distribution.SplitJobPhaseBackfill { + return s.copySplitJobPhase(ctx, job, distribution.SplitJobExportPhaseBackfill, 0, job.SnapshotTS) + } + return s.copySplitJobPhase(ctx, job, distribution.SplitJobExportPhaseDeltaCopy, job.DeltaFloor, job.FenceTS) +} + +func (s *DistributionServer) runSplitJobControlPhase(ctx context.Context, job distribution.SplitJob) error { + switch job.Phase { + case distribution.SplitJobPhasePlanned: + return s.beginSplitJobBackfill(ctx, job) + case distribution.SplitJobPhaseFence: + return s.finalizeSplitJobFence(ctx, job) + case distribution.SplitJobPhaseCutover: + return s.cutoverSplitJob(ctx, job) + case distribution.SplitJobPhaseCleanup: + return s.cleanupSplitJob(ctx, job) + case distribution.SplitJobPhaseAbandoning: + return s.abandonSplitJob(ctx, job) + case distribution.SplitJobPhaseNone, + distribution.SplitJobPhaseBackfill, + distribution.SplitJobPhaseDeltaCopy, + distribution.SplitJobPhaseDone, + distribution.SplitJobPhaseFailed, + distribution.SplitJobPhaseAbandoned: + return nil + } + return errors.New("split job phase is not runnable") +} + +func (s *DistributionServer) cleanupSplitJob(ctx context.Context, job distribution.SplitJob) error { + if !job.TargetPromotionDone { + return s.promoteSplitJobTargetAndComplete(ctx, job) + } + if !bytes.Equal(job.TargetClearedDescriptorAckCursor, splitCleanupDoneCursor) { + if err := s.cleanupSplitJobTargetProofs(ctx, job); err != nil { + return err + } + return nil + } + if !bytes.Equal(job.SourceCutoverAckCursor, splitCleanupDoneCursor) { + if err := s.ackSplitJobSourceRouteRemoval(ctx, job); err != nil { + return err + } + return nil + } + if !bytes.Equal(job.SourceReadDrainCursor, splitCleanupDoneCursor) { + if err := s.ackSplitJobSourceReadDrain(ctx, job); err != nil { + return err + } + return nil + } + if !bytes.Equal(job.Cursor, splitCleanupDoneCursor) { + return s.cleanupSplitJobSourceData(ctx, job) + } + if job.SourceRetentionPinTS != math.MaxUint64 { + return s.cleanupSplitJobSourceProofs(ctx, job) + } + return s.finishSplitJobHistory(ctx, job, distribution.SplitJobPhaseDone) +} + +func (s *DistributionServer) splitJobMigrationClients(ctx context.Context, job distribution.SplitJob) (SplitMigrationClient, SplitMigrationClient, []byte, error) { + if s.splitMigrationClientFactory == nil { + return nil, nil, nil, errors.New("split migration client factory is not configured") + } + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return nil, nil, nil, err + } + sourceGroupID, routeEnd, ok := splitJobSourceSibling(snapshot.Routes, job) + if !ok { + if parent, found := findRouteByID(snapshot.Routes, job.SourceRouteID); found { + sourceGroupID = parent.GroupID + routeEnd = distribution.CloneBytes(parent.End) + ok = true + } + } + if !ok { + return nil, nil, nil, errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + } + source, target, err := s.splitMigrationClientFactory(ctx, job, sourceGroupID) + if err != nil { + return nil, nil, nil, errors.WithStack(err) + } + if source == nil || target == nil { + return nil, nil, nil, errors.New("split migration source or target client is nil") + } + return source, target, routeEnd, nil +} + +func (s *DistributionServer) cleanupSplitJobTargetProofs(ctx context.Context, job distribution.SplitJob) error { + _, target, routeEnd, err := s.splitJobMigrationClients(ctx, job) + if err != nil { + return err + } + descriptorCursor, descriptorComplete, err := s.syncSplitMigrationVoterBarrier(ctx, job.TargetGroupID, job.TargetClearedDescriptorAckCursor, target, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED, + RouteStart: job.SplitKey, + RouteEnd: routeEnd, + ExpectedCatalogVersion: job.CutoverVersion, + ExpectedGroupId: job.TargetGroupID, + MinWriteTsExclusive: job.FenceTS, + }) + if err != nil { + return err + } + if !descriptorComplete { + return s.persistSplitJobTargetCleanupCursor(ctx, job.JobID, descriptorCursor) + } + if _, err := target.CleanupMigration(ctx, &pb.CleanupMigrationRequest{ + JobId: job.JobID, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA, + }); err != nil { + return errors.WithStack(err) + } + _, metadataComplete, err := s.syncSplitMigrationVoterBarrier(ctx, job.TargetGroupID, nil, target, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, + }) + if err != nil { + return err + } + if !metadataComplete { + return splitMigrationVoterBarrierIncompleteError(job.JobID, pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED) + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseCleanup && current.TargetPromotionDone { + current.TargetClearedDescriptorAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func (s *DistributionServer) persistSplitJobTargetCleanupCursor(ctx context.Context, jobID uint64, cursor []byte) error { + return s.updateSplitJobViaCoordinator(ctx, jobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseCleanup { + current.TargetClearedDescriptorAckCursor = distribution.CloneBytes(cursor) + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func (s *DistributionServer) ackSplitJobSourceRouteRemoval(ctx context.Context, job distribution.SplitJob) error { + source, _, routeEnd, err := s.splitJobMigrationClients(ctx, job) + if err != nil { + return err + } + sourceGroupID, err := s.splitJobSourceGroupID(ctx, job) + if err != nil { + return err + } + cursor, complete, err := s.syncSplitMigrationVoterBarrier(ctx, sourceGroupID, job.SourceCutoverAckCursor, source, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_ROUTE_REMOVED, + RouteStart: job.SplitKey, + RouteEnd: routeEnd, + ExpectedCatalogVersion: job.CutoverVersion, + ExpectedGroupId: job.TargetGroupID, + }) + if err != nil { + return err + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseCleanup { + current.SourceCutoverAckCursor = distribution.CloneBytes(cursor) + if complete { + current.SourceCutoverAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) + } + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func (s *DistributionServer) ackSplitJobSourceReadDrain(ctx context.Context, job distribution.SplitJob) error { + source, _, _, err := s.splitJobMigrationClients(ctx, job) + if err != nil { + return err + } + sourceGroupID, err := s.splitJobSourceGroupID(ctx, job) + if err != nil { + return err + } + notBefore := int64(job.PromotionCompletedTS>>kv.HLCLogicalBits) + defaultSplitMigrationReadFenceGrace.Milliseconds() //nolint:gosec // HLC physical millis fit int64. + cursor, complete, err := s.syncSplitMigrationVoterBarrier(ctx, sourceGroupID, job.SourceReadDrainCursor, source, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED, + ReadDrainNotBeforeMs: notBefore, + }) + if err != nil { + return err + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseCleanup { + current.SourceReadDrainCursor = distribution.CloneBytes(cursor) + if complete { + current.SourceReadDrainCursor = distribution.CloneBytes(splitCleanupDoneCursor) + } + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func (s *DistributionServer) splitJobSourceGroupID(ctx context.Context, job distribution.SplitJob) (uint64, error) { + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return 0, err + } + groupID, _, ok := splitJobSourceSibling(snapshot.Routes, job) + if !ok { + if parent, found := findRouteByID(snapshot.Routes, job.SourceRouteID); found { + return parent.GroupID, nil + } + return 0, errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + } + return groupID, nil +} + +func (s *DistributionServer) cleanupSplitJobSourceData(ctx context.Context, job distribution.SplitJob) error { + source, _, routeEnd, err := s.splitJobMigrationClients(ctx, job) + if err != nil { + return err + } + brackets, err := distribution.PlanExportBrackets(job.SplitKey, routeEnd) + if err != nil { + return errors.WithStack(err) + } + index, cursor, done, err := decodeSplitCleanupCursor(job.Cursor, len(brackets)) + if err != nil || done { + return err + } + bracket := brackets[index] + resp, err := source.CleanupMigration(ctx, splitMigrationCleanupRequest(job, routeEnd, bracket, cursor, job.FenceTS)) + if err != nil { + return errors.WithStack(err) + } + encoded, err := nextSplitCleanupCursor(resp, cursor, index, len(brackets)) + if err != nil { + return err + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseCleanup && bytes.Equal(current.Cursor, job.Cursor) { + current.Cursor = encoded + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func nextSplitCleanupCursor(resp *pb.CleanupMigrationResponse, cursor []byte, index, bracketCount int) ([]byte, error) { + if !resp.GetDone() { + next := resp.GetNextCursor() + if len(next) == 0 || bytes.Equal(next, cursor) { + return nil, errors.New("split migration source cleanup made no cursor progress") + } + return encodeSplitCleanupCursor(index, next, false), nil + } + nextIndex := index + 1 + return encodeSplitCleanupCursor(nextIndex, nil, nextIndex >= bracketCount), nil +} + +func splitMigrationCleanupRequest(job distribution.SplitJob, routeEnd []byte, bracket distribution.MigrationBracket, cursor []byte, maxCommitTS uint64) *pb.CleanupMigrationRequest { + return &pb.CleanupMigrationRequest{ + JobId: job.JobID, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS, + RangeStart: bracket.Start, + RangeEnd: bracket.End, + Cursor: cursor, + MaxCommitTs: maxCommitTS, + MaxVersions: defaultSplitPromotionMaxVersions, + MaxBytes: defaultSplitPromotionMaxBytes, + MaxScannedBytes: defaultSplitPromotionMaxScanned, + KeyFamily: bracket.Family, + RouteStart: job.SplitKey, + RouteEnd: routeEnd, + ExcludeKnownInternal: bracket.ExcludeKnownInternal, + ExcludePrefixes: bracket.ExcludePrefixes, + RequiresRouteKeyCheck: bracket.RequiresRouteKeyCheck, + RequiresDecodedS3: bracket.RequiresDecodedS3, + } +} + +func (s *DistributionServer) cleanupSplitJobSourceProofs(ctx context.Context, job distribution.SplitJob) error { + if err := s.cleanupSplitJobSourceMetadata(ctx, job); err != nil { + return err + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseCleanup { + current.SourceCutoverAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) + current.SourceRetentionPinTS = math.MaxUint64 + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func encodeSplitCleanupCursor(index int, cursor []byte, done bool) []byte { + if done { + return distribution.CloneBytes(splitCleanupDoneCursor) + } + out := make([]byte, splitCleanupCursorHeaderBytes+len(cursor)) + out[0] = splitCleanupCursorVersion + binary.BigEndian.PutUint32(out[1:], uint32(index)) //nolint:gosec // bracket count is bounded. + copy(out[splitCleanupCursorHeaderBytes:], cursor) + return out +} + +func decodeSplitCleanupCursor(raw []byte, bracketCount int) (int, []byte, bool, error) { + if len(raw) == 0 { + return 0, nil, bracketCount == 0, nil + } + if bytes.Equal(raw, splitCleanupDoneCursor) { + return bracketCount, nil, true, nil + } + if len(raw) < splitCleanupCursorHeaderBytes || raw[0] != splitCleanupCursorVersion { + return 0, nil, false, errors.New("invalid split cleanup cursor") + } + index := int(binary.BigEndian.Uint32(raw[1:splitCleanupCursorHeaderBytes])) + if index < 0 || index >= bracketCount { + return 0, nil, false, errors.New("split cleanup cursor bracket is out of range") + } + return index, distribution.CloneBytes(raw[splitCleanupCursorHeaderBytes:]), false, nil +} + +func (s *DistributionServer) abandonSplitJob(ctx context.Context, job distribution.SplitJob) error { + if err := s.rollbackAbandonedSplitJobFence(ctx, job); err != nil { + return err + } + if !bytes.Equal(job.TargetClearedDescriptorAckCursor, splitCleanupDoneCursor) { + return s.cleanupAbandonedSplitJobTarget(ctx, job) + } + if !bytes.Equal(job.SourceCutoverAckCursor, splitCleanupDoneCursor) { + return s.cleanupAbandonedSplitJobSource(ctx, job) + } + return s.finishSplitJobHistory(ctx, job, distribution.SplitJobPhaseAbandoned) +} + +func (s *DistributionServer) cleanupAbandonedSplitJobTarget(ctx context.Context, job distribution.SplitJob) error { + _, target, _, err := s.splitJobMigrationClients(ctx, job) + if err != nil { + return err + } + _, cursor, done, err := decodeSplitCleanupCursor(job.TargetClearedDescriptorAckCursor, 1) + if err != nil { + return err + } + if !done { + complete, err := s.cleanupAbandonedSplitJobTargetVersions(ctx, target, job, cursor) + if err != nil || !complete { + return err + } + } + if _, err := target.CleanupMigration(ctx, &pb.CleanupMigrationRequest{ + JobId: job.JobID, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA, + }); err != nil { + return errors.WithStack(err) + } + _, metadataComplete, err := s.syncSplitMigrationVoterBarrier(ctx, job.TargetGroupID, nil, target, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, + }) + if err != nil { + return err + } + if !metadataComplete { + return splitMigrationVoterBarrierIncompleteError(job.JobID, pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED) + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseAbandoning { + current.TargetClearedDescriptorAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func (s *DistributionServer) cleanupAbandonedSplitJobTargetVersions( + ctx context.Context, + target SplitMigrationClient, + job distribution.SplitJob, + cursor []byte, +) (bool, error) { + prefix := distribution.MigrationStagedDataKeyPrefix(job.JobID) + resp, err := target.CleanupMigration(ctx, &pb.CleanupMigrationRequest{ + JobId: job.JobID, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS, + RangeStart: prefix, + RangeEnd: store.PrefixScanEnd(prefix), + Cursor: cursor, + MaxCommitTs: math.MaxUint64, + MaxVersions: defaultSplitPromotionMaxVersions, + MaxBytes: defaultSplitPromotionMaxBytes, + MaxScannedBytes: defaultSplitPromotionMaxScanned, + KeyFamily: distribution.MigrationFamilyUser, + }) + if err != nil { + return false, errors.WithStack(err) + } + if resp.GetDone() { + return true, nil + } + if len(resp.GetNextCursor()) == 0 || bytes.Equal(resp.GetNextCursor(), cursor) { + return false, errors.New("split migration abandon cleanup made no cursor progress") + } + encoded := encodeSplitCleanupCursor(0, resp.GetNextCursor(), false) + err = s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseAbandoning && bytes.Equal(current.TargetClearedDescriptorAckCursor, job.TargetClearedDescriptorAckCursor) { + current.TargetClearedDescriptorAckCursor = encoded + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) + return false, err +} + +func (s *DistributionServer) cleanupAbandonedSplitJobSource(ctx context.Context, job distribution.SplitJob) error { + if err := s.cleanupSplitJobSourceMetadata(ctx, job); err != nil { + return err + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseAbandoning { + current.SourceCutoverAckCursor = distribution.CloneBytes(splitCleanupDoneCursor) + current.SourceRetentionPinTS = math.MaxUint64 + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func (s *DistributionServer) cleanupSplitJobSourceMetadata(ctx context.Context, job distribution.SplitJob) error { + source, _, _, err := s.splitJobMigrationClients(ctx, job) + if err != nil { + return err + } + if _, err := source.CleanupMigration(ctx, &pb.CleanupMigrationRequest{ + JobId: job.JobID, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA, + }); err != nil { + return errors.WithStack(err) + } + sourceGroupID, err := s.splitJobSourceGroupID(ctx, job) + if err != nil { + return err + } + _, metadataComplete, err := s.syncSplitMigrationVoterBarrier(ctx, sourceGroupID, nil, source, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED, + }) + if err != nil { + return err + } + if !metadataComplete { + return splitMigrationVoterBarrierIncompleteError(job.JobID, pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED) + } + return nil +} + +func (s *DistributionServer) rollbackAbandonedSplitJobFence(ctx context.Context, job distribution.SplitJob) error { + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return err + } + fenced := abandonedSplitJobFencedRoute(snapshot.Routes, job) + if fenced == nil { + return nil + } + if snapshot.Version == math.MaxUint64 { + return errors.New("split migration catalog version overflow") + } + fenced.State = distribution.RouteStateActive + encoded, err := distribution.EncodeRouteDescriptorForCatalogWrite(*fenced, s.catalog.AllowsRouteDescriptorV2Writes()) + if err != nil { + return errors.WithStack(err) + } + nextVersion := snapshot.Version + 1 + if err := s.commitAbandonedSplitJobFenceRollback(ctx, snapshot, *fenced, encoded, nextVersion); err != nil { + return err + } + loaded, err := s.loadCatalogSnapshotAtLeastVersion(ctx, nextVersion) + if err != nil { + return err + } + return s.applyEngineSnapshot(loaded) +} + +func abandonedSplitJobFencedRoute(routes []distribution.RouteDescriptor, job distribution.SplitJob) *distribution.RouteDescriptor { + for i := range routes { + route := &routes[i] + if route.ParentRouteID == job.SourceRouteID && bytes.Equal(route.Start, job.SplitKey) && route.State == distribution.RouteStateWriteFenced { + return route + } + } + return nil +} + +func (s *DistributionServer) commitAbandonedSplitJobFenceRollback( + ctx context.Context, + snapshot distribution.CatalogSnapshot, + fenced distribution.RouteDescriptor, + encoded []byte, + nextVersion uint64, +) error { + if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: []*kv.Elem[kv.OP]{ + {Op: kv.Put, Key: distribution.CatalogRouteKey(fenced.RouteID), Value: encoded}, + {Op: kv.Put, Key: distribution.CatalogVersionKey(), Value: distribution.EncodeCatalogVersion(nextVersion)}, + }, + IsTxn: true, + StartTS: snapshot.ReadTS, + ReadKeys: [][]byte{ + distribution.CatalogRouteKey(fenced.RouteID), + distribution.CatalogVersionKey(), + }, + }); err != nil { + return splitJobCoordinatorStatusError(err) + } + return nil +} + +func (s *DistributionServer) finishSplitJobHistory(ctx context.Context, job distribution.SplitJob, terminalPhase distribution.SplitJobPhase) error { + current, readTS, err := s.catalog.LiveSplitJobForUpdate(ctx, job.JobID) + if err != nil { + return splitJobCatalogStatusError(err) + } + if current.Phase != job.Phase { + return nil + } + nowMs := time.Now().UnixMilli() + next := distribution.CloneSplitJob(current) + next.Phase = terminalPhase + next.RetryPhase = distribution.SplitJobPhaseNone + next.AbandonFromPhase = distribution.SplitJobPhaseNone + next.TerminalAtMs = nowMs + next.UpdatedAtMs = nowMs + encoded, err := distribution.EncodeSplitJob(next) + if err != nil { + return splitJobCatalogStatusError(err) + } + liveKey := distribution.CatalogSplitJobKey(job.JobID) + historyKey := distribution.CatalogSplitJobHistoryKey(nowMs, job.JobID) + if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: []*kv.Elem[kv.OP]{ + {Op: kv.Put, Key: historyKey, Value: encoded}, + {Op: kv.Del, Key: liveKey}, + }, + IsTxn: true, + StartTS: readTS, + ReadKeys: [][]byte{liveKey, historyKey}, + }); err != nil { + return splitJobCoordinatorStatusError(err) + } + return nil +} + +func (s *DistributionServer) gcSplitJobHistory(ctx context.Context, jobs []distribution.SplitJob, now time.Time) error { + s.mu.Lock() + if !s.splitJobHistoryGCLast.IsZero() && now.Sub(s.splitJobHistoryGCLast) < splitJobHistoryGCInterval { + s.mu.Unlock() + return nil + } + s.mu.Unlock() + + keys := splitJobHistoryGCKeys(jobs, now) + if len(keys) > 0 { + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return err + } + elems := make([]*kv.Elem[kv.OP], 0, len(keys)) + for _, key := range keys { + elems = append(elems, &kv.Elem[kv.OP]{Op: kv.Del, Key: key}) + } + if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: elems, + IsTxn: true, + StartTS: snapshot.ReadTS, + ReadKeys: keys, + }); err != nil { + return splitJobCoordinatorStatusError(err) + } + } + s.mu.Lock() + s.splitJobHistoryGCLast = now + s.mu.Unlock() + return nil +} + +func splitJobHistoryGCKeys(jobs []distribution.SplitJob, now time.Time) [][]byte { + history := make([]distribution.SplitJob, 0, len(jobs)) + for _, job := range jobs { + if (job.Phase == distribution.SplitJobPhaseDone || job.Phase == distribution.SplitJobPhaseAbandoned) && job.TerminalAtMs > 0 { + history = append(history, job) + } + } + sort.Slice(history, func(i, j int) bool { + if history[i].TerminalAtMs == history[j].TerminalAtMs { + return history[i].JobID < history[j].JobID + } + return history[i].TerminalAtMs < history[j].TerminalAtMs + }) + cutoff := now.Add(-splitJobHistoryTTL).UnixMilli() + excess := len(history) - splitJobHistoryLimit + keys := make([][]byte, 0, min(len(history), splitJobHistoryDeleteBatchLimit)) + for i, job := range history { + if job.TerminalAtMs >= cutoff && i >= excess { + continue + } + keys = append(keys, distribution.CatalogSplitJobHistoryKey(job.TerminalAtMs, job.JobID)) + if len(keys) == splitJobHistoryDeleteBatchLimit { + break + } + } + return keys +} + +func splitJobSourceSibling(routes []distribution.RouteDescriptor, job distribution.SplitJob) (uint64, []byte, bool) { + var sourceGroupID uint64 + var routeEnd []byte + for _, route := range routes { + if route.ParentRouteID != job.SourceRouteID { + continue + } + if bytes.Equal(route.Start, job.SplitKey) { + routeEnd = distribution.CloneBytes(route.End) + continue + } + if bytes.Equal(route.End, job.SplitKey) { + sourceGroupID = route.GroupID + } + } + return sourceGroupID, routeEnd, sourceGroupID != 0 +} + +func (s *DistributionServer) beginSplitJobBackfill(ctx context.Context, job distribution.SplitJob) error { + snapshot, parent, err := s.splitJobSourceRoute(ctx, job) + if err != nil { + return err + } + source, _, err := s.splitMigrationClientFactory(ctx, job, parent.GroupID) + if err != nil { + return errors.WithStack(err) + } + if source == nil { + return errors.New("split migration source client is nil") + } + ackCursor, complete, err := s.armSplitJobSourceTracker(ctx, job, snapshot, parent, source) + if err != nil { + return err + } + if !complete { + return s.persistSplitJobBackfillAck(ctx, job.JobID, ackCursor) + } + return s.openSplitJobBackfill(ctx, job, source) +} + +const initialMigrationRetentionPinTS = uint64(1) + +func (s *DistributionServer) armSplitJobSourceTracker( + ctx context.Context, + job distribution.SplitJob, + snapshot distribution.CatalogSnapshot, + parent distribution.RouteDescriptor, + source SplitMigrationClient, +) ([]byte, bool, error) { + // Arm the write tracker and its no-prune retention pin before choosing the + // snapshot boundary. This closes the window where a low-ts write could land + // after timestamp selection without being represented in the delta floor. + if _, err := applySplitMigrationControl(ctx, source, job, parent.End, snapshot.Version+1, initialMigrationRetentionPinTS, false, false, true, initialMigrationRetentionPinTS); err != nil { + return nil, false, err + } + return s.syncSplitMigrationVoterBarrier(ctx, parent.GroupID, job.FenceAckCursor, source, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED, + RouteStart: job.SplitKey, + RouteEnd: parent.End, + ExpectedCatalogVersion: snapshot.Version + 1, + MigrationJobId: job.JobID, + MinWriteTsExclusive: initialMigrationRetentionPinTS, + TrackWrites: true, + RetentionPinTs: initialMigrationRetentionPinTS, + }) +} + +func (s *DistributionServer) persistSplitJobBackfillAck(ctx context.Context, jobID uint64, ackCursor []byte) error { + return s.updateSplitJobViaCoordinator(ctx, jobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhasePlanned { + current.FenceAckCursor = ackCursor + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func (s *DistributionServer) openSplitJobBackfill(ctx context.Context, job distribution.SplitJob, source SplitMigrationClient) error { + issued, err := source.IssueMigrationTimestamp(ctx, &pb.IssueMigrationTimestampRequest{}) + if err != nil { + return errors.WithStack(err) + } + snapshotTS := issued.GetTimestamp() + if snapshotTS == 0 || snapshotTS <= issued.GetLastCommitTs() { + return errors.New("split migration source issued an invalid snapshot timestamp") + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase != distribution.SplitJobPhasePlanned { + return current, nil + } + current.Phase = distribution.SplitJobPhaseBackfill + current.SnapshotTS = snapshotTS + current.WriteTrackerArmed = true + current.SourceRetentionPinTS = initialMigrationRetentionPinTS + current.FenceAckCursor = nil + current.UpdatedAtMs = time.Now().UnixMilli() + return current, nil + }) +} + +func (s *DistributionServer) copySplitJobPhase( + ctx context.Context, + job distribution.SplitJob, + exportPhase distribution.SplitJobExportPhase, + minCommitTS uint64, + maxCommitTS uint64, +) error { + if maxCommitTS == 0 { + return errors.New("split migration copy upper timestamp is zero") + } + _, sourceRoute, err := s.splitJobSourceRoute(ctx, job) + if err != nil { + return err + } + brackets, err := distribution.PlanExportBrackets(job.SplitKey, sourceRoute.End) + if err != nil { + return errors.WithStack(err) + } + progressIndex, bracket, ok := nextSplitJobBracket(job, brackets, exportPhase) + if !ok { + return s.advanceCompletedCopyPhase(ctx, job, exportPhase) + } + source, target, err := s.splitMigrationClientFactory(ctx, job, sourceRoute.GroupID) + if err != nil { + return errors.WithStack(err) + } + if source == nil || target == nil { + return errors.New("split migration source or target client is nil") + } + progress := job.BracketProgress[progressIndex] + stream, err := source.ExportRangeVersions(ctx, &pb.ExportRangeVersionsRequest{ + RangeStart: bracket.Start, + RangeEnd: bracket.End, + MaxCommitTs: maxCommitTS, + MinCommitTs: minCommitTS, + Cursor: progress.Cursor, + ChunkBytes: defaultSplitMigrationChunkBytes, + RouteStart: job.SplitKey, + RouteEnd: sourceRoute.End, + MaxScannedBytes: defaultSplitMigrationMaxScannedBytes, + KeyFamily: bracket.Family, + ExcludeKnownInternal: bracket.ExcludeKnownInternal, + ExcludePrefixes: bracket.ExcludePrefixes, + }) + if err != nil { + return errors.WithStack(err) + } + return s.copySplitJobStream(ctx, job, progressIndex, bracket, progress, stream, target) +} + +func (s *DistributionServer) copySplitJobStream( + ctx context.Context, + job distribution.SplitJob, + progressIndex int, + bracket distribution.MigrationBracket, + progress distribution.SplitJobBracketProgress, + stream grpc.ServerStreamingClient[pb.ExportRangeVersionsResponse], + target SplitMigrationClient, +) error { + for { + resp, recvErr := stream.Recv() + if errors.Is(recvErr, io.EOF) { + return nil + } + if recvErr != nil { + return errors.WithStack(recvErr) + } + nextCursor := distribution.CloneBytes(resp.GetNextCursor()) + batchSeq := progress.LastAckedBatchSeq + 1 + importResp, importErr := target.ImportRangeVersions(ctx, &pb.ImportRangeVersionsRequest{ + JobId: job.JobID, + Versions: resp.GetVersions(), + Cursor: nextCursor, + BracketId: bracket.BracketID, + BatchSeq: batchSeq, + }) + if importErr != nil { + return errors.WithStack(importErr) + } + if !bytes.Equal(importResp.GetAckedCursor(), nextCursor) { + return errors.New("split migration import acknowledged a different cursor") + } + progress.Cursor = nextCursor + progress.Done = resp.GetDone() + progress.AcceptedRows += uint64(len(resp.GetVersions())) + progress.LastAckedBatchSeq = batchSeq + for _, version := range resp.GetVersions() { + if version.GetCommitTs() > job.MaxImportedTS { + job.MaxImportedTS = version.GetCommitTs() + } + } + job.BracketProgress[progressIndex] = progress + job.Cursor = nextCursor + job.UpdatedAtMs = time.Now().UnixMilli() + if err := s.persistSplitJobCopyProgress(ctx, job); err != nil { + return err + } + if progress.Done { + return nil + } + } +} + +func nextSplitJobBracket( + job distribution.SplitJob, + brackets []distribution.MigrationBracket, + exportPhase distribution.SplitJobExportPhase, +) (int, distribution.MigrationBracket, bool) { + byID := make(map[uint64]distribution.MigrationBracket, len(brackets)) + for _, bracket := range brackets { + byID[bracket.BracketID] = bracket + } + for i, progress := range job.BracketProgress { + if progress.ExportPhase != exportPhase || progress.Done { + continue + } + bracket, found := byID[progress.BracketID] + if found && bracket.Family == progress.Family { + return i, bracket, true + } + } + return 0, distribution.MigrationBracket{}, false +} + +func (s *DistributionServer) persistSplitJobCopyProgress(ctx context.Context, next distribution.SplitJob) error { + return s.updateSplitJobViaCoordinator(ctx, next.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase != next.Phase { + return current, nil + } + return distribution.CloneSplitJob(next), nil + }) +} + +func (s *DistributionServer) advanceCompletedCopyPhase(ctx context.Context, job distribution.SplitJob, phase distribution.SplitJobExportPhase) error { + switch phase { + case distribution.SplitJobExportPhaseBackfill: + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseBackfill { + current.Phase = distribution.SplitJobPhaseFence + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) + case distribution.SplitJobExportPhaseDeltaCopy: + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseDeltaCopy { + current.Phase = distribution.SplitJobPhaseCutover + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) + case distribution.SplitJobExportPhaseNone: + return errors.New("unknown split migration export phase") + } + return errors.New("unknown split migration export phase") +} + +func (s *DistributionServer) finalizeSplitJobFence(ctx context.Context, job distribution.SplitJob) error { + snapshot, sourceRoute, err := s.splitJobSourceRoute(ctx, job) + if err != nil { + return err + } + snapshot, sourceRoute, err = s.ensureSplitJobCatalogFence(ctx, job, snapshot, sourceRoute) + if err != nil { + return err + } + source, minAdmittedTS, ackCursor, complete, err := s.applySplitJobFenceBarrier(ctx, job, snapshot, sourceRoute) + if err != nil { + return err + } + if !complete { + return s.persistSplitJobFenceAck(ctx, job.JobID, ackCursor) + } + pendingLocks, err := splitJobPendingLocks(ctx, source, job.SplitKey, sourceRoute.End) + if err != nil { + return err + } + if pendingLocks { + return nil + } + issued, err := source.IssueMigrationTimestamp(ctx, &pb.IssueMigrationTimestampRequest{}) + if err != nil { + return errors.WithStack(err) + } + if issued.GetTimestamp() == 0 || issued.GetTimestamp() <= issued.GetLastCommitTs() { + return errors.New("split migration source issued an invalid fence timestamp") + } + return s.commitSplitJobFenceState(ctx, job, snapshot.Version, minAdmittedTS, ackCursor, issued.GetTimestamp()) +} + +func (s *DistributionServer) applySplitJobFenceBarrier( + ctx context.Context, + job distribution.SplitJob, + snapshot distribution.CatalogSnapshot, + sourceRoute distribution.RouteDescriptor, +) (SplitMigrationClient, uint64, []byte, bool, error) { + source, _, err := s.splitMigrationClientFactory(ctx, job, sourceRoute.GroupID) + if err != nil { + return nil, 0, nil, false, errors.WithStack(err) + } + minAdmittedTS, err := applySplitMigrationControl( + ctx, + source, + job, + sourceRoute.End, + snapshot.Version, + job.SnapshotTS, + true, + false, + true, + 1, + ) + if err != nil { + return nil, 0, nil, false, err + } + ackCursor, complete, err := s.syncSplitMigrationVoterBarrier(ctx, sourceRoute.GroupID, job.FenceAckCursor, source, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED, + RouteStart: job.SplitKey, + RouteEnd: sourceRoute.End, + ExpectedCatalogVersion: snapshot.Version, + MigrationJobId: job.JobID, + MinWriteTsExclusive: job.SnapshotTS, + SourceWriteFence: true, + TrackWrites: true, + RetentionPinTs: 1, + }) + if err != nil { + return nil, 0, nil, false, err + } + return source, minAdmittedTS, ackCursor, complete, nil +} + +func (s *DistributionServer) persistSplitJobFenceAck(ctx context.Context, jobID uint64, ackCursor []byte) error { + return s.updateSplitJobViaCoordinator(ctx, jobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase == distribution.SplitJobPhaseFence { + current.FenceAckCursor = distribution.CloneBytes(ackCursor) + current.UpdatedAtMs = time.Now().UnixMilli() + } + return current, nil + }) +} + +func (s *DistributionServer) commitSplitJobFenceState(ctx context.Context, job distribution.SplitJob, fenceCatalogVersion, minAdmittedTS uint64, ackCursor []byte, fenceTS uint64) error { + deltaFloor := job.SnapshotTS + if minAdmittedTS > 0 && minAdmittedTS-1 < deltaFloor { + deltaFloor = minAdmittedTS - 1 + } + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase != distribution.SplitJobPhaseFence { + return current, nil + } + current.Phase = distribution.SplitJobPhaseDeltaCopy + current.PostFenceDrainCompleted = true + current.SnapshotMinAdmittedTS = minAdmittedTS + current.DeltaFloor = deltaFloor + current.FenceTS = fenceTS + current.FenceCatalogVersion = fenceCatalogVersion + current.FenceAckCursor = distribution.CloneBytes(ackCursor) + current.SourceRetentionPinTS = 1 + for i := range current.BracketProgress { + current.BracketProgress[i].ExportPhase = distribution.SplitJobExportPhaseDeltaCopy + current.BracketProgress[i].Cursor = nil + current.BracketProgress[i].Done = false + current.BracketProgress[i].ScannedBytes = 0 + current.BracketProgress[i].AcceptedRows = 0 + } + current.UpdatedAtMs = time.Now().UnixMilli() + return current, nil + }) +} + +func (s *DistributionServer) ensureSplitJobCatalogFence( + ctx context.Context, + job distribution.SplitJob, + snapshot distribution.CatalogSnapshot, + sourceRoute distribution.RouteDescriptor, +) (distribution.CatalogSnapshot, distribution.RouteDescriptor, error) { + if sourceRoute.RouteID != job.SourceRouteID { + return snapshot, sourceRoute, nil + } + leftID, rightID, err := s.allocateChildRouteIDs(ctx, snapshot.ReadTS, snapshot.Routes) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.RouteDescriptor{}, err + } + left, right := splitCatalogRoutes(sourceRoute, job.SplitKey, leftID, rightID, job.FenceTS) + right.State = distribution.RouteStateWriteFenced + snapshot, err = s.saveSplitResultViaCoordinator(ctx, snapshot.ReadTS, snapshot.Version, sourceRoute.RouteID, nil, left, right) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.RouteDescriptor{}, err + } + if err := s.applyEngineSnapshot(snapshot); err != nil { + return distribution.CatalogSnapshot{}, distribution.RouteDescriptor{}, err + } + return snapshot, right, nil +} + +func splitJobPendingLocks(ctx context.Context, source SplitMigrationClient, routeStart []byte, routeEnd []byte) (bool, error) { + resp, err := source.ProbeMigrationLocks(ctx, &pb.ProbeMigrationLocksRequest{ + RouteStart: routeStart, + RouteEnd: routeEnd, + Limit: defaultSplitMigrationLockProbeLimit, + }) + if err != nil { + return false, errors.WithStack(err) + } + return resp.GetPendingCount() != 0, nil +} + +func (s *DistributionServer) cutoverSplitJob(ctx context.Context, job distribution.SplitJob) error { + snapshot, sourceRoute, err := s.splitJobSourceRoute(ctx, job) + if err != nil { + return err + } + if snapshot.Version == math.MaxUint64 { + return errors.New("split migration catalog version overflow") + } + expectedVersion := snapshot.Version + 1 + if job.CutoverVersion == 0 || job.CutoverVersion != expectedVersion { + return s.armSplitJobCutoverWitness(ctx, job, expectedVersion) + } + targetCursor, sourceCursor, targetComplete, sourceComplete, err := s.applySplitJobCutoverBarriers(ctx, job, sourceRoute, expectedVersion) + if err != nil { + return err + } + if splitJobCutoverBarrierPending(job, targetComplete, sourceComplete) { + return s.persistSplitJobCutoverAcks(ctx, job.JobID, targetCursor, sourceCursor, targetComplete, sourceComplete) + } + return s.commitSplitJobCutover(ctx, snapshot, sourceRoute, job, expectedVersion) +} + +func (s *DistributionServer) applySplitJobCutoverBarriers( + ctx context.Context, + job distribution.SplitJob, + sourceRoute distribution.RouteDescriptor, + expectedVersion uint64, +) ([]byte, []byte, bool, bool, error) { + source, target, err := s.splitMigrationClientFactory(ctx, job, sourceRoute.GroupID) + if err != nil { + return nil, nil, false, false, errors.WithStack(err) + } + if _, err := applySplitMigrationControl(ctx, target, job, sourceRoute.End, expectedVersion, job.FenceTS, false, false, false, 0); err != nil { + return nil, nil, false, false, err + } + if _, err := applySplitMigrationControl(ctx, source, job, sourceRoute.End, expectedVersion, job.FenceTS, true, true, false, 1); err != nil { + return nil, nil, false, false, err + } + targetCursor, targetComplete, err := s.syncSplitMigrationVoterBarrier(ctx, job.TargetGroupID, job.TargetStagedReadinessAckCursor, target, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED, + RouteStart: job.SplitKey, + RouteEnd: sourceRoute.End, + ExpectedCatalogVersion: expectedVersion, + MigrationJobId: job.JobID, + MinWriteTsExclusive: job.FenceTS, + }) + if err != nil { + return nil, nil, false, false, err + } + sourceCursor, sourceComplete, err := s.syncSplitMigrationVoterBarrier(ctx, sourceRoute.GroupID, job.SourceCutoverReadFenceAckCursor, source, &pb.ProbeMigrationStateRequest{ + JobId: job.JobID, + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED, + RouteStart: job.SplitKey, + RouteEnd: sourceRoute.End, + ExpectedCatalogVersion: expectedVersion, + MigrationJobId: job.JobID, + MinWriteTsExclusive: job.FenceTS, + SourceWriteFence: true, + SourceReadFence: true, + RetentionPinTs: 1, + }) + if err != nil { + return nil, nil, false, false, err + } + return targetCursor, sourceCursor, targetComplete, sourceComplete, nil +} + +func splitJobCutoverBarrierPending(job distribution.SplitJob, targetComplete, sourceComplete bool) bool { + return !targetComplete || !sourceComplete || + job.TargetStagedReadinessState != distribution.SplitJobBarrierArmed || + job.CutoverReadFenceState != distribution.SplitJobBarrierArmed +} + +func (s *DistributionServer) armSplitJobCutoverWitness(ctx context.Context, job distribution.SplitJob, expectedVersion uint64) error { + return s.updateSplitJobViaCoordinator(ctx, job.JobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase != distribution.SplitJobPhaseCutover { + return current, nil + } + current.CutoverVersion = expectedVersion + current.CutoverReadFenceState = distribution.SplitJobBarrierArming + current.TargetStagedReadinessState = distribution.SplitJobBarrierArming + current.SourceCutoverReadFenceAckCursor = nil + current.TargetStagedReadinessAckCursor = nil + current.UpdatedAtMs = time.Now().UnixMilli() + return current, nil + }) +} + +func (s *DistributionServer) persistSplitJobCutoverAcks( + ctx context.Context, + jobID uint64, + targetCursor []byte, + sourceCursor []byte, + targetComplete bool, + sourceComplete bool, +) error { + return s.updateSplitJobViaCoordinator(ctx, jobID, func(current distribution.SplitJob) (distribution.SplitJob, error) { + if current.Phase != distribution.SplitJobPhaseCutover { + return current, nil + } + current.TargetStagedReadinessAckCursor = distribution.CloneBytes(targetCursor) + current.SourceCutoverReadFenceAckCursor = distribution.CloneBytes(sourceCursor) + current.TargetStagedReadinessState = distribution.SplitJobBarrierArming + if targetComplete { + current.TargetStagedReadinessState = distribution.SplitJobBarrierArmed + } + current.CutoverReadFenceState = distribution.SplitJobBarrierArming + if sourceComplete { + current.CutoverReadFenceState = distribution.SplitJobBarrierArmed + } + current.UpdatedAtMs = time.Now().UnixMilli() + return current, nil + }) +} + +func applySplitMigrationControl( + ctx context.Context, + client SplitMigrationClient, + job distribution.SplitJob, + routeEnd []byte, + expectedVersion uint64, + minWriteTS uint64, + sourceWriteFence bool, + sourceReadFence bool, + trackWrites bool, + retentionPinTS uint64, +) (uint64, error) { + if client == nil { + return 0, errors.New("split migration control client is nil") + } + resp, err := client.ApplyTargetStagedReadiness(ctx, splitMigrationControlRequest( + job, + routeEnd, + expectedVersion, + minWriteTS, + sourceWriteFence, + sourceReadFence, + trackWrites, + retentionPinTS, + )) + if err != nil { + return 0, errors.WithStack(err) + } + return resp.GetMinAdmittedTs(), nil +} + +func splitMigrationControlRequest( + job distribution.SplitJob, + routeEnd []byte, + expectedVersion uint64, + minWriteTS uint64, + sourceWriteFence bool, + sourceReadFence bool, + trackWrites bool, + retentionPinTS uint64, +) *pb.TargetStagedReadinessRequest { + return &pb.TargetStagedReadinessRequest{ + JobId: job.JobID, + RouteStart: job.SplitKey, + RouteEnd: routeEnd, + ExpectedCutoverVersion: expectedVersion, + MigrationJobId: job.JobID, + MinWriteTsExclusive: minWriteTS, + Armed: true, + SourceWriteFence: sourceWriteFence, + SourceReadFence: sourceReadFence, + RetentionPinTs: retentionPinTS, + TrackWrites: trackWrites, + } +} + +func probeMigrationControlRequest(req *pb.TargetStagedReadinessRequest) *pb.ProbeMigrationStateRequest { + return &pb.ProbeMigrationStateRequest{ + JobId: req.GetJobId(), + Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED, + RouteStart: distribution.CloneBytes(req.GetRouteStart()), + RouteEnd: distribution.CloneBytes(req.GetRouteEnd()), + ExpectedCatalogVersion: req.GetExpectedCutoverVersion(), + MigrationJobId: req.GetMigrationJobId(), + MinWriteTsExclusive: req.GetMinWriteTsExclusive(), + SourceWriteFence: req.GetSourceWriteFence(), + SourceReadFence: req.GetSourceReadFence(), + TrackWrites: req.GetTrackWrites(), + RetentionPinTs: req.GetRetentionPinTs(), + } +} + +func splitJobCapabilityRegressionRequiresFailClosed(job distribution.SplitJob) bool { + return splitJobCapabilityRegressionSourceGuardRequired(job) || + splitJobCapabilityRegressionTargetGuardRequired(job) +} + +func splitJobCapabilityRegressionSourceGuardRequired(job distribution.SplitJob) bool { + switch job.Phase { + case distribution.SplitJobPhasePlanned: + return job.WriteTrackerArmed || + job.SnapshotTS != 0 || + job.SourceRetentionPinTS != 0 || + len(job.FenceAckCursor) != 0 + case distribution.SplitJobPhaseBackfill, + distribution.SplitJobPhaseFence, + distribution.SplitJobPhaseDeltaCopy, + distribution.SplitJobPhaseCutover: + return true + case distribution.SplitJobPhaseCleanup: + return job.SourceRetentionPinTS != math.MaxUint64 + case distribution.SplitJobPhaseNone, + distribution.SplitJobPhaseDone, + distribution.SplitJobPhaseFailed, + distribution.SplitJobPhaseAbandoning, + distribution.SplitJobPhaseAbandoned: + return false + } + return false +} + +func splitJobCapabilityRegressionTargetGuardRequired(job distribution.SplitJob) bool { + switch job.Phase { + case distribution.SplitJobPhasePlanned: + return job.WriteTrackerArmed || + job.SnapshotTS != 0 || + job.SourceRetentionPinTS != 0 || + len(job.FenceAckCursor) != 0 + case distribution.SplitJobPhaseBackfill, + distribution.SplitJobPhaseFence, + distribution.SplitJobPhaseDeltaCopy, + distribution.SplitJobPhaseCutover: + return true + case distribution.SplitJobPhaseCleanup: + return !bytes.Equal(job.TargetClearedDescriptorAckCursor, splitCleanupDoneCursor) + case distribution.SplitJobPhaseNone, + distribution.SplitJobPhaseDone, + distribution.SplitJobPhaseFailed, + distribution.SplitJobPhaseAbandoning, + distribution.SplitJobPhaseAbandoned: + return false + } + return false +} + +type splitJobMigrationControlTargets struct { + snapshot distribution.CatalogSnapshot + sourceGroupID uint64 + routeEnd []byte + source SplitMigrationClient + target SplitMigrationClient +} + +func (s *DistributionServer) splitJobCapabilityControlTargets(ctx context.Context, job distribution.SplitJob) (splitJobMigrationControlTargets, error) { + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return splitJobMigrationControlTargets{}, err + } + sourceGroupID, routeEnd, ok := splitJobSourceSibling(snapshot.Routes, job) + if !ok { + if parent, found := findRouteByID(snapshot.Routes, job.SourceRouteID); found { + sourceGroupID = parent.GroupID + routeEnd = distribution.CloneBytes(parent.End) + ok = true + } + } + if !ok { + return splitJobMigrationControlTargets{}, errors.WithStack(distribution.ErrMigrationSourceRouteChanged) + } + source, target, err := s.splitMigrationClientFactory(ctx, job, sourceGroupID) + if err != nil { + return splitJobMigrationControlTargets{}, errors.WithStack(err) + } + if source == nil || target == nil { + return splitJobMigrationControlTargets{}, errors.New("split migration source or target client is nil") + } + return splitJobMigrationControlTargets{ + snapshot: snapshot, + sourceGroupID: sourceGroupID, + routeEnd: routeEnd, + source: source, + target: target, + }, nil +} + +func (s *DistributionServer) ensureSplitJobCapabilityRegressionFailClosed(ctx context.Context, job distribution.SplitJob) (bool, error) { + targets, err := s.splitJobCapabilityControlTargets(ctx, job) + if err != nil { + return false, err + } + expectedVersion := splitJobCapabilityRegressionExpectedVersion(job, targets.snapshot.Version) + minWriteTS := splitJobCapabilityRegressionMinWriteTS(job) + retentionPinTS := splitJobCapabilityRegressionRetentionPinTS(job) + sourceComplete := true + if splitJobCapabilityRegressionSourceGuardRequired(job) { + sourceReq := splitMigrationControlRequest(job, targets.routeEnd, expectedVersion, minWriteTS, true, true, true, retentionPinTS) + var err error + sourceComplete, err = s.applyAndProbeSplitMigrationControl(ctx, targets.sourceGroupID, targets.source, sourceReq) + if err != nil { + return false, err + } + } + targetComplete := true + if splitJobCapabilityRegressionTargetGuardRequired(job) { + targetReq := splitMigrationControlRequest(job, targets.routeEnd, expectedVersion, minWriteTS, false, false, false, 0) + var err error + targetComplete, err = s.applyAndProbeSplitMigrationControl(ctx, job.TargetGroupID, targets.target, targetReq) + if err != nil { + return false, err + } + } + return sourceComplete && targetComplete, nil +} + +func (s *DistributionServer) restoreSplitJobCapabilityRegressionGuards(ctx context.Context, job distribution.SplitJob) (bool, error) { + targets, err := s.splitJobCapabilityControlTargets(ctx, job) + if err != nil { + return false, err + } + if splitJobCapabilityRegressionSourceGuardRequired(job) { + sourceReq, ok := splitJobNormalSourceControlRequest(job, targets.routeEnd, targets.snapshot.Version) + if ok { + complete, err := s.applyAndProbeSplitMigrationControl(ctx, targets.sourceGroupID, targets.source, sourceReq) + if err != nil || !complete { + return complete, err + } + } + } + if splitJobCapabilityRegressionTargetGuardRequired(job) { + targetReq, ok := splitJobNormalTargetControlRequest(job, targets.routeEnd) + if ok { + complete, err := s.applyAndProbeSplitMigrationControl(ctx, job.TargetGroupID, targets.target, targetReq) + if err != nil || !complete { + return complete, err + } + } + } + return true, nil +} + +func (s *DistributionServer) applyAndProbeSplitMigrationControl( + ctx context.Context, + groupID uint64, + client SplitMigrationClient, + req *pb.TargetStagedReadinessRequest, +) (bool, error) { + if client == nil { + return false, errors.New("split migration control client is nil") + } + if _, err := client.ApplyTargetStagedReadiness(ctx, req); err != nil { + return false, errors.WithStack(err) + } + _, complete, err := s.syncSplitMigrationVoterBarrier(ctx, groupID, nil, client, probeMigrationControlRequest(req)) + return complete, err +} + +func splitJobCapabilityRegressionExpectedVersion(job distribution.SplitJob, snapshotVersion uint64) uint64 { + switch { + case job.CutoverVersion != 0: + return job.CutoverVersion + case job.FenceCatalogVersion != 0: + return job.FenceCatalogVersion + case snapshotVersion < math.MaxUint64: + return snapshotVersion + 1 + default: + return snapshotVersion + } +} + +func splitJobCapabilityRegressionMinWriteTS(job distribution.SplitJob) uint64 { + switch { + case job.FenceTS != 0: + return job.FenceTS + case job.SnapshotTS != 0: + return job.SnapshotTS + default: + return initialMigrationRetentionPinTS + } +} + +func splitJobCapabilityRegressionRetentionPinTS(job distribution.SplitJob) uint64 { + if job.SourceRetentionPinTS != 0 && job.SourceRetentionPinTS != math.MaxUint64 { + return job.SourceRetentionPinTS + } + return initialMigrationRetentionPinTS +} + +func splitJobNormalSourceControlRequest(job distribution.SplitJob, routeEnd []byte, snapshotVersion uint64) (*pb.TargetStagedReadinessRequest, bool) { + switch job.Phase { + case distribution.SplitJobPhasePlanned, distribution.SplitJobPhaseBackfill: + return splitMigrationControlRequest( + job, + routeEnd, + splitJobCapabilityRegressionExpectedVersion(job, snapshotVersion), + initialMigrationRetentionPinTS, + false, + false, + true, + initialMigrationRetentionPinTS, + ), true + case distribution.SplitJobPhaseFence, distribution.SplitJobPhaseDeltaCopy: + expectedVersion := job.FenceCatalogVersion + if expectedVersion == 0 { + expectedVersion = splitJobCapabilityRegressionExpectedVersion(job, snapshotVersion) + } + return splitMigrationControlRequest(job, routeEnd, expectedVersion, job.SnapshotTS, true, false, true, initialMigrationRetentionPinTS), true + case distribution.SplitJobPhaseCutover, distribution.SplitJobPhaseCleanup: + if job.FenceTS == 0 || job.CutoverVersion == 0 { + return nil, false + } + return splitMigrationControlRequest(job, routeEnd, job.CutoverVersion, job.FenceTS, true, true, false, initialMigrationRetentionPinTS), true + case distribution.SplitJobPhaseNone, + distribution.SplitJobPhaseDone, + distribution.SplitJobPhaseFailed, + distribution.SplitJobPhaseAbandoning, + distribution.SplitJobPhaseAbandoned: + return nil, false + } + return nil, false +} + +func splitJobNormalTargetControlRequest(job distribution.SplitJob, routeEnd []byte) (*pb.TargetStagedReadinessRequest, bool) { + if job.CutoverVersion == 0 || job.FenceTS == 0 { + return nil, false + } + switch job.Phase { + case distribution.SplitJobPhaseCutover, distribution.SplitJobPhaseCleanup: + return splitMigrationControlRequest(job, routeEnd, job.CutoverVersion, job.FenceTS, false, false, false, 0), true + case distribution.SplitJobPhaseNone, + distribution.SplitJobPhasePlanned, + distribution.SplitJobPhaseBackfill, + distribution.SplitJobPhaseFence, + distribution.SplitJobPhaseDeltaCopy, + distribution.SplitJobPhaseDone, + distribution.SplitJobPhaseFailed, + distribution.SplitJobPhaseAbandoning, + distribution.SplitJobPhaseAbandoned: + return nil, false + } + return nil, false +} + +func (s *DistributionServer) splitJobSourceRoute( + ctx context.Context, + job distribution.SplitJob, +) (distribution.CatalogSnapshot, distribution.RouteDescriptor, error) { + snapshot, err := s.loadCatalogSnapshot(ctx) + if err != nil { + return distribution.CatalogSnapshot{}, distribution.RouteDescriptor{}, err + } + if parent, found := findRouteByID(snapshot.Routes, job.SourceRouteID); found { + return snapshot, parent, nil + } + for _, route := range snapshot.Routes { + if route.ParentRouteID == job.SourceRouteID && bytes.Equal(route.Start, job.SplitKey) { + return snapshot, distribution.CloneRouteDescriptor(route), nil + } + } + return distribution.CatalogSnapshot{}, distribution.RouteDescriptor{}, splitJobCatalogStatusError(distribution.ErrMigrationSourceRouteChanged) +} + +func (s *DistributionServer) commitSplitJobCutover( + ctx context.Context, + snapshot distribution.CatalogSnapshot, + right distribution.RouteDescriptor, + job distribution.SplitJob, + nextVersion uint64, +) error { + right.GroupID = job.TargetGroupID + right.State = distribution.RouteStateActive + right.StagedVisibilityActive = true + right.MigrationJobID = job.JobID + right.MinWriteTSExclusive = job.FenceTS + encodedRoute, err := distribution.EncodeRouteDescriptorForCatalogWrite(right, s.catalog.AllowsRouteDescriptorV2Writes()) + if err != nil { + return errors.WithStack(err) + } + nextJob := distribution.CloneSplitJob(job) + nextJob.Phase = distribution.SplitJobPhaseCleanup + nextJob.CutoverVersion = nextVersion + nextJob.CutoverReadFenceState = distribution.SplitJobBarrierArmed + nextJob.TargetStagedReadinessState = distribution.SplitJobBarrierArmed + nextJob.Cursor = nil + nextJob.UpdatedAtMs = time.Now().UnixMilli() + encodedJob, err := distribution.EncodeSplitJob(nextJob) + if err != nil { + return errors.WithStack(err) + } + versionKey := distribution.CatalogVersionKey() + routeKey := distribution.CatalogRouteKey(right.RouteID) + jobKey := distribution.CatalogSplitJobKey(job.JobID) + if _, err := s.coordinator.Dispatch(ctx, &kv.OperationGroup[kv.OP]{ + Elems: []*kv.Elem[kv.OP]{ + {Op: kv.Put, Key: routeKey, Value: encodedRoute}, + {Op: kv.Put, Key: versionKey, Value: distribution.EncodeCatalogVersion(nextVersion)}, + {Op: kv.Put, Key: jobKey, Value: encodedJob}, + }, + IsTxn: true, + StartTS: snapshot.ReadTS, + ReadKeys: [][]byte{routeKey, versionKey, jobKey}, + }); err != nil { + if errors.Is(err, store.ErrWriteConflict) { + return grpcStatusError(codes.Aborted, errDistributionCatalogConflict.Error()) + } + return errors.WithStack(err) + } + updated, err := s.loadCatalogSnapshotAtLeastVersion(ctx, nextVersion) + if err != nil { + return err + } + return s.applyEngineSnapshot(updated) +} diff --git a/adapter/split_job_voter_ack_test.go b/adapter/split_job_voter_ack_test.go new file mode 100644 index 000000000..08a12acc7 --- /dev/null +++ b/adapter/split_job_voter_ack_test.go @@ -0,0 +1,67 @@ +package adapter + +import ( + "context" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" +) + +func TestSplitMigrationVoterBarrierReopensForMembershipChanges(t *testing.T) { + t.Parallel() + + ready := func(*pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + return &pb.ProbeMigrationStateResponse{Ready: true}, nil + } + blocked := func(*pb.ProbeMigrationStateRequest) (*pb.ProbeMigrationStateResponse, error) { + return &pb.ProbeMigrationStateResponse{}, nil + } + n1 := &splitMigrationClientStub{probeFn: ready} + n2 := &splitMigrationClientStub{probeFn: ready} + n3 := &splitMigrationClientStub{probeFn: blocked} + voters := []SplitMigrationVoter{ + {ID: "n1", Address: "n1:50051", Client: n1}, + {ID: "n2", Address: "n2:50051", Client: n2}, + } + server := &DistributionServer{splitMigrationVoterFactory: func(context.Context, uint64) ([]SplitMigrationVoter, error) { + return voters, nil + }} + req := &pb.ProbeMigrationStateRequest{JobId: 7, Kind: pb.MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED} + + cursor, complete, err := server.syncSplitMigrationVoterBarrier(context.Background(), 1, nil, nil, req) + require.NoError(t, err) + require.True(t, complete) + acks, err := decodeSplitVoterAckCursor(cursor) + require.NoError(t, err) + require.Len(t, acks, 2) + + voters = append(voters, SplitMigrationVoter{ID: "n3", Address: "n3:50051", Client: n3}) + cursor, complete, err = server.syncSplitMigrationVoterBarrier(context.Background(), 1, cursor, nil, req) + require.NoError(t, err) + require.False(t, complete) + acks, err = decodeSplitVoterAckCursor(cursor) + require.NoError(t, err) + require.Len(t, acks, 3) + require.False(t, acks[2].acked) + + n3.probeFn = ready + _, complete, err = server.syncSplitMigrationVoterBarrier(context.Background(), 1, cursor, nil, req) + require.NoError(t, err) + require.True(t, complete) + + n2.probeFn = blocked + voters[1].Address = "n2:50052" + _, complete, err = server.syncSplitMigrationVoterBarrier(context.Background(), 1, cursor, nil, req) + require.NoError(t, err) + require.False(t, complete, "an address change must be treated as a new voter endpoint") +} + +func TestSplitVoterAckCursorRejectsCorruption(t *testing.T) { + t.Parallel() + + _, err := decodeSplitVoterAckCursor([]byte{99}) + require.Error(t, err) + _, err = decodeSplitVoterAckCursor([]byte{splitVoterAckCursorVersion, 1, 4, 'n'}) + require.Error(t, err) +} diff --git a/cmd/elastickv-split/main.go b/cmd/elastickv-split/main.go index 767359c71..787f1e901 100644 --- a/cmd/elastickv-split/main.go +++ b/cmd/elastickv-split/main.go @@ -42,6 +42,9 @@ var ( routeID = flag.Uint64("route-id", 0, "RouteID of the route to split (required)") splitKey = flag.String("split-key", "", "Split key — must lie strictly inside the route's [Start, End) range; rejected if == Start or == End by validateSplitKey (required)") expectedVersion = flag.Uint64("expected-version", 0, "Expected catalog version for OCC; obtain by calling ListRoutes first (required, must be >= 1 — catalog version is 1-based)") + targetGroupID = flag.Uint64("target-group-id", 0, "Target Raft group for an asynchronous cross-group migration; zero uses the existing same-group split") + abandonJobID = flag.Uint64("abandon-job-id", 0, "Abandon a pre-cutover migration job; mutually exclusive with split flags") + getJobID = flag.Uint64("get-job-id", 0, "Print one migration job; mutually exclusive with mutation flags") ) func main() { @@ -76,7 +79,16 @@ func run() error { rpcCtx, rpcCancel := context.WithTimeout(context.Background(), rpcTimeout) defer rpcCancel() + if *getJobID != 0 { + return runGetSplitJob(rpcCtx, client) + } + if *abandonJobID != 0 { + return runAbandonSplitJob(rpcCtx, client) + } + if *targetGroupID != 0 { + return runStartSplitMigration(rpcCtx, client) + } req := &pb.SplitRangeRequest{ ExpectedCatalogVersion: *expectedVersion, RouteId: *routeID, @@ -90,7 +102,51 @@ func run() error { return nil } +func runGetSplitJob(ctx context.Context, client pb.DistributionClient) error { + resp, err := client.GetSplitJob(ctx, &pb.GetSplitJobRequest{JobId: *getJobID}) + if err != nil { + return errors.Wrap(err, "GetSplitJob") + } + job := resp.GetJob() + fmt.Printf("job_id: %d\nphase: %s\n", job.GetJobId(), job.GetPhase()) + return nil +} + +func runAbandonSplitJob(ctx context.Context, client pb.DistributionClient) error { + if _, err := client.AbandonSplitJob(ctx, &pb.AbandonSplitJobRequest{JobId: *abandonJobID}); err != nil { + return errors.Wrap(err, "AbandonSplitJob") + } + fmt.Printf("abandoned_job_id: %d\n", *abandonJobID) + return nil +} + +func runStartSplitMigration(ctx context.Context, client pb.DistributionClient) error { + resp, err := client.StartSplitMigration(ctx, &pb.StartSplitMigrationRequest{ + ExpectedCatalogVersion: *expectedVersion, + RouteId: *routeID, + SplitKey: []byte(*splitKey), + TargetGroupId: *targetGroupID, + }) + if err != nil { + return errors.Wrap(err, "StartSplitMigration") + } + fmt.Printf("catalog_version: %d\njob_id: %d\n", resp.GetCatalogVersion(), resp.GetJobId()) + return nil +} + func validateFlags() error { + if *getJobID != 0 { + if splitMutationFlagsSet() || *abandonJobID != 0 { + return errors.New("--get-job-id is mutually exclusive with mutation flags") + } + return nil + } + if *abandonJobID != 0 { + if splitMutationFlagsSet() { + return errors.New("--abandon-job-id is mutually exclusive with split flags") + } + return nil + } switch { case *routeID == 0: return errors.New("--route-id is required and must be > 0") @@ -102,6 +158,10 @@ func validateFlags() error { return nil } +func splitMutationFlagsSet() bool { + return *routeID != 0 || *splitKey != "" || *expectedVersion != 0 || *targetGroupID != 0 +} + func printResponse(resp *pb.SplitRangeResponse) { fmt.Printf("catalog_version: %d\n", resp.GetCatalogVersion()) printRoute("left: ", resp.GetLeft()) diff --git a/cmd/elastickv-split/main_test.go b/cmd/elastickv-split/main_test.go index bc9d40769..903d7e171 100644 --- a/cmd/elastickv-split/main_test.go +++ b/cmd/elastickv-split/main_test.go @@ -48,6 +48,34 @@ func TestValidateFlags_AcceptsAllPresent(t *testing.T) { require.NoError(t, validateFlags()) } +func TestValidateFlags_AcceptsAbandonOnly(t *testing.T) { + resetFlags(t) + *abandonJobID = 42 + require.NoError(t, validateFlags()) +} + +func TestValidateFlags_RejectsAbandonWithSplitFlags(t *testing.T) { + resetFlags(t) + *abandonJobID = 42 + *routeID = 7 + err := validateFlags() + require.ErrorContains(t, err, "mutually exclusive") +} + +func TestValidateFlags_AcceptsGetOnly(t *testing.T) { + resetFlags(t) + *getJobID = 42 + require.NoError(t, validateFlags()) +} + +func TestValidateFlags_RejectsGetWithMutationFlags(t *testing.T) { + resetFlags(t) + *getJobID = 42 + *abandonJobID = 42 + err := validateFlags() + require.ErrorContains(t, err, "mutually exclusive") +} + // resetFlags reinitialises the package-level flag pointers around // each test so subtests don't leak state through them. We rebind // the globals directly rather than re-parsing argv because the @@ -58,12 +86,15 @@ func resetFlags(t *testing.T) { // init). errors.New rather than t.Fatalf so a future // refactor that drops the package-level flags surfaces here // rather than silently no-oping the resets. - if routeID == nil || splitKey == nil || expectedVersion == nil || address == nil { + if routeID == nil || splitKey == nil || expectedVersion == nil || targetGroupID == nil || abandonJobID == nil || getJobID == nil || address == nil { t.Fatal(errors.New("package-level flag pointers were not initialised")) } *routeID = 0 *splitKey = "" *expectedVersion = 0 + *targetGroupID = 0 + *abandonJobID = 0 + *getJobID = 0 *address = "127.0.0.1:50051" _ = flag.CommandLine // touch to avoid unused import on future trims } diff --git a/distribution/migration_promotion_complete.go b/distribution/migration_promotion_complete.go new file mode 100644 index 000000000..aa4487a9c --- /dev/null +++ b/distribution/migration_promotion_complete.go @@ -0,0 +1,350 @@ +package distribution + +import ( + "bytes" + "context" + "math" + + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" +) + +var ( + ErrMigrationPromotionNotReady = errors.New("migration target promotion is not ready") + ErrMigrationPromotionTargetAbsent = errors.New("migration target promotion route is missing") +) + +// TargetPromotionCompletion is the result of applying the default-group +// promotion-complete transition to a SplitJob and its catalog routes. +type TargetPromotionCompletion struct { + Job SplitJob + Routes []RouteDescriptor + Changed bool + ClearedRouteIDs []uint64 +} + +// CompleteTargetPromotionState clears the target route's staged visibility +// fields after target-local promotion has completed. It deliberately preserves +// MinWriteTSExclusive; the timestamp floor remains a durable route invariant +// after the staged/live merge is no longer needed. +func CompleteTargetPromotionState(job SplitJob, routes []RouteDescriptor, nowMs int64) (TargetPromotionCompletion, error) { + normalized, err := normalizePromotionCompletionInput(job, routes) + if err != nil { + return TargetPromotionCompletion{}, err + } + + out := TargetPromotionCompletion{ + Job: CloneSplitJob(job), + Routes: normalized, + } + if out.Job.TargetPromotionDone { + return completeAlreadyPromotedTarget(out, nowMs) + } + + cleared, err := clearTargetPromotionRoutes(job, out.Routes) + if err != nil { + return TargetPromotionCompletion{}, err + } + if len(cleared) == 0 { + return TargetPromotionCompletion{}, errors.WithStack(ErrMigrationPromotionTargetAbsent) + } + + out.Changed = true + out.ClearedRouteIDs = cleared + out.Job.TargetPromotionDone = true + out.Job.UpdatedAtMs = nowMs + return out, nil +} + +func completeAlreadyPromotedTarget(out TargetPromotionCompletion, nowMs int64) (TargetPromotionCompletion, error) { + if !targetClearedDescriptorPresent(out.Job, out.Routes) { + return TargetPromotionCompletion{}, errors.WithStack(ErrMigrationPromotionTargetAbsent) + } + // Preserve compatibility for jobs terminalized by older binaries, but do + // not move a current CLEANUP job to DONE until source and target cleanup + // proofs have been removed by the runner. + if out.Job.Phase == SplitJobPhaseDone && out.Job.TerminalAtMs <= 0 { + out.Changed = true + out.Job.TerminalAtMs = nowMs + } + if out.Changed { + out.Job.UpdatedAtMs = nowMs + } + return out, nil +} + +func normalizePromotionCompletionInput(job SplitJob, routes []RouteDescriptor) ([]RouteDescriptor, error) { + if err := validateSplitJob(job); err != nil { + return nil, err + } + if job.Phase == SplitJobPhaseDone && job.TargetPromotionDone { + return normalizeRoutes(routes) + } + if job.Phase != SplitJobPhaseCleanup { + return nil, errors.WithStack(ErrMigrationPromotionNotReady) + } + return normalizeRoutes(routes) +} + +func clearTargetPromotionRoutes(job SplitJob, routes []RouteDescriptor) ([]uint64, error) { + cleared := make([]uint64, 0, 1) + for i := range routes { + route := &routes[i] + if route.MigrationJobID != job.JobID { + continue + } + if !route.StagedVisibilityActive || + route.GroupID != job.TargetGroupID || + route.ParentRouteID != job.SourceRouteID || + !bytes.Equal(route.Start, job.SplitKey) { + return nil, errors.WithStack(ErrMigrationInvalidRoute) + } + route.StagedVisibilityActive = false + route.MigrationJobID = 0 + cleared = append(cleared, route.RouteID) + } + return cleared, nil +} + +// CompleteSplitJobTargetPromotion applies the promotion-complete catalog CAS: +// route descriptor staged fields are cleared, catalog version is bumped, and +// the SplitJob witness is updated in the same MVCC batch. +func (s *CatalogStore) CompleteSplitJobTargetPromotion( + ctx context.Context, + expectedVersion uint64, + expected SplitJob, + nowMs int64, +) (CatalogSnapshot, SplitJob, error) { + if err := ensureCatalogStore(s); err != nil { + return CatalogSnapshot{}, SplitJob{}, err + } + ctx = contextOrBackground(ctx) + + readTS, currentVersion, routes, currentJob, alreadyApplied, err := s.loadPromotionCompleteInputs(ctx, expectedVersion, expected) + if err != nil { + return CatalogSnapshot{}, SplitJob{}, err + } + if alreadyApplied { + return CatalogSnapshot{ + Version: currentVersion, + Routes: cloneRouteDescriptors(routes), + ReadTS: readTS, + }, currentJob, nil + } + completion, err := CompleteTargetPromotionState(expected, routes, nowMs) + if err != nil { + return CatalogSnapshot{}, SplitJob{}, err + } + if !completion.Changed { + return CatalogSnapshot{ + Version: currentVersion, + Routes: cloneRouteDescriptors(completion.Routes), + ReadTS: readTS, + }, completion.Job, nil + } + + plan, mutations, commitTS, err := s.buildPromotionCompleteMutations(ctx, readTS, expectedVersion, expected.JobID, &completion) + if err != nil { + return CatalogSnapshot{}, SplitJob{}, err + } + if err := s.applyPromotionCompleteMutations(ctx, plan, mutations, expected.JobID, commitTS); err != nil { + return CatalogSnapshot{}, SplitJob{}, err + } + + return CatalogSnapshot{ + Version: plan.nextVersion, + Routes: cloneRouteDescriptors(completion.Routes), + }, completion.Job, nil +} + +func (s *CatalogStore) loadPromotionCompleteInputs(ctx context.Context, expectedVersion uint64, expected SplitJob) (uint64, uint64, []RouteDescriptor, SplitJob, bool, error) { + expectedRaw, err := EncodeSplitJob(expected) + if err != nil { + return 0, 0, nil, SplitJob{}, false, err + } + readTS := s.store.LastCommitTS() + currentVersion, err := s.versionAt(ctx, readTS) + if err != nil { + return 0, 0, nil, SplitJob{}, false, err + } + raw, currentJob, err := s.livePromotionCompleteJobAt(ctx, expected.JobID, readTS, expectedVersion, currentVersion) + if err != nil { + return 0, 0, nil, SplitJob{}, false, err + } + if currentVersion != expectedVersion || !bytes.Equal(raw, expectedRaw) { + return s.resolvePromotionCompleteInputConflict(ctx, readTS, currentVersion, expectedVersion, expected, expectedRaw, currentJob) + } + routes, err := s.routesAt(ctx, readTS) + if err != nil { + return 0, 0, nil, SplitJob{}, false, err + } + return readTS, currentVersion, routes, currentJob, false, nil +} + +func (s *CatalogStore) livePromotionCompleteJobAt(ctx context.Context, jobID uint64, ts uint64, expectedVersion uint64, currentVersion uint64) ([]byte, SplitJob, error) { + raw, err := s.store.GetAt(ctx, CatalogSplitJobKey(jobID), ts) + if err != nil { + if errors.Is(err, store.ErrKeyNotFound) { + return nil, SplitJob{}, promotionCompleteMissingJobError(expectedVersion, currentVersion) + } + return nil, SplitJob{}, errors.WithStack(err) + } + currentJob, err := DecodeSplitJob(raw) + if err != nil { + return nil, SplitJob{}, err + } + if currentJob.JobID != jobID { + return nil, SplitJob{}, errors.WithStack(ErrCatalogSplitJobKeyIDMismatch) + } + return raw, currentJob, nil +} + +func promotionCompleteMissingJobError(expectedVersion uint64, currentVersion uint64) error { + if currentVersion != expectedVersion { + return errors.WithStack(ErrCatalogVersionMismatch) + } + return errors.WithStack(ErrCatalogSplitJobConflict) +} + +func (s *CatalogStore) resolvePromotionCompleteInputConflict( + ctx context.Context, + readTS uint64, + currentVersion uint64, + expectedVersion uint64, + expected SplitJob, + expectedRaw []byte, + currentJob SplitJob, +) (uint64, uint64, []RouteDescriptor, SplitJob, bool, error) { + alreadyApplied, routes, err := s.promotionCompleteAlreadyAppliedAt(ctx, readTS, expected, expectedRaw, currentJob) + if err != nil { + return 0, 0, nil, SplitJob{}, false, err + } + if alreadyApplied { + return readTS, currentVersion, routes, currentJob, true, nil + } + if currentVersion != expectedVersion { + return 0, 0, nil, SplitJob{}, false, errors.WithStack(ErrCatalogVersionMismatch) + } + return 0, 0, nil, SplitJob{}, false, errors.WithStack(ErrCatalogSplitJobConflict) +} + +func (s *CatalogStore) promotionCompleteAlreadyAppliedAt(ctx context.Context, ts uint64, expected SplitJob, expectedRaw []byte, current SplitJob) (bool, []RouteDescriptor, error) { + matches, err := promotionCompleteJobMatchesExpected(expected, expectedRaw, current) + if err != nil || !matches { + return false, nil, err + } + routes, err := s.routesAt(ctx, ts) + if err != nil { + return false, nil, err + } + if !targetClearedDescriptorPresent(current, routes) { + return false, nil, errors.WithStack(ErrMigrationPromotionTargetAbsent) + } + return true, routes, nil +} + +func promotionCompleteJobMatchesExpected(expected SplitJob, expectedRaw []byte, current SplitJob) (bool, error) { + if expected.TargetPromotionDone || + !current.TargetPromotionDone || + current.PromotionCompletedTS == 0 || + (current.Phase != expected.Phase && current.Phase != SplitJobPhaseDone) { + return false, nil + } + normalized := CloneSplitJob(current) + normalized.Phase = expected.Phase + normalized.TargetPromotionDone = expected.TargetPromotionDone + normalized.PromotionCompletedTS = expected.PromotionCompletedTS + normalized.TerminalAtMs = expected.TerminalAtMs + normalized.UpdatedAtMs = expected.UpdatedAtMs + raw, err := EncodeSplitJob(normalized) + if err != nil { + return false, err + } + return bytes.Equal(raw, expectedRaw), nil +} + +func (s *CatalogStore) buildPromotionCompleteMutations( + ctx context.Context, + readTS uint64, + expectedVersion uint64, + jobID uint64, + completion *TargetPromotionCompletion, +) (savePlan, []*store.KVPairMutation, uint64, error) { + if expectedVersion == math.MaxUint64 { + return savePlan{}, nil, 0, errors.WithStack(ErrCatalogVersionOverflow) + } + minCommitTS := readTS + 1 + if minCommitTS == 0 { + return savePlan{}, nil, 0, errors.WithStack(ErrCatalogVersionOverflow) + } + + plan := savePlan{ + readTS: readTS, + minCommitTS: minCommitTS, + nextVersion: expectedVersion + 1, + routes: completion.Routes, + } + mutations, err := s.buildSaveMutations(ctx, &plan) + if err != nil { + return savePlan{}, nil, 0, err + } + commitTS, err := s.commitTSForApply(plan.minCommitTS) + if err != nil { + return savePlan{}, nil, 0, err + } + if completion.Job.PromotionCompletedTS == 0 { + completion.Job.PromotionCompletedTS = commitTS + } + encodedJob, err := EncodeSplitJob(completion.Job) + if err != nil { + return savePlan{}, nil, 0, err + } + jobMutations, err := s.buildSplitJobPutMutations(ctx, readTS, CatalogSplitJobKey(jobID), encodedJob, jobID) + if err != nil { + return savePlan{}, nil, 0, err + } + return plan, append(mutations, jobMutations...), commitTS, nil +} + +func (s *CatalogStore) applyPromotionCompleteMutations(ctx context.Context, plan savePlan, mutations []*store.KVPairMutation, jobID uint64, commitTS uint64) error { + readKeys := [][]byte{ + CatalogVersionKey(), + CatalogSplitJobKey(jobID), + } + if err := s.store.ApplyMutations(ctx, mutations, readKeys, plan.readTS, commitTS); err != nil { + if errors.Is(err, store.ErrWriteConflict) { + return s.promotionCompleteWriteConflict(ctx, plan) + } + return errors.WithStack(err) + } + return nil +} + +func (s *CatalogStore) promotionCompleteWriteConflict(ctx context.Context, plan savePlan) error { + currentVersion, err := s.versionAt(ctx, s.store.LastCommitTS()) + if err != nil { + return err + } + if currentVersion != plan.nextVersion-1 { + return errors.WithStack(ErrCatalogVersionMismatch) + } + return errors.WithStack(ErrCatalogSplitJobConflict) +} + +func targetClearedDescriptorPresent(job SplitJob, routes []RouteDescriptor) bool { + for _, route := range routes { + if route.GroupID != job.TargetGroupID { + continue + } + if route.StagedVisibilityActive || route.MigrationJobID != 0 { + continue + } + if route.ParentRouteID != job.SourceRouteID { + continue + } + if bytes.Equal(route.Start, job.SplitKey) { + return true + } + } + return false +} diff --git a/distribution/migration_promotion_complete_test.go b/distribution/migration_promotion_complete_test.go new file mode 100644 index 000000000..96bfc7e57 --- /dev/null +++ b/distribution/migration_promotion_complete_test.go @@ -0,0 +1,258 @@ +package distribution + +import ( + "context" + "testing" + + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "github.com/stretchr/testify/require" +) + +func TestCompleteTargetPromotionStateClearsStagedFieldsAndRetainsFloor(t *testing.T) { + t.Parallel() + + job := promotionCompleteTestJob() + routes := promotionCompleteTestRoutes() + + result, err := CompleteTargetPromotionState(job, routes, 1000) + require.NoError(t, err) + require.True(t, result.Changed) + require.Equal(t, []uint64{3}, result.ClearedRouteIDs) + require.True(t, result.Job.TargetPromotionDone) + require.Zero(t, result.Job.PromotionCompletedTS) + require.Equal(t, int64(1000), result.Job.UpdatedAtMs) + require.Zero(t, result.Job.TerminalAtMs) + require.Equal(t, SplitJobPhaseCleanup, result.Job.Phase) + + target := routeByID(t, result.Routes, 3) + require.False(t, target.StagedVisibilityActive) + require.Equal(t, uint64(0), target.MigrationJobID) + require.Equal(t, uint64(777), target.MinWriteTSExclusive) + require.Equal(t, uint64(42), target.ParentRouteID) + + badRoutes := promotionCompleteTestRoutes() + badRoutes[1].ParentRouteID = 777 + _, err = CompleteTargetPromotionState(job, badRoutes, 1000) + require.ErrorIs(t, err, ErrMigrationInvalidRoute) +} + +func TestCompleteTargetPromotionStateAcceptsAlreadyClearedDescriptor(t *testing.T) { + t.Parallel() + + job := promotionCompleteTestJob() + job.TargetPromotionDone = true + job.PromotionCompletedTS = 900 + job.UpdatedAtMs = 1000 + routes := promotionCompleteTestRoutes() + routes[1].StagedVisibilityActive = false + routes[1].MigrationJobID = 0 + + result, err := CompleteTargetPromotionState(job, routes, 1100) + require.NoError(t, err) + require.False(t, result.Changed) + require.Equal(t, SplitJobPhaseCleanup, result.Job.Phase) + require.Equal(t, uint64(900), result.Job.PromotionCompletedTS) + require.Equal(t, int64(1000), result.Job.UpdatedAtMs) + require.Zero(t, result.Job.TerminalAtMs) + + target := routeByID(t, result.Routes, 3) + require.False(t, target.StagedVisibilityActive) + require.Equal(t, uint64(0), target.MigrationJobID) + require.Equal(t, uint64(777), target.MinWriteTSExclusive) +} + +func TestCatalogStoreCompleteSplitJobTargetPromotionCommitsRouteAndJobTogether(t *testing.T) { + t.Parallel() + + ctx := context.Background() + cs := NewCatalogStore(store.NewMVCCStore(), WithCatalogRouteDescriptorV2Writes(true)) + saved, err := cs.Save(ctx, 0, promotionCompleteTestRoutes()) + require.NoError(t, err) + job := promotionCompleteTestJob() + require.NoError(t, cs.CreateSplitJob(ctx, job)) + before, err := cs.Snapshot(ctx) + require.NoError(t, err) + + snapshot, completed, err := cs.CompleteSplitJobTargetPromotion(ctx, saved.Version, job, 1000) + require.NoError(t, err) + require.Equal(t, saved.Version+1, snapshot.Version) + require.True(t, completed.TargetPromotionDone) + require.Equal(t, SplitJobPhaseCleanup, completed.Phase) + require.Greater(t, completed.PromotionCompletedTS, before.ReadTS) + require.Equal(t, int64(1000), completed.UpdatedAtMs) + + loadedJob, found, err := cs.SplitJob(ctx, job.JobID) + require.NoError(t, err) + require.True(t, found) + assertSplitJobEqual(t, completed, loadedJob) + + loaded, err := cs.Snapshot(ctx) + require.NoError(t, err) + require.Equal(t, saved.Version+1, loaded.Version) + target := routeByID(t, loaded.Routes, 3) + require.False(t, target.StagedVisibilityActive) + require.Equal(t, uint64(0), target.MigrationJobID) + require.Equal(t, uint64(777), target.MinWriteTSExclusive) +} + +func TestCatalogStoreCompleteSplitJobTargetPromotionRetryAfterCommit(t *testing.T) { + t.Parallel() + + ctx := context.Background() + cs := NewCatalogStore(store.NewMVCCStore(), WithCatalogRouteDescriptorV2Writes(true)) + saved, err := cs.Save(ctx, 0, promotionCompleteTestRoutes()) + require.NoError(t, err) + job := promotionCompleteTestJob() + require.NoError(t, cs.CreateSplitJob(ctx, job)) + + firstSnapshot, firstCompleted, err := cs.CompleteSplitJobTargetPromotion(ctx, saved.Version, job, 1000) + require.NoError(t, err) + + retrySnapshot, retryCompleted, err := cs.CompleteSplitJobTargetPromotion(ctx, saved.Version, job, 2000) + require.NoError(t, err) + require.Equal(t, firstSnapshot.Version, retrySnapshot.Version) + assertSplitJobEqual(t, firstCompleted, retryCompleted) + + loaded, err := cs.Snapshot(ctx) + require.NoError(t, err) + require.Equal(t, firstSnapshot.Version, loaded.Version) +} + +func TestCatalogStoreCompleteSplitJobTargetPromotionIsIdempotentAfterClearedDescriptor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + cs := NewCatalogStore(store.NewMVCCStore(), WithCatalogRouteDescriptorV2Writes(true)) + routes := promotionCompleteTestRoutes() + routes[1].StagedVisibilityActive = false + routes[1].MigrationJobID = 0 + saved, err := cs.Save(ctx, 0, routes) + require.NoError(t, err) + job := promotionCompleteTestJob() + job.TargetPromotionDone = true + job.Phase = SplitJobPhaseCleanup + job.PromotionCompletedTS = 900 + job.UpdatedAtMs = 1000 + require.NoError(t, cs.CreateSplitJob(ctx, job)) + + snapshot, completed, err := cs.CompleteSplitJobTargetPromotion(ctx, saved.Version, job, 1100) + require.NoError(t, err) + require.Equal(t, saved.Version, snapshot.Version) + require.Equal(t, SplitJobPhaseCleanup, completed.Phase) + require.Equal(t, uint64(900), completed.PromotionCompletedTS) + require.Equal(t, int64(1000), completed.UpdatedAtMs) + require.Zero(t, completed.TerminalAtMs) + + loaded, err := cs.Snapshot(ctx) + require.NoError(t, err) + require.Equal(t, saved.Version, loaded.Version) +} + +func TestCompleteTargetPromotionStateBackfillsMissingTerminalTime(t *testing.T) { + t.Parallel() + + job := promotionCompleteTestJob() + job.TargetPromotionDone = true + job.Phase = SplitJobPhaseDone + routes := promotionCompleteTestRoutes() + routes[1].StagedVisibilityActive = false + routes[1].MigrationJobID = 0 + + result, err := CompleteTargetPromotionState(job, routes, 1200) + require.NoError(t, err) + require.True(t, result.Changed) + require.Equal(t, int64(1200), result.Job.TerminalAtMs) + require.Equal(t, int64(1200), result.Job.UpdatedAtMs) +} + +func TestCatalogStoreCompleteSplitJobTargetPromotionRejectsStaleInputs(t *testing.T) { + t.Parallel() + + ctx := context.Background() + cs := NewCatalogStore(store.NewMVCCStore(), WithCatalogRouteDescriptorV2Writes(true)) + saved, err := cs.Save(ctx, 0, promotionCompleteTestRoutes()) + require.NoError(t, err) + job := promotionCompleteTestJob() + require.NoError(t, cs.CreateSplitJob(ctx, job)) + + _, _, err = cs.CompleteSplitJobTargetPromotion(ctx, saved.Version+1, job, 1000) + require.True(t, errors.Is(err, ErrCatalogVersionMismatch), "got %v", err) + + advanced := job + advanced.Cursor = []byte("advanced") + advanced.UpdatedAtMs++ + require.NoError(t, cs.SaveSplitJob(ctx, job, advanced)) + + _, _, err = cs.CompleteSplitJobTargetPromotion(ctx, saved.Version, job, 1000) + require.True(t, errors.Is(err, ErrCatalogSplitJobConflict), "got %v", err) +} + +func TestCatalogStoreCompleteSplitJobTargetPromotionPreservesVersionConflict(t *testing.T) { + t.Parallel() + + ctx := context.Background() + cs := NewCatalogStore(store.NewMVCCStore(), WithCatalogRouteDescriptorV2Writes(true)) + saved, err := cs.Save(ctx, 0, promotionCompleteTestRoutes()) + require.NoError(t, err) + job := promotionCompleteTestJob() + require.NoError(t, cs.CreateSplitJob(ctx, job)) + + readTS, _, routes, _, _, err := cs.loadPromotionCompleteInputs(ctx, saved.Version, job) + require.NoError(t, err) + completion, err := CompleteTargetPromotionState(job, routes, 1000) + require.NoError(t, err) + plan, mutations, commitTS, err := cs.buildPromotionCompleteMutations(ctx, readTS, saved.Version, job.JobID, &completion) + require.NoError(t, err) + + _, err = cs.Save(ctx, saved.Version, promotionCompleteTestRoutes()) + require.NoError(t, err) + + err = cs.applyPromotionCompleteMutations(ctx, plan, mutations, job.JobID, commitTS) + require.True(t, errors.Is(err, ErrCatalogVersionMismatch), "got %v", err) +} + +func promotionCompleteTestJob() SplitJob { + return SplitJob{ + JobID: 99, + SourceRouteID: 42, + SplitKey: []byte("m"), + TargetGroupID: 2, + Phase: SplitJobPhaseCleanup, + } +} + +func promotionCompleteTestRoutes() []RouteDescriptor { + return []RouteDescriptor{ + { + RouteID: 2, + Start: []byte(""), + End: []byte("m"), + GroupID: 1, + State: RouteStateActive, + ParentRouteID: 42, + }, + { + RouteID: 3, + Start: []byte("m"), + End: nil, + GroupID: 2, + State: RouteStateActive, + ParentRouteID: 42, + StagedVisibilityActive: true, + MigrationJobID: 99, + MinWriteTSExclusive: 777, + }, + } +} + +func routeByID(t *testing.T, routes []RouteDescriptor, routeID uint64) RouteDescriptor { + t.Helper() + for _, route := range routes { + if route.RouteID == routeID { + return route + } + } + t.Fatalf("route %d not found", routeID) + return RouteDescriptor{} +} diff --git a/distribution/split_job_catalog.go b/distribution/split_job_catalog.go index 2470a1252..b09fb26c9 100644 --- a/distribution/split_job_catalog.go +++ b/distribution/split_job_catalog.go @@ -36,6 +36,7 @@ var ( ErrCatalogSplitJobConflict = errors.New("catalog split job conflict") ErrCatalogSplitJobTerminalRequired = errors.New("catalog split job terminal state is required") ErrSplitJobOverlap = errors.New("split job overlaps requested route") + ErrTooManyInFlightSplitJobs = errors.New("too many in-flight split jobs") ) // SplitJobPhase is the durable phase of a split migration job. @@ -167,6 +168,7 @@ type SplitJob struct { StartedAtMs int64 UpdatedAtMs int64 TerminalAtMs int64 + CapabilityRegressed bool } // CatalogNextSplitJobIDKey returns the reserved key used for split-job ID allocation. @@ -588,12 +590,12 @@ func (p SplitJobPhase) retryable() bool { func (p SplitJobPhase) abandonable() bool { switch p { - case SplitJobPhaseBackfill, + case SplitJobPhasePlanned, + SplitJobPhaseBackfill, SplitJobPhaseFence, SplitJobPhaseDeltaCopy: return true case SplitJobPhaseNone, - SplitJobPhasePlanned, SplitJobPhaseCutover, SplitJobPhaseCleanup, SplitJobPhaseDone, @@ -905,9 +907,15 @@ func CloneSplitJob(job SplitJob) SplitJob { StartedAtMs: job.StartedAtMs, UpdatedAtMs: job.UpdatedAtMs, TerminalAtMs: job.TerminalAtMs, + CapabilityRegressed: job.CapabilityRegressed, } } +// SplitJobToProto converts a catalog SplitJob into its wire representation. +func SplitJobToProto(job SplitJob) *pb.SplitJob { + return splitJobToProto(job) +} + func splitJobToProto(job SplitJob) *pb.SplitJob { job = CloneSplitJob(job) return &pb.SplitJob{ @@ -944,6 +952,7 @@ func splitJobToProto(job SplitJob) *pb.SplitJob { StartedAtMs: job.StartedAtMs, UpdatedAtMs: job.UpdatedAtMs, TerminalAtMs: job.TerminalAtMs, + CapabilityRegressed: job.CapabilityRegressed, } } @@ -1009,6 +1018,7 @@ func splitJobFromProto(msg *pb.SplitJob) (SplitJob, error) { StartedAtMs: msg.GetStartedAtMs(), UpdatedAtMs: msg.GetUpdatedAtMs(), TerminalAtMs: msg.GetTerminalAtMs(), + CapabilityRegressed: msg.GetCapabilityRegressed(), }, nil } diff --git a/distribution/split_job_catalog_test.go b/distribution/split_job_catalog_test.go index d7ef808e4..18deb2679 100644 --- a/distribution/split_job_catalog_test.go +++ b/distribution/split_job_catalog_test.go @@ -119,6 +119,7 @@ func TestSplitJobCodecValidatesRestartPhases(t *testing.T) { } for _, phase := range []SplitJobPhase{ + SplitJobPhasePlanned, SplitJobPhaseBackfill, SplitJobPhaseFence, SplitJobPhaseDeltaCopy, @@ -338,6 +339,120 @@ func TestCatalogStoreSaveSplitJobRejectsStaleExpectedJob(t *testing.T) { assertSplitJobEqual(t, advanced, got) } +func TestCatalogStoreRetrySplitJobUsesDurableRetryPhase(t *testing.T) { + cs := NewCatalogStore(store.NewMVCCStore()) + ctx := context.Background() + job := sampleSplitJob(18) + job.Phase = SplitJobPhaseFailed + job.RetryPhase = SplitJobPhaseDeltaCopy + job.LastError = "transient" + + if err := cs.CreateSplitJob(ctx, job); err != nil { + t.Fatalf("create split job: %v", err) + } + retried, err := cs.RetrySplitJob(ctx, job.JobID, 1200) + if err != nil { + t.Fatalf("retry split job: %v", err) + } + if retried.Phase != SplitJobPhaseDeltaCopy || retried.RetryPhase != SplitJobPhaseNone { + t.Fatalf("unexpected retry transition: %+v", retried) + } + if retried.AbandonFromPhase != SplitJobPhaseNone || retried.LastError != "" || retried.UpdatedAtMs != 1200 { + t.Fatalf("unexpected retry witness cleanup: %+v", retried) + } + got, found, err := cs.SplitJob(ctx, job.JobID) + if err != nil { + t.Fatalf("load retried split job: %v", err) + } + if !found { + t.Fatal("expected retried split job") + } + assertSplitJobEqual(t, retried, got) +} + +func TestCatalogStoreRetrySplitJobRejectsMissingRetryWitness(t *testing.T) { + cs := NewCatalogStore(store.NewMVCCStore()) + ctx := context.Background() + job := sampleSplitJob(19) + job.Phase = SplitJobPhaseFailed + job.RetryPhase = SplitJobPhaseNone + + if err := cs.CreateSplitJob(ctx, job); !errors.Is(err, ErrCatalogInvalidSplitJobPhase) { + t.Fatalf("expected ErrCatalogInvalidSplitJobPhase, got %v", err) + } +} + +func TestCatalogStoreBeginSplitJobAbandonRecordsPreCutoverPhase(t *testing.T) { + cs := NewCatalogStore(store.NewMVCCStore()) + ctx := context.Background() + job := sampleSplitJob(20) + job.Phase = SplitJobPhaseFailed + job.RetryPhase = SplitJobPhaseFence + job.AbandonFromPhase = SplitJobPhaseNone + + if err := cs.CreateSplitJob(ctx, job); err != nil { + t.Fatalf("create split job: %v", err) + } + abandoning, err := cs.BeginSplitJobAbandon(ctx, job.JobID, 1300) + if err != nil { + t.Fatalf("begin split job abandon: %v", err) + } + if abandoning.Phase != SplitJobPhaseAbandoning || + abandoning.RetryPhase != SplitJobPhaseNone || + abandoning.AbandonFromPhase != SplitJobPhaseFence || + abandoning.UpdatedAtMs != 1300 { + t.Fatalf("unexpected abandon transition: %+v", abandoning) + } + got, found, err := cs.SplitJob(ctx, job.JobID) + if err != nil { + t.Fatalf("load abandoning split job: %v", err) + } + if !found { + t.Fatal("expected abandoning split job") + } + assertSplitJobEqual(t, abandoning, got) +} + +func TestCatalogStoreBeginSplitJobAbandonAllowsPlannedPhase(t *testing.T) { + cs := NewCatalogStore(store.NewMVCCStore()) + ctx := context.Background() + job := sampleSplitJob(22) + job.Phase = SplitJobPhasePlanned + job.RetryPhase = SplitJobPhaseNone + job.AbandonFromPhase = SplitJobPhaseNone + + if err := cs.CreateSplitJob(ctx, job); err != nil { + t.Fatalf("create split job: %v", err) + } + abandoning, err := cs.BeginSplitJobAbandon(ctx, job.JobID, 1400) + if err != nil { + t.Fatalf("begin split job abandon: %v", err) + } + if abandoning.Phase != SplitJobPhaseAbandoning || + abandoning.AbandonFromPhase != SplitJobPhasePlanned || + abandoning.UpdatedAtMs != 1400 { + t.Fatalf("unexpected planned abandon transition: %+v", abandoning) + } + if _, err := EncodeSplitJob(abandoning); err != nil { + t.Fatalf("encode planned abandon transition: %v", err) + } +} + +func TestCatalogStoreBeginSplitJobAbandonRejectsPostCutoverPhases(t *testing.T) { + cs := NewCatalogStore(store.NewMVCCStore()) + ctx := context.Background() + job := sampleSplitJob(21) + job.Phase = SplitJobPhaseCleanup + job.RetryPhase = SplitJobPhaseNone + + if err := cs.CreateSplitJob(ctx, job); err != nil { + t.Fatalf("create split job: %v", err) + } + if _, err := cs.BeginSplitJobAbandon(ctx, job.JobID, 1300); !errors.Is(err, ErrCatalogSplitJobCannotAbandon) { + t.Fatalf("expected ErrCatalogSplitJobCannotAbandon, got %v", err) + } +} + func TestCatalogStoreListSplitJobsIncludesLiveAndHistory(t *testing.T) { cs := NewCatalogStore(store.NewMVCCStore()) ctx := context.Background() diff --git a/distribution/split_job_lifecycle.go b/distribution/split_job_lifecycle.go new file mode 100644 index 000000000..f2427a4e0 --- /dev/null +++ b/distribution/split_job_lifecycle.go @@ -0,0 +1,147 @@ +package distribution + +import ( + "bytes" + "context" + + "github.com/cockroachdb/errors" +) + +var ( + ErrCatalogSplitJobNotFound = errors.New("catalog split job is not found") + ErrCatalogSplitJobCannotRetry = errors.New("catalog split job cannot be retried") + ErrCatalogSplitJobCannotAbandon = errors.New("catalog split job cannot be abandoned") +) + +// RetrySplitJobState returns the durable state transition for RetrySplitJob. +// The retry target must come from the recorded RetryPhase witness; callers must +// not infer a phase from cursors, timestamps, or diagnostics after restart. +func RetrySplitJobState(job SplitJob, nowMs int64) (SplitJob, error) { + out := CloneSplitJob(job) + if out.Phase != SplitJobPhaseFailed || !splitJobRetryPhaseAllowed(out.RetryPhase) { + return SplitJob{}, errors.WithStack(ErrCatalogSplitJobCannotRetry) + } + out.Phase = out.RetryPhase + out.RetryPhase = SplitJobPhaseNone + out.AbandonFromPhase = SplitJobPhaseNone + out.LastError = "" + out.UpdatedAtMs = nowMs + return out, nil +} + +// BeginSplitJobAbandon records the durable ABANDONING witness before any +// pre-CUTOVER cleanup side effect is removed. +func BeginSplitJobAbandon(job SplitJob, nowMs int64) (SplitJob, error) { + out := CloneSplitJob(job) + from, ok := splitJobAbandonFromPhase(out) + if !ok { + return SplitJob{}, errors.WithStack(ErrCatalogSplitJobCannotAbandon) + } + if out.Phase == SplitJobPhaseAbandoning { + return out, nil + } + out.Phase = SplitJobPhaseAbandoning + out.RetryPhase = SplitJobPhaseNone + out.AbandonFromPhase = from + out.UpdatedAtMs = nowMs + return out, nil +} + +// RetrySplitJob CASes a FAILED live job back to its recorded retry phase. +func (s *CatalogStore) RetrySplitJob(ctx context.Context, jobID uint64, nowMs int64) (SplitJob, error) { + expected, _, err := s.LiveSplitJobForUpdate(ctx, jobID) + if err != nil { + return SplitJob{}, err + } + next, err := RetrySplitJobState(expected, nowMs) + if err != nil { + return SplitJob{}, err + } + return next, s.SaveSplitJob(ctx, expected, next) +} + +// BeginSplitJobAbandon CASes a live pre-CUTOVER job into ABANDONING. +func (s *CatalogStore) BeginSplitJobAbandon(ctx context.Context, jobID uint64, nowMs int64) (SplitJob, error) { + expected, _, err := s.LiveSplitJobForUpdate(ctx, jobID) + if err != nil { + return SplitJob{}, err + } + next, err := BeginSplitJobAbandon(expected, nowMs) + if err != nil { + return SplitJob{}, err + } + if SplitJobsEquivalent(expected, next) { + return next, nil + } + return next, s.SaveSplitJob(ctx, expected, next) +} + +// LiveSplitJobForUpdate reads the latest live split job and the snapshot +// timestamp used for the read. RPC callers use readTS as a CAS StartTS when +// proposing the update through Raft. +func (s *CatalogStore) LiveSplitJobForUpdate(ctx context.Context, jobID uint64) (SplitJob, uint64, error) { + if err := ensureCatalogStore(s); err != nil { + return SplitJob{}, 0, err + } + if jobID == 0 { + return SplitJob{}, 0, errors.WithStack(ErrCatalogSplitJobIDRequired) + } + ctx = contextOrBackground(ctx) + readTS := s.store.LastCommitTS() + job, found, err := s.liveSplitJobAt(ctx, jobID, readTS) + if err != nil { + return SplitJob{}, 0, err + } + if !found { + return SplitJob{}, 0, errors.WithStack(ErrCatalogSplitJobNotFound) + } + return job, readTS, nil +} + +func splitJobRetryPhaseAllowed(phase SplitJobPhase) bool { + switch phase { + case SplitJobPhaseBackfill, SplitJobPhaseFence, SplitJobPhaseDeltaCopy, SplitJobPhaseCutover, SplitJobPhaseCleanup: + return true + case SplitJobPhaseNone, SplitJobPhasePlanned, SplitJobPhaseDone, SplitJobPhaseFailed, SplitJobPhaseAbandoning, SplitJobPhaseAbandoned: + return false + } + return false +} + +func splitJobAbandonFromPhase(job SplitJob) (SplitJobPhase, bool) { + switch job.Phase { + case SplitJobPhasePlanned, SplitJobPhaseBackfill, SplitJobPhaseFence, SplitJobPhaseDeltaCopy: + return job.Phase, true + case SplitJobPhaseFailed: + if splitJobAbandonRetryPhaseAllowed(job.RetryPhase) { + return job.RetryPhase, true + } + case SplitJobPhaseAbandoning: + if splitJobAbandonRetryPhaseAllowed(job.AbandonFromPhase) { + return job.AbandonFromPhase, true + } + case SplitJobPhaseNone, SplitJobPhaseCutover, SplitJobPhaseCleanup, SplitJobPhaseDone, SplitJobPhaseAbandoned: + } + return SplitJobPhaseNone, false +} + +func splitJobAbandonRetryPhaseAllowed(phase SplitJobPhase) bool { + switch phase { + case SplitJobPhasePlanned, SplitJobPhaseBackfill, SplitJobPhaseFence, SplitJobPhaseDeltaCopy: + return true + case SplitJobPhaseNone, SplitJobPhaseCutover, SplitJobPhaseCleanup, SplitJobPhaseDone, SplitJobPhaseFailed, SplitJobPhaseAbandoning, SplitJobPhaseAbandoned: + return false + } + return false +} + +// SplitJobsEquivalent reports whether two split jobs encode to the same +// durable catalog value. +func SplitJobsEquivalent(left, right SplitJob) bool { + leftRaw, leftErr := EncodeSplitJob(left) + rightRaw, rightErr := EncodeSplitJob(right) + if leftErr != nil || rightErr != nil { + return false + } + return bytes.Equal(leftRaw, rightRaw) +} diff --git a/docs/design/2026_02_18_partial_hotspot_shard_split.md b/docs/design/2026_02_18_partial_hotspot_shard_split.md index e2c26ca01..ee8a0b5b8 100644 --- a/docs/design/2026_02_18_partial_hotspot_shard_split.md +++ b/docs/design/2026_02_18_partial_hotspot_shard_split.md @@ -293,6 +293,14 @@ Add RPCs: 2. Job phases: BACKFILL/FENCE/DELTA/CUTOVER 3. Manual split with target-group relocation +Status: implemented. The production runner advances durable cross-group jobs +through BACKFILL, FENCE, DELTA_COPY, CUTOVER, CLEANUP, and `DONE`; current-voter +readiness and cleanup barriers protect leadership changes; and the deterministic +Jepsen workload covers the split alongside leader-kill and partition packages. +The completed M2 contract is recorded in +[`2026_06_11_implemented_hotspot_split_milestone2_migration.md`](2026_06_11_implemented_hotspot_split_milestone2_migration.md) +and its final runner/lifecycle slice landed in PR #1096. + ### Milestone 3: Automation 1. Access aggregation diff --git a/docs/design/2026_06_11_proposed_hotspot_split_milestone2_migration.md b/docs/design/2026_06_11_implemented_hotspot_split_milestone2_migration.md similarity index 99% rename from docs/design/2026_06_11_proposed_hotspot_split_milestone2_migration.md rename to docs/design/2026_06_11_implemented_hotspot_split_milestone2_migration.md index 1b5592f7f..80b7716dd 100644 --- a/docs/design/2026_06_11_proposed_hotspot_split_milestone2_migration.md +++ b/docs/design/2026_06_11_implemented_hotspot_split_milestone2_migration.md @@ -1,12 +1,14 @@ # Hotspot Shard Split — Milestone 2: Migration Plane -Status: Proposed +Status: Implemented Author: bootjp Date: 2026-06-11 Parent: [2026_02_18_partial_hotspot_shard_split.md](2026_02_18_partial_hotspot_shard_split.md). M1 (control plane) is as-built in [2026_02_18_implemented_hotspot_split_milestone1_pr.md](2026_02_18_implemented_hotspot_split_milestone1_pr.md). +Current implementation status: implemented. The M2 stack provides the durable SplitJob catalog, versioned route descriptors, resumable range export/import, staged/live visibility, source write and read fences, current-voter readiness and cleanup barriers, constant-time route cutover, incremental target promotion, bounded source/target cleanup, terminal `DONE` history, and the deterministic cross-group Jepsen workload. PR #1096 completed the production runner and lifecycle wiring. Automatic hotspot detection and split scheduling remain M3 scope; the broader route-shuffle and fault campaign remains M4 scope. + ## 1. Background M1 landed the control plane: @@ -1241,7 +1243,7 @@ Phased into reviewable PRs, each lands behind its own doc-or-test gate: | M2-PR5 | Coordinator + FSM FENCE rejection (`ErrRouteWriteFenced`) with point and `DEL_PREFIX` range-footprint checks + route-faithful txn-lock drain (§3.2a.0a) + same-group `SplitRange` overlap rejection while a SplitJob is live | fsm + coordinator unit + `migrator_lock_drain_test` + `catalog_test` overlap red controls | | M2-PR6 | Cross-group end-to-end: ExportRangeVersions / ImportVersions server-side handlers + migrator BACKFILL/DELTA_COPY + raw-candidate staged/live merge read path in both scan directions and `LatestCommitTS` + §7.2.2e source-side cutover read-fence arm with every-current-source-voter ACK, membership-epoch re-ACK, catalog-version waiter, route-key-normalized scan ownership checks, and server-stamped RawKV read versions | integration incl. delete/TTL DELTA_COPY, Redis list/hash/set/zset/stream migration, HLC restart/fence-floor cases + `kv/fsm_cutover_read_fence_test.go` | | M2-PR7 | Target-local `PromotionState` + background promoter + ordered default-group promotion-complete CAS retaining `min_write_ts_exclusive` + source/target cleanup before DONE history move; readiness guard accepts matching cleared descriptors while retained; `AbandonSplitJob` with durable `ABANDONING` cleanup witness + CLEANUP GC + Jepsen split workload | `kv/fsm_promote_staged_test.go` raw hidden-version/LatestCommitTS merge + jepsen suite | -| M2-PR8 | Rename `*_proposed_*` → `*_partial_*` after PR1 ships; update parent partial doc M2 status; rename to `*_implemented_*` after PR7 | | +| M2-PR8 | Rename `*_proposed_*` → `*_partial_*` after PR1 ships; update parent partial doc M2 status; rename to `*_implemented_*` after PR7 | Completed after the runner, voter barriers, cleanup lifecycle, route publication, and Jepsen workload landed in PR #1096. | Each PR follows the five-lens self-review and is gated by its tests + `make lint`. @@ -1281,9 +1283,12 @@ This is independent of the existing rolling-upgrade protocol for unrelated subsy ## 14. Lifecycle -This document begins as `*_proposed_*`. Per CLAUDE.md: +This document is `*_implemented_*` because the M2 cross-group migration plane is complete as a central subsystem. The production runner now resumes durable jobs across leadership changes, drives every phase through `CLEANUP`, waits for current source and target voter proofs, publishes the target route, removes bounded migration state, and moves the job to `DONE` history. The deterministic Jepsen split workload exercises the cross-group operation alongside leader-kill and partition fault packages. + +The remaining work is intentionally outside M2's central scope: -- Rename to `*_partial_*` after M2-PR1 lands; track per-PR landing under §11. -- Rename to `*_implemented_*` after M2-PR7 ships and the parent partial doc's M2 row is checked off. +- M3 owns automatic hotspot detection, target selection, and split scheduling. +- M4 owns the broader route-shuffle nemesis matrix and production-scale fault campaigns. +- Reverse migration after CUTOVER and concurrent migration jobs remain future extensions. `git mv` is used so history follows. diff --git a/docs/design/2026_06_12_proposed_scaling_roadmap.md b/docs/design/2026_06_12_proposed_scaling_roadmap.md index 2174a3183..6d3fa34ac 100644 --- a/docs/design/2026_06_12_proposed_scaling_roadmap.md +++ b/docs/design/2026_06_12_proposed_scaling_roadmap.md @@ -343,9 +343,9 @@ control-plane (`*_proposed_*` doc TBD).** - M1 standalone but doesn't enable cross-region writes — it just makes Raft survive cross-WAN partition. -- M2 depends on the M2 hotspot-split migration contract - (`2026_06_11_proposed_hotspot_split_milestone2_migration.md`) - being implemented so the monotone-merge primitive exists. +- M2's hotspot-split migration dependency is implemented in + [2026_06_11_implemented_hotspot_split_milestone2_migration.md](2026_06_11_implemented_hotspot_split_milestone2_migration.md), including + the monotone-merge primitive. - M3 depends on M1's region-aware membership and M2's per-region ceiling. - M4 depends on M2 and M3. @@ -538,7 +538,7 @@ the ceiling shape: Composability invariant: **every monotone-merge happens via the same `SetPhysicalCeiling` + `Observe` primitive**. The M2 hotspot-split contract (§6.2.1 of -`2026_06_11_proposed_hotspot_split_milestone2_migration.md`) is the +[2026_06_11_implemented_hotspot_split_milestone2_migration.md](2026_06_11_implemented_hotspot_split_milestone2_migration.md)) is the reference implementation; per-region and per-group merges reuse it. ### 7.2 Capability bits diff --git a/docs/design/2026_06_23_proposed_scaling_roadmap.md b/docs/design/2026_06_23_proposed_scaling_roadmap.md index fe74eda05..210d52c14 100644 --- a/docs/design/2026_06_23_proposed_scaling_roadmap.md +++ b/docs/design/2026_06_23_proposed_scaling_roadmap.md @@ -98,9 +98,10 @@ memory each group's private cache/memtable pins. - **Range split — distribute a range across groups.** Same-group split shipped in M1 (`distribution/`). Cross-group migration (the part that - actually relocates data and reduces per-node volume) is **PR #945** - (`docs/design/2026_06_11_proposed_hotspot_split_milestone2_migration.md`, - branch `docs/hotspot-split-m2-proposal`): a resumable `SplitJob` with + actually relocates data and reduces per-node volume) is implemented by the + M2 stack recorded in + [2026_06_11_implemented_hotspot_split_milestone2_migration.md](2026_06_11_implemented_hotspot_split_milestone2_migration.md): + a resumable `SplitJob` with `PLANNED → BACKFILL → FENCE → DELTA_COPY → CUTOVER → CLEANUP → DONE` phases driven by a migrator on the default-group leader. M2 is the required ownership-migration mechanism, but it reduces per-node bytes only when the diff --git a/jepsen/src/elastickv/db.clj b/jepsen/src/elastickv/db.clj index 369d33bc9..a650285f2 100644 --- a/jepsen/src/elastickv/db.clj +++ b/jepsen/src/elastickv/db.clj @@ -16,6 +16,8 @@ (def ^:private pid-file "/var/run/elastickv.pid") (def ^:private server-bin (str bin-dir "/elastickv")) (def ^:private raftadmin-bin (str bin-dir "/raftadmin")) +(def ^:private split-bin (str bin-dir "/elastickv-split")) +(def ^:private list-routes-bin (str bin-dir "/elastickv-list-routes")) (def ^:private build-dir ;; local (control node) directory for built binaries @@ -38,7 +40,9 @@ "GOPATH" "/home/vagrant/go" "GOCACHE" "/home/vagrant/.cache/go-build"})] (doseq [[out-cmd args] [["elastickv" ["go" "build" "-o" (str build-dir "/elastickv") "./cmd/server"]] - ["raftadmin" ["go" "build" "-o" (str build-dir "/raftadmin") "./cmd/raftadmin"]]]] + ["raftadmin" ["go" "build" "-o" (str build-dir "/raftadmin") "./cmd/raftadmin"]] + ["elastickv-split" ["go" "build" "-o" (str build-dir "/elastickv-split") "./cmd/elastickv-split"]] + ["elastickv-list-routes" ["go" "build" "-o" (str build-dir "/elastickv-list-routes") "./cmd/elastickv-list-routes"]]]] (let [{:keys [exit err]} (apply sh/sh (concat args [:env env :dir root]))] (when-not (zero? exit) (throw (ex-info (str "failed to build " out-cmd) {:err err}))))))) @@ -58,7 +62,7 @@ (c/on node (c/su (c/exec :mkdir :-p bin-dir) - (doseq [bin ["elastickv" "raftadmin"]] + (doseq [bin ["elastickv" "raftadmin" "elastickv-split" "elastickv-list-routes"]] (c/upload (str build-dir "/" bin) (str bin-dir "/" bin)) (c/exec :chmod "755" (str bin-dir "/" bin)))))) @@ -104,7 +108,7 @@ (build-raft-service-map nodes grpc-port dynamo-port raft-groups)) (defn- start-node! - [test node {:keys [bootstrap-node grpc-port redis-port dynamo-port s3-port sqs-port sqs-region data-dir raft-groups shard-ranges raft-engine server-env]}] + [test node {:keys [bootstrap-node grpc-port redis-port dynamo-port s3-port sqs-port sqs-region data-dir raft-groups shard-ranges raft-engine server-env migration-enabled]}] (when (and (seq raft-groups) (> (count raft-groups) 1) (nil? shard-ranges)) @@ -137,11 +141,16 @@ (seq raft-groups) (conj "--raftGroups" (build-raft-groups-arg node raft-groups)) (seq shard-ranges) (conj "--shardRanges" shard-ranges) bootstrap? (conj "--raftBootstrap")) + effective-server-env (cond-> (or server-env {}) + migration-enabled + (merge {"ELASTICKV_ENABLE_MIGRATION_IMPORT_OPCODE" "true" + "ELASTICKV_ENABLE_MIGRATION_PROMOTE_OPCODE" "true" + "ELASTICKV_ENABLE_MIGRATION_CLEANUP_OPCODE" "true"})) daemon-opts (cond-> {:chdir bin-dir :logfile log-file :pidfile pid-file :background? true} - (seq server-env) (assoc :env server-env))] + (seq effective-server-env) (assoc :env effective-server-env))] (c/on node (c/su (c/exec :mkdir :-p data-dir) diff --git a/jepsen/src/elastickv/jepsen_test.clj b/jepsen/src/elastickv/jepsen_test.clj index 9de017df0..39d2c3c5f 100644 --- a/jepsen/src/elastickv/jepsen_test.clj +++ b/jepsen/src/elastickv/jepsen_test.clj @@ -6,6 +6,7 @@ [elastickv.dynamodb-types-workload :as dynamodb-types-workload] [elastickv.s3-workload :as s3-workload] [elastickv.sqs-htfifo-workload :as sqs-htfifo-workload] + [elastickv.split-workload :as split-workload] [jepsen.cli :as cli])) (defn elastickv-test @@ -28,6 +29,10 @@ ([] (elastickv-zset-safety-test {})) ([opts] (zset-safety-workload/elastickv-zset-safety-test opts))) +(defn elastickv-split-test + ([] (elastickv-split-test {})) + ([opts] (split-workload/elastickv-split-test opts))) + (def ^:private test-fns "Map of user-facing test names to their constructor fns. The first positional CLI arg selects which workload runs; if absent or unknown, @@ -36,7 +41,8 @@ {"elastickv-test" elastickv-test "elastickv-zset-safety-test" elastickv-zset-safety-test "elastickv-dynamodb-test" elastickv-dynamodb-test - "elastickv-s3-test" elastickv-s3-test}) + "elastickv-s3-test" elastickv-s3-test + "elastickv-split-test" elastickv-split-test}) (defn elastickv-sqs-htfifo-test "HT-FIFO Jepsen test (PR 7b). Run via the workload's own -main: diff --git a/jepsen/src/elastickv/split_workload.clj b/jepsen/src/elastickv/split_workload.clj new file mode 100644 index 000000000..b69fe89ed --- /dev/null +++ b/jepsen/src/elastickv/split_workload.clj @@ -0,0 +1,252 @@ +(ns elastickv.split-workload + "Deterministic M2 cross-group split workload over two Redis registers." + (:gen-class) + (:require [clojure.java.shell :as shell] + [clojure.tools.logging :refer [info warn]] + [elastickv.cli :as cli] + [elastickv.composed1-nemesis :as routes] + [elastickv.db :as ekdb] + [jepsen [checker :as checker] + [client :as client] + [generator :as gen] + [independent :as independent] + [net :as net]] + [jepsen.checker.timeline :as timeline] + [jepsen.control :as control] + [jepsen.db :as jdb] + [jepsen.nemesis :as nemesis] + [jepsen.nemesis.combined :as combined] + [jepsen.os :as os] + [jepsen.os.debian :as debian] + [knossos.model :as model] + [taoensso.carmine :as car])) + +(def default-nodes ["n1" "n2" "n3" "n4" "n5"]) +(def ^:private split-key "m2-split") +(def ^:private register-keys ["m2-left" "m2-target"]) + +(defn- helper-path [name] + (str (System/getProperty "user.dir") "/target/elastickv-jepsen/" name)) + +(defrecord SplitRegisterClient [node->port conn-spec] + client/Client + (open! [this test node] + (assoc this :conn-spec + {:pool {} + :spec {:host (or (:redis-host test) (name node)) + :port (get node->port node 6379) + :timeout-ms 10000}})) + (close! [this _test] (assoc this :conn-spec nil)) + (setup! [this _test] + (doseq [k register-keys] + (car/wcar conn-spec (car/del k))) + this) + (teardown! [this _test] this) + (invoke! [_this _test op] + (try + (let [[k value] (:value op) + redis-key (nth register-keys k)] + (case (:f op) + :write (do (car/wcar conn-spec (car/set redis-key (str value))) + (assoc op :type :ok)) + :read (let [raw (car/wcar conn-spec (car/get redis-key)) + value (when raw (Long/parseLong (str raw)))] + (assoc op :type :ok :value (independent/tuple k value))))) + (catch java.net.ConnectException _ + (assoc op :type :info :error :connection-refused)) + (catch java.net.SocketTimeoutException _ + (assoc op :type :info :error :socket-timeout)) + (catch Throwable t + (assoc op :type :info :error (or (.getMessage t) (str t))))))) + +(defn split-register-workload [opts] + (let [client (->SplitRegisterClient + (or (:node->port opts) (zipmap default-nodes (repeat 6379))) + nil) + max-writes (or (:max-writes-per-key opts) 100)] + {:client client + :generator (independent/concurrent-generator + 2 + (range (count register-keys)) + (fn [_] + (gen/mix [(map (fn [v] {:f :write :value v}) (range max-writes)) + (repeat max-writes {:f :read})]))) + :checker (independent/checker + (checker/compose + {:linear (checker/linearizable {:model (model/register) + :algorithm :competition}) + :timeline (timeline/html)}))})) + +(defn- grpc-address [opts test] + (or (:grpc-host-port opts) + (:grpc-host-port test) + (let [node (name (first (:nodes test))) + groups (:raft-groups test) + port (if (seq groups) + (get groups (first (sort (keys groups)))) + (or (:grpc-port test) 50051))] + (str node ":" port)))) + +(defn- run-helper! [bin & args] + (let [result (apply shell/sh bin args)] + (when-not (zero? (:exit result)) + (throw (ex-info (str bin " failed") result))) + (:out result))) + +(defn- current-plan [opts test] + (let [address (grpc-address opts test) + snapshot (routes/parse-routes-json + (run-helper! (or (:list-routes-bin opts) (helper-path "elastickv-list-routes")) + "--address" address)) + source (routes/route-containing-key (:routes snapshot) split-key) + groups (sort (keys (:raft-groups test))) + target (or (:target-group-id opts) + (first (remove #{(:raft-group-id source)} groups)))] + (when-not (and source target) + (throw (ex-info "split workload requires an active source route and a different target group" + {:source source :groups groups}))) + {:address address + :catalog-version (:catalog-version snapshot) + :route-id (:route-id source) + :target-group-id target})) + +(defn- start-split! [opts test] + (let [{:keys [address catalog-version route-id target-group-id] :as plan} + (current-plan opts test) + output (run-helper! (or (:split-bin opts) (helper-path "elastickv-split")) + "--address" address + "--route-id" (str route-id) + "--split-key" split-key + "--expected-version" (str catalog-version) + "--target-group-id" (str target-group-id)) + job-id (some->> (re-find #"job_id:\s*(\d+)" output) second Long/parseLong)] + (when-not job-id + (throw (ex-info "split helper did not return job_id" {:output output :plan plan}))) + (assoc plan :job-id job-id :output output))) + +(defn- job-phase [opts address job-id] + (let [output (run-helper! (or (:split-bin opts) (helper-path "elastickv-split")) + "--address" address "--get-job-id" (str job-id))] + (some->> (re-find #"phase:\s*(\S+)" output) second))) + +(defn- await-phase! [opts address job-id wanted timeout-ms] + (let [deadline (+ (System/currentTimeMillis) timeout-ms)] + (loop [] + (let [phase (job-phase opts address job-id)] + (cond + (contains? wanted phase) phase + (> (System/currentTimeMillis) deadline) + (throw (ex-info "timed out waiting for split job phase" + {:job-id job-id :phase phase :wanted wanted})) + :else (do (Thread/sleep 100) (recur))))))) + +(defn split-nemesis [opts] + (let [active (atom nil)] + (reify + nemesis/Reflection + (fs [_] #{:start-cross-group-split :verify-cross-group-split}) + nemesis/Nemesis + (setup! [this _test] this) + (invoke! [_ test op] + (try + (case (:f op) + :start-cross-group-split + (let [plan (start-split! opts test)] + (reset! active plan) + (if (:abandon-at-fence opts) + (do + (await-phase! opts (:address plan) (:job-id plan) + #{"SPLIT_JOB_PHASE_FENCE"} 60000) + (run-helper! (or (:split-bin opts) (helper-path "elastickv-split")) + "--address" (:address plan) + "--abandon-job-id" (str (:job-id plan)))) + (info "started M2 split job" (:job-id plan))) + (assoc op :value plan)) + + :verify-cross-group-split + (let [{:keys [address job-id] :as plan} @active + wanted (if (:abandon-at-fence opts) + #{"SPLIT_JOB_PHASE_ABANDONED"} + #{"SPLIT_JOB_PHASE_DONE"}) + phase (await-phase! opts address job-id wanted 120000)] + (assoc op :value (assoc plan :phase phase))) + + op) + (catch Throwable t + (warn t "M2 split nemesis failed") + (assoc op :type :fail :error (or (.getMessage t) (str t)))))) + (teardown! [this _test] this)))) + +(defn split-package [opts] + {:generator (gen/phases + (gen/sleep (or (:split-at-seconds opts) 5)) + (gen/once {:type :info :f :start-cross-group-split}) + (gen/sleep (or (:split-verify-after-seconds opts) 20)) + (gen/once {:type :info :f :verify-cross-group-split})) + :final-generator nil + :nemesis (split-nemesis opts) + :perf #{{:name "cross-group-split" + :fs #{:start-cross-group-split :verify-cross-group-split} + :start #{:start-cross-group-split} + :stop #{:verify-cross-group-split} + :color "#4A90E2"}}}) + +(defn elastickv-split-test + ([] (elastickv-split-test {})) + ([opts] + (let [nodes (or (:nodes opts) default-nodes) + ports (or (:redis-ports opts) (repeat (count nodes) (or (:redis-port opts) 6379))) + node->port (or (:node->port opts) (cli/ports->node-map ports nodes)) + local? (:local opts) + db (if local? jdb/noop + (ekdb/db {:grpc-port (or (:grpc-port opts) 50051) + :redis-port node->port + :raft-groups (:raft-groups opts) + :shard-ranges (:shard-ranges opts) + :migration-enabled true})) + fault-package (when-not local? + (combined/nemesis-package + {:db db + :faults (cli/normalize-faults (or (:faults opts) [:partition :kill])) + :interval (or (:fault-interval opts) 10)})) + package (combined/compose-packages + (cond-> [(split-package opts)] fault-package (conj fault-package))) + workload (split-register-workload (assoc opts :node->port node->port))] + (merge workload + {:name (or (:name opts) "elastickv-m2-cross-group-split") + :nodes nodes + :db db + :raft-groups (:raft-groups opts) + :grpc-port (:grpc-port opts) + :grpc-host-port (:grpc-host-port opts) + :redis-host (:redis-host opts) + :os (if local? os/noop debian/os) + :net (if local? net/noop net/iptables) + :ssh (merge {:username "vagrant" + :private-key-path "/home/vagrant/.ssh/id_rsa" + :strict-host-key-checking false} + (when local? {:dummy true}) (:ssh opts)) + :remote control/ssh + :nemesis (:nemesis package) + :final-generator nil + :concurrency (or (:concurrency opts) 6) + :generator (->> (:generator workload) + (gen/nemesis (:generator package)) + (gen/stagger (/ (double (or (:rate opts) 10)))) + (gen/time-limit (or (:time-limit opts) 60)))})))) + +(def split-cli-opts + [[nil "--redis-port PORT" "Redis port." :default 6379 :parse-fn #(Integer/parseInt %)] + [nil "--target-group-id ID" "Target Raft group." :parse-fn #(Long/parseLong %)] + [nil "--grpc-host-port HOST:PORT" "Distribution gRPC address."] + [nil "--split-bin PATH" "Path to elastickv-split."] + [nil "--list-routes-bin PATH" "Path to elastickv-list-routes."] + [nil "--split-at-seconds SECONDS" "Workload time to start migration." :default 5 :parse-fn #(Integer/parseInt %)] + [nil "--split-verify-after-seconds SECONDS" "Delay before verifying terminal state." :default 20 :parse-fn #(Integer/parseInt %)] + [nil "--abandon-at-fence" "Abandon after observing FENCE." :default false]]) + +(defn -main [& args] + (cli/run-workload! args + (into cli/common-cli-opts split-cli-opts) + #(cli/parse-common-opts % nil) + elastickv-split-test)) diff --git a/jepsen/test/elastickv/split_workload_test.clj b/jepsen/test/elastickv/split_workload_test.clj new file mode 100644 index 000000000..de6cf89c2 --- /dev/null +++ b/jepsen/test/elastickv/split_workload_test.clj @@ -0,0 +1,46 @@ +(ns elastickv.split-workload-test + (:require [clojure.java.shell :as shell] + [clojure.test :refer :all] + [elastickv.split-workload :as split] + [jepsen.nemesis :as nemesis])) + +(defn- routes-json [] + (str "{\"catalog_version\":7,\"routes\":[" + "{\"route_id\":100,\"raft_group_id\":1,\"start\":\"\",\"end\":\"\",\"state\":\"ROUTE_STATE_ACTIVE\"}" + "]}")) + +(deftest split-package-is-deterministic + (let [package (split/split-package {:split-at-seconds 1 + :split-verify-after-seconds 2})] + (is (some? (:generator package))) + (is (some? (:nemesis package))) + (is (= #{:start-cross-group-split :verify-cross-group-split} + (nemesis/fs (:nemesis package)))))) + +(deftest split-register-keys-cover-both-routes + (let [split-key (var-get (ns-resolve 'elastickv.split-workload 'split-key)) + register-keys (var-get (ns-resolve 'elastickv.split-workload 'register-keys))] + (is (some #(neg? (compare % split-key)) register-keys)) + (is (some #(not (neg? (compare % split-key))) register-keys)))) + +(deftest split-nemesis-starts-cross-group-job + (let [calls (atom [])] + (with-redefs [shell/sh (fn [& args] + (swap! calls conj args) + (if (.contains (first args) "list-routes") + {:exit 0 :out (routes-json) :err ""} + {:exit 0 :out "catalog_version: 8\njob_id: 44\n" :err ""}))] + (let [result (nemesis/invoke! + (split/split-nemesis + {:list-routes-bin "elastickv-list-routes" + :split-bin "elastickv-split" + :target-group-id 2 + :grpc-host-port "n1:50051"}) + {:nodes ["n1"] :raft-groups {1 50051, 2 50052}} + {:type :info :f :start-cross-group-split})] + (is (= 44 (get-in result [:value :job-id]))) + (is (= [["elastickv-list-routes" "--address" "n1:50051"] + ["elastickv-split" "--address" "n1:50051" + "--route-id" "100" "--split-key" "m2-split" + "--expected-version" "7" "--target-group-id" "2"]] + @calls)))))) diff --git a/kv/compactor.go b/kv/compactor.go index c13c7d836..8700f92bb 100644 --- a/kv/compactor.go +++ b/kv/compactor.go @@ -234,6 +234,13 @@ func (c *FSMCompactor) compactRuntime(ctx context.Context, runtime FSMCompactRun if !ok { return nil } + safeMinTS, err := migrationRetentionMinTS(ctx, runtime.Store, safeMinTS) + if err != nil { + return errors.Wrapf(err, "read migration retention pins for group %d", runtime.GroupID) + } + if safeMinTS <= retention.MinRetainedTS() { + return nil + } compactCtx, cancel := c.compactContext(ctx, status) defer cancel() @@ -253,6 +260,23 @@ func (c *FSMCompactor) compactRuntime(ctx context.Context, runtime FSMCompactRun return nil } +func migrationRetentionMinTS(ctx context.Context, st store.MVCCStore, safeMinTS uint64) (uint64, error) { + reader, ok := st.(store.MigrationTargetReadinessReader) + if !ok { + return safeMinTS, nil + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return 0, errors.WithStack(err) + } + for _, state := range states { + if state.Armed && state.RetentionPinTS > 0 && state.RetentionPinTS < safeMinTS { + safeMinTS = state.RetentionPinTS + } + } + return safeMinTS, nil +} + func (c *FSMCompactor) compactionRuntimeReady(runtime FSMCompactRuntime) (store.RetentionController, raftengine.Status, bool) { if runtime.StatusReader == nil || runtime.Store == nil { return nil, raftengine.Status{}, false diff --git a/kv/compactor_test.go b/kv/compactor_test.go index b6417a8e0..c6ae90f0f 100644 --- a/kv/compactor_test.go +++ b/kv/compactor_test.go @@ -56,6 +56,15 @@ type metricsCapturingStore struct { metrics *pebble.Metrics } +type migrationPinnedCompactionStore struct { + deadlineCapturingStore + states []store.TargetStagedReadinessState +} + +func (s *migrationPinnedCompactionStore) MigrationTargetReadinessStates(context.Context) ([]store.TargetStagedReadinessState, error) { + return s.states, nil +} + func (s *metricsCapturingStore) Metrics() *pebble.Metrics { return s.metrics } @@ -146,6 +155,69 @@ func TestFSMCompactorRespectsPinnedTimestamp(t *testing.T) { require.Equal(t, []byte("v20"), val) } +func TestFSMCompactorRespectsMigrationRetentionPin(t *testing.T) { + t.Parallel() + + st := &migrationPinnedCompactionStore{ + deadlineCapturingStore: deadlineCapturingStore{lastCommitTS: ^uint64(0)}, + states: []store.TargetStagedReadinessState{{ + JobID: 1, + MigrationJobID: 1, + MinWriteTSExclusive: 1, + Armed: true, + RetentionPinTS: 42, + }}, + } + compactor := NewFSMCompactor( + []FSMCompactRuntime{{ + GroupID: 1, + StatusReader: fakeRaftStatus{status: raftengine.Status{ + State: raftengine.StateFollower, + AppliedIndex: 1, + CommitIndex: 1, + }}, + Store: st, + }}, + WithFSMCompactorInterval(time.Hour), + WithFSMCompactorRetentionWindow(time.Millisecond), + ) + + require.NoError(t, compactor.SyncOnce(context.Background())) + require.True(t, st.compactCalled) + require.Equal(t, uint64(42), st.compactMinTS) +} + +func TestFSMCompactorIgnoresDisarmedMigrationRetentionPin(t *testing.T) { + t.Parallel() + + st := &migrationPinnedCompactionStore{ + deadlineCapturingStore: deadlineCapturingStore{lastCommitTS: ^uint64(0)}, + states: []store.TargetStagedReadinessState{{ + JobID: 1, + MigrationJobID: 1, + MinWriteTSExclusive: 1, + RetentionPinTS: 42, + }}, + } + compactor := NewFSMCompactor( + []FSMCompactRuntime{{ + GroupID: 1, + StatusReader: fakeRaftStatus{status: raftengine.Status{ + State: raftengine.StateFollower, + AppliedIndex: 1, + CommitIndex: 1, + }}, + Store: st, + }}, + WithFSMCompactorInterval(time.Hour), + WithFSMCompactorRetentionWindow(time.Millisecond), + ) + + require.NoError(t, compactor.SyncOnce(context.Background())) + require.True(t, st.compactCalled) + require.Greater(t, st.compactMinTS, uint64(42)) +} + func TestFSMCompactorSkipsLaggingRuntime(t *testing.T) { st := store.NewMVCCStore() ctx := context.Background() diff --git a/kv/fsm.go b/kv/fsm.go index 55a95216d..607d46c63 100644 --- a/kv/fsm.go +++ b/kv/fsm.go @@ -129,9 +129,12 @@ type RouteSnapshot interface { // OwnerOf returns the Raft group ID that owned key at this // snapshot's version. (0, false) when no route covered key. OwnerOf(key []byte) (uint64, bool) - // RouteOf returns the complete route descriptor covering key. + // RouteOf returns the complete route descriptor covering key at this + // snapshot's version. Used by apply-time target-readiness checks to + // prove staged/cleared descriptor state before mutating the store. RouteOf(key []byte) (distribution.Route, bool) - // IntersectingRoutes returns every route intersecting [start, end). + // IntersectingRoutes returns the route descriptors that intersect + // [start, end) at this snapshot's version. A nil end denotes +infinity. IntersectingRoutes(start, end []byte) []distribution.Route // WriteFencedForKey reports whether key is currently inside a // WriteFenced route in this snapshot. @@ -139,6 +142,12 @@ type RouteSnapshot interface { // WriteFencedIntersects reports whether [start, end) intersects // any WriteFenced route in this snapshot. WriteFencedIntersects(start, end []byte) bool + // WriteFloorForKey returns the post-migration write timestamp floor + // for key, when the current route retains one. + WriteFloorForKey(key []byte) (uint64, bool) + // WriteFloorIntersects returns the maximum post-migration write + // timestamp floor across routes intersecting [start, end). + WriteFloorIntersects(start, end []byte) (uint64, bool) } // SetApplyIndex implements raftengine.ApplyIndexAware. The engine @@ -381,6 +390,10 @@ func (f *kvFSM) applyReservedOpcode(ctx context.Context, data []byte) (any, bool return f.applyMigrationImport(ctx, data[1:]), true case data[0] == raftEncodeMigrationPromote: return f.applyMigrationPromote(ctx, data[1:]), true + case data[0] == raftEncodeTargetReadiness: + return f.applyTargetStagedReadiness(ctx, data[1:]), true + case data[0] == raftEncodeMigrationCleanup: + return f.applyMigrationCleanup(ctx, data[1:]), true case data[0] >= fsmwire.OpEncryptionMin && data[0] <= fsmwire.OpEncryptionMax: return f.applyEncryption(f.pendingApplyIdx, data[0], data[1:]), true default: @@ -420,6 +433,13 @@ const ( // data promotion chunk. Every target voter atomically copies staged MVCC // versions into the live keyspace and removes the promoted staged rows. raftEncodeMigrationPromote byte = 0x0b + // raftEncodeTargetReadiness carries the target-local staged-readiness + // guard. It must be replicated through the target Raft group before the + // migration controller can treat a target as fail-closed for cutover. + raftEncodeTargetReadiness byte = 0x0c + // raftEncodeMigrationCleanup carries one bounded source/target physical + // cleanup chunk or a per-job proof-record cleanup. + raftEncodeMigrationCleanup byte = 0x0d ) func decodeRaftRequests(data []byte) ([]*pb.Request, error) { @@ -521,6 +541,9 @@ func (f *kvFSM) handleRawRequest(ctx context.Context, r *pb.Request, commitTS ui if err := f.validateRawMutationsForApply(ctx, r, commitTS); err != nil { return err } + if err := f.recordMigrationWrite(ctx, r.Mutations, commitTS); err != nil { + return err + } muts, err := toStoreMutations(r.Mutations) if err != nil { @@ -553,14 +576,20 @@ func (f *kvFSM) validateRawMutationForApply(ctx context.Context, mut *pb.Mutatio if isTxnInternalKey(mut.Key) { return errors.WithStack(ErrInvalidRequest) } + if err := f.verifySourceWriteFenceForRange(ctx, mut.Key, nextScanCursor(mut.Key)); err != nil { + return err + } if _, bypass := writeFenceBypassKeys[string(mut.Key)]; !bypass { - if err := f.verifyRouteNotFencedForKey(mut.Key); err != nil { - return err - } - if err := f.verifyRouteWriteTimestampFloorForKey(mut.Key, commitTS); err != nil { + if err := f.verifyRouteNotFencedForKey(ctx, mut.Key); err != nil { return err } } + if err := f.verifyTargetReadinessForRange(ctx, mut.Key, nextScanCursor(mut.Key)); err != nil { + return err + } + if err := f.verifyRouteWriteFloorForKey(mut.Key, commitTS); err != nil { + return err + } if err := f.assertNoConflictingTxnLock(ctx, mut.Key, nil, 0); err != nil { return err } @@ -581,10 +610,23 @@ func extractDelPrefix(muts []*pb.Mutation) (bool, []byte) { // handleDelPrefix delegates prefix deletion to the store. Transaction-internal // keys are always excluded to preserve transactional integrity. func (f *kvFSM) handleDelPrefix(ctx context.Context, prefix []byte, commitTS uint64) error { - if err := f.verifyRouteNotFencedForPrefix(prefix); err != nil { + if err := f.verifyRouteNotFencedForPrefix(ctx, prefix); err != nil { + return err + } + if err := f.verifyTargetReadinessForPrefix(ctx, prefix); err != nil { return err } - if err := f.verifyRouteWriteTimestampFloorForPrefix(prefix, commitTS); err != nil { + if err := f.verifyRouteWriteFloorForPrefix(prefix, commitTS); err != nil { + return err + } + if err := f.recordMigrationWrite(ctx, []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: prefix}}, commitTS); err != nil { + return err + } + routes := f.stagedVisibilityRoutesForPrefix(prefix) + deleteStagedPrefix := func(stagedPrefix []byte, stagedExcludePrefix []byte) error { + return f.store.DeletePrefixAtRaftAt(ctx, stagedPrefix, stagedExcludePrefix, commitTS, 0) + } + if err := deleteStagedVisibilityPrefixes(routes, prefix, txnCommonPrefix, deleteStagedPrefix); err != nil { return err } if err := f.store.DeletePrefixAtRaftAt(ctx, prefix, txnCommonPrefix, commitTS, f.pendingApplyIdx); err != nil { @@ -594,7 +636,44 @@ func (f *kvFSM) handleDelPrefix(ctx context.Context, prefix []byte, commitTS uin return nil } -func (f *kvFSM) verifyRouteNotFencedForKey(key []byte) error { +func (f *kvFSM) stagedVisibilityRoutesForPrefix(prefix []byte) []distribution.Route { + if f.routes == nil { + return nil + } + snap, ok := f.routes.Current() + if !ok { + return nil + } + start, end := routePrefixRange(prefix) + routes := snap.IntersectingRoutes(start, end) + out := make([]distribution.Route, 0, len(routes)) + for _, route := range routes { + if route.GroupID == f.shardGroupID && routeHasStagedVisibility(route) { + out = append(out, route) + } + } + return out +} + +func (f *kvFSM) verifyRouteNotFencedForMutations(ctx context.Context, muts []*pb.Mutation, bypassKeys map[string]struct{}) error { + for _, mut := range muts { + if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { + continue + } + if err := f.verifySourceWriteFenceForRange(ctx, mut.Key, nextScanCursor(mut.Key)); err != nil { + return err + } + if _, bypass := bypassKeys[string(mut.Key)]; bypass { + continue + } + if err := f.verifyRouteNotFencedForKey(ctx, mut.Key); err != nil { + return err + } + } + return nil +} + +func (f *kvFSM) verifyRouteNotFencedForKey(_ context.Context, key []byte) error { if f.routes == nil { return nil } @@ -612,7 +691,11 @@ func (f *kvFSM) verifyRouteNotFencedForKey(key []byte) error { return nil } -func (f *kvFSM) verifyRouteNotFencedForPrefix(prefix []byte) error { +func (f *kvFSM) verifyRouteNotFencedForPrefix(ctx context.Context, prefix []byte) error { + start, end := routePrefixRange(prefix) + if err := f.verifySourceWriteFenceForRouteRange(ctx, start, end); err != nil { + return err + } if f.routes == nil { return nil } @@ -620,83 +703,247 @@ func (f *kvFSM) verifyRouteNotFencedForPrefix(prefix []byte) error { if !ok { return nil } - start, end := routePrefixRange(prefix) if !snap.WriteFencedIntersects(start, end) { return nil } return errors.Wrapf(ErrRouteWriteFenced, "prefix %q route range [%q,%q)", prefix, start, end) } -func (f *kvFSM) verifyRouteWriteTimestampFloorForKey(key []byte, commitTS uint64) error { - if f.routes == nil || commitTS == 0 { - return nil - } - snap, ok := f.routes.Current() +func (f *kvFSM) verifySourceWriteFenceForRange(ctx context.Context, start []byte, end []byte) error { + routeStart, routeEnd := readinessRouteRangeForScan(start, end) + return f.verifySourceWriteFenceForRouteRange(ctx, routeStart, routeEnd) +} + +func (f *kvFSM) verifySourceWriteFenceForRouteRange(ctx context.Context, routeStart []byte, routeEnd []byte) error { + reader, ok := f.store.(store.MigrationTargetReadinessReader) if !ok { return nil } - if start, end, ok := s3BucketAuxiliaryRouteRange(key); ok { - for _, route := range snap.IntersectingRoutes(start, end) { - if err := verifyRouteWriteTimestampFloorForRange(route, key, start, end, commitTS); err != nil { - return err - } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return errors.WithStack(err) + } + for _, state := range states { + if state.Armed && state.SourceWriteFence && routeRangeIntersects(routeStart, routeEnd, state.RouteStart, state.RouteEnd) { + return errors.Wrapf(ErrRouteWriteFenced, "source migration fence job %d route range [%q,%q)", state.JobID, routeStart, routeEnd) } - return nil } - rkey := routeKey(key) - if route, ok := snap.RouteOf(rkey); ok { - if err := verifyRouteWriteTimestampFloorForRoute(route, key, commitTS); err != nil { + return nil +} + +func (f *kvFSM) verifyRouteWriteFloorForMutations(muts []*pb.Mutation, commitTS uint64) error { + for _, mut := range muts { + if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { + continue + } + if err := f.verifyRouteWriteFloorForKey(mut.Key, commitTS); err != nil { return err } } return nil } -func (f *kvFSM) verifyRouteWriteTimestampFloorsForMutations(muts []*pb.Mutation, writeFenceBypassKeys [][]byte, commitTS uint64) error { - bypassKeys := writeFenceBypassKeySet(writeFenceBypassKeys) +func (f *kvFSM) verifyTargetReadinessForMutations(ctx context.Context, muts []*pb.Mutation) error { for _, mut := range muts { if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { continue } - if _, bypass := bypassKeys[string(mut.Key)]; bypass { + if err := f.verifyTargetReadinessForRange(ctx, mut.Key, nextScanCursor(mut.Key)); err != nil { + return err + } + } + return nil +} + +func (f *kvFSM) verifyTargetReadinessForReadKeys(ctx context.Context, keys [][]byte) error { + for _, key := range keys { + if isTxnInternalKey(key) { continue } - if err := f.verifyRouteWriteTimestampFloorForKey(mut.Key, commitTS); err != nil { + if err := f.verifySourceReadFenceForRange(ctx, key, nextScanCursor(key)); err != nil { + return err + } + if err := f.verifyTargetReadinessForRange(ctx, key, nextScanCursor(key)); err != nil { return err } } return nil } -func (f *kvFSM) verifyRouteWriteTimestampFloorForPrefix(prefix []byte, commitTS uint64) error { - if f.routes == nil || commitTS == 0 { +func (f *kvFSM) verifySourceReadFenceForRange(ctx context.Context, start []byte, end []byte) error { + reader, ok := f.store.(store.MigrationTargetReadinessReader) + if !ok { return nil } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return errors.WithStack(err) + } + routeStart, routeEnd := readinessRouteRangeForScan(start, end) + if sourceReadFenceApplies(states, routeStart, routeEnd) { + return errors.WithStack(ErrRouteCutoverPending) + } + return nil +} + +func (f *kvFSM) verifyTargetReadinessForTxnFootprint(ctx context.Context, muts []*pb.Mutation, readKeys [][]byte, primaryKey []byte) error { + if len(primaryKey) != 0 && !isTxnInternalKey(primaryKey) { + if err := f.verifyTargetReadinessForRange(ctx, primaryKey, nextScanCursor(primaryKey)); err != nil { + return err + } + } + if err := f.verifyTargetReadinessForReadKeys(ctx, readKeys); err != nil { + return err + } + return f.verifyTargetReadinessForMutations(ctx, muts) +} + +func (f *kvFSM) verifyTargetReadinessForPrefix(ctx context.Context, prefix []byte) error { + start, end := routePrefixRange(prefix) + return f.verifyTargetReadinessForRouteRange(ctx, start, end) +} + +func (f *kvFSM) verifyTargetReadinessForRange(ctx context.Context, start []byte, end []byte) error { + routeStart, routeEnd := readinessRouteRange(start, end) + return f.verifyTargetReadinessForRouteRange(ctx, routeStart, routeEnd) +} + +func (f *kvFSM) verifyTargetReadinessForRouteRange(ctx context.Context, routeStart []byte, routeEnd []byte) error { + _, err := f.targetReadyRoutesForRouteRange(ctx, routeStart, routeEnd) + return err +} + +func (f *kvFSM) targetReadyRoutesForRange(ctx context.Context, start []byte, end []byte) ([]distribution.Route, error) { + routeStart, routeEnd := readinessRouteRange(start, end) + return f.targetReadyRoutesForRouteRange(ctx, routeStart, routeEnd) +} + +func (f *kvFSM) targetReadyRoutesForRouteRange(ctx context.Context, routeStart []byte, routeEnd []byte) ([]distribution.Route, error) { + routes, catalogVersion, proof := f.currentShardRoutesForRouteRange(routeStart, routeEnd) + + reader, ok := f.store.(store.MigrationTargetReadinessReader) + if !ok { + return routes, nil + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return nil, errors.WithStack(err) + } + if len(states) == 0 { + return routes, nil + } + if targetReadinessStatesSatisfied(states, routes, routeStart, routeEnd, f.shardGroupID, catalogVersion, proof) { + return routes, nil + } + return nil, errors.WithStack(ErrRouteCutoverPending) +} + +func (f *kvFSM) currentShardRoutesForRouteRange(routeStart []byte, routeEnd []byte) ([]distribution.Route, uint64, bool) { + if f.routes == nil { + return nil, 0, false + } snap, ok := f.routes.Current() if !ok { - return nil + return nil, 0, false } - start, end := routePrefixRange(prefix) - for _, route := range snap.IntersectingRoutes(start, end) { - if err := verifyRouteWriteTimestampFloorForRange(route, prefix, start, end, commitTS); err != nil { - return err + routes := make([]distribution.Route, 0) + for _, route := range snap.IntersectingRoutes(routeStart, routeEnd) { + if route.GroupID == f.shardGroupID { + routes = append(routes, route) + } + } + return routes, snap.Version(), true +} + +func targetReadinessStatesSatisfied( + states []store.TargetStagedReadinessState, + routes []distribution.Route, + routeStart []byte, + routeEnd []byte, + groupID uint64, + catalogVersion uint64, + proof bool, +) bool { + for _, ready := range states { + if !ready.Armed || ready.SourceWriteFence || ready.SourceReadFence || ready.TrackWrites || + !routeRangeIntersects(routeStart, routeEnd, ready.RouteStart, ready.RouteEnd) { + continue + } + if !proof || !routesSatisfyTargetReadiness(routes, ready, groupID, catalogVersion) { + return false + } + } + return true +} + +func routesSatisfyTargetReadiness(routes []distribution.Route, ready store.TargetStagedReadinessState, groupID uint64, catalogVersion uint64) bool { + matched := false + for _, route := range routes { + if route.GroupID != groupID { + continue + } + if !routeRangeIntersects(route.Start, route.End, ready.RouteStart, ready.RouteEnd) { + continue + } + matched = true + if !routeSatisfiesTargetReadiness(route, ready, catalogVersion) { + return false } } + return matched +} + +func (f *kvFSM) verifyRouteWriteFloorForKey(key []byte, commitTS uint64) error { + if f.routes == nil { + return nil + } + snap, ok := f.routes.Current() + if !ok { + return nil + } + if err := f.verifyS3BucketAuxiliaryRouteWriteFloor(snap, key, commitTS); err != nil { + return err + } + if _, _, ok := s3BucketAuxiliaryRouteRange(key); ok { + return nil + } + rkey := routeKey(key) + floor, ok := snap.WriteFloorForKey(rkey) + if ok && commitTS != 0 && commitTS <= floor { + return errors.Wrapf(ErrRouteWriteBelowFloor, "commit_ts %d <= floor %d for key %q routeKey %q", commitTS, floor, key, rkey) + } return nil } -func verifyRouteWriteTimestampFloorForRoute(route distribution.Route, key []byte, commitTS uint64) error { - if route.MinWriteTSExclusive == 0 || commitTS > route.MinWriteTSExclusive { +func (f *kvFSM) verifyS3BucketAuxiliaryRouteWriteFloor(snap RouteSnapshot, key []byte, commitTS uint64) error { + if commitTS == 0 { return nil } - return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q routeKey %q commit_ts=%d floor=%d", key, routeKey(key), commitTS, route.MinWriteTSExclusive) + start, end, ok := s3BucketAuxiliaryRouteRange(key) + if !ok { + return nil + } + floor, ok := snap.WriteFloorIntersects(start, end) + if !ok || commitTS > floor { + return nil + } + return errors.Wrapf(ErrRouteWriteBelowFloor, "commit_ts %d <= floor %d for key %q route range [%q,%q)", commitTS, floor, key, start, end) } -func verifyRouteWriteTimestampFloorForRange(route distribution.Route, key, start, end []byte, commitTS uint64) error { - if route.MinWriteTSExclusive == 0 || commitTS > route.MinWriteTSExclusive { +func (f *kvFSM) verifyRouteWriteFloorForPrefix(prefix []byte, commitTS uint64) error { + if f.routes == nil || commitTS == 0 { + return nil + } + snap, ok := f.routes.Current() + if !ok { return nil } - return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q route range [%q,%q) commit_ts=%d floor=%d", key, start, end, commitTS, route.MinWriteTSExclusive) + start, end := routePrefixRange(prefix) + floor, ok := snap.WriteFloorIntersects(start, end) + if !ok || commitTS > floor { + return nil + } + return errors.Wrapf(ErrRouteWriteBelowFloor, "commit_ts %d <= floor %d for prefix %q route range [%q,%q)", commitTS, floor, prefix, start, end) } func routePrefixRange(prefix []byte) ([]byte, []byte) { @@ -1152,7 +1399,7 @@ func (f *kvFSM) validateConflicts(ctx context.Context, muts []*pb.Mutation, star } seen[keyStr] = struct{}{} - latest, exists, err := f.store.LatestCommitTS(ctx, mut.Key) + latest, exists, err := f.latestCommitTSForTargetReadyKey(ctx, mut.Key) if err != nil { return errors.WithStack(err) } @@ -1163,6 +1410,64 @@ func (f *kvFSM) validateConflicts(ctx context.Context, muts []*pb.Mutation, star return nil } +func (f *kvFSM) validateReadConflicts(ctx context.Context, keys [][]byte, startTS uint64) error { + seen := make(map[string]struct{}, len(keys)) + for _, key := range keys { + if len(key) == 0 || isTxnInternalKey(key) { + continue + } + keyStr := string(key) + if _, ok := seen[keyStr]; ok { + continue + } + seen[keyStr] = struct{}{} + + latest, exists, err := f.latestCommitTSForTargetReadyKey(ctx, key) + if err != nil { + return errors.WithStack(err) + } + if exists && latest > startTS { + return errors.WithStack(store.NewWriteConflictError(key)) + } + } + return nil +} + +func (f *kvFSM) latestCommitTSForTargetReadyKey(ctx context.Context, key []byte) (uint64, bool, error) { + routes, err := f.targetReadyRoutesForRange(ctx, key, nextScanCursor(key)) + if err != nil { + return 0, false, err + } + liveTS, liveExists, err := f.store.LatestCommitTS(ctx, key) + if err != nil { + return 0, false, errors.WithStack(err) + } + latest, exists := liveTS, liveExists + for _, route := range routes { + if !routeHasStagedVisibility(route) { + continue + } + stagedTS, stagedExists, err := f.store.LatestCommitTS(ctx, distribution.MigrationStagedDataKey(route.MigrationJobID, key)) + if err != nil { + return 0, false, errors.WithStack(err) + } + if stagedExists && (!exists || stagedTS > latest) { + latest, exists = stagedTS, true + } + } + return latest, exists, nil +} + +func (f *kvFSM) validateTxnConflicts(ctx context.Context, muts []*pb.Mutation, readKeys [][]byte, startTS uint64) error { + if err := f.validateConflicts(ctx, muts, startTS); err != nil { + return errors.WithStack(err) + } + if err := f.validateReadConflicts(ctx, readKeys, startTS); err != nil { + return errors.WithStack(err) + } + return nil +} + func uniqueMutations(muts []*pb.Mutation) ([]*pb.Mutation, error) { if len(muts) == 0 { return []*pb.Mutation{}, nil @@ -1204,22 +1509,35 @@ func (f *kvFSM) handlePrepareRequest(ctx context.Context, r *pb.Request) error { } startTS := r.Ts - uniq, err := f.uniqueMutationsAboveFloor(muts, r.GetWriteFenceBypassKeys(), startTS) + floorTS := startTS + if meta.CommitTS != 0 { + floorTS = meta.CommitTS + } + if err := f.verifyTargetReadinessForTxnFootprint(ctx, muts, r.ReadKeys, nil); err != nil { + return err + } + uniq, err := f.uniqueMutationsNotFenced(ctx, muts, r.GetWriteFenceBypassKeys(), floorTS) if err != nil { return err } - if err := f.validateConflicts(ctx, uniq, startTS); err != nil { - return errors.WithStack(err) + if err := f.validateTxnConflicts(ctx, uniq, r.ReadKeys, startTS); err != nil { + return err } expireAt := txnLockExpireAt(meta.LockTTLms) - storeMuts, err := f.buildPrepareStoreMutations(ctx, uniq, meta.PrimaryKey, startTS, expireAt) + storeMuts, err := f.buildPrepareStoreMutations(ctx, uniq, meta.PrimaryKey, startTS, expireAt, meta.CommitTS) if err != nil { return err } + return f.recordAndApplyPrepare(ctx, uniq, storeMuts, r.ReadKeys, floorTS, startTS) +} - if err := f.store.ApplyMutationsRaftAt(ctx, storeMuts, r.ReadKeys, startTS, startTS, f.pendingApplyIdx); err != nil { +func (f *kvFSM) recordAndApplyPrepare(ctx context.Context, muts []*pb.Mutation, storeMuts []*store.KVPairMutation, readKeys [][]byte, floorTS, startTS uint64) error { + if err := f.recordMigrationWrite(ctx, muts, floorTS); err != nil { + return err + } + if err := f.store.ApplyMutationsRaftAt(ctx, storeMuts, readKeys, startTS, startTS, f.pendingApplyIdx); err != nil { return errors.WithStack(err) } return nil @@ -1266,7 +1584,7 @@ func (f *kvFSM) handleOnePhaseTxnRequest(ctx context.Context, r *pb.Request, com // applying this log entry. The retention-window > max-retry-latency // invariant prevents the rare case where a real never-landed retry // arrives with PrevCommitTS below pebble's compacted floor. - dedup, err := f.dedupProbeOnePhase(ctx, meta) + dedup, err := f.dedupProbeOnePhase(ctx, meta, muts, r.ReadKeys) if err != nil { return err } @@ -1274,13 +1592,18 @@ func (f *kvFSM) handleOnePhaseTxnRequest(ctx context.Context, r *pb.Request, com return nil } - uniq, err := f.uniqueMutationsAboveFloor(muts, r.GetWriteFenceBypassKeys(), commitTS) + uniq, storeMuts, err := f.onePhaseStoreMutations( + ctx, + muts, + r.ReadKeys, + r.GetWriteFenceBypassKeys(), + startTS, + commitTS, + ) if err != nil { return err } - - storeMuts, err := f.buildOnePhaseStoreMutations(ctx, uniq) - if err != nil { + if err := f.recordMigrationWrite(ctx, uniq, commitTS); err != nil { return err } if err := f.store.ApplyMutationsRaftAt(ctx, storeMuts, r.ReadKeys, startTS, commitTS, f.pendingApplyIdx); err != nil { @@ -1290,45 +1613,63 @@ func (f *kvFSM) handleOnePhaseTxnRequest(ctx context.Context, r *pb.Request, com return nil } -func uniqueTxnMutations(muts []*pb.Mutation) ([]*pb.Mutation, error) { - uniq, err := uniqueMutations(muts) +func (f *kvFSM) onePhaseStoreMutations( + ctx context.Context, + muts []*pb.Mutation, + readKeys [][]byte, + writeFenceBypassKeys [][]byte, + startTS uint64, + commitTS uint64, +) ([]*pb.Mutation, []*store.KVPairMutation, error) { + uniq, err := f.uniqueMutationsNotFenced(ctx, muts, writeFenceBypassKeys, commitTS) if err != nil { - return nil, err + return nil, nil, err } - return uniq, nil + if err := f.validateTxnConflicts(ctx, uniq, readKeys, startTS); err != nil { + return nil, nil, err + } + storeMuts, err := f.buildOnePhaseStoreMutations(ctx, uniq) + if err != nil { + return nil, nil, err + } + return uniq, storeMuts, nil } -func (f *kvFSM) uniqueMutationsAboveFloor(muts []*pb.Mutation, writeFenceBypassKeys [][]byte, commitTS uint64) ([]*pb.Mutation, error) { +func (f *kvFSM) uniqueMutationsNotFenced( + ctx context.Context, + muts []*pb.Mutation, + writeFenceBypassKeys [][]byte, + commitTS uint64, +) ([]*pb.Mutation, error) { uniq, err := uniqueMutations(muts) if err != nil { return nil, err } - if err := f.verifyRouteWriteTimestampFloorsForMutations(uniq, writeFenceBypassKeys, commitTS); err != nil { + if err := f.verifyRouteNotFencedForMutations(ctx, uniq, writeFenceBypassKeySet(writeFenceBypassKeys)); err != nil { return nil, err } - return uniq, nil -} - -func (f *kvFSM) uniqueTxnMutationsAboveFloor(muts []*pb.Mutation, writeFenceBypassKeys [][]byte, commitTS uint64) ([]*pb.Mutation, error) { - uniq, err := uniqueTxnMutations(muts) - if err != nil { + if err := f.verifyTargetReadinessForMutations(ctx, uniq); err != nil { return nil, err } - if err := f.verifyRouteWriteTimestampFloorsForMutations(uniq, writeFenceBypassKeys, commitTS); err != nil { + if err := f.verifyRouteWriteFloorForMutations(uniq, commitTS); err != nil { return nil, err } return uniq, nil } -// dedupProbeOnePhase decides whether handleOnePhaseTxnRequest should no-op -// because the entry is a retry whose prior attempt already landed. Extracted -// to keep handleOnePhaseTxnRequest under the cyclop budget; the determinism -// rationale lives at the call site. +// dedupProbeOnePhase first fail-closes the target readiness footprint, then +// decides whether handleOnePhaseTxnRequest should no-op because the entry is a +// retry whose prior attempt already landed. Extracted to keep +// handleOnePhaseTxnRequest under the cyclop budget; the determinism rationale +// lives at the call site. // // Returns (true, nil) → the entry must no-op (prior attempt landed). // Returns (false, nil) → fall through to normal apply. // Returns (false, err) → propagate err; apply must not proceed. -func (f *kvFSM) dedupProbeOnePhase(ctx context.Context, meta TxnMeta) (bool, error) { +func (f *kvFSM) dedupProbeOnePhase(ctx context.Context, meta TxnMeta, muts []*pb.Mutation, readKeys [][]byte) (bool, error) { + if err := f.verifyTargetReadinessForTxnFootprint(ctx, muts, readKeys, meta.PrimaryKey); err != nil { + return false, err + } if meta.PrevCommitTS == 0 { return false, nil } @@ -1405,7 +1746,7 @@ func (f *kvFSM) handleCommitRequest(ctx context.Context, r *pb.Request) error { if err != nil { return err } - uniq, err := f.uniqueTxnMutationsAboveFloor(muts, r.GetWriteFenceBypassKeys(), commitTS) + uniq, err := f.uniqueMutationsAboveWriteFloor(ctx, muts, commitTS) if err != nil { return err } @@ -1413,9 +1754,16 @@ func (f *kvFSM) handleCommitRequest(ctx context.Context, r *pb.Request) error { if err != nil { return err } + return f.recordAndApplyCommit(ctx, storeMuts, uniq, applyStartTS, commitTS) +} + +func (f *kvFSM) recordAndApplyCommit(ctx context.Context, storeMuts []*store.KVPairMutation, uniq []*pb.Mutation, applyStartTS, commitTS uint64) error { if len(storeMuts) == 0 { return nil } + if err := f.recordMigrationWrite(ctx, uniq, commitTS); err != nil { + return err + } if err := f.applyCommitWithIdempotencyFallback(ctx, storeMuts, uniq, applyStartTS, commitTS); err != nil { return err } @@ -1522,7 +1870,7 @@ func (f *kvFSM) handleAbortRequest(ctx context.Context, r *pb.Request, abortTS u // shouldClearAbortKey (lock-missing ⇒ nothing to do) and for the // rollback-marker Put in appendRollbackRecord. - uniq, err := uniqueMutations(muts) + uniq, err := f.uniqueAbortCleanupMutations(ctx, muts) if err != nil { return err } @@ -1542,10 +1890,28 @@ func (f *kvFSM) handleAbortRequest(ctx context.Context, r *pb.Request, abortTS u return errors.WithStack(f.store.ApplyMutationsRaftAt(ctx, storeMuts, nil, startTS, abortTS, f.pendingApplyIdx)) } -func (f *kvFSM) buildPrepareStoreMutations(ctx context.Context, muts []*pb.Mutation, primaryKey []byte, startTS, expireAt uint64) ([]*store.KVPairMutation, error) { +func (f *kvFSM) uniqueMutationsAboveWriteFloor(ctx context.Context, muts []*pb.Mutation, commitTS uint64) ([]*pb.Mutation, error) { + uniq, err := uniqueMutations(muts) + if err != nil { + return nil, err + } + if err := f.verifyTargetReadinessForMutations(ctx, uniq); err != nil { + return nil, err + } + if err := f.verifyRouteWriteFloorForMutations(uniq, commitTS); err != nil { + return nil, err + } + return uniq, nil +} + +func (f *kvFSM) uniqueAbortCleanupMutations(_ context.Context, muts []*pb.Mutation) ([]*pb.Mutation, error) { + return uniqueMutations(muts) +} + +func (f *kvFSM) buildPrepareStoreMutations(ctx context.Context, muts []*pb.Mutation, primaryKey []byte, startTS, expireAt, commitTS uint64) ([]*store.KVPairMutation, error) { storeMuts := make([]*store.KVPairMutation, 0, len(muts)*txnPrepareStoreMutationFactor) for _, mut := range muts { - preparedMuts, err := f.prepareTxnMutation(ctx, mut, primaryKey, startTS, expireAt) + preparedMuts, err := f.prepareTxnMutation(ctx, mut, primaryKey, startTS, expireAt, commitTS) if err != nil { return nil, err } @@ -1678,7 +2044,7 @@ func (f *kvFSM) txnCommitTS(ctx context.Context, primaryKey []byte, startTS uint return commitTS, true, nil } -func (f *kvFSM) prepareTxnMutation(ctx context.Context, mut *pb.Mutation, primaryKey []byte, startTS, expireAt uint64) ([]*store.KVPairMutation, error) { +func (f *kvFSM) prepareTxnMutation(ctx context.Context, mut *pb.Mutation, primaryKey []byte, startTS, expireAt, commitTS uint64) ([]*store.KVPairMutation, error) { if err := f.assertNoConflictingTxnLock(ctx, mut.Key, primaryKey, startTS); err != nil { return nil, err } @@ -1688,6 +2054,7 @@ func (f *kvFSM) prepareTxnMutation(ctx context.Context, mut *pb.Mutation, primar TTLExpireAt: expireAt, PrimaryKey: primaryKey, IsPrimaryKey: bytes.Equal(mut.Key, primaryKey), + CommitTS: commitTS, }) intent, err := txnIntentFromPBMutation(mut, startTS) if err != nil { diff --git a/kv/fsm_migration_cleanup.go b/kv/fsm_migration_cleanup.go new file mode 100644 index 000000000..9444fb39d --- /dev/null +++ b/kv/fsm_migration_cleanup.go @@ -0,0 +1,93 @@ +package kv + +import ( + "bytes" + "context" + + "github.com/bootjp/elastickv/distribution" + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/protobuf/proto" +) + +// MarshalMigrationCleanupCommand encodes a bounded cleanup operation for Raft. +func MarshalMigrationCleanupCommand(req *pb.CleanupMigrationRequest) ([]byte, error) { + if req == nil { + return nil, errors.WithStack(ErrInvalidRequest) + } + b, err := proto.Marshal(req) + if err != nil { + return nil, errors.WithStack(err) + } + if len(b) >= maxMarshaledCommandSize { + return nil, errors.New("marshaled migration cleanup request too large") + } + return prependByte(raftEncodeMigrationCleanup, b), nil +} + +func (f *kvFSM) applyMigrationCleanup(ctx context.Context, data []byte) any { + req := &pb.CleanupMigrationRequest{} + if err := proto.Unmarshal(data, req); err != nil { + return errors.WithStack(err) + } + cleaner, ok := f.store.(store.MigrationCleaner) + if !ok { + return errors.WithStack(store.ErrNotSupported) + } + if req.GetMode() == pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA { + return errors.WithStack(cleaner.ClearMigrationState(ctx, req.GetJobId(), f.pendingApplyIdx)) + } + result, err := cleaner.CleanupVersions(ctx, migrationCleanupOptionsFromProto(req, f.pendingApplyIdx)) + if err != nil { + return errors.WithStack(err) + } + return result +} + +func migrationCleanupOptionsFromProto(req *pb.CleanupMigrationRequest, appliedIndex uint64) store.CleanupVersionsOptions { + maxVersions := int(req.GetMaxVersions()) + if maxVersions <= 0 { + maxVersions = defaultMigrationPromoteMaxVersions + } + maxBytes := req.GetMaxBytes() + if maxBytes == 0 { + maxBytes = defaultMigrationPromoteMaxBytes + } + maxScannedBytes := req.GetMaxScannedBytes() + if maxScannedBytes == 0 { + maxScannedBytes = defaultMigrationPromoteMaxScannedBytes + } + bracket := distribution.MigrationBracket{ + Family: req.GetKeyFamily(), + Start: bytes.Clone(req.GetRangeStart()), + End: bytes.Clone(req.GetRangeEnd()), + ExcludePrefixes: cloneMigrationByteSlices(req.GetExcludePrefixes()), + ExcludeKnownInternal: req.GetExcludeKnownInternal(), + RequiresRouteKeyCheck: req.GetRequiresRouteKeyCheck(), + RequiresDecodedS3: req.GetRequiresDecodedS3(), + } + return store.CleanupVersionsOptions{ + JobID: req.GetJobId(), + AppliedIndex: appliedIndex, + StartKey: bytes.Clone(req.GetRangeStart()), + EndKey: bytes.Clone(req.GetRangeEnd()), + Cursor: bytes.Clone(req.GetCursor()), + MaxCommitTS: req.GetMaxCommitTs(), + MaxVersions: maxVersions, + MaxBytes: maxBytes, + MaxScannedBytes: maxScannedBytes, + KeyFamily: req.GetKeyFamily(), + AcceptVersion: func(key, value []byte) bool { + return bracket.ContainsRoutedVersion(key, value, req.GetRouteStart(), req.GetRouteEnd(), routeKey) + }, + } +} + +func cloneMigrationByteSlices(in [][]byte) [][]byte { + out := make([][]byte, len(in)) + for i := range in { + out[i] = bytes.Clone(in[i]) + } + return out +} diff --git a/kv/fsm_migration_cleanup_test.go b/kv/fsm_migration_cleanup_test.go new file mode 100644 index 000000000..70aa9ee74 --- /dev/null +++ b/kv/fsm_migration_cleanup_test.go @@ -0,0 +1,42 @@ +package kv + +import ( + "context" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestApplyMigrationCleanupCommandDeletesBoundedVersions(t *testing.T) { + ctx := context.Background() + st := store.NewMVCCStore() + require.NoError(t, st.PutAt(ctx, []byte("a"), []byte("old"), 10, 0)) + require.NoError(t, st.PutAt(ctx, []byte("a"), []byte("new"), 20, 0)) + fsm, ok := NewKvFSMWithHLC(st, NewHLC()).(*kvFSM) + require.True(t, ok) + + cmd, err := MarshalMigrationCleanupCommand(&pb.CleanupMigrationRequest{ + JobId: 1, + Mode: pb.MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS, + RangeStart: []byte("a"), + RangeEnd: []byte("b"), + MaxCommitTs: 10, + MaxVersions: 16, + MaxBytes: 1 << 20, + MaxScannedBytes: 1 << 20, + KeyFamily: 1, + }) + require.NoError(t, err) + result, ok := fsm.applyMigrationCleanup(ctx, cmd[1:]).(store.CleanupVersionsResult) + require.True(t, ok) + require.True(t, result.Done) + require.Equal(t, uint64(1), result.DeletedRows) + + value, err := st.GetAt(ctx, []byte("a"), 20) + require.NoError(t, err) + require.Equal(t, []byte("new"), value) + _, err = st.GetAt(ctx, []byte("a"), 10) + require.ErrorIs(t, err, store.ErrKeyNotFound) +} diff --git a/kv/fsm_migration_fence_test.go b/kv/fsm_migration_fence_test.go index 1eee26301..43997b922 100644 --- a/kv/fsm_migration_fence_test.go +++ b/kv/fsm_migration_fence_test.go @@ -1,6 +1,7 @@ package kv import ( + "bytes" "context" "testing" @@ -8,9 +9,39 @@ import ( "github.com/bootjp/elastickv/internal/s3keys" pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" "github.com/stretchr/testify/require" ) +type deletePrefixIndexRecordingStore struct { + store.MVCCStore + + deletePrefixIndexes []uint64 + deletePrefixPrefixes [][]byte +} + +func (s *deletePrefixIndexRecordingStore) DeletePrefixAtRaftAt(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS, appliedIndex uint64) error { + s.deletePrefixIndexes = append(s.deletePrefixIndexes, appliedIndex) + s.deletePrefixPrefixes = append(s.deletePrefixPrefixes, append([]byte(nil), prefix...)) + return s.MVCCStore.DeletePrefixAtRaftAt(ctx, prefix, excludePrefix, commitTS, appliedIndex) +} + +func (s *deletePrefixIndexRecordingStore) MigrationTargetReadinessStates(ctx context.Context) ([]store.TargetStagedReadinessState, error) { + reader, ok := s.MVCCStore.(store.MigrationTargetReadinessReader) + if !ok { + return nil, nil + } + return reader.MigrationTargetReadinessStates(ctx) +} + +func (s *deletePrefixIndexRecordingStore) ApplyTargetStagedReadiness(ctx context.Context, state store.TargetStagedReadinessState) error { + writer, ok := s.MVCCStore.(store.MigrationTargetReadinessWriter) + if !ok { + return store.ErrNotSupported + } + return writer.ApplyTargetStagedReadiness(ctx, state) +} + func newWriteFencedFSM(t *testing.T) *kvFSM { t.Helper() @@ -27,20 +58,201 @@ func newWriteFloorFSM(t *testing.T) *kvFSM { engine := distribution.NewEngine() applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{ - {RouteID: 1, Start: []byte(""), End: nil, GroupID: 1, State: distribution.RouteStateActive, MinWriteTSExclusive: 100}, + { + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }, }) return newComposed1FSM(t, engine, 1) } -func newFirstRouteWriteFencedFSM(t *testing.T) *kvFSM { +func newTargetReadinessFSM(t *testing.T, route distribution.RouteDescriptor) *kvFSM { t.Helper() engine := distribution.NewEngine() - applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{ - {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateWriteFenced}, - {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 1, State: distribution.RouteStateActive}, + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{route}) + fsm := newComposed1FSM(t, engine, route.GroupID) + writer, ok := fsm.store.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(context.Background(), store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + })) + return fsm +} + +func applyTargetReadinessToFSM(t *testing.T, fsm *kvFSM, state store.TargetStagedReadinessState) { + t.Helper() + + writer, ok := fsm.store.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(context.Background(), state)) +} + +func applySourceMigrationControlToFSM(t *testing.T, fsm *kvFSM, readFence bool) { + t.Helper() + applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{ + JobID: 10, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + MigrationJobID: 10, + MinWriteTSExclusive: 50, + Armed: true, + SourceWriteFence: true, + SourceReadFence: readFence, + RetentionPinTS: 40, }) - return newComposed1FSM(t, engine, 1) +} + +func newMigrationWriteTrackerFSM(t *testing.T) *kvFSM { + t.Helper() + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + fsm := newComposed1FSM(t, engine, 1) + applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{ + JobID: 11, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + MigrationJobID: 11, + MinWriteTSExclusive: 1, + Armed: true, + RetentionPinTS: 1, + TrackWrites: true, + }) + return fsm +} + +func migrationTrackerMinimum(t *testing.T, fsm *kvFSM) uint64 { + t.Helper() + reader, ok := fsm.store.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(context.Background()) + require.NoError(t, err) + require.Len(t, states, 1) + return states[0].MinAdmittedTS +} + +func TestFSMMigrationWriteTrackerCoversAllAdmissionPaths(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newMigrationWriteTrackerFSM(t) + require.NoError(t, fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("raw")}}, + }, 80)) + require.Equal(t, uint64(80), migrationTrackerMinimum(t, fsm)) + + require.NoError(t, fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("m")}}, + }, 70)) + require.Equal(t, uint64(70), migrationTrackerMinimum(t, fsm)) + + require.NoError(t, fsm.handleTxnRequest(ctx, &pb.Request{ + IsTxn: true, + Ts: 50, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("o"), CommitTS: 60})}, + {Op: pb.Op_PUT, Key: []byte("o"), Value: []byte("one-phase")}, + }, + }, 60)) + require.Equal(t, uint64(60), migrationTrackerMinimum(t, fsm)) + + require.NoError(t, fsm.handleTxnRequest(ctx, &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 40, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("p"), CommitTS: 55, LockTTLms: defaultTxnLockTTLms})}, + {Op: pb.Op_PUT, Key: []byte("p"), Value: []byte("prepared")}, + }, + }, 40)) + require.Equal(t, uint64(55), migrationTrackerMinimum(t, fsm)) +} + +func TestFSMMigrationWriteTrackerRecordsCommitPreparedBeforeArm(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + fsm := newComposed1FSM(t, engine, 1) + startTS, commitTS := uint64(40), uint64(60) + meta := &pb.Mutation{ + Op: pb.Op_PUT, + Key: []byte(txnMetaPrefix), + Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: []byte("n"), + CommitTS: commitTS, + LockTTLms: defaultTxnLockTTLms, + }), + } + mutation := &pb.Mutation{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("value")} + require.NoError(t, fsm.handleTxnRequest(ctx, &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: startTS, + Mutations: []*pb.Mutation{meta, mutation}, + }, startTS)) + + applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{ + JobID: 12, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + MigrationJobID: 12, + MinWriteTSExclusive: 1, + Armed: true, + RetentionPinTS: 1, + TrackWrites: true, + }) + require.NoError(t, fsm.handleTxnRequest(ctx, &pb.Request{ + IsTxn: true, + Phase: pb.Phase_COMMIT, + Ts: startTS, + Mutations: []*pb.Mutation{meta, mutation}, + }, commitTS)) + require.Equal(t, commitTS, migrationTrackerMinimum(t, fsm)) +} + +func newReadinessReadKeyFSM(t *testing.T) *kvFSM { + t.Helper() + + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte("a"), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: []byte("z"), GroupID: 1, State: distribution.RouteStateActive}, + }) + fsm := newComposed1FSM(t, engine, 1) + applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + return fsm } func s3BucketAuxiliaryFenceRoutes(bucket string, rawGroupID, fencedGroupID uint64) []distribution.RouteDescriptor { @@ -61,7 +273,7 @@ func newS3BucketAuxiliaryWriteFencedFSM(t *testing.T, bucket string) *kvFSM { return newComposed1FSM(t, engine, 1) } -func TestFSMRejectsCurrentWriteFencedRawPointWrite(t *testing.T) { +func TestFSMRejectsRawPointWriteOnWriteFencedRoute(t *testing.T) { t.Parallel() fsm := newWriteFencedFSM(t) @@ -69,34 +281,16 @@ func TestFSMRejectsCurrentWriteFencedRawPointWrite(t *testing.T) { Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}}, }, 10) require.ErrorIs(t, err, ErrRouteWriteFenced) -} - -func TestFSMRejectsCurrentWriteFencedEmptyRawPointWrite(t *testing.T) { - t.Parallel() - - fsm := newFirstRouteWriteFencedFSM(t) - err := fsm.handleRawRequest(context.Background(), &pb.Request{ - Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte(""), Value: []byte("v")}}, - }, 10) - require.ErrorIs(t, err, ErrRouteWriteFenced) -} - -func TestFSMRejectsObservedWriteFencedRawPointWrite(t *testing.T) { - t.Parallel() - fsm := newWriteFencedFSM(t) - err := fsm.handleRawRequest(context.Background(), &pb.Request{ - ObservedRouteVersion: 1, - Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}}, - }, 10) - require.ErrorIs(t, err, ErrRouteWriteFenced) + _, getErr := fsm.store.GetAt(context.Background(), []byte("z"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) } func TestFSMWriteFenceBypassAllowsMarkedRawPointWrite(t *testing.T) { t.Parallel() - fsm := newFirstRouteWriteFencedFSM(t) - key := []byte("!sqs|msg|data|p|partitioned-key") + fsm := newWriteFencedFSM(t) + key := []byte("n") err := fsm.handleRawRequest(context.Background(), &pb.Request{ WriteFenceBypassKeys: [][]byte{key}, Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("v")}}, @@ -108,16 +302,54 @@ func TestFSMWriteFenceBypassAllowsMarkedRawPointWrite(t *testing.T) { require.Equal(t, []byte("v"), got) } -func TestFSMWriteFenceBypassAllowsRawWriteBelowBypassedRouteFloor(t *testing.T) { +func TestFSMDurableSourceFenceRejectsRawWriteDespiteCatalogBypass(t *testing.T) { t.Parallel() - fsm := newWriteFloorFSM(t) - key := []byte("!sqs|msg|data|p|partitioned-key") + fsm := newWriteFencedFSM(t) + applySourceMigrationControlToFSM(t, fsm, false) + key := []byte("n") err := fsm.handleRawRequest(context.Background(), &pb.Request{ WriteFenceBypassKeys: [][]byte{key}, Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("v")}}, - }, 10) - require.NoError(t, err) + }, 60) + require.ErrorIs(t, err, ErrRouteWriteFenced) +} + +func TestFSMDurableSourceFenceRejectsPrefixWrite(t *testing.T) { + t.Parallel() + + fsm := newWriteFencedFSM(t) + applySourceMigrationControlToFSM(t, fsm, false) + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("m")}}, + }, 60) + require.ErrorIs(t, err, ErrRouteWriteFenced) +} + +func TestFSMDurableSourceFenceRejectsPrefixWriteOnActiveRoute(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("m"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}) + fsm := newComposed1FSM(t, engine, 1) + applySourceMigrationControlToFSM(t, fsm, false) + require.NoError(t, fsm.store.PutAt(ctx, []byte("m/key"), []byte("v"), 1, 0)) + + err := fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("m")}}, + }, 60) + require.ErrorIs(t, err, ErrRouteWriteFenced) + + got, getErr := fsm.store.GetAt(ctx, []byte("m/key"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) } func TestFSMWriteFenceBypassAllowsPinnedTxnOnNonOwningGroup(t *testing.T) { @@ -133,22 +365,67 @@ func TestFSMWriteFenceBypassAllowsPinnedTxnOnNonOwningGroup(t *testing.T) { err := fsm.handleTxnRequest(context.Background(), &pb.Request{ IsTxn: true, Phase: pb.Phase_PREPARE, - Ts: 10, + Ts: 101, ObservedRouteVersion: 1, WriteFenceBypassKeys: [][]byte{key}, Mutations: []*pb.Mutation{ {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: key, LockTTLms: defaultTxnLockTTLms})}, {Op: pb.Op_DEL, Key: key}, }, - }, 10) + }, 101) require.NoError(t, err) } +func TestFSMWriteFenceBypassAllowsPinnedOnePhaseTxnOnNonOwningGroup(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateWriteFenced}, + }) + fsm := newComposed1FSM(t, engine, 1) + key := []byte("z") + err := fsm.handleTxnRequest(context.Background(), &pb.Request{ + IsTxn: true, + Ts: 10, + ObservedRouteVersion: 1, + WriteFenceBypassKeys: [][]byte{key}, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: key})}, + {Op: pb.Op_PUT, Key: key, Value: []byte("v")}, + }, + }, 20) + require.NoError(t, err) + + got, err := fsm.store.GetAt(context.Background(), key, 20) + require.NoError(t, err) + require.Equal(t, []byte("v"), got) +} + +func TestFSMDurableSourceFenceRejectsPinnedOnePhaseTxn(t *testing.T) { + t.Parallel() + + fsm := newWriteFencedFSM(t) + applySourceMigrationControlToFSM(t, fsm, false) + key := []byte("n") + err := fsm.handleTxnRequest(context.Background(), &pb.Request{ + IsTxn: true, + Ts: 50, + WriteFenceBypassKeys: [][]byte{key}, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: key})}, + {Op: pb.Op_PUT, Key: key, Value: []byte("v")}, + }, + }, 60) + require.ErrorIs(t, err, ErrRouteWriteFenced) +} + func TestFSMWriteFenceBypassDoesNotAllowDelPrefix(t *testing.T) { t.Parallel() - fsm := newFirstRouteWriteFencedFSM(t) - prefix := []byte("!sqs|msg|data|p|") + fsm := newWriteFencedFSM(t) + prefix := []byte("m") err := fsm.handleRawRequest(context.Background(), &pb.Request{ WriteFenceBypassKeys: [][]byte{prefix}, Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: prefix}}, @@ -156,56 +433,84 @@ func TestFSMWriteFenceBypassDoesNotAllowDelPrefix(t *testing.T) { require.ErrorIs(t, err, ErrRouteWriteFenced) } -func TestFSMRejectsCurrentWriteFenceAfterObservedActiveRawPointWrite(t *testing.T) { +func TestFSMRejectsRawPointWriteWithoutTargetReadinessProof(t *testing.T) { t.Parallel() - engine := distribution.NewEngine() - applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{ - {RouteID: 1, Start: []byte(""), End: nil, GroupID: 1, State: distribution.RouteStateActive}, - }) - fsm := newComposed1FSM(t, engine, 1) - applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{ - {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, - {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 1, State: distribution.RouteStateWriteFenced}, + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, }) + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}}, + }, 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + _, getErr := fsm.store.GetAt(context.Background(), []byte("b"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + +func TestFSMAllowsRawPointWriteWithClearedTargetReadinessProof(t *testing.T) { + t.Parallel() + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }) err := fsm.handleRawRequest(context.Background(), &pb.Request{ - ObservedRouteVersion: 1, - Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}}, - }, 10) - require.ErrorIs(t, err, ErrRouteWriteFenced) + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}}, + }, 101) + require.NoError(t, err) + + got, getErr := fsm.store.GetAt(context.Background(), []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) } -func TestFSMRejectsCurrentWriteFencedUnpinnedPrepare(t *testing.T) { +func TestFSMRejectsTargetReadinessProofFromAnotherGroup(t *testing.T) { t.Parallel() engine := distribution.NewEngine() applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{ - {RouteID: 1, Start: []byte(""), End: nil, GroupID: 1, State: distribution.RouteStateActive}, + { + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 2, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }, }) fsm := newComposed1FSM(t, engine, 1) - applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{ - {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, - {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 1, State: distribution.RouteStateWriteFenced}, - }) + writer, ok := fsm.store.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(context.Background(), store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + })) - err := fsm.handleTxnRequest(context.Background(), &pb.Request{ - IsTxn: true, - Phase: pb.Phase_PREPARE, - Ts: 10, - Mutations: []*pb.Mutation{ - {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), LockTTLms: defaultTxnLockTTLms})}, - {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}, - }, - }, 10) - require.ErrorIs(t, err, ErrRouteWriteFenced) + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}}, + }, 101) + require.ErrorIs(t, err, ErrRouteCutoverPending) } -func TestFSMRejectsCurrentWriteFencedS3BucketAuxiliaryPointWrite(t *testing.T) { +func TestFSMRejectsS3BucketAuxiliaryPointWriteOnWriteFencedRoute(t *testing.T) { t.Parallel() ctx := context.Background() - const bucket = "bucket-b" + const bucket = "bucket-a" fsm := newS3BucketAuxiliaryWriteFencedFSM(t, bucket) for _, key := range [][]byte{ @@ -216,74 +521,110 @@ func TestFSMRejectsCurrentWriteFencedS3BucketAuxiliaryPointWrite(t *testing.T) { Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("v")}}, }, 10) require.ErrorIs(t, err, ErrRouteWriteFenced) + + _, getErr := fsm.store.GetAt(ctx, key, ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) } } -func TestFSMRejectsObservedWriteFencedS3BucketAuxiliaryPointWrite(t *testing.T) { +func TestFSMRejectsS3BucketAuxiliaryPointWriteBelowRouteFloor(t *testing.T) { t.Parallel() - const bucket = "bucket-b" - fsm := newS3BucketAuxiliaryWriteFencedFSM(t, bucket) + ctx := context.Background() + const bucket = "bucket-a" + routeStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) + routeEnd := prefixScanEnd(routeStart) + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: routeStart, + End: routeEnd, + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }}) + fsm := newComposed1FSM(t, engine, 1) - err := fsm.handleRawRequest(context.Background(), &pb.Request{ - ObservedRouteVersion: 1, - Mutations: []*pb.Mutation{ - {Op: pb.Op_PUT, Key: s3keys.BucketGenerationKey(bucket), Value: []byte("v")}, - }, - }, 10) - require.ErrorIs(t, err, ErrRouteWriteFenced) + err := fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: s3keys.BucketMetaKey(bucket), Value: []byte("v")}}, + }, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) } -func TestFSMComposed1UsesS3BucketAuxiliaryRouteOwner(t *testing.T) { +func TestFSMIgnoresRawRouteFloorForS3BucketAuxiliaryPointWrite(t *testing.T) { t.Parallel() - const bucket = "bucket-b" + ctx := context.Background() + const bucket = "bucket-a" key := s3keys.BucketMetaKey(bucket) - engine := distribution.NewEngine() - applyComposed1Snapshot(t, engine, 1, s3BucketAuxiliaryStagedRoutes(bucket, 3, 4)) - fsm := newComposed1FSM(t, engine, 4) + auxStart, auxEnd, ok := s3BucketAuxiliaryRouteRange(key) + require.True(t, ok) + rawStart := []byte(s3keys.BucketMetaPrefix) + rawEnd := prefixScanEnd(rawStart) + require.Less(t, bytes.Compare(auxEnd, rawStart), 0) - err := fsm.verifyComposed1(&pb.Request{ - IsTxn: true, - Phase: pb.Phase_PREPARE, - Ts: 10, - ObservedRouteVersion: 1, - Mutations: []*pb.Mutation{ - {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: key, LockTTLms: defaultTxnLockTTLms})}, - {Op: pb.Op_PUT, Key: key, Value: []byte("meta")}, + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: auxStart, + End: auxEnd, + GroupID: 1, + State: distribution.RouteStateActive, + }, + { + RouteID: 2, + Start: rawStart, + End: rawEnd, + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, }, }) + fsm := newComposed1FSM(t, engine, 1) + + err := fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("v")}}, + }, 100) require.NoError(t, err) + got, err := fsm.store.GetAt(ctx, key, 100) + require.NoError(t, err) + require.Equal(t, []byte("v"), got) } -func TestFSMIgnoresRawRouteFloorForS3BucketAuxiliaryWrite(t *testing.T) { +func TestFSMRejectsS3BucketAuxiliaryPointWriteWithoutTargetReadinessProof(t *testing.T) { t.Parallel() + ctx := context.Background() const bucket = "bucket-a" - key := s3keys.BucketMetaKey(bucket) + routeStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) + routeEnd := prefixScanEnd(routeStart) engine := distribution.NewEngine() - routes := s3BucketAuxiliaryFenceRoutes(bucket, 1, 1) - routes[1].State = distribution.RouteStateActive - routes[2].MinWriteTSExclusive = ^uint64(0) - applyComposed1Snapshot(t, engine, 1, routes) - - rawRoute, ok := engine.GetRoute(routeKey(key)) - require.True(t, ok) - require.Equal(t, ^uint64(0), rawRoute.MinWriteTSExclusive) - auxStart, auxEnd, ok := s3BucketAuxiliaryRouteRange(key) - require.True(t, ok) - auxRoutes := engine.GetIntersectingRoutes(auxStart, auxEnd) - require.NotEmpty(t, auxRoutes) - require.Zero(t, auxRoutes[0].MinWriteTSExclusive) - + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: routeStart, + End: routeEnd, + GroupID: 1, + State: distribution.RouteStateActive, + }}) fsm := newComposed1FSM(t, engine, 1) - err := fsm.handleRawRequest(context.Background(), &pb.Request{ - Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: key, Value: []byte("meta")}}, - }, 100) - require.NoError(t, err) + applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: routeStart, + RouteEnd: routeEnd, + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + + err := fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: s3keys.BucketGenerationKey(bucket), Value: []byte("v")}}, + }, 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) } -func TestFSMRejectsCurrentWriteFencedDelPrefix(t *testing.T) { +func TestFSMRejectsDelPrefixIntersectingWriteFencedRoute(t *testing.T) { t.Parallel() fsm := newWriteFencedFSM(t) @@ -293,22 +634,94 @@ func TestFSMRejectsCurrentWriteFencedDelPrefix(t *testing.T) { Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("z")}}, }, 10) require.ErrorIs(t, err, ErrRouteWriteFenced) + + got, getErr := fsm.store.GetAt(context.Background(), []byte("z"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) } -func TestFSMRejectsObservedWriteFencedDelPrefix(t *testing.T) { +func TestFSMRejectsDelPrefixWithoutTargetReadinessProof(t *testing.T) { t.Parallel() - fsm := newWriteFencedFSM(t) - require.NoError(t, fsm.store.PutAt(context.Background(), []byte("z"), []byte("v"), 1, 0)) + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + require.NoError(t, fsm.store.PutAt(context.Background(), []byte("b"), []byte("v"), 1, 0)) err := fsm.handleRawRequest(context.Background(), &pb.Request{ - ObservedRouteVersion: 1, - Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("z")}}, - }, 10) - require.ErrorIs(t, err, ErrRouteWriteFenced) + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("b")}}, + }, 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + got, getErr := fsm.store.GetAt(context.Background(), []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestFSMDelPrefixTombstonesStagedVisibilityRows(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }) + rawKey := []byte("b") + stagedKey := distribution.MigrationStagedDataKey(9, rawKey) + require.NoError(t, fsm.store.PutAt(ctx, stagedKey, []byte("staged"), 110, 0)) + + err := fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: rawKey}}, + }, 120) + require.NoError(t, err) + + _, err = fsm.store.GetAt(ctx, stagedKey, 130) + require.ErrorIs(t, err, store.ErrKeyNotFound) +} + +func TestFSMDelPrefixAdvancesApplyIndexOnlyOnLiveDelete(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }) + recording := &deletePrefixIndexRecordingStore{MVCCStore: fsm.store} + fsm.store = recording + fsm.SetApplyIndex(55) + + rawKey := []byte("b") + stagedKey := distribution.MigrationStagedDataKey(9, rawKey) + require.NoError(t, fsm.store.PutAt(ctx, stagedKey, []byte("staged"), 110, 0)) + + err := fsm.handleRawRequest(ctx, &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: rawKey}}, + }, 120) + require.NoError(t, err) + + require.Equal(t, []uint64{0, 55}, recording.deletePrefixIndexes) + require.Equal(t, stagedKey, recording.deletePrefixPrefixes[0]) + require.Equal(t, rawKey, recording.deletePrefixPrefixes[1]) } -func TestFSMRejectsCurrentWriteFencedFullRangeDelPrefix(t *testing.T) { +func TestFSMRejectsFullRangeDelPrefixWhenRouteIsWriteFenced(t *testing.T) { t.Parallel() fsm := newWriteFencedFSM(t) @@ -318,9 +731,42 @@ func TestFSMRejectsCurrentWriteFencedFullRangeDelPrefix(t *testing.T) { Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: nil}}, }, 10) require.ErrorIs(t, err, ErrRouteWriteFenced) + + got, getErr := fsm.store.GetAt(context.Background(), []byte("z"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestFSMRejectsRawPointWriteBelowRouteFloor(t *testing.T) { + t.Parallel() + + fsm := newWriteFloorFSM(t) + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}}, + }, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + _, getErr := fsm.store.GetAt(context.Background(), []byte("b"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + +func TestFSMRejectsDelPrefixBelowRouteFloor(t *testing.T) { + t.Parallel() + + fsm := newWriteFloorFSM(t) + require.NoError(t, fsm.store.PutAt(context.Background(), []byte("b"), []byte("v"), 1, 0)) + + err := fsm.handleRawRequest(context.Background(), &pb.Request{ + Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("b")}}, + }, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + got, getErr := fsm.store.GetAt(context.Background(), []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) } -func TestFSMRejectsCurrentWriteFencedBroadInternalDelPrefix(t *testing.T) { +func TestFSMRejectsBroadInternalDelPrefixWhenRouteIsWriteFenced(t *testing.T) { t.Parallel() fsm := newWriteFencedFSM(t) @@ -331,130 +777,367 @@ func TestFSMRejectsCurrentWriteFencedBroadInternalDelPrefix(t *testing.T) { Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("!redis|")}}, }, 10) require.ErrorIs(t, err, ErrRouteWriteFenced) + + got, getErr := fsm.store.GetAt(context.Background(), key, ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestFSMRejectsOnePhaseTxnBelowRouteFloor(t *testing.T) { + t.Parallel() + + fsm := newWriteFloorFSM(t) + err := fsm.handleTxnRequest(context.Background(), onePhaseReq(10, 100, 0, []byte("b"), []byte("v")), 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + _, getErr := fsm.store.GetAt(context.Background(), []byte("b"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) } -func TestFSMRejectsCurrentWriteFencedPrepareButAllowsAbort(t *testing.T) { +func TestFSMOnePhaseDedupChecksTargetReadinessBeforeNoOp(t *testing.T) { t.Parallel() ctx := context.Background() - fsm := newWriteFencedFSM(t) - prepare := &pb.Request{ - IsTxn: true, - Phase: pb.Phase_PREPARE, - Ts: 10, + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + require.NoError(t, fsm.store.PutAt(ctx, []byte("b"), []byte("landed"), 20, 0)) + + err := fsm.handleTxnRequest(ctx, onePhaseReq(30, 40, 20, []byte("b"), []byte("retry")), 40) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + landedAt40, probeErr := fsm.store.CommittedVersionAt(ctx, []byte("b"), 40) + require.NoError(t, probeErr) + require.False(t, landedAt40) +} + +func TestFSMOnePhaseTxnChecksReadKeysForTargetReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newReadinessReadKeyFSM(t) + req := onePhaseReq(10, 20, 0, []byte("b"), []byte("v")) + req.ReadKeys = [][]byte{[]byte("n")} + + err := fsm.handleTxnRequest(ctx, req, 20) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + _, getErr := fsm.store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + +func TestFSMPrepareTxnChecksReadKeysForTargetReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newReadinessReadKeyFSM(t) + req := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 10, + ReadKeys: [][]byte{[]byte("n")}, Mutations: []*pb.Mutation{ - {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), LockTTLms: defaultTxnLockTTLms})}, - {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}, + { + Op: pb.Op_PUT, + Key: []byte(txnMetaPrefix), + Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: []byte("b"), + LockTTLms: defaultTxnLockTTLms, + }), + }, + {Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}, }, } - require.ErrorIs(t, fsm.handleTxnRequest(ctx, prepare, 10), ErrRouteWriteFenced) - abort := &pb.Request{ + err := fsm.handleTxnRequest(ctx, req, 10) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + _, getErr := fsm.store.GetAt(ctx, txnLockKey([]byte("b")), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + +func TestFSMOnePhaseTxnChecksSourceReadFenceForReadKeys(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 1, []distribution.RouteDescriptor{{ + RouteID: 1, Start: []byte("a"), End: []byte("z"), GroupID: 1, State: distribution.RouteStateActive, + }}) + fsm := newComposed1FSM(t, engine, 1) + applySourceMigrationControlToFSM(t, fsm, true) + req := onePhaseReq(10, 20, 0, []byte("b"), []byte("v")) + req.ReadKeys = [][]byte{[]byte("n")} + + err := fsm.handleTxnRequest(ctx, req, 20) + require.ErrorIs(t, err, ErrRouteCutoverPending) + _, getErr := fsm.store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) +} + +func TestFSMPrepareTxnSkipsRemotePrimaryForTargetReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + applyComposed1Snapshot(t, engine, 2, []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte("a"), End: []byte("m"), GroupID: 2, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: []byte("z"), GroupID: 1, State: distribution.RouteStateActive}, + }) + fsm := newComposed1FSM(t, engine, 1) + applyTargetReadinessToFSM(t, fsm, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("m"), + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + + err := fsm.handleTxnRequest(ctx, &pb.Request{ IsTxn: true, - Phase: pb.Phase_ABORT, - Ts: 11, + Phase: pb.Phase_PREPARE, + Ts: 10, Mutations: []*pb.Mutation{ - {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), CommitTS: 11})}, - {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}, + { + Op: pb.Op_PUT, + Key: []byte(txnMetaPrefix), + Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: []byte("b"), + LockTTLms: defaultTxnLockTTLms, + }), + }, + {Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("v")}, }, - } - err := fsm.handleTxnRequest(ctx, abort, 11) - require.NotErrorIs(t, err, ErrRouteWriteFenced, "ABORT must keep the narrow cleanup lane open") + }, 10) + require.NoError(t, err) + + _, getErr := fsm.store.GetAt(ctx, txnLockKey([]byte("n")), ^uint64(0)) + require.NoError(t, getErr) } -func TestFSMRejectsObservedWriteFencedPrepareButAllowsAbort(t *testing.T) { +func TestFSMPrepareTxnChecksStagedVisibilityWriteConflicts(t *testing.T) { t.Parallel() ctx := context.Background() - fsm := newWriteFencedFSM(t) - prepare := &pb.Request{ - IsTxn: true, - Phase: pb.Phase_PREPARE, - Ts: 10, - ObservedRouteVersion: 1, - Mutations: []*pb.Mutation{ - {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), LockTTLms: defaultTxnLockTTLms})}, - {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}, - }, - } - require.ErrorIs(t, fsm.handleTxnRequest(ctx, prepare, 10), ErrRouteWriteFenced) + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }) + require.NoError(t, fsm.store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("b")), []byte("staged"), 20, 0)) - abort := &pb.Request{ - IsTxn: true, - Phase: pb.Phase_ABORT, - Ts: 11, - ObservedRouteVersion: 1, + err := fsm.handleTxnRequest(ctx, &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 10, Mutations: []*pb.Mutation{ - {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), CommitTS: 11})}, - {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}, + { + Op: pb.Op_PUT, + Key: []byte(txnMetaPrefix), + Value: EncodeTxnMeta(TxnMeta{ + PrimaryKey: []byte("b"), + CommitTS: 120, + LockTTLms: defaultTxnLockTTLms, + }), + }, + {Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}, }, - } - require.NotErrorIs(t, fsm.handleTxnRequest(ctx, abort, 11), ErrRouteWriteFenced) + }, 10) + require.ErrorIs(t, err, store.ErrWriteConflict) + + _, getErr := fsm.store.GetAt(ctx, txnLockKey([]byte("b")), ^uint64(0)) + require.ErrorIs(t, getErr, store.ErrKeyNotFound) } -func TestFSMRejectsRawPointWriteAtMigrationTimestampFloorDuringApply(t *testing.T) { +func TestFSMOnePhaseTxnChecksStagedVisibilityReadConflicts(t *testing.T) { t.Parallel() ctx := context.Background() - fsm := newWriteFloorFSM(t) - err := fsm.handleRawRequest(ctx, &pb.Request{ - Mutations: []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("replayed")}}, - }, 100) - require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow) - _, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }) + require.NoError(t, fsm.store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("n")), []byte("staged"), 20, 0)) + + req := onePhaseReq(10, 120, 0, []byte("b"), []byte("v")) + req.ReadKeys = [][]byte{[]byte("n")} + + err := fsm.handleTxnRequest(ctx, req, 120) + require.ErrorIs(t, err, store.ErrWriteConflict) + + _, getErr := fsm.store.GetAt(ctx, []byte("b"), ^uint64(0)) require.ErrorIs(t, getErr, store.ErrKeyNotFound) } -func TestFSMRejectsDelPrefixAtMigrationTimestampFloorDuringApply(t *testing.T) { +func TestFSMPrepareUsesCommitTSForRouteFloorWhenPresent(t *testing.T) { t.Parallel() ctx := context.Background() fsm := newWriteFloorFSM(t) - require.NoError(t, fsm.store.PutAt(ctx, []byte("z"), []byte("v"), 10, 0)) - - err := fsm.handleRawRequest(ctx, &pb.Request{ - Mutations: []*pb.Mutation{{Op: pb.Op_DEL_PREFIX, Key: []byte("z")}}, - }, 100) - require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow) + commitTS := uint64(101) + primaryKey := []byte("b") + err := fsm.handleTxnRequest(ctx, &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: 50, + Mutations: []*pb.Mutation{ + { + Op: pb.Op_PUT, + Key: []byte(txnMetaPrefix), + Value: EncodeTxnMeta(TxnMeta{PrimaryKey: primaryKey, LockTTLms: defaultTxnLockTTLms, CommitTS: commitTS}), + }, + {Op: pb.Op_PUT, Key: primaryKey, Value: []byte("v")}, + }, + }, 50) + require.NoError(t, err) - got, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) - require.NoError(t, getErr) - require.Equal(t, []byte("v"), got) + rawLock, err := fsm.store.GetAt(ctx, txnLockKey(primaryKey), ^uint64(0)) + require.NoError(t, err) + lock, err := decodeTxnLock(rawLock) + require.NoError(t, err) + require.Equal(t, commitTS, lock.CommitTS) } -func TestFSMRejectsOnePhaseTxnAtMigrationTimestampFloorDuringApply(t *testing.T) { +func TestFSMPrepareWithoutCommitTSUsesStartTSForRouteFloor(t *testing.T) { t.Parallel() - ctx := context.Background() fsm := newWriteFloorFSM(t) - req := &pb.Request{ + err := fsm.handleTxnRequest(context.Background(), &pb.Request{ IsTxn: true, - Phase: pb.Phase_NONE, - Ts: 90, + Phase: pb.Phase_PREPARE, + Ts: 100, Mutations: []*pb.Mutation{ - {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), CommitTS: 100})}, - {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("low")}, + { + Op: pb.Op_PUT, + Key: []byte(txnMetaPrefix), + Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("b"), LockTTLms: defaultTxnLockTTLms}), + }, + {Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("v")}, }, - } - err := fsm.handleTxnRequest(ctx, req, 100) - require.ErrorIs(t, err, ErrRouteWriteTimestampTooLow) - _, getErr := fsm.store.GetAt(ctx, []byte("z"), ^uint64(0)) - require.ErrorIs(t, getErr, store.ErrKeyNotFound) + }, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) } -func TestFSMRejectsPrepareAtMigrationTimestampFloorDuringApply(t *testing.T) { +func TestFSMRejectsPrepareOnWriteFencedRouteButAllowsAbort(t *testing.T) { t.Parallel() ctx := context.Background() - fsm := newWriteFloorFSM(t) + fsm := newWriteFencedFSM(t) prepare := &pb.Request{ IsTxn: true, Phase: pb.Phase_PREPARE, - Ts: 90, + Ts: 10, Mutations: []*pb.Mutation{ {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), LockTTLms: defaultTxnLockTTLms})}, {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}, }, } - require.ErrorIs(t, fsm.handleTxnRequest(ctx, prepare, 90), ErrRouteWriteTimestampTooLow) + require.ErrorIs(t, fsm.handleTxnRequest(ctx, prepare, 10), ErrRouteWriteFenced) + + abort := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_ABORT, + Ts: 11, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("z"), CommitTS: 11})}, + {Op: pb.Op_PUT, Key: []byte("z"), Value: []byte("v")}, + }, + } + err := fsm.handleTxnRequest(ctx, abort, 11) + require.False(t, errors.Is(err, ErrRouteWriteFenced), "ABORT must keep the narrow cleanup lane open") +} + +func TestFSMAbortCleanupBypassesRetainedWriteFloor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newWriteFloorFSM(t) + startTS := uint64(10) + abortTS := uint64(11) + primaryKey := []byte("b") + require.NoError(t, fsm.store.PutAt(ctx, txnLockKey(primaryKey), encodeTxnLock(txnLock{ + StartTS: startTS, + PrimaryKey: primaryKey, + IsPrimaryKey: true, + }), startTS, 0)) + require.NoError(t, fsm.store.PutAt(ctx, txnIntentKey(primaryKey), encodeTxnIntent(txnIntent{ + StartTS: startTS, + Op: txnIntentOpPut, + Value: []byte("v"), + }), startTS, 0)) + + abort := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_ABORT, + Ts: startTS, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: primaryKey, CommitTS: abortTS})}, + {Op: pb.Op_PUT, Key: primaryKey}, + }, + } + require.NoError(t, fsm.handleTxnRequest(ctx, abort, abortTS)) + + _, lockErr := fsm.store.GetAt(ctx, txnLockKey(primaryKey), ^uint64(0)) + require.ErrorIs(t, lockErr, store.ErrKeyNotFound) + _, intentErr := fsm.store.GetAt(ctx, txnIntentKey(primaryKey), ^uint64(0)) + require.ErrorIs(t, intentErr, store.ErrKeyNotFound) +} + +func TestFSMAbortCleanupBypassesTargetReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + fsm := newTargetReadinessFSM(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + startTS := uint64(10) + abortTS := uint64(11) + primaryKey := []byte("b") + require.NoError(t, fsm.store.PutAt(ctx, txnLockKey(primaryKey), encodeTxnLock(txnLock{ + StartTS: startTS, + PrimaryKey: primaryKey, + IsPrimaryKey: true, + }), startTS, 0)) + require.NoError(t, fsm.store.PutAt(ctx, txnIntentKey(primaryKey), encodeTxnIntent(txnIntent{ + StartTS: startTS, + Op: txnIntentOpPut, + Value: []byte("v"), + }), startTS, 0)) + + abort := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_ABORT, + Ts: startTS, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: primaryKey, CommitTS: abortTS})}, + {Op: pb.Op_PUT, Key: primaryKey}, + }, + } + require.NoError(t, fsm.handleTxnRequest(ctx, abort, abortTS)) + + _, lockErr := fsm.store.GetAt(ctx, txnLockKey(primaryKey), ^uint64(0)) + require.ErrorIs(t, lockErr, store.ErrKeyNotFound) + _, intentErr := fsm.store.GetAt(ctx, txnIntentKey(primaryKey), ^uint64(0)) + require.ErrorIs(t, intentErr, store.ErrKeyNotFound) } diff --git a/kv/fsm_migration_readiness.go b/kv/fsm_migration_readiness.go new file mode 100644 index 000000000..f9e7adc81 --- /dev/null +++ b/kv/fsm_migration_readiness.go @@ -0,0 +1,166 @@ +package kv + +import ( + "bytes" + "context" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/cockroachdb/errors" + "google.golang.org/protobuf/proto" +) + +// MarshalTargetStagedReadinessCommand encodes a target-group readiness guard +// as a Raft FSM command. The target Internal RPC handler uses this instead of +// mutating its local store directly so an acknowledged guard has been applied +// by the target group's voters. +func MarshalTargetStagedReadinessCommand(req *pb.TargetStagedReadinessRequest) ([]byte, error) { + if req == nil { + return nil, errors.WithStack(ErrInvalidRequest) + } + b, err := proto.Marshal(req) + if err != nil { + return nil, errors.WithStack(err) + } + if len(b) >= maxMarshaledCommandSize { + return nil, errors.New("marshaled target readiness request too large") + } + return prependByte(raftEncodeTargetReadiness, b), nil +} + +func (f *kvFSM) applyTargetStagedReadiness(ctx context.Context, data []byte) any { + req := &pb.TargetStagedReadinessRequest{} + if err := proto.Unmarshal(data, req); err != nil { + return errors.WithStack(err) + } + writer, ok := f.store.(store.MigrationTargetReadinessWriter) + if !ok { + return errors.WithStack(store.ErrNotSupported) + } + state := targetStagedReadinessStateFromProto(req) + state = f.preserveMigrationTrackerMinimum(ctx, state) + if err := applyTargetStagedReadinessAt(ctx, writer, state, f.pendingApplyIdx); err != nil { + return errors.WithStack(err) + } + return nil +} + +func (f *kvFSM) preserveMigrationTrackerMinimum(ctx context.Context, state store.TargetStagedReadinessState) store.TargetStagedReadinessState { + if !state.TrackWrites { + return state + } + reader, ok := f.store.(store.MigrationTargetReadinessReader) + if !ok { + return state + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return state + } + for _, current := range states { + if current.JobID == state.JobID && (state.MinAdmittedTS == 0 || current.MinAdmittedTS < state.MinAdmittedTS) { + state.MinAdmittedTS = current.MinAdmittedTS + } + } + return state +} + +func (f *kvFSM) recordMigrationWrite(ctx context.Context, muts []*pb.Mutation, commitTS uint64) error { + if commitTS == 0 { + return nil + } + reader, writer, ok := f.migrationReadinessStore() + if !ok { + return nil + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return errors.WithStack(err) + } + // Tracker updates share the Raft entry with the following user mutation. + // Leave metaAppliedIndex to the data batch so replay cannot skip a write if + // the process stops after only the tracker record is persisted. + return recordMigrationWriteInStates(ctx, writer, states, muts, commitTS, 0) +} + +func (f *kvFSM) migrationReadinessStore() (store.MigrationTargetReadinessReader, store.MigrationTargetReadinessWriter, bool) { + reader, ok := f.store.(store.MigrationTargetReadinessReader) + if !ok { + return nil, nil, false + } + writer, ok := f.store.(store.MigrationTargetReadinessWriter) + if !ok { + return nil, nil, false + } + return reader, writer, true +} + +func recordMigrationWriteInStates(ctx context.Context, writer store.MigrationTargetReadinessWriter, states []store.TargetStagedReadinessState, muts []*pb.Mutation, commitTS uint64, appliedIndex uint64) error { + for _, state := range states { + if !state.Armed || !state.TrackWrites || !migrationMutationsIntersect(muts, state.RouteStart, state.RouteEnd) { + continue + } + if state.MinAdmittedTS != 0 && state.MinAdmittedTS <= commitTS { + continue + } + state.MinAdmittedTS = commitTS + if err := applyTargetStagedReadinessAt(ctx, writer, state, appliedIndex); err != nil { + return errors.WithStack(err) + } + } + return nil +} + +func applyTargetStagedReadinessAt(ctx context.Context, writer store.MigrationTargetReadinessWriter, state store.TargetStagedReadinessState, appliedIndex uint64) error { + if raftWriter, ok := writer.(store.MigrationTargetReadinessRaftWriter); ok { + if err := raftWriter.ApplyTargetStagedReadinessAt(ctx, state, appliedIndex); err != nil { + return errors.WithStack(err) + } + return nil + } + if err := writer.ApplyTargetStagedReadiness(ctx, state); err != nil { + return errors.WithStack(err) + } + return nil +} + +func migrationMutationsIntersect(muts []*pb.Mutation, routeStart []byte, routeEnd []byte) bool { + for _, mut := range muts { + if mut == nil || isTxnInternalKey(mut.Key) { + continue + } + var start, end []byte + if mut.GetOp() == pb.Op_DEL_PREFIX { + start, end = routePrefixRange(mut.Key) + } else { + if len(mut.Key) == 0 { + continue + } + start, end = readinessRouteRangeForScan(mut.Key, nextScanCursor(mut.Key)) + } + if routeRangeIntersects(start, end, routeStart, routeEnd) { + return true + } + } + return false +} + +func targetStagedReadinessStateFromProto(req *pb.TargetStagedReadinessRequest) store.TargetStagedReadinessState { + if req == nil { + return store.TargetStagedReadinessState{} + } + return store.TargetStagedReadinessState{ + JobID: req.GetJobId(), + RouteStart: bytes.Clone(req.GetRouteStart()), + RouteEnd: bytes.Clone(req.GetRouteEnd()), + ExpectedCutoverVersion: req.GetExpectedCutoverVersion(), + MigrationJobID: req.GetMigrationJobId(), + MinWriteTSExclusive: req.GetMinWriteTsExclusive(), + Armed: req.GetArmed(), + SourceWriteFence: req.GetSourceWriteFence(), + SourceReadFence: req.GetSourceReadFence(), + RetentionPinTS: req.GetRetentionPinTs(), + TrackWrites: req.GetTrackWrites(), + MinAdmittedTS: req.GetMinAdmittedTs(), + } +} diff --git a/kv/fsm_migration_readiness_test.go b/kv/fsm_migration_readiness_test.go new file mode 100644 index 000000000..dbb358d46 --- /dev/null +++ b/kv/fsm_migration_readiness_test.go @@ -0,0 +1,156 @@ +package kv + +import ( + "context" + "testing" + + pb "github.com/bootjp/elastickv/proto" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" +) + +func TestApplyTargetStagedReadinessCommandPersistsGuard(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + fsm := &kvFSM{store: st} + + cmd, err := MarshalTargetStagedReadinessCommand(&pb.TargetStagedReadinessRequest{ + JobId: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobId: 7, + MinWriteTsExclusive: 100, + Armed: true, + }) + require.NoError(t, err) + require.Nil(t, fsm.Apply(cmd)) + + reader, ok := st.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []store.TargetStagedReadinessState{{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + }}, states) +} + +func TestApplyTargetStagedReadinessCommandPersistsAppliedIndex(t *testing.T) { + t.Parallel() + + st, err := store.NewPebbleStore(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + fsm := &kvFSM{store: st, pendingApplyIdx: 77} + + cmd, err := MarshalTargetStagedReadinessCommand(&pb.TargetStagedReadinessRequest{ + JobId: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 3, + MigrationJobId: 7, + MinWriteTsExclusive: 100, + Armed: true, + }) + require.NoError(t, err) + require.Nil(t, fsm.Apply(cmd)) + + appliedReader, ok := st.(interface { + LastAppliedIndex() (uint64, bool, error) + }) + require.True(t, ok) + idx, present, err := appliedReader.LastAppliedIndex() + require.NoError(t, err) + require.True(t, present) + require.Equal(t, uint64(77), idx) +} + +func TestRecordMigrationWriteDoesNotAdvanceAppliedIndexBeforeDataWrite(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, err := store.NewPebbleStore(t.TempDir()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, st.Close()) }) + writer, ok := st.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, store.TargetStagedReadinessState{ + JobID: 11, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + MigrationJobID: 11, + MinWriteTSExclusive: 1, + Armed: true, + TrackWrites: true, + RetentionPinTS: 1, + MinAdmittedTS: 90, + })) + fsm := &kvFSM{store: st, pendingApplyIdx: 77} + + err = fsm.recordMigrationWrite(ctx, []*pb.Mutation{{Op: pb.Op_PUT, Key: []byte("n"), Value: []byte("v")}}, 50) + require.NoError(t, err) + reader, ok := st.(store.MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Len(t, states, 1) + require.Equal(t, uint64(50), states[0].MinAdmittedTS) + + appliedReader, ok := st.(interface { + LastAppliedIndex() (uint64, bool, error) + }) + require.True(t, ok) + idx, present, err := appliedReader.LastAppliedIndex() + require.NoError(t, err) + require.False(t, present) + require.Zero(t, idx) + + err = st.ApplyMutationsRaftAt(ctx, []*store.KVPairMutation{{ + Op: store.OpTypePut, + Key: []byte("n"), + Value: []byte("v"), + }}, nil, 50, 50, 77) + require.NoError(t, err) + idx, present, err = appliedReader.LastAppliedIndex() + require.NoError(t, err) + require.True(t, present) + require.Equal(t, uint64(77), idx) +} + +func TestRecordMigrationWriteTracksEmptyDelPrefix(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st := store.NewMVCCStore() + writer, ok := st.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + reader, ok := st.(store.MigrationTargetReadinessReader) + require.True(t, ok) + + err := recordMigrationWriteInStates(ctx, writer, []store.TargetStagedReadinessState{{ + JobID: 11, + RouteStart: []byte("m"), + RouteEnd: []byte("n"), + ExpectedCutoverVersion: 2, + MigrationJobID: 11, + MinWriteTSExclusive: 100, + Armed: true, + TrackWrites: true, + RetentionPinTS: 40, + MinAdmittedTS: 90, + }}, []*pb.Mutation{{Op: pb.Op_DEL_PREFIX}}, 50, 0) + require.NoError(t, err) + + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Len(t, states, 1) + require.Equal(t, uint64(50), states[0].MinAdmittedTS) +} diff --git a/kv/migrator_lock_drain.go b/kv/migrator_lock_drain.go index ef7a7e531..62adb9e05 100644 --- a/kv/migrator_lock_drain.go +++ b/kv/migrator_lock_drain.go @@ -18,6 +18,7 @@ type TxnLockDrainEntry struct { TTLExpireAt uint64 PrimaryKey []byte IsPrimaryKey bool + CommitTS uint64 } // PendingTxnLocksInRoute scans the txn-lock namespace and filters each lock by @@ -88,5 +89,6 @@ func txnLockDrainEntry(kvp *store.KVPair, filter func([]byte) bool) (TxnLockDrai TTLExpireAt: lock.TTLExpireAt, PrimaryKey: bytes.Clone(lock.PrimaryKey), IsPrimaryKey: lock.IsPrimaryKey, + CommitTS: lock.CommitTS, }, true, nil } diff --git a/kv/migrator_lock_drain_test.go b/kv/migrator_lock_drain_test.go index d790e85c0..d30647c00 100644 --- a/kv/migrator_lock_drain_test.go +++ b/kv/migrator_lock_drain_test.go @@ -27,6 +27,7 @@ func TestPendingTxnLocksInRouteFiltersLocksByRouteKey(t *testing.T) { TTLExpireAt: 99, PrimaryKey: itemKey, IsPrimaryKey: true, + CommitTS: 101, }) require.NoError(t, st.PutAt(ctx, txnLockKey(itemKey), lock, 1, 0)) require.NoError(t, st.PutAt(ctx, txnLockKey(outsideKey), encodeTxnLock(txnLock{ @@ -40,6 +41,7 @@ func TestPendingTxnLocksInRouteFiltersLocksByRouteKey(t *testing.T) { require.Equal(t, itemKey, pending[0].UserKey) require.Equal(t, uint64(11), pending[0].StartTS) require.Equal(t, uint64(99), pending[0].TTLExpireAt) + require.Equal(t, uint64(101), pending[0].CommitTS) require.True(t, pending[0].IsPrimaryKey) require.Equal(t, txnLockKey(itemKey), pending[0].LockKey) } diff --git a/kv/route_history.go b/kv/route_history.go index 8bc50c58e..94eec1076 100644 --- a/kv/route_history.go +++ b/kv/route_history.go @@ -82,3 +82,21 @@ func (s distributionRouteSnapshot) WriteFencedIntersects(start, end []byte) bool } return false } + +func (s distributionRouteSnapshot) WriteFloorForKey(key []byte) (uint64, bool) { + route, ok := s.snap.RouteOf(key) + if !ok || route.MinWriteTSExclusive == 0 { + return 0, false + } + return route.MinWriteTSExclusive, true +} + +func (s distributionRouteSnapshot) WriteFloorIntersects(start, end []byte) (uint64, bool) { + var maxFloor uint64 + for _, route := range s.snap.IntersectingRoutes(start, end) { + if route.MinWriteTSExclusive > maxFloor { + maxFloor = route.MinWriteTSExclusive + } + } + return maxFloor, maxFloor > 0 +} diff --git a/kv/shard_store.go b/kv/shard_store.go index 032930e54..0be1ed981 100644 --- a/kv/shard_store.go +++ b/kv/shard_store.go @@ -35,9 +35,11 @@ type ShardStore struct { var ( ErrCrossShardMutationBatchNotSupported = errors.New("cross-shard mutation batches are not supported") + ErrRouteCutoverPending = errors.New("route cutover pending") ErrExplicitGroupStagedVisibilityUnresolved = errors.New("explicit group read cannot resolve staged visibility route") ErrReadRouteVersionUnavailable = errors.New("read route version is not locally available") ErrFilesystemPlacementTargetNotFound = errors.New("filesystem placement target group has no routable home slot") + ErrRouteWriteBelowFloor = ErrRouteWriteTimestampTooLow ) // NewShardStore creates a sharded MVCC store wrapper. @@ -258,6 +260,11 @@ func isLinearizableRaftLeader(ctx context.Context, engine raftengine.LeaderView) } func (s *ShardStore) leaderGetAt(ctx context.Context, g *ShardGroup, route distribution.Route, key []byte, ts uint64) ([]byte, error) { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { + return nil, err + } if !isTxnInternalKey(key) { if err := s.maybeResolveTxnLock(ctx, g, key, ts); err != nil { return nil, err @@ -267,6 +274,11 @@ func (s *ShardStore) leaderGetAt(ctx context.Context, g *ShardGroup, route distr } func (s *ShardStore) localGetAt(ctx context.Context, g *ShardGroup, route distribution.Route, key []byte, ts uint64) ([]byte, error) { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { + return nil, err + } if routeHasStagedVisibility(route) { return s.getAtWithStagedVisibility(ctx, g, route, key, ts) } @@ -281,6 +293,197 @@ func routeHasStagedVisibility(route distribution.Route) bool { return route.StagedVisibilityActive && route.MigrationJobID != 0 } +func routeSatisfiesTargetReadiness(route distribution.Route, ready store.TargetStagedReadinessState, catalogVersion uint64) bool { + if !routeRangeIntersects(route.Start, route.End, ready.RouteStart, ready.RouteEnd) { + return false + } + if route.MinWriteTSExclusive < ready.MinWriteTSExclusive { + return false + } + if route.StagedVisibilityActive { + return route.MigrationJobID == ready.MigrationJobID + } + if route.MigrationJobID != 0 { + return false + } + return ready.ExpectedCutoverVersion == 0 || catalogVersion >= ready.ExpectedCutoverVersion +} + +func (s *ShardStore) targetReadyRouteForRange(ctx context.Context, g *ShardGroup, route distribution.Route, start []byte, end []byte) (distribution.Route, error) { + routeStart, routeEnd := readinessRouteRangeForScan(start, end) + return s.targetReadyRouteForRouteRange(ctx, g, route, routeStart, routeEnd) +} + +func (s *ShardStore) verifyTargetReadinessForRange(ctx context.Context, g *ShardGroup, route distribution.Route, start []byte, end []byte) error { + _, err := s.targetReadyRouteForRange(ctx, g, route, start, end) + return err +} + +func (s *ShardStore) targetReadyRouteForRouteRange(ctx context.Context, g *ShardGroup, route distribution.Route, routeStart []byte, routeEnd []byte) (distribution.Route, error) { + routes, err := s.targetReadyRoutesForRouteRange(ctx, g, route, routeStart, routeEnd) + if err != nil { + return route, err + } + if len(routes) == 1 { + return routes[0], nil + } + return route, nil +} + +func (s *ShardStore) targetReadyRoutesForRouteRange(ctx context.Context, g *ShardGroup, route distribution.Route, routeStart []byte, routeEnd []byte) ([]distribution.Route, error) { + if g == nil || g.Store == nil { + return []distribution.Route{route}, nil + } + reader, ok := g.Store.(store.MigrationTargetReadinessReader) + if !ok { + return []distribution.Route{route}, nil + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return nil, errors.WithStack(err) + } + if sourceReadFenceApplies(states, routeStart, routeEnd) { + return nil, errors.WithStack(ErrRouteCutoverPending) + } + applicable := targetReadinessApplicableStates(states, route, routeStart, routeEnd) + if len(applicable) == 0 { + return []distribution.Route{route}, nil + } + + proofRoutes, catalogVersion, ok := s.readinessProofRoutes(route, routeStart, routeEnd) + if !ok { + return nil, errors.WithStack(ErrRouteCutoverPending) + } + if !readinessProofSatisfiesStates(applicable, proofRoutes, route.GroupID, catalogVersion) { + return nil, errors.WithStack(ErrRouteCutoverPending) + } + return proofRoutes, nil +} + +func targetReadinessApplicableStates( + states []store.TargetStagedReadinessState, + route distribution.Route, + routeStart []byte, + routeEnd []byte, +) []store.TargetStagedReadinessState { + applicable := make([]store.TargetStagedReadinessState, 0, len(states)) + for _, ready := range states { + if ready.SourceWriteFence || ready.SourceReadFence || ready.TrackWrites { + continue + } + if targetReadinessAppliesToRoute(route, routeStart, routeEnd, ready) { + applicable = append(applicable, ready) + } + } + return applicable +} + +func sourceReadFenceApplies(states []store.TargetStagedReadinessState, routeStart []byte, routeEnd []byte) bool { + for _, state := range states { + if state.Armed && state.SourceReadFence && routeRangeIntersects(routeStart, routeEnd, state.RouteStart, state.RouteEnd) { + return true + } + } + return false +} + +func readinessProofSatisfiesStates( + states []store.TargetStagedReadinessState, + routes []distribution.Route, + groupID uint64, + catalogVersion uint64, +) bool { + for _, ready := range states { + if !routesSatisfyTargetReadiness(routes, ready, groupID, catalogVersion) { + return false + } + } + return true +} + +func (s *ShardStore) verifyTargetReadinessForRouteRange(ctx context.Context, g *ShardGroup, route distribution.Route, routeStart []byte, routeEnd []byte) error { + _, err := s.targetReadyRouteForRouteRange(ctx, g, route, routeStart, routeEnd) + return err +} + +func (s *ShardStore) readinessProofRoutes(route distribution.Route, routeStart []byte, routeEnd []byte) ([]distribution.Route, uint64, bool) { + if s == nil || s.engine == nil { + return []distribution.Route{route}, 0, true + } + snap, ok := s.engine.Current() + if !ok { + return nil, 0, false + } + routes := snap.IntersectingRoutes(routeStart, routeEnd) + proof := routes[:0] + for _, candidate := range routes { + if candidate.GroupID != route.GroupID { + continue + } + if (route.Start != nil || route.End != nil || route.RouteID != 0) && + !routeRangeIntersects(candidate.Start, candidate.End, route.Start, route.End) { + continue + } + proof = append(proof, candidate) + } + return proof, snap.Version(), len(proof) > 0 +} + +func targetReadinessAppliesToRoute(route distribution.Route, routeStart []byte, routeEnd []byte, ready store.TargetStagedReadinessState) bool { + if !ready.Armed || !routeRangeIntersects(routeStart, routeEnd, ready.RouteStart, ready.RouteEnd) { + return false + } + if route.RouteID != 0 && !routeRangeIntersects(route.Start, route.End, ready.RouteStart, ready.RouteEnd) { + return false + } + return true +} + +func readinessRouteRange(start []byte, end []byte) ([]byte, []byte) { + if routeStart, routeEnd, ok := s3BucketAuxiliaryRouteRange(start); ok && (end == nil || bytes.Equal(end, nextScanCursor(start))) { + return routeStart, routeEnd + } + routeStart := routeKey(start) + if end == nil { + return routeStart, nil + } + routeEnd := routeKey(end) + if bytes.Compare(routeEnd, routeStart) <= 0 { + routeEnd = nextScanCursor(routeStart) + } + return routeStart, routeEnd +} + +func readinessRouteRangeForScan(start []byte, end []byte) ([]byte, []byte) { + if routeStart, routeEnd, ok := s3keys.ManifestScanRouteBounds(start, end); ok { + return routeStart, routeEnd + } + if s3BucketAuxiliaryScanBounds(start, end) { + return s3BucketAuxiliaryScanRouteRange(start, end) + } + if routeStart, routeEnd, ok := fskeys.ChunkScanRouteBounds(start, end); ok { + return routeStart, routeEnd + } + return readinessRouteRange(start, end) +} + +func verifyRouteWriteFloor(route distribution.Route, commitTS uint64) error { + if route.MinWriteTSExclusive == 0 || commitTS == 0 || commitTS > route.MinWriteTSExclusive { + return nil + } + return errors.Wrapf(ErrRouteWriteBelowFloor, "commit_ts %d <= floor %d", commitTS, route.MinWriteTSExclusive) +} + +func routeRangeIntersects(aStart, aEnd, bStart, bEnd []byte) bool { + if aEnd != nil && bytes.Compare(aEnd, bStart) <= 0 { + return false + } + if bEnd != nil && bytes.Compare(bEnd, aStart) <= 0 { + return false + } + return true +} + func (s *ShardStore) routeForExplicitGroupKey(groupID uint64, key []byte) (distribution.Route, error) { fallback := distribution.Route{GroupID: groupID} if s == nil || s.engine == nil { @@ -331,6 +534,7 @@ func latestMVCCVersionAt(ctx context.Context, st store.MVCCStore, key []byte, ts StartKey: key, EndKey: prefixScanEnd(key), MaxCommitTSInclusive: ts, + ReadTS: ts, MaxVersions: 1, MaxScannedBytes: 0, MinCommitTSExclusive: 0, @@ -415,19 +619,20 @@ func (s *ShardStore) ExistsAt(ctx context.Context, key []byte, ts uint64) (bool, // pending.length fast-path during churn. Mirrors LeaderRoutedStore's fix // for codex P1 #796. func (s *ShardStore) CommittedVersionAt(ctx context.Context, key []byte, commitTS uint64) (bool, error) { - g, ok := s.groupForKey(key) + route, g, ok := s.routeAndGroupForKey(key) if !ok || g.Store == nil { return false, nil } + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { + return false, err + } // engineForGroup may be nil in test fixtures that wire ShardStore // without raft; preserve the existing local-only fallback there. engine := engineForGroup(g) if engine == nil { - exists, err := g.Store.CommittedVersionAt(ctx, key, commitTS) - if err != nil { - return false, errors.WithStack(err) - } - return exists, nil + return committedVersionAtForRoute(ctx, g.Store, route, key, commitTS) } if !isLinearizableRaftLeader(ctx, engine) && !tryEngineLinearizableFence(ctx, engine) { // Not the linearizable leader for this group AND the ReadIndex @@ -437,7 +642,23 @@ func (s *ShardStore) CommittedVersionAt(ctx context.Context, key []byte, commitT // serialization. return false, nil } - exists, err := g.Store.CommittedVersionAt(ctx, key, commitTS) + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { + return false, err + } + return committedVersionAtForRoute(ctx, g.Store, route, key, commitTS) +} + +func committedVersionAtForRoute(ctx context.Context, st store.MVCCStore, route distribution.Route, key []byte, commitTS uint64) (bool, error) { + exists, err := st.CommittedVersionAt(ctx, key, commitTS) + if err != nil { + return false, errors.WithStack(err) + } + if exists || !routeHasStagedVisibility(route) { + return exists, nil + } + stagedKey := distribution.MigrationStagedDataKey(route.MigrationJobID, key) + exists, err = st.CommittedVersionAt(ctx, stagedKey, commitTS) if err != nil { return false, errors.WithStack(err) } @@ -575,6 +796,10 @@ func (s *ShardStore) scanExplicitGroupAtWithReadFence(ctx context.Context, group if err != nil { return nil, err } + readinessStart, readinessEnd := readinessRouteRangeForScan(start, end) + if err := s.verifyExplicitGroupRoutesForRange(ctx, groupID, routes, readinessStart, readinessEnd); err != nil { + return nil, err + } routeFilterPresent := routeScanBoundsPresent(routeStart, routeEnd) dedupeByKey := s3BucketAuxiliaryScanBounds(start, end) if !clampToRoutes && !routeFilterPresent { @@ -2158,6 +2383,11 @@ func (s *ShardStore) scanRouteAtDirectionPhysicalLimit( return nil, false, nil } markRouteGroup := shouldMarkRouteGroupOnScan(start, false, nil, nil) + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, start, end) + if err != nil { + return nil, false, err + } if engineForGroup(g) == nil { if routeHasStagedVisibility(route) { @@ -2172,11 +2402,7 @@ func (s *ShardStore) scanRouteAtDirectionPhysicalLimit( } if isLinearizableRaftLeader(ctx, engineForGroup(g)) { - if routeHasStagedVisibility(route) { - kvs, err := s.scanRouteAtLeader(ctx, g, route, start, end, visibleLimit, ts, reverse) - return markScanRouteGroup(kvs, route.GroupID, markRouteGroup), false, err - } - kvs, limitReached, err := s.scanRouteAtLeaderPhysicalLimit(ctx, g, route, start, end, visibleLimit, physicalLimit, ts, reverse) + kvs, limitReached, err := s.scanReadyLeaderPhysicalLimit(ctx, g, route, start, end, visibleLimit, physicalLimit, ts, reverse) return markScanRouteGroup(kvs, route.GroupID, markRouteGroup), limitReached, err } @@ -2185,6 +2411,28 @@ func (s *ShardStore) scanRouteAtDirectionPhysicalLimit( return nil, true, nil } +func (s *ShardStore) scanReadyLeaderPhysicalLimit( + ctx context.Context, + g *ShardGroup, + route distribution.Route, + start []byte, + end []byte, + visibleLimit int, + physicalLimit int, + ts uint64, + reverse bool, +) ([]*store.KVPair, bool, error) { + route, err := s.targetReadyRouteForRange(ctx, g, route, start, end) + if err != nil { + return nil, false, err + } + if routeHasStagedVisibility(route) { + kvs, err := s.scanRouteAtLeader(ctx, g, route, start, end, visibleLimit, ts, reverse) + return kvs, false, err + } + return s.scanRouteAtLeaderPhysicalLimit(ctx, g, route, start, end, visibleLimit, physicalLimit, ts, reverse) +} + func scanLocalPhysicalLimit( ctx context.Context, st store.MVCCStore, @@ -2235,6 +2483,11 @@ func (s *ShardStore) scanRouteLocal( ts uint64, reverse bool, ) ([]*store.KVPair, error) { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, start, end) + if err != nil { + return nil, err + } if routeHasStagedVisibility(route) { return s.scanRouteWithStagedVisibility(ctx, g, route, start, end, limit, ts, reverse) } @@ -2257,6 +2510,11 @@ func (s *ShardStore) scanRouteAtLeaderPhysicalLimit( ts uint64, reverse bool, ) ([]*store.KVPair, bool, error) { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, start, end) + if err != nil { + return nil, false, err + } kvs, limitReached, err := scanLocalPhysicalLimit(ctx, g.Store, start, end, visibleLimit, physicalLimit, ts, reverse) if err != nil { return nil, limitReached, errors.WithStack(err) @@ -2280,10 +2538,12 @@ func (s *ShardStore) scanRouteAtLeader( ts uint64, reverse bool, ) ([]*store.KVPair, error) { - var ( - kvs []*store.KVPair - err error - ) + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, start, end) + if err != nil { + return nil, err + } + var kvs []*store.KVPair switch { case routeHasStagedVisibility(route): kvs, err = s.scanRouteWithStagedVisibility(ctx, g, route, start, end, limit, ts, reverse) @@ -2982,9 +3242,14 @@ func clampScanEnd(end []byte, routeEnd []byte) []byte { func (s *ShardStore) PutAt(ctx context.Context, key []byte, value []byte, commitTS uint64, expireAt uint64) error { route, g, ok := s.routeAndGroupForKey(key) - if !ok || g.Store == nil { + if !ok || g == nil || g.Store == nil { return store.ErrNotSupported } + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { + return err + } if err := ensureRouteWriteTimestampFloor(route, key, commitTS); err != nil { return err } @@ -2996,9 +3261,14 @@ func (s *ShardStore) PutAt(ctx context.Context, key []byte, value []byte, commit func (s *ShardStore) DeleteAt(ctx context.Context, key []byte, commitTS uint64) error { route, g, ok := s.routeAndGroupForKey(key) - if !ok || g.Store == nil { + if !ok || g == nil || g.Store == nil { return store.ErrNotSupported } + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { + return err + } if err := ensureRouteWriteTimestampFloor(route, key, commitTS); err != nil { return err } @@ -3010,9 +3280,14 @@ func (s *ShardStore) DeleteAt(ctx context.Context, key []byte, commitTS uint64) func (s *ShardStore) PutWithTTLAt(ctx context.Context, key []byte, value []byte, commitTS uint64, expireAt uint64) error { route, g, ok := s.routeAndGroupForKey(key) - if !ok || g.Store == nil { + if !ok || g == nil || g.Store == nil { return store.ErrNotSupported } + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { + return err + } if err := ensureRouteWriteTimestampFloor(route, key, commitTS); err != nil { return err } @@ -3024,9 +3299,14 @@ func (s *ShardStore) PutWithTTLAt(ctx context.Context, key []byte, value []byte, func (s *ShardStore) ExpireAt(ctx context.Context, key []byte, expireAt uint64, commitTS uint64) error { route, g, ok := s.routeAndGroupForKey(key) - if !ok || g.Store == nil { + if !ok || g == nil || g.Store == nil { return store.ErrNotSupported } + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { + return err + } if err := ensureRouteWriteTimestampFloor(route, key, commitTS); err != nil { return err } @@ -3075,6 +3355,11 @@ func (s *ShardStore) LatestCommitTSWithReadFence(ctx context.Context, key []byte } func (s *ShardStore) localLatestCommitTS(ctx context.Context, g *ShardGroup, route distribution.Route, key []byte) (uint64, bool, error) { + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { + return 0, false, err + } liveTS, liveExists, err := g.Store.LatestCommitTS(ctx, key) if err != nil { return 0, false, errors.WithStack(err) @@ -3771,7 +4056,7 @@ func (s *ShardStore) ApplyMutations(ctx context.Context, mutations []*store.KVPa if err != nil || group == nil { return err } - if err := s.ensureMutationWriteTimestampFloors(mutations, commitTS); err != nil { + if err := s.verifyMutationRoutes(ctx, mutations, readKeys, commitTS); err != nil { return err } readKeys = s.readKeysWithStagedVisibilityAliases(group, readKeys) @@ -3779,6 +4064,40 @@ func (s *ShardStore) ApplyMutations(ctx context.Context, mutations []*store.KVPa return errors.WithStack(group.Store.ApplyMutations(ctx, mutations, readKeys, startTS, commitTS)) } +func (s *ShardStore) verifyMutationRoutes(ctx context.Context, mutations []*store.KVPairMutation, readKeys [][]byte, commitTS uint64) error { + for _, mut := range mutations { + if err := s.verifyMutationWriteRoute(ctx, mut.Key, commitTS); err != nil { + return err + } + } + for _, key := range readKeys { + route, g, ok := s.routeAndGroupForKey(key) + if !ok || g == nil || g.Store == nil { + return store.ErrNotSupported + } + if err := s.verifyTargetReadinessForRange(ctx, g, route, key, nextScanCursor(key)); err != nil { + return err + } + } + return nil +} + +func (s *ShardStore) verifyMutationWriteRoute(ctx context.Context, key []byte, commitTS uint64) error { + route, g, ok := s.routeAndGroupForKey(key) + if !ok || g == nil || g.Store == nil { + return store.ErrNotSupported + } + var err error + route, err = s.targetReadyRouteForRange(ctx, g, route, key, nextScanCursor(key)) + if err != nil { + return err + } + if err := ensureRouteWriteTimestampFloor(route, key, commitTS); err != nil { + return err + } + return s.ensureS3BucketAuxiliaryWriteTimestampFloor(key, commitTS) +} + // ApplyMutationsRaft is the raft-apply variant; see store.MVCCStore for the // durability contract. Only the FSM may call this method. func (s *ShardStore) ApplyMutationsRaft(ctx context.Context, mutations []*store.KVPairMutation, readKeys [][]byte, startTS, commitTS uint64) error { @@ -3786,9 +4105,6 @@ func (s *ShardStore) ApplyMutationsRaft(ctx context.Context, mutations []*store. if err != nil || group == nil { return err } - if err := s.ensureMutationWriteTimestampFloors(mutations, commitTS); err != nil { - return err - } readKeys = s.readKeysWithStagedVisibilityAliases(group, readKeys) readKeys = s.readKeysWithStagedVisibilityMutationAliases(group, readKeys, mutations) return errors.WithStack(group.Store.ApplyMutationsRaft(ctx, mutations, readKeys, startTS, commitTS)) @@ -3802,9 +4118,6 @@ func (s *ShardStore) ApplyMutationsRaftAt(ctx context.Context, mutations []*stor if err != nil || group == nil { return err } - if err := s.ensureMutationWriteTimestampFloors(mutations, commitTS); err != nil { - return err - } readKeys = s.readKeysWithStagedVisibilityAliases(group, readKeys) readKeys = s.readKeysWithStagedVisibilityMutationAliases(group, readKeys, mutations) return errors.WithStack(group.Store.ApplyMutationsRaftAt(ctx, mutations, readKeys, startTS, commitTS, appliedIndex)) @@ -3817,28 +4130,6 @@ func ensureRouteWriteTimestampFloor(route distribution.Route, key []byte, commit return errors.Wrapf(ErrRouteWriteTimestampTooLow, "key %q routeKey %q commit_ts=%d floor=%d", key, routeKey(key), commitTS, route.MinWriteTSExclusive) } -func (s *ShardStore) ensureMutationWriteTimestampFloors(mutations []*store.KVPairMutation, commitTS uint64) error { - if commitTS == 0 { - return nil - } - for _, mut := range mutations { - if mut == nil || len(mut.Key) == 0 || isTxnInternalKey(mut.Key) { - continue - } - route, _, ok := s.routeAndGroupForKey(mut.Key) - if !ok { - return store.ErrNotSupported - } - if err := ensureRouteWriteTimestampFloor(route, mut.Key, commitTS); err != nil { - return err - } - if err := s.ensureS3BucketAuxiliaryWriteTimestampFloor(mut.Key, commitTS); err != nil { - return err - } - } - return nil -} - func (s *ShardStore) ensureS3BucketAuxiliaryWriteTimestampFloor(key []byte, commitTS uint64) error { if s == nil || s.engine == nil || commitTS == 0 { return nil @@ -3935,70 +4226,22 @@ func (s *ShardStore) resolveSingleShardGroup(mutations []*store.KVPairMutation) // DeletePrefixAt applies a prefix delete to every shard in the store. func (s *ShardStore) DeletePrefixAt(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS uint64) error { - if err := s.ensurePrefixWriteTimestampFloors(prefix, commitTS); err != nil { + routes, err := s.verifyPrefixDeleteRoutes(ctx, prefix, commitTS) + if err != nil { return err } - for _, g := range s.groups { - if g == nil || g.Store == nil { - continue - } - if err := g.Store.DeletePrefixAt(ctx, prefix, excludePrefix, commitTS); err != nil { - return errors.WithStack(err) - } - } - for _, del := range s.stagedVisibilityPrefixDeletes(prefix, excludePrefix) { - if err := del.group.Store.DeletePrefixAt(ctx, del.prefix, del.excludePrefix, commitTS); err != nil { - return errors.WithStack(err) - } - } - return nil -} - -type stagedVisibilityPrefixDelete struct { - group *ShardGroup - prefix []byte - excludePrefix []byte -} - -func (s *ShardStore) stagedVisibilityPrefixDeletes(prefix []byte, excludePrefix []byte) []stagedVisibilityPrefixDelete { - if s == nil || s.engine == nil { - return nil - } - start, end := routePrefixRange(prefix) - routes := s.engine.GetIntersectingRoutes(start, end) - out := make([]stagedVisibilityPrefixDelete, 0, len(routes)) - seen := make(map[string]struct{}, len(routes)) - for _, route := range routes { - if !routeHasStagedVisibility(route) { - continue - } - g := s.groups[route.GroupID] + for groupID, g := range s.groups { if g == nil || g.Store == nil { continue } - stagedPrefix := distribution.MigrationStagedDataKey(route.MigrationJobID, prefix) - var stagedExclude []byte - if excludePrefix != nil { - stagedExclude = distribution.MigrationStagedDataKey(route.MigrationJobID, excludePrefix) + deleteStagedPrefix := func(stagedPrefix []byte, stagedExcludePrefix []byte) error { + return g.Store.DeletePrefixAt(ctx, stagedPrefix, stagedExcludePrefix, commitTS) } - dedupeKey := string(stagedPrefix) + "\x00" + string(stagedExclude) - if _, ok := seen[dedupeKey]; ok { - continue + if err := deleteStagedVisibilityPrefixes(routesForGroupID(routes, groupID), prefix, excludePrefix, deleteStagedPrefix); err != nil { + return err } - seen[dedupeKey] = struct{}{} - out = append(out, stagedVisibilityPrefixDelete{group: g, prefix: stagedPrefix, excludePrefix: stagedExclude}) - } - return out -} - -func (s *ShardStore) ensurePrefixWriteTimestampFloors(prefix []byte, commitTS uint64) error { - if s == nil || s.engine == nil || commitTS == 0 { - return nil - } - start, end := routePrefixRange(prefix) - for _, route := range s.engine.GetIntersectingRoutes(start, end) { - if route.MinWriteTSExclusive != 0 && commitTS <= route.MinWriteTSExclusive { - return errors.Wrapf(ErrRouteWriteTimestampTooLow, "prefix %q route range [%q,%q) commit_ts=%d floor=%d", prefix, start, end, commitTS, route.MinWriteTSExclusive) + if err := g.Store.DeletePrefixAt(ctx, prefix, excludePrefix, commitTS); err != nil { + return errors.WithStack(err) } } return nil @@ -4006,19 +4249,21 @@ func (s *ShardStore) ensurePrefixWriteTimestampFloors(prefix []byte, commitTS ui // DeletePrefixAtRaft is the raft-apply variant of DeletePrefixAt. func (s *ShardStore) DeletePrefixAtRaft(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS uint64) error { - if err := s.ensurePrefixWriteTimestampFloors(prefix, commitTS); err != nil { + routes, err := s.verifyPrefixDeleteRoutes(ctx, prefix, commitTS) + if err != nil { return err } - for _, g := range s.groups { + for groupID, g := range s.groups { if g == nil || g.Store == nil { continue } - if err := g.Store.DeletePrefixAtRaft(ctx, prefix, excludePrefix, commitTS); err != nil { - return errors.WithStack(err) + deleteStagedPrefix := func(stagedPrefix []byte, stagedExcludePrefix []byte) error { + return g.Store.DeletePrefixAtRaft(ctx, stagedPrefix, stagedExcludePrefix, commitTS) } - } - for _, del := range s.stagedVisibilityPrefixDeletes(prefix, excludePrefix) { - if err := del.group.Store.DeletePrefixAtRaft(ctx, del.prefix, del.excludePrefix, commitTS); err != nil { + if err := deleteStagedVisibilityPrefixes(routesForGroupID(routes, groupID), prefix, excludePrefix, deleteStagedPrefix); err != nil { + return err + } + if err := g.Store.DeletePrefixAtRaft(ctx, prefix, excludePrefix, commitTS); err != nil { return errors.WithStack(err) } } @@ -4040,13 +4285,20 @@ func (s *ShardStore) DeletePrefixAtRaft(ctx context.Context, prefix []byte, excl // is the receiver only when an aggregate (admin / coordinator) path // is replaying a global FLUSHALL, which is not raft-applied. func (s *ShardStore) DeletePrefixAtRaftAt(ctx context.Context, prefix []byte, excludePrefix []byte, commitTS, appliedIndex uint64) error { - if err := s.ensurePrefixWriteTimestampFloors(prefix, commitTS); err != nil { + routes, err := s.verifyPrefixDeleteRoutes(ctx, prefix, commitTS) + if err != nil { return err } - for _, g := range s.groups { + for groupID, g := range s.groups { if g == nil || g.Store == nil { continue } + deleteStagedPrefix := func(stagedPrefix []byte, stagedExcludePrefix []byte) error { + return g.Store.DeletePrefixAtRaftAt(ctx, stagedPrefix, stagedExcludePrefix, commitTS, 0) + } + if err := deleteStagedVisibilityPrefixes(routesForGroupID(routes, groupID), prefix, excludePrefix, deleteStagedPrefix); err != nil { + return err + } // Pass appliedIndex through to every group. In the // single-group call-path (the production raft-apply case) // this is correct: appliedIndex IS that group's raft entry @@ -4061,8 +4313,72 @@ func (s *ShardStore) DeletePrefixAtRaftAt(ctx context.Context, prefix []byte, ex return errors.WithStack(err) } } - for _, del := range s.stagedVisibilityPrefixDeletes(prefix, excludePrefix) { - if err := del.group.Store.DeletePrefixAtRaftAt(ctx, del.prefix, del.excludePrefix, commitTS, appliedIndex); err != nil { + return nil +} + +func (s *ShardStore) verifyPrefixDeleteRoutes(ctx context.Context, prefix []byte, commitTS uint64) ([]distribution.Route, error) { + if s == nil || s.engine == nil { + return nil, nil + } + routeStart, routeEnd := routePrefixRange(prefix) + routes := s.engine.GetIntersectingRoutes(routeStart, routeEnd) + if len(routes) == 0 { + return nil, errors.WithStack(ErrRouteCutoverPending) + } + verified := make([]distribution.Route, 0, len(routes)) + for _, route := range routes { + g, ok := s.groupForID(route.GroupID) + if !ok || g == nil || g.Store == nil { + return nil, store.ErrNotSupported + } + proofRoutes, err := s.verifyPrefixDeleteRoute(ctx, g, route, routeStart, routeEnd, commitTS) + if err != nil { + return nil, err + } + verified = append(verified, proofRoutes...) + } + return verified, nil +} + +func (s *ShardStore) verifyPrefixDeleteRoute(ctx context.Context, g *ShardGroup, route distribution.Route, routeStart []byte, routeEnd []byte, commitTS uint64) ([]distribution.Route, error) { + proofRoutes, err := s.targetReadyRoutesForRouteRange(ctx, g, route, routeStart, routeEnd) + if err != nil { + return nil, err + } + for _, proofRoute := range proofRoutes { + if err := verifyRouteWriteFloor(proofRoute, commitTS); err != nil { + return nil, err + } + } + return proofRoutes, nil +} + +func routesForGroupID(routes []distribution.Route, groupID uint64) []distribution.Route { + out := make([]distribution.Route, 0, len(routes)) + for _, route := range routes { + if route.GroupID == groupID { + out = append(out, route) + } + } + return out +} + +func deleteStagedVisibilityPrefixes(routes []distribution.Route, prefix []byte, excludePrefix []byte, deletePrefix func([]byte, []byte) error) error { + seen := make(map[uint64]struct{}) + for _, route := range routes { + if !routeHasStagedVisibility(route) { + continue + } + if _, ok := seen[route.MigrationJobID]; ok { + continue + } + seen[route.MigrationJobID] = struct{}{} + stagedPrefix := distribution.MigrationStagedDataKey(route.MigrationJobID, prefix) + var stagedExcludePrefix []byte + if len(excludePrefix) > 0 { + stagedExcludePrefix = distribution.MigrationStagedDataKey(route.MigrationJobID, excludePrefix) + } + if err := deletePrefix(stagedPrefix, stagedExcludePrefix); err != nil { return errors.WithStack(err) } } @@ -4254,6 +4570,19 @@ func (s *ShardStore) groupForKey(key []byte) (*ShardGroup, bool) { return g, ok } +func (s *ShardStore) verifyExplicitGroupRoutesForRange(ctx context.Context, groupID uint64, routes []distribution.Route, routeStart []byte, routeEnd []byte) error { + g, ok := s.groupForID(groupID) + if !ok || g == nil || g.Store == nil { + return store.ErrNotSupported + } + for _, route := range routes { + if err := s.verifyTargetReadinessForRouteRange(ctx, g, route, routeStart, routeEnd); err != nil { + return err + } + } + return nil +} + func (s *ShardStore) routeAndGroupForKey(key []byte) (distribution.Route, *ShardGroup, bool) { route, g, _, ok := s.routeAndGroupForKeyWithVersion(key) return route, g, ok diff --git a/kv/shard_store_test.go b/kv/shard_store_test.go index 143e5351d..f6ca420ab 100644 --- a/kv/shard_store_test.go +++ b/kv/shard_store_test.go @@ -3,6 +3,7 @@ package kv import ( "bytes" "context" + "errors" "fmt" "sync" "testing" @@ -50,6 +51,84 @@ func newStagedVisibilityShardStore(t *testing.T) (*ShardStore, *ShardGroup) { return NewShardStore(engine, map[uint64]*ShardGroup{1: group}), group } +func applyTargetReadiness(t *testing.T, group *ShardGroup) { + t.Helper() + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) +} + +func applyTargetReadinessState(t *testing.T, group *ShardGroup, state store.TargetStagedReadinessState) { + t.Helper() + writer, ok := group.Store.(store.MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(context.Background(), state)) +} + +func newReadinessShardStore(t *testing.T, route distribution.RouteDescriptor) (*ShardStore, *ShardGroup) { + t.Helper() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{route}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + return NewShardStore(engine, map[uint64]*ShardGroup{route.GroupID: group}), group +} + +type readinessFenceEngine struct { + state raftengine.State + onLinearizableRead func() +} + +func (e *readinessFenceEngine) State() raftengine.State { + if e.state != "" { + return e.state + } + return raftengine.StateFollower +} + +func (e *readinessFenceEngine) Leader() raftengine.LeaderInfo { + return raftengine.LeaderInfo{ID: "leader"} +} + +func (e *readinessFenceEngine) VerifyLeader(context.Context) error { + return nil +} + +func (e *readinessFenceEngine) LinearizableRead(context.Context) (uint64, error) { + if e.onLinearizableRead != nil { + e.onLinearizableRead() + } + return 1, nil +} + +func (e *readinessFenceEngine) Propose(context.Context, []byte) (*raftengine.ProposalResult, error) { + return nil, errors.New("unexpected propose") +} + +func (e *readinessFenceEngine) ProposeAdmin(context.Context, []byte) (*raftengine.ProposalResult, error) { + return nil, errors.New("unexpected propose admin") +} + +func (e *readinessFenceEngine) Status() raftengine.Status { + return raftengine.Status{State: e.State()} +} + +func (e *readinessFenceEngine) Configuration(context.Context) (raftengine.Configuration, error) { + return raftengine.Configuration{}, nil +} + +func (e *readinessFenceEngine) Close() error { + return nil +} + func newStagedVisibilityPebbleShardStore(t *testing.T) (*ShardStore, *ShardGroup) { t.Helper() @@ -100,6 +179,283 @@ func TestShardStoreGetAt_MergesStagedVisibility(t *testing.T) { require.ErrorIs(t, err, store.ErrKeyNotFound) } +func TestShardStoreCommittedVersionAtChecksStagedVisibilityKey(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + rawKey := []byte("k") + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, rawKey), []byte("staged"), 77, 0)) + + landed, err := st.CommittedVersionAt(ctx, rawKey, 77) + require.NoError(t, err) + require.True(t, landed) +} + +func TestShardStoreCommittedVersionAtChecksTargetReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("k"), []byte("live"), 77, 0)) + + landed, err := st.CommittedVersionAt(ctx, []byte("k"), 77) + require.False(t, landed) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreSourceReadFenceRejectsPointRead(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 10, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + MigrationJobID: 10, + MinWriteTSExclusive: 50, + Armed: true, + SourceWriteFence: true, + SourceReadFence: true, + RetentionPinTS: 40, + }) + require.NoError(t, group.Store.PutAt(ctx, []byte("n"), []byte("live"), 60, 0)) + + _, err := st.GetAt(ctx, []byte("n"), 60) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + _, err = st.ScanAt(ctx, []byte("m"), []byte("z"), 60, 0) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreSourceWriteFenceKeepsReadsAvailable(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 10, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + MigrationJobID: 10, + MinWriteTSExclusive: 50, + Armed: true, + SourceWriteFence: true, + RetentionPinTS: 40, + }) + require.NoError(t, group.Store.PutAt(ctx, []byte("n"), []byte("live"), 60, 0)) + + got, err := st.GetAt(ctx, []byte("n"), 60) + require.NoError(t, err) + require.Equal(t, []byte("live"), got) +} + +func TestShardStoreCommittedVersionAtRechecksTargetReadinessAfterFence(t *testing.T) { + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + group.Engine = &readinessFenceEngine{ + onLinearizableRead: func() { + applyTargetReadiness(t, group) + }, + } + require.NoError(t, group.Store.PutAt(ctx, []byte("k"), []byte("live"), 77, 0)) + + landed, err := st.CommittedVersionAt(ctx, []byte("k"), 77) + require.False(t, landed) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreTargetReadinessFailsClosedWithoutDescriptorProof(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadiness(t, group) + + _, err := st.GetAt(ctx, []byte("k"), 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + err = st.PutAt(ctx, []byte("k"), []byte("v"), 120, 0) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreTargetReadinessNormalizesInternalKeyRange(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadiness(t, group) + + itemKey := store.ListItemKey([]byte("b"), 0) + require.NoError(t, group.Store.PutAt(ctx, itemKey, []byte("item"), 120, 0)) + + _, err := st.GetAt(ctx, itemKey, 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreTargetReadinessAcceptsStagedDescriptor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + applyTargetReadiness(t, group) + rawKey := []byte("k") + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, rawKey), []byte("staged"), 120, 0)) + + got, err := st.GetAt(ctx, rawKey, 130) + require.NoError(t, err) + require.Equal(t, []byte("staged"), got) +} + +func TestShardStoreTargetReadinessAcceptsClearedDescriptorAndRetainsFloor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("k"), []byte("live"), 120, 0)) + + got, err := st.GetAt(ctx, []byte("k"), 130) + require.NoError(t, err) + require.Equal(t, []byte("live"), got) + + err = st.PutAt(ctx, []byte("k"), []byte("low"), 100, 0) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + err = st.ExpireAt(ctx, []byte("k"), 200, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + err = st.ApplyMutations(ctx, []*store.KVPairMutation{{ + Op: store.OpTypePut, + Key: []byte("n"), + Value: []byte("low"), + }}, nil, 0, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + err = st.PutAt(ctx, []byte("k"), []byte("ok"), 101, 0) + require.NoError(t, err) + + err = st.ApplyMutations(ctx, []*store.KVPairMutation{{ + Op: store.OpTypePut, + Key: []byte("n"), + Value: []byte("ok"), + }}, nil, 0, 101) + require.NoError(t, err) +} + +func TestShardStoreTargetReadinessRejectsClearedDescriptorBeforeCutoverVersion(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("k"), []byte("live"), 120, 0)) + + _, err := st.GetAt(ctx, []byte("k"), 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreTargetReadinessUsesSingleCatalogSnapshotProof(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }}, + })) + staleRoute, ok := engine.GetRoute([]byte("b")) + require.True(t, ok) + group := &ShardGroup{Store: store.NewMVCCStore()} + shards := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("live-old"), 80, 0)) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("b")), []byte("staged-new"), 120, 0)) + + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }}, + })) + + got, err := shards.localGetAt(ctx, group, staleRoute, []byte("b"), 130) + require.NoError(t, err) + require.Equal(t, []byte("staged-new"), got) +} + func TestShardStoreGetAt_MergesStagedVisibilityPebbleExactKey(t *testing.T) { t.Parallel() @@ -116,6 +472,224 @@ func TestShardStoreGetAt_MergesStagedVisibilityPebbleExactKey(t *testing.T) { require.Equal(t, []byte("staged-new"), got) } +func TestShardStoreDeletePrefixAtUsesReadinessProofRouteFloor(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + shards := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("live"), 80, 0)) + + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }}, + })) + + err := shards.DeletePrefixAt(ctx, []byte("b"), nil, 100) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + + got, getErr := group.Store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("live"), got) +} + +func TestShardStoreDeletePrefixAtUsesReadinessProofRouteForStagedCleanup(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + shards := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + applyTargetReadiness(t, group) + stagedKey := distribution.MigrationStagedDataKey(9, []byte("b")) + require.NoError(t, group.Store.PutAt(ctx, stagedKey, []byte("staged"), 120, 0)) + + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + }}, + })) + + require.NoError(t, shards.DeletePrefixAt(ctx, []byte("b"), nil, 130)) + + _, err := group.Store.GetAt(ctx, stagedKey, 140) + require.ErrorIs(t, err, store.ErrKeyNotFound) + _, err = shards.GetAt(ctx, []byte("b"), 140) + require.ErrorIs(t, err, store.ErrKeyNotFound) +} + +func TestShardStoreExplicitGroupReadUsesRouteProofForReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 42, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("live"), 120, 0)) + require.NoError(t, group.Store.PutAt(ctx, []byte("c"), []byte("scan"), 121, 0)) + + got, err := st.GetGroupAt(ctx, 42, []byte("b"), 130) + require.NoError(t, err) + require.Equal(t, []byte("live"), got) + + kvs, err := st.ScanGroupAt(ctx, 42, []byte("a"), []byte("z"), 10, 130) + require.NoError(t, err) + require.Len(t, kvs, 2) +} + +func TestShardStoreScanGroupAtChecksEveryCoveredRoute(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 42, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }, + { + RouteID: 2, + Start: []byte("m"), + End: []byte("z"), + GroupID: 42, + State: distribution.RouteStateActive, + }, + }, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{42: group}) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("left"), 120, 0)) + require.NoError(t, group.Store.PutAt(ctx, []byte("x"), []byte("right"), 120, 0)) + + _, err := st.ScanGroupAt(ctx, 42, []byte("a"), []byte("z"), 10, 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreScanGroupAtClampsSingleMatchedRoute(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("m"), + End: []byte("z"), + GroupID: 42, + State: distribution.RouteStateActive, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{42: group}) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("outside"), 7, 0)) + require.NoError(t, group.Store.PutAt(ctx, []byte("n"), []byte("inside"), 7, 0)) + + kvs, err := st.ScanGroupAt(ctx, 42, []byte("a"), []byte("z"), 10, 7) + require.NoError(t, err) + require.Equal(t, []*store.KVPair{{Key: []byte("n"), Value: []byte("inside")}}, kvs) +} + +func TestShardStoreScanGroupAtSplitsCoveredRoutesForStagedVisibility(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 42, + State: distribution.RouteStateActive, + }, + { + RouteID: 2, + Start: []byte("m"), + End: []byte("z"), + GroupID: 42, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 10, + }, + }, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{42: group}) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("left-live"), 110, 0)) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(10, []byte("x")), []byte("right-staged"), 120, 0)) + + kvs, err := st.ScanGroupAt(ctx, 42, []byte("a"), []byte("z"), 10, 130) + require.NoError(t, err) + require.Equal(t, []*store.KVPair{ + {Key: []byte("b"), Value: []byte("left-live")}, + {Key: []byte("x"), Value: []byte("right-staged")}, + }, kvs) +} + func TestShardStoreGetAt_MergesStagedVisibilityForS3BucketAuxiliary(t *testing.T) { t.Parallel() @@ -159,87 +733,229 @@ func TestShardStoreGetAt_MergesStagedVisibilityForS3BucketAuxiliary(t *testing.T } } -func TestShardStoreS3BucketAuxiliaryScanFiltersStagedRoutesToBucketRange(t *testing.T) { +func TestShardStorePhysicalLimitScanChecksTargetReadiness(t *testing.T) { t.Parallel() ctx := context.Background() - const ( - bucketA = "bucket-a" - bucketB = "bucket-b" - bucketC = "bucket-c" - ) - routeStartA := s3keys.RoutePrefixForBucketAnyGeneration(bucketA) - routeEndA := prefixScanEnd(routeStartA) - routeStartB := s3keys.RoutePrefixForBucketAnyGeneration(bucketB) - routeEndB := prefixScanEnd(routeStartB) - require.Less(t, bytes.Compare(routeStartA, routeStartB), 0) + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadiness(t, group) + + userKey := []byte("b") + start := store.ListItemKey(userKey, 0) + end := store.ListItemKey(userKey, 2) + require.NoError(t, group.Store.PutAt(ctx, start, []byte("v"), 120, 0)) + + _, _, err := st.ScanAtPhysicalLimit(ctx, start, end, 10, 10, 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStorePhysicalLimitScanRechecksTargetReadinessAfterFence(t *testing.T) { + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + group.Engine = &readinessFenceEngine{ + state: raftengine.StateLeader, + onLinearizableRead: func() { + applyTargetReadiness(t, group) + }, + } + + userKey := []byte("b") + start := store.ListItemKey(userKey, 0) + end := store.ListItemKey(userKey, 2) + require.NoError(t, group.Store.PutAt(ctx, start, []byte("v"), 120, 0)) + + _, _, err := st.ScanAtPhysicalLimit(ctx, start, end, 10, 10, 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreS3ManifestScanChecksTargetReadinessInRouteSpace(t *testing.T) { + t.Parallel() + + ctx := context.Background() + start := s3keys.ObjectManifestScanStart("bucket-a", 1, "z/") + end := prefixScanEnd(start) + routeStart, routeEnd, ok := s3keys.ManifestScanRouteBounds(start, end) + require.True(t, ok) engine := distribution.NewEngine() require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ - Version: 1, - Routes: []distribution.RouteDescriptor{ - {RouteID: 1, Start: []byte(""), End: routeStartA, GroupID: 1, State: distribution.RouteStateActive}, - {RouteID: 2, Start: routeStartA, End: routeEndA, GroupID: 1, State: distribution.RouteStateActive, StagedVisibilityActive: true, MigrationJobID: 9}, - {RouteID: 3, Start: routeEndA, End: routeStartB, GroupID: 1, State: distribution.RouteStateActive}, - {RouteID: 4, Start: routeStartB, End: routeEndB, GroupID: 1, State: distribution.RouteStateActive, StagedVisibilityActive: true, MigrationJobID: 10}, - {RouteID: 5, Start: routeEndB, End: nil, GroupID: 1, State: distribution.RouteStateActive}, - }, + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: routeStart, + End: routeEnd, + GroupID: 1, + State: distribution.RouteStateActive, + }}, })) group := &ShardGroup{Store: store.NewMVCCStore()} st := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: routeStart, + RouteEnd: routeEnd, + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) - keyA := s3keys.BucketMetaKey(bucketA) - keyB := s3keys.BucketMetaKey(bucketB) - keyC := s3keys.BucketMetaKey(bucketC) - require.NoError(t, group.Store.PutAt(ctx, keyA, []byte("live-a"), 10, 0)) - require.NoError(t, group.Store.PutAt(ctx, keyB, []byte("live-b"), 10, 0)) - require.NoError(t, group.Store.PutAt(ctx, keyC, []byte("live-c"), 10, 0)) + key := s3keys.ObjectManifestKey("bucket-a", 1, "z/object-0") + require.NoError(t, group.Store.PutAt(ctx, key, []byte("manifest"), 120, 0)) - exactA, err := st.ScanAt(ctx, keyA, prefixScanEnd(keyA), 10, 20) - require.NoError(t, err) - require.Equal(t, []*store.KVPair{{Key: keyA, Value: []byte("live-a")}}, exactA) - - all, err := st.ScanAt(ctx, []byte(s3keys.BucketMetaPrefix), prefixScanEnd([]byte(s3keys.BucketMetaPrefix)), 10, 20) - require.NoError(t, err) - require.Equal(t, []*store.KVPair{ - {Key: keyA, Value: []byte("live-a")}, - {Key: keyB, Value: []byte("live-b")}, - {Key: keyC, Value: []byte("live-c")}, - }, all) - - reverseAll, err := st.ReverseScanAt(ctx, []byte(s3keys.BucketMetaPrefix), prefixScanEnd([]byte(s3keys.BucketMetaPrefix)), 10, 20) - require.NoError(t, err) - require.Equal(t, []*store.KVPair{ - {Key: keyC, Value: []byte("live-c")}, - {Key: keyB, Value: []byte("live-b")}, - {Key: keyA, Value: []byte("live-a")}, - }, reverseAll) + _, err := st.ScanAt(ctx, start, end, 10, 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) + _, _, err = st.ScanAtPhysicalLimit(ctx, start, end, 10, 10, 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) } -func TestShardStoreRouteBoundedS3BucketAuxiliaryScanKeepsStagedRows(t *testing.T) { +func TestShardStoreS3BucketAuxiliaryScanChecksSourceReadFenceInRouteSpace(t *testing.T) { t.Parallel() ctx := context.Background() const bucket = "bucket-a" + start := []byte(s3keys.BucketMetaPrefix) + end := prefixScanEnd(start) routeStart := s3keys.RoutePrefixForBucketAnyGeneration(bucket) routeEnd := prefixScanEnd(routeStart) + engine := distribution.NewEngine() require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ - Version: 1, + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: routeStart, + End: routeEnd, + GroupID: 42, + State: distribution.RouteStateActive, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{42: group}) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: routeStart, + RouteEnd: routeEnd, + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + SourceWriteFence: true, + SourceReadFence: true, + RetentionPinTS: 90, + }) + require.NoError(t, group.Store.PutAt(ctx, s3keys.BucketMetaKey(bucket), []byte("meta"), 120, 0)) + + _, err := st.ScanGroupAt(ctx, 42, start, end, 10, 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreFilesystemChunkScanChecksSourceReadFenceInRouteSpace(t *testing.T) { + t.Parallel() + + ctx := context.Background() + const homeSlot uint64 = 11 + const inode uint64 = 22 + start := fskeys.ChunkPrefix(homeSlot, inode) + end := prefixScanEnd(start) + routeStart := fskeys.ChunkRouteKey(homeSlot, inode) + routeEnd := prefixScanEnd(routeStart) + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: routeStart, + End: routeEnd, + GroupID: 42, + State: distribution.RouteStateActive, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{42: group}) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: routeStart, + RouteEnd: routeEnd, + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + SourceWriteFence: true, + SourceReadFence: true, + RetentionPinTS: 90, + }) + require.NoError(t, group.Store.PutAt(ctx, fskeys.ChunkKey(homeSlot, inode, 0), []byte("chunk"), 120, 0)) + + _, err := st.ScanAt(ctx, start, end, 10, 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) + _, err = st.ScanGroupAt(ctx, 42, start, end, 10, 130) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestShardStoreTargetReadinessSkipsNonOverlappingGuardForScannedRoute(t *testing.T) { + t.Parallel() + + ctx := context.Background() + start := s3keys.ObjectManifestScanStart("bucket-a", 1, "a/") + end := s3keys.ObjectManifestScanStart("bucket-a", 1, "z/") + routeStart, routeEnd, ok := s3keys.ManifestScanRouteBounds(start, end) + require.True(t, ok) + routeBoundary, _, ok := s3keys.ManifestScanRouteBounds( + s3keys.ObjectManifestScanStart("bucket-a", 1, "m/"), + end, + ) + require.True(t, ok) + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, Routes: []distribution.RouteDescriptor{ - {RouteID: 1, Start: []byte(""), End: routeStart, GroupID: 1, State: distribution.RouteStateActive}, - {RouteID: 2, Start: routeStart, End: routeEnd, GroupID: 1, State: distribution.RouteStateActive, StagedVisibilityActive: true, MigrationJobID: 9}, - {RouteID: 3, Start: routeEnd, End: nil, GroupID: 1, State: distribution.RouteStateActive}, + { + RouteID: 1, + Start: routeStart, + End: routeBoundary, + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }, + { + RouteID: 2, + Start: routeBoundary, + End: routeEnd, + GroupID: 1, + State: distribution.RouteStateActive, + }, }, })) group := &ShardGroup{Store: store.NewMVCCStore()} st := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) - key := s3keys.BucketMetaKey(bucket) - require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, key), []byte("staged"), 20, 0)) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: routeStart, + RouteEnd: routeBoundary, + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) - kvs, err := st.ScanAtWithReadFence(ctx, []byte(s3keys.BucketMetaPrefix), prefixScanEnd([]byte(s3keys.BucketMetaPrefix)), 10, 25, false, 0, 1, routeStart, routeEnd) + kvs, err := st.ScanAt(ctx, start, end, 10, 130) require.NoError(t, err) - require.Equal(t, []*store.KVPair{{Key: key, Value: []byte("staged")}}, kvs) + require.Empty(t, kvs) } func TestShardStoreRejectsS3BucketAuxiliaryWriteAtMigrationTimestampFloor(t *testing.T) { @@ -293,6 +1009,176 @@ func TestShardStoreRejectsS3BucketAuxiliaryWriteAtMigrationTimestampFloor(t *tes } } +func TestShardStoreExplicitGroupS3ManifestScanUsesRouteProofForReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + start := s3keys.ObjectManifestScanStart("bucket-a", 1, "z/") + end := prefixScanEnd(start) + routeStart, routeEnd, ok := s3keys.ManifestScanRouteBounds(start, end) + require.True(t, ok) + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: routeStart, + End: routeEnd, + GroupID: 42, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{42: group}) + applyTargetReadinessState(t, group, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: routeStart, + RouteEnd: routeEnd, + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + + key := s3keys.ObjectManifestKey("bucket-a", 1, "z/object-0") + require.NoError(t, group.Store.PutAt(ctx, key, []byte("manifest"), 120, 0)) + + kvs, err := st.ScanGroupAt(ctx, 42, start, end, 10, 130) + require.NoError(t, err) + require.Len(t, kvs, 1) + require.Equal(t, key, kvs[0].Key) +} + +func TestShardStoreDeletePrefixAtChecksTargetReadinessBeforeDelete(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("v"), 1, 0)) + + err := st.DeletePrefixAt(ctx, []byte("b"), nil, 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + got, getErr := group.Store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestShardStoreDeletePrefixAtRaftChecksTargetReadinessBeforeDelete(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("v"), 1, 0)) + + err := st.DeletePrefixAtRaft(ctx, []byte("b"), nil, 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + got, getErr := group.Store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestShardStoreDeletePrefixAtRaftAtChecksTargetReadinessBeforeDelete(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newReadinessShardStore(t, distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }) + applyTargetReadiness(t, group) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("v"), 1, 0)) + + err := st.DeletePrefixAtRaftAt(ctx, []byte("b"), nil, 120, 10) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + got, getErr := group.Store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestShardStoreDeletePrefixAtFailsClosedWithoutRouteProof(t *testing.T) { + t.Parallel() + + ctx := context.Background() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("m"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + }}, + })) + group := &ShardGroup{Store: store.NewMVCCStore()} + st := NewShardStore(engine, map[uint64]*ShardGroup{1: group}) + require.NoError(t, group.Store.PutAt(ctx, []byte("b"), []byte("v"), 1, 0)) + + err := st.DeletePrefixAt(ctx, []byte("b"), nil, 120) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + got, getErr := group.Store.GetAt(ctx, []byte("b"), ^uint64(0)) + require.NoError(t, getErr) + require.Equal(t, []byte("v"), got) +} + +func TestShardStoreDeletePrefixAtTombstonesStagedVisibilityRows(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + rawKey := []byte("b") + stagedKey := distribution.MigrationStagedDataKey(9, rawKey) + require.NoError(t, group.Store.PutAt(ctx, stagedKey, []byte("staged"), 120, 0)) + + require.NoError(t, st.DeletePrefixAt(ctx, rawKey, nil, 130)) + + _, err := st.GetAt(ctx, rawKey, 140) + require.ErrorIs(t, err, store.ErrKeyNotFound) + _, err = group.Store.GetAt(ctx, stagedKey, 140) + require.ErrorIs(t, err, store.ErrKeyNotFound) +} + +func TestShardStoreDeletePrefixAtRaftAtAdvancesApplyIndexOnlyOnLiveDelete(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + recording := &deletePrefixIndexRecordingStore{MVCCStore: group.Store} + group.Store = recording + rawKey := []byte("b") + stagedKey := distribution.MigrationStagedDataKey(9, rawKey) + require.NoError(t, group.Store.PutAt(ctx, stagedKey, []byte("staged"), 120, 0)) + + require.NoError(t, st.DeletePrefixAtRaftAt(ctx, rawKey, nil, 130, 88)) + + require.Equal(t, []uint64{0, 88}, recording.deletePrefixIndexes) + require.Equal(t, stagedKey, recording.deletePrefixPrefixes[0]) + require.Equal(t, rawKey, recording.deletePrefixPrefixes[1]) +} + func TestShardStoreStagedVisibilityReadTSCompacted(t *testing.T) { t.Parallel() @@ -394,6 +1280,23 @@ func TestShardStoreScanAt_FiltersStagedShadowRowsFromLiveCandidates(t *testing.T require.Equal(t, []*store.KVPair{{Key: rawKey, Value: []byte("staged")}}, kvs) } +func TestShardStoreStagedVisibilityPreservesCompactionErrors(t *testing.T) { + t.Parallel() + + ctx := context.Background() + st, group := newStagedVisibilityShardStore(t) + retention, ok := group.Store.(store.RetentionController) + require.True(t, ok) + require.NoError(t, group.Store.PutAt(ctx, distribution.MigrationStagedDataKey(9, []byte("b")), []byte("staged"), 10, 0)) + retention.SetMinRetainedTS(20) + + _, err := st.GetAt(ctx, []byte("b"), 15) + require.ErrorIs(t, err, store.ErrReadTSCompacted) + + _, err = st.ScanAt(ctx, []byte("a"), []byte("z"), 10, 15) + require.ErrorIs(t, err, store.ErrReadTSCompacted) +} + func TestShardStoreScanAt_PreservesNonStagedRoutesDuringBroadStagedVisibilityScan(t *testing.T) { t.Parallel() @@ -617,13 +1520,6 @@ func TestShardStoreExplicitGroupReads_MergeStagedVisibility(t *testing.T) { {Key: []byte("b"), Value: []byte("staged-b")}, {Key: []byte("c"), Value: []byte("staged-c")}, }, kvs) - - kvs, err = st.ScanAtWithReadFence(ctx, []byte("a"), []byte("z"), 10, 35, false, 1, 0, []byte("a"), []byte("z")) - require.NoError(t, err) - require.Equal(t, []*store.KVPair{ - {Key: []byte("b"), Value: []byte("staged-b")}, - {Key: []byte("c"), Value: []byte("staged-c")}, - }, kvs) } func TestShardStoreExplicitGroupReads_FailClosedWhenRouteMovedToStagedGroup(t *testing.T) { @@ -878,18 +1774,18 @@ func TestShardStoreRejectsWritesAtMigrationTimestampFloor(t *testing.T) { require.NoError(t, st.PutAt(ctx, []byte("k"), []byte("ok"), 101, 0)) } -func TestShardStoreRaftApplyRejectsMigrationTimestampFloor(t *testing.T) { +func TestShardStoreRaftApplySkipsPointMigrationTimestampFloorButGuardsPrefixDeletes(t *testing.T) { t.Parallel() ctx := context.Background() st, _ := newStagedVisibilityShardStore(t) - require.ErrorIs(t, st.ApplyMutationsRaft(ctx, []*store.KVPairMutation{ + require.NoError(t, st.ApplyMutationsRaft(ctx, []*store.KVPairMutation{ {Op: store.OpTypePut, Key: []byte("k-raft"), Value: []byte("v")}, - }, nil, 90, 100), ErrRouteWriteTimestampTooLow) - require.ErrorIs(t, st.ApplyMutationsRaftAt(ctx, []*store.KVPairMutation{ + }, nil, 90, 100)) + require.NoError(t, st.ApplyMutationsRaftAt(ctx, []*store.KVPairMutation{ {Op: store.OpTypePut, Key: []byte("k-raft-at"), Value: []byte("v")}, - }, nil, 90, 100, 1), ErrRouteWriteTimestampTooLow) + }, nil, 90, 100, 1)) require.ErrorIs(t, st.DeletePrefixAtRaft(ctx, []byte("k-raft"), nil, 100), ErrRouteWriteTimestampTooLow) require.ErrorIs(t, st.DeletePrefixAtRaftAt(ctx, []byte("k-raft-at"), nil, 100, 2), ErrRouteWriteTimestampTooLow) } @@ -1030,26 +1926,6 @@ func TestShardStoreScanAt_RoutesListDeltaScansByUserKey(t *testing.T) { } } -func TestShardStoreScanAt_BroadLegacyListDeltaScansAllRoutes(t *testing.T) { - t.Parallel() - - ctx := context.Background() - st := newTwoRouteShardStoreForScanTest() - deltaValue := store.MarshalListMetaDelta(store.ListMetaDelta{LenDelta: 1}) - leftKey := legacyListMetaDeltaKey([]byte("left-list"), 10, 1) - rightKey := legacyListMetaDeltaKey([]byte("right-list"), 11, 1) - require.NoError(t, st.groups[1].Store.PutAt(ctx, leftKey, deltaValue, 1, 0)) - require.NoError(t, st.groups[2].Store.PutAt(ctx, rightKey, deltaValue, 1, 0)) - - kvs, err := st.ScanAt(ctx, []byte(store.LegacyListMetaDeltaPrefix), store.PrefixScanEnd([]byte(store.LegacyListMetaDeltaPrefix)), 10, ^uint64(0)) - require.NoError(t, err) - require.Len(t, kvs, 2) - require.Equal(t, leftKey, kvs[0].Key) - require.Equal(t, uint64(1), kvs[0].RouteGroupID) - require.Equal(t, rightKey, kvs[1].Key) - require.Equal(t, uint64(2), kvs[1].RouteGroupID) -} - func TestShardStoreScanAt_RoutesWideColumnScansByUserKey(t *testing.T) { t.Parallel() @@ -1136,34 +2012,6 @@ func TestShardStoreScanGroupAt_DoesNotClampRouteMappedRawBounds(t *testing.T) { require.Equal(t, []*store.KVPair{{Key: key, Value: []byte("msg-2")}}, kvs) } -func TestShardStoreScanGroupAt_DeduplicatesRouteMappedSameGroupSplits(t *testing.T) { - t.Parallel() - - ctx := context.Background() - engine := distribution.NewEngine() - routeEnd := prefixScanEnd(sqsGlobalRouteKey) - split := append(bytes.Clone(sqsGlobalRouteKey), 'm') - require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ - Version: 1, - Routes: []distribution.RouteDescriptor{ - {RouteID: 1, Start: sqsGlobalRouteKey, End: split, GroupID: 42, State: distribution.RouteStateActive}, - {RouteID: 2, Start: split, End: routeEnd, GroupID: 42, State: distribution.RouteStateActive}, - }, - })) - groups := map[uint64]*ShardGroup{ - 42: {Store: store.NewMVCCStore()}, - } - st := NewShardStore(engine, groups) - - start := []byte("!sqs|msg|vis|p|") - key := []byte("!sqs|msg|vis|p|orders|partition-2") - require.NoError(t, groups[42].Store.PutAt(ctx, key, []byte("msg-2"), 7, 0)) - - kvs, err := st.ScanGroupAt(ctx, 42, start, prefixScanEnd(start), 10, 7) - require.NoError(t, err) - require.Equal(t, []*store.KVPair{{Key: key, Value: []byte("msg-2")}}, kvs) -} - func TestShardStoreGetGroupAt_UsesExplicitGroup(t *testing.T) { t.Parallel() diff --git a/kv/shard_store_txn_lock_test.go b/kv/shard_store_txn_lock_test.go index 6158f60fd..165a23479 100644 --- a/kv/shard_store_txn_lock_test.go +++ b/kv/shard_store_txn_lock_test.go @@ -114,6 +114,36 @@ func TestShardStoreGetAt_ReturnsTxnLockedForPendingLock(t *testing.T) { require.True(t, errors.Is(err, ErrTxnLocked), "expected ErrTxnLocked, got %v", err) } +func TestShardStoreLeaderGetAtChecksReadinessBeforeResolvingPendingLock(t *testing.T) { + t.Parallel() + + ctx := context.Background() + route := distribution.RouteDescriptor{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + } + shardStore, group := newReadinessShardStore(t, route) + applyTargetReadiness(t, group) + + lockTS := uint64(1) + key := []byte("b") + require.NoError(t, group.Store.PutAt(ctx, txnLockKey(key), encodeTxnLock(txnLock{ + StartTS: lockTS, + PrimaryKey: key, + }), lockTS, 0)) + + resolvedRoute, _, ok := shardStore.routeAndGroupForKey(key) + require.True(t, ok) + _, err := shardStore.leaderGetAt(ctx, group, resolvedRoute, key, ^uint64(0)) + require.ErrorIs(t, err, ErrRouteCutoverPending) + + _, lockErr := group.Store.GetAt(ctx, txnLockKey(key), ^uint64(0)) + require.NoError(t, lockErr) +} + func TestShardStoreGetAt_ReturnsTxnLockedForPendingCrossShardTxn(t *testing.T) { t.Parallel() @@ -671,6 +701,104 @@ func TestShardStoreScanAt_ResolvesCommittedSecondaryLockWithoutCommittedValue(t require.Equal(t, []byte("v2"), kvs[0].Value) } +func TestShardStorePhysicalLimitScanPreservesRouteProofDuringLockResolution(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte("a"), + End: []byte("z"), + GroupID: 1, + State: distribution.RouteStateActive, + MinWriteTSExclusive: 100, + }}, + })) + + st1 := store.NewMVCCStore() + fsm := NewKvFSMWithHLC(st1, NewHLC(), WithRouteHistory(WrapDistributionEngine(engine), 1)) + applyFSM, ok := fsm.(*kvFSM) + require.True(t, ok) + txn := &localApplyTransactional{fsm: applyFSM} + + groups := map[uint64]*ShardGroup{ + 1: {Store: st1, Txn: txn}, + } + shardStore := NewShardStore(engine, groups) + applyTargetReadiness(t, groups[1]) + + startTS := uint64(101) + commitTS := uint64(120) + primaryKey := []byte("b") + secondaryKey := []byte("c") + require.NoError(t, st1.PutAt(ctx, secondaryKey, []byte("old"), 1, 0)) + prepare := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_PREPARE, + Ts: startTS, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: primaryKey, LockTTLms: defaultTxnLockTTLms})}, + {Op: pb.Op_PUT, Key: primaryKey, Value: []byte("vp")}, + {Op: pb.Op_PUT, Key: secondaryKey, Value: []byte("vs")}, + }, + } + _, err := groups[1].Txn.Commit(ctx, []*pb.Request{prepare}) + require.NoError(t, err) + commitPrimary := &pb.Request{ + IsTxn: true, + Phase: pb.Phase_COMMIT, + Ts: startTS, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(txnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: primaryKey, CommitTS: commitTS})}, + {Op: pb.Op_PUT, Key: primaryKey}, + }, + } + _, err = groups[1].Txn.Commit(ctx, []*pb.Request{commitPrimary}) + require.NoError(t, err) + + route, ok := engine.GetRoute(secondaryKey) + require.True(t, ok) + kvs, _, err := shardStore.scanRouteAtLeaderPhysicalLimit(ctx, groups[1], route, []byte(""), nil, 10, 10, commitTS, false) + require.NoError(t, err) + got := map[string]string{} + for _, kvp := range kvs { + got[string(kvp.Key)] = string(kvp.Value) + } + require.Equal(t, "vs", got[string(secondaryKey)]) + + _, lockErr := st1.GetAt(ctx, txnLockKey(secondaryKey), ^uint64(0)) + require.ErrorIs(t, lockErr, store.ErrKeyNotFound) +} + +type localApplyTransactional struct { + fsm *kvFSM +} + +func (t *localApplyTransactional) Commit(ctx context.Context, reqs []*pb.Request) (*TransactionResponse, error) { + for _, req := range reqs { + if err := t.fsm.handleTxnRequest(ctx, req, txnRequestResolveTS(req)); err != nil { + return nil, err + } + } + return &TransactionResponse{}, nil +} + +func (t *localApplyTransactional) Abort(ctx context.Context, reqs []*pb.Request) (*TransactionResponse, error) { + return t.Commit(ctx, reqs) +} + +func txnRequestResolveTS(req *pb.Request) uint64 { + meta, _, err := extractTxnMeta(req.GetMutations()) + if err == nil && meta.CommitTS != 0 { + return meta.CommitTS + } + return req.GetTs() +} + func TestShardStoreScanAt_FiltersTxnInternalKeysWithoutRaft(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index 9954c1785..411d4c563 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -1162,6 +1162,9 @@ func (c *ShardedCoordinator) dispatchDelPrefixBroadcast(ctx context.Context, isT if err := c.rejectWriteTimestampFloorDelPrefixes(elems, ts); err != nil { return nil, err } + if err := c.rejectDelPrefixesWithoutTargetReadinessProof(ctx, elems, ts); err != nil { + return nil, err + } requests := make([]*pb.Request, 0, len(elems)) for _, elem := range elems { requests = append(requests, &pb.Request{ @@ -1280,6 +1283,22 @@ func (c *ShardedCoordinator) rejectWriteFencedDelPrefixes(elems []*Elem[OP]) err return nil } +func (c *ShardedCoordinator) rejectDelPrefixesWithoutTargetReadinessProof(ctx context.Context, elems []*Elem[OP], commitTS uint64) error { + shards, ok := c.store.(*ShardStore) + if !ok || shards == nil { + return nil + } + for _, elem := range elems { + if elem == nil { + continue + } + if _, err := shards.verifyPrefixDeleteRoutes(ctx, elem.Key, commitTS); err != nil { + return errors.Wrapf(err, "prefix %q", elem.Key) + } + } + return nil +} + func (c *ShardedCoordinator) rejectWriteTimestampFloorPointElems(elems []*Elem[OP], commitTS uint64) error { if c == nil || c.engine == nil || commitTS == 0 { return nil @@ -1595,7 +1614,7 @@ type preparedGroup struct { } func (c *ShardedCoordinator) prewriteTxn(ctx context.Context, startTS, commitTS uint64, primaryKey []byte, grouped map[uint64][]*pb.Mutation, gids []uint64, groupedReadKeys map[uint64][][]byte, observedRouteVersion uint64, bypassKeysByGroup map[uint64][][]byte) ([]preparedGroup, error) { - prepareMeta := txnMetaMutation(primaryKey, defaultTxnLockTTLms, 0) + prepareMeta := txnMetaMutation(primaryKey, defaultTxnLockTTLms, commitTS) prepared := make([]preparedGroup, 0, len(gids)) for _, gid := range gids { @@ -2419,6 +2438,9 @@ func (c *ShardedCoordinator) validateReadKeysOnShard(ctx context.Context, gid ui return errors.WithStack(err) } for _, key := range keys { + if err := c.verifyTargetReadinessForReadKeyOnShard(ctx, gid, g, key); err != nil { + return err + } ts, exists, err := c.latestCommitTSForReadKeyOnShard(ctx, gid, g, key) if err != nil { return errors.WithStack(err) @@ -2430,6 +2452,46 @@ func (c *ShardedCoordinator) validateReadKeysOnShard(ctx context.Context, gid ui return nil } +func (c *ShardedCoordinator) verifyTargetReadinessForReadKeyOnShard(ctx context.Context, gid uint64, g *ShardGroup, key []byte) error { + reader, ok := g.Store.(store.MigrationTargetReadinessReader) + if !ok { + return nil + } + states, err := reader.MigrationTargetReadinessStates(ctx) + if err != nil { + return errors.WithStack(err) + } + if len(states) == 0 { + return nil + } + routeStart, routeEnd := readinessRouteRange(key, nextScanCursor(key)) + if sourceReadFenceApplies(states, routeStart, routeEnd) { + return errors.WithStack(ErrRouteCutoverPending) + } + routes, catalogVersion, proof := c.currentShardRoutesForRouteRange(gid, routeStart, routeEnd) + if targetReadinessStatesSatisfied(states, routes, routeStart, routeEnd, gid, catalogVersion, proof) { + return nil + } + return errors.WithStack(ErrRouteCutoverPending) +} + +func (c *ShardedCoordinator) currentShardRoutesForRouteRange(gid uint64, routeStart []byte, routeEnd []byte) ([]distribution.Route, uint64, bool) { + if c == nil || c.engine == nil { + return nil, 0, false + } + snap, ok := c.engine.Current() + if !ok { + return nil, 0, false + } + routes := make([]distribution.Route, 0) + for _, route := range snap.IntersectingRoutes(routeStart, routeEnd) { + if route.GroupID == gid { + routes = append(routes, route) + } + } + return routes, snap.Version(), true +} + func (c *ShardedCoordinator) latestCommitTSForReadKeyOnShard(ctx context.Context, gid uint64, g *ShardGroup, key []byte) (uint64, bool, error) { liveTS, liveExists, err := g.Store.LatestCommitTS(ctx, key) if err != nil { diff --git a/kv/sharded_coordinator_del_prefix_test.go b/kv/sharded_coordinator_del_prefix_test.go index b53583056..91bf9dd89 100644 --- a/kv/sharded_coordinator_del_prefix_test.go +++ b/kv/sharded_coordinator_del_prefix_test.go @@ -584,6 +584,70 @@ func TestShardedCoordinatorRejectsDelPrefixIntersectingWriteFencedRoute(t *testi require.Empty(t, g2Txn.requests) } +func TestShardedCoordinatorRejectsDelPrefixBelowRouteFloorBeforeBroadcast(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateActive, MinWriteTSExclusive: ^uint64(0)}, + }, + })) + + g1Txn := &recordingTransactional{} + g2Txn := &recordingTransactional{} + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {Txn: g1Txn}, + 2: {Txn: g2Txn}, + }, 1, NewHLC(), nil) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: DelPrefix, Key: []byte("z")}}, + }) + require.ErrorIs(t, err, ErrRouteWriteBelowFloor) + require.Empty(t, g1Txn.requests) + require.Empty(t, g2Txn.requests) +} + +func TestShardedCoordinatorRejectsDelPrefixWithoutTargetReadinessProofBeforeBroadcast(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte(""), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateActive}, + }, + })) + + g1Txn := &recordingTransactional{} + g2Txn := &recordingTransactional{} + g1 := &ShardGroup{Txn: g1Txn, Store: store.NewMVCCStore()} + g2 := &ShardGroup{Txn: g2Txn, Store: store.NewMVCCStore()} + applyTargetReadinessState(t, g2, store.TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("m"), + RouteEnd: nil, + ExpectedCutoverVersion: 2, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + }) + groups := map[uint64]*ShardGroup{1: g1, 2: g2} + shardStore := NewShardStore(engine, groups) + coord := NewShardedCoordinator(engine, groups, 1, NewHLC(), shardStore) + + _, err := coord.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{{Op: DelPrefix, Key: []byte("z")}}, + }) + require.ErrorIs(t, err, ErrRouteCutoverPending) + require.Empty(t, g1Txn.requests) + require.Empty(t, g2Txn.requests) +} + func TestShardedCoordinatorRejectsFullRangeDelPrefixWhenRouteIsWriteFenced(t *testing.T) { t.Parallel() diff --git a/kv/sharded_coordinator_txn_test.go b/kv/sharded_coordinator_txn_test.go index 96614f4b3..445b2b6af 100644 --- a/kv/sharded_coordinator_txn_test.go +++ b/kv/sharded_coordinator_txn_test.go @@ -445,8 +445,8 @@ func TestShardedCoordinatorDispatchTxn_CrossShardPhasesAndCommitIndex(t *testing require.Equal(t, []byte("b"), prepareMeta2.PrimaryKey) require.Equal(t, defaultTxnLockTTLms, prepareMeta1.LockTTLms) require.Equal(t, defaultTxnLockTTLms, prepareMeta2.LockTTLms) - require.Zero(t, prepareMeta1.CommitTS) - require.Zero(t, prepareMeta2.CommitTS) + require.Greater(t, prepareMeta1.CommitTS, startTS) + require.Equal(t, prepareMeta1.CommitTS, prepareMeta2.CommitTS) require.Equal(t, pb.Phase_COMMIT, g1Commit.Phase) require.Equal(t, pb.Phase_COMMIT, g2Commit.Phase) @@ -465,7 +465,7 @@ func TestShardedCoordinatorDispatchTxn_CrossShardPhasesAndCommitIndex(t *testing require.Equal(t, []byte("b"), commitMeta2.PrimaryKey) require.Zero(t, commitMeta1.LockTTLms) require.Zero(t, commitMeta2.LockTTLms) - require.Greater(t, commitMeta1.CommitTS, startTS) + require.Equal(t, prepareMeta1.CommitTS, commitMeta1.CommitTS) require.Equal(t, commitMeta1.CommitTS, commitMeta2.CommitTS) require.Equal(t, commitMeta1.CommitTS, binary.BigEndian.Uint64(g1Prepare.Mutations[1].Value[4:12])) require.Equal(t, commitMeta1.CommitTS, binary.BigEndian.Uint64(g2Prepare.Mutations[1].Value[4:12])) @@ -551,6 +551,10 @@ func TestShardedCoordinatorDispatchTxn_UsesProvidedCommitTS(t *testing.T) { commitMeta2 := requestTxnMeta(t, g2Txn.requests[1]) require.Equal(t, commitTS, commitMeta1.CommitTS) require.Equal(t, commitTS, commitMeta2.CommitTS) + prepareMeta1 := requestTxnMeta(t, g1Txn.requests[0]) + prepareMeta2 := requestTxnMeta(t, g2Txn.requests[0]) + require.Equal(t, commitTS, prepareMeta1.CommitTS) + require.Equal(t, commitTS, prepareMeta2.CommitTS) } func TestShardedCoordinatorDispatchTxn_RejectsMigrationTimestampFloor(t *testing.T) { @@ -734,6 +738,7 @@ func TestGroupReadKeysByShardID_RoutesS3BucketAuxiliaryToStagedOwner(t *testing. type stubMVCCStore struct { store.MVCCStore latestTS map[string]uint64 + readiness []store.TargetStagedReadinessState returnErr error } @@ -745,6 +750,10 @@ func (s *stubMVCCStore) LatestCommitTS(_ context.Context, key []byte) (uint64, b return ts, ok, nil } +func (s *stubMVCCStore) MigrationTargetReadinessStates(_ context.Context) ([]store.TargetStagedReadinessState, error) { + return s.readiness, nil +} + // noopEngine satisfies raftengine.Engine for unit tests. // LinearizableRead returns immediately (simulates an already-up-to-date FSM). type noopEngine struct{} @@ -831,6 +840,141 @@ func TestValidateReadOnlyShards_NoConflictWhenKeyUnchanged(t *testing.T) { require.NoError(t, err) } +func TestValidateReadOnlyShards_FailsClosedOnTargetReadiness(t *testing.T) { + t.Parallel() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + }, + { + RouteID: 2, + Start: []byte("m"), + End: nil, + GroupID: 2, + State: distribution.RouteStateActive, + }, + }, + })) + + readOnlyStore := &stubMVCCStore{ + latestTS: map[string]uint64{ + "x": 5, + }, + readiness: []store.TargetStagedReadinessState{ + { + JobID: 7, + RouteStart: []byte("m"), + RouteEnd: nil, + ExpectedCutoverVersion: 2, + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + }, + }, + } + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {}, + 2: {Store: readOnlyStore, Engine: noopEngine{}}, + }, 1, NewHLC(), nil) + + groupedReadKeys := map[uint64][][]byte{ + 2: {[]byte("x")}, + } + err := coord.validateReadOnlyShards(context.Background(), groupedReadKeys, []uint64{1}, 10) + require.Error(t, err) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestValidateReadOnlyShards_FailsClosedOnSourceReadFence(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + engine.UpdateRoute([]byte("a"), []byte("m"), 1) + engine.UpdateRoute([]byte("m"), nil, 2) + readOnlyStore := &stubMVCCStore{ + latestTS: map[string]uint64{"x": 5}, + readiness: []store.TargetStagedReadinessState{{ + JobID: 7, + RouteStart: []byte("m"), + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + SourceWriteFence: true, + SourceReadFence: true, + RetentionPinTS: 10, + }}, + } + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {}, + 2: {Store: readOnlyStore, Engine: noopEngine{}}, + }, 1, NewHLC(), nil) + + err := coord.validateReadOnlyShards(context.Background(), map[uint64][][]byte{ + 2: {[]byte("x")}, + }, []uint64{1}, 10) + require.ErrorIs(t, err, ErrRouteCutoverPending) +} + +func TestValidateReadOnlyShards_ChecksStagedVisibilityLatestCommitTS(t *testing.T) { + t.Parallel() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{ + { + RouteID: 1, + Start: []byte("a"), + End: []byte("m"), + GroupID: 1, + State: distribution.RouteStateActive, + }, + { + RouteID: 2, + Start: []byte("m"), + End: []byte("z"), + GroupID: 2, + State: distribution.RouteStateActive, + StagedVisibilityActive: true, + MigrationJobID: 7, + }, + }, + })) + + readOnlyStore := &stubMVCCStore{ + latestTS: map[string]uint64{ + "x": 5, + string(distribution.MigrationStagedDataKey(7, []byte("x"))): 20, + }, + readiness: []store.TargetStagedReadinessState{ + { + JobID: 7, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 7, + Armed: true, + }, + }, + } + coord := NewShardedCoordinator(engine, map[uint64]*ShardGroup{ + 1: {}, + 2: {Store: readOnlyStore, Engine: noopEngine{}}, + }, 1, NewHLC(), nil) + + groupedReadKeys := map[uint64][][]byte{ + 2: {[]byte("x")}, + } + err := coord.validateReadOnlyShards(context.Background(), groupedReadKeys, []uint64{1}, 10) + require.ErrorIs(t, err, store.ErrWriteConflict) +} + func TestValidateReadOnlyShards_PropagatesStoreError(t *testing.T) { t.Parallel() engine := distribution.NewEngine() diff --git a/kv/txn_codec.go b/kv/txn_codec.go index 679340559..82ef6ac33 100644 --- a/kv/txn_codec.go +++ b/kv/txn_codec.go @@ -20,7 +20,13 @@ const ( txnReadChunkSize = 4096 ) -const txnLockFlagPrimary byte = 0x01 +const ( + txnLockFlagPrimary byte = 0x01 + txnLockFlagCommitTS byte = 0x02 + txnLockKnownFlags = txnLockFlagPrimary | txnLockFlagCommitTS +) + +const txnLockPrimaryOffset = 1 + uint64FieldSize + uint64FieldSize + 1 + uint64FieldSize const ( txnMetaFlagLockTTL byte = 0x01 @@ -225,11 +231,15 @@ type txnLock struct { TTLExpireAt uint64 PrimaryKey []byte IsPrimaryKey bool + CommitTS uint64 } func encodeTxnLock(l txnLock) []byte { - // version(1) + StartTS(8) + TTLExpireAt(8) + flags(1) + primaryLen(8) + primaryKey + // version(1) + StartTS(8) + TTLExpireAt(8) + flags(1) + primaryLen(8) + primaryKey + optional fields size := 1 + uint64FieldSize + uint64FieldSize + 1 + uint64FieldSize + len(l.PrimaryKey) + if l.CommitTS != 0 { + size += uint64FieldSize + } b := make([]byte, size) b[0] = txnLockVersion binary.BigEndian.PutUint64(b[1:], l.StartTS) @@ -238,9 +248,16 @@ func encodeTxnLock(l txnLock) []byte { if l.IsPrimaryKey { flags |= txnLockFlagPrimary } + if l.CommitTS != 0 { + flags |= txnLockFlagCommitTS + } b[17] = flags binary.BigEndian.PutUint64(b[18:], uint64(len(l.PrimaryKey))) - copy(b[26:], l.PrimaryKey) + copy(b[txnLockPrimaryOffset:], l.PrimaryKey) + offset := txnLockPrimaryOffset + len(l.PrimaryKey) + if flags&txnLockFlagCommitTS != 0 { + binary.BigEndian.PutUint64(b[offset:], l.CommitTS) + } return b } @@ -252,32 +269,58 @@ func decodeTxnLock(b []byte) (txnLock, error) { return txnLock{}, errors.WithStack(errors.Newf("txn lock: unsupported version %d", b[0])) } r := bytes.NewReader(b[1:]) - var startTS uint64 - var ttlExpireAt uint64 - if err := binary.Read(r, binary.BigEndian, &startTS); err != nil { - return txnLock{}, errors.WithStack(err) + lock, flags, err := decodeTxnLockRequired(r) + if err != nil { + return txnLock{}, err + } + return decodeTxnLockOptional(lock, flags, r) +} + +func decodeTxnLockRequired(r *bytes.Reader) (txnLock, byte, error) { + startTS, err := readTxnUint64(r, "txn lock: start ts truncated") + if err != nil { + return txnLock{}, 0, err } - if err := binary.Read(r, binary.BigEndian, &ttlExpireAt); err != nil { - return txnLock{}, errors.WithStack(err) + ttlExpireAt, err := readTxnUint64(r, "txn lock: ttl expire at truncated") + if err != nil { + return txnLock{}, 0, err } flags, err := r.ReadByte() if err != nil { - return txnLock{}, errors.WithStack(err) + return txnLock{}, 0, errors.WithStack(err) } - var primaryLen uint64 - if err := binary.Read(r, binary.BigEndian, &primaryLen); err != nil { - return txnLock{}, errors.WithStack(err) + if flags&^txnLockKnownFlags != 0 { + return txnLock{}, 0, errors.WithStack(errors.Newf("txn lock: unsupported flags 0x%02x", flags)) + } + primaryLen, err := readTxnUint64(r, "txn lock: primary key length truncated") + if err != nil { + return txnLock{}, 0, err } primaryKey, err := readTxnField(r, primaryLen, "txn lock: primary key truncated") if err != nil { - return txnLock{}, err + return txnLock{}, 0, err } - return txnLock{ + lock := txnLock{ StartTS: startTS, TTLExpireAt: ttlExpireAt, PrimaryKey: primaryKey, IsPrimaryKey: (flags & txnLockFlagPrimary) != 0, - }, nil + } + return lock, flags, nil +} + +func decodeTxnLockOptional(lock txnLock, flags byte, r *bytes.Reader) (txnLock, error) { + if flags&txnLockFlagCommitTS != 0 { + commitTS, rerr := readTxnUint64(r, "txn lock: commit ts truncated") + if rerr != nil { + return txnLock{}, rerr + } + lock.CommitTS = commitTS + } + if r.Len() != 0 { + return txnLock{}, errors.WithStack(errors.Newf("txn lock: unexpected trailing bytes %d", r.Len())) + } + return lock, nil } type txnIntent struct { diff --git a/kv/txn_codec_test.go b/kv/txn_codec_test.go index 6e1a900a9..a52224e3e 100644 --- a/kv/txn_codec_test.go +++ b/kv/txn_codec_test.go @@ -1,6 +1,7 @@ package kv import ( + "encoding/binary" "testing" "github.com/stretchr/testify/require" @@ -127,3 +128,53 @@ func TestDecodeTxnMetaV2RejectsTrailingBytes(t *testing.T) { _, err := DecodeTxnMeta(encoded) require.ErrorContains(t, err, "unexpected trailing bytes") } + +func TestTxnLockCommitTSRoundTrip(t *testing.T) { + t.Parallel() + + lock := txnLock{ + StartTS: 11, + TTLExpireAt: 99, + PrimaryKey: []byte("primary"), + IsPrimaryKey: true, + CommitTS: 101, + } + encoded := encodeTxnLock(lock) + + require.Equal(t, txnLockVersion, encoded[0]) + require.Equal(t, txnLockFlagPrimary|txnLockFlagCommitTS, encoded[17]) + require.Equal(t, lock.CommitTS, binary.BigEndian.Uint64(encoded[len(encoded)-uint64FieldSize:])) + + decoded, err := decodeTxnLock(encoded) + require.NoError(t, err) + require.Equal(t, lock, decoded) +} + +func TestTxnLockLegacyRoundTripOmitsCommitTS(t *testing.T) { + t.Parallel() + + lock := txnLock{ + StartTS: 11, + TTLExpireAt: 99, + PrimaryKey: []byte("primary"), + IsPrimaryKey: true, + } + encoded := encodeTxnLock(lock) + + require.Equal(t, txnLockFlagPrimary, encoded[17]) + require.Len(t, encoded, 1+uint64FieldSize+uint64FieldSize+1+uint64FieldSize+len(lock.PrimaryKey)) + + decoded, err := decodeTxnLock(encoded) + require.NoError(t, err) + require.Equal(t, lock, decoded) +} + +func TestDecodeTxnLockRejectsInvalidOptionalCommitTS(t *testing.T) { + t.Parallel() + + encoded := encodeTxnLock(txnLock{StartTS: 11, PrimaryKey: []byte("primary")}) + encoded[17] |= txnLockFlagCommitTS + + _, err := decodeTxnLock(encoded) + require.ErrorContains(t, err, "commit ts truncated") +} diff --git a/main.go b/main.go index 6c08a6bd2..010fa753b 100644 --- a/main.go +++ b/main.go @@ -59,6 +59,7 @@ const ( defaultFilesystemRootMode = 0o755 defaultFilesystemPlacementScanInterval = 30 * time.Second defaultFilesystemLeaseReapInterval = 30 * time.Second + splitMigrationCapabilityProbeTimeout = 2 * time.Second ) func newRaftFactory(engineType raftEngineType, coldStartObs raftengine.ColdStartObserver) (raftengine.Factory, error) { @@ -558,12 +559,35 @@ func run() error { startKeyVizFlusher(runCtx, eg, sampler) startKeyVizLeaderTermPublisher(runCtx, eg, sampler, runtimes) startMemoryWatchdog(runCtx, eg, cancel) + splitMigrationConnCache := &kv.GRPCConnCache{} + cleanup.Add(func() { _ = splitMigrationConnCache.Close() }) distServer := adapter.NewDistributionServer( cfg.engine, distCatalog, adapter.WithDistributionCoordinator(coordinate), adapter.WithDistributionActiveTimestampTracker(readTracker), adapter.WithDistributionFilesystemObserver(metricsRegistry.FileSystemObserver()), + adapter.WithDistributionKnownRaftGroups(shardGroupIDs(shardGroups)...), + adapter.WithSplitMigrationCapabilityGate(newSplitMigrationCapabilityGate( + splitMigrationCapabilityPeerSourceForRuntimes(runtimes), + splitMigrationCapabilityProbeTimeout, + nil, + splitMigrationLocalReadinessGate, + )), + adapter.WithSplitPromotionClientFactory(splitPromotionClientFactory( + splitPromotionTargetLeaderResolver(shardGroups), + splitMigrationConnCache, + )), + adapter.WithSplitMigrationClientFactory(splitMigrationClientFactory( + splitMigrationGroupLeaderResolver(shardGroups), + splitMigrationConnCache, + )), + adapter.WithSplitMigrationVoterFactory(splitMigrationVoterFactory( + shardGroups, + splitMigrationConnCache, + )), + adapter.WithSplitJobRunnerReadinessGate(splitMigrationLocalReadinessGate), + adapter.WithSplitJobRunnerReady(), ) startMonitoringCollectors(runCtx, metricsRegistry, runtimes, clock) startFSMCompactorIfEnabled(runCtx, eg, runtimes, readTracker) @@ -929,6 +953,152 @@ func cloneRaftServers(in []raftengine.Server) []raftengine.Server { return append([]raftengine.Server(nil), in...) } +func shardGroupIDs(groups map[uint64]*kv.ShardGroup) []uint64 { + ids := make([]uint64, 0, len(groups)) + for id := range groups { + ids = append(ids, id) + } + slices.Sort(ids) + return ids +} + +type splitMigrationCapabilityPeer struct { + ID string + Address string +} + +type splitMigrationCapabilityPeerSource func(context.Context) ([]splitMigrationCapabilityPeer, error) + +type splitMigrationCapabilityProbe func(context.Context, string) error + +func splitMigrationCapabilityPeerSourceForRuntimes(runtimes []*raftGroupRuntime) splitMigrationCapabilityPeerSource { + return func(ctx context.Context) ([]splitMigrationCapabilityPeer, error) { + if len(runtimes) == 0 { + return nil, errors.New("raft group runtimes are not configured") + } + peers := make([]splitMigrationCapabilityPeer, 0) + seen := make(map[string]struct{}) + for _, rt := range runtimes { + if rt == nil { + return nil, errors.New("raft group runtime is not configured") + } + engine := rt.snapshotEngine() + if engine == nil { + return nil, errors.Errorf("raft group %d engine is not configured", rt.spec.id) + } + cfg, err := engine.Configuration(ctx) + if err != nil { + return nil, errors.Wrapf(err, "raft group %d configuration", rt.spec.id) + } + groupPeers := splitMigrationCapabilityPeersFromConfiguration(cfg) + if len(groupPeers) == 0 { + return nil, errors.Errorf("raft group %d configuration has no split migration capability peers", rt.spec.id) + } + for _, peer := range groupPeers { + key := peer.ID + "\x00" + peer.Address + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + peers = append(peers, peer) + } + } + return peers, nil + } +} + +func splitMigrationCapabilityPeersFromConfiguration(cfg raftengine.Configuration) []splitMigrationCapabilityPeer { + servers := cfg.Servers + if len(servers) == 0 { + return nil + } + peers := make([]splitMigrationCapabilityPeer, 0, len(servers)) + seen := make(map[string]struct{}, len(servers)) + for _, server := range servers { + key := server.ID + "\x00" + server.Address + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + id := server.ID + if id == "" { + id = server.Address + } + peers = append(peers, splitMigrationCapabilityPeer{ + ID: id, + Address: server.Address, + }) + } + return peers +} + +func newSplitMigrationCapabilityGate( + source splitMigrationCapabilityPeerSource, + timeout time.Duration, + probe splitMigrationCapabilityProbe, + localGate adapter.SplitMigrationCapabilityGate, +) adapter.SplitMigrationCapabilityGate { + if probe == nil { + probe = probeSplitMigrationCapabilityPeer + } + return func(ctx context.Context) error { + if localGate != nil { + if err := localGate(ctx); err != nil { + return status.Errorf(codes.FailedPrecondition, "split migration local readiness is not available: %v", err) + } + } + if source == nil { + return status.Error(codes.FailedPrecondition, "split migration capability peers are not configured") + } + peers, err := source(ctx) + if err != nil { + return status.Errorf(codes.FailedPrecondition, "split migration capability peers are not available: %v", err) + } + if len(peers) == 0 { + return status.Error(codes.FailedPrecondition, "split migration capability peers are not configured") + } + for _, peer := range peers { + peerCtx := ctx + cancel := func() {} + if timeout > 0 { + var cancelCtx context.CancelFunc + peerCtx, cancelCtx = context.WithTimeout(ctx, timeout) + cancel = cancelCtx + } + err := probe(peerCtx, peer.Address) + cancel() + if err != nil { + return status.Errorf(codes.FailedPrecondition, "split migration capability peer %s is not ready: %v", peer.ID, err) + } + } + return nil + } +} + +func probeSplitMigrationCapabilityPeer(ctx context.Context, address string) error { + if strings.TrimSpace(address) == "" { + return errors.New("empty split migration capability peer address") + } + conn, err := grpc.NewClient(address, internalutil.GRPCDialOptions()...) + if err != nil { + return errors.Wrapf(err, "dial split migration capability peer %s", address) + } + defer func() { + _ = conn.Close() + }() + resp, err := pb.NewDistributionClient(conn).GetSplitMigrationCapability(ctx, &pb.GetSplitMigrationCapabilityRequest{}) + if err != nil { + return errors.Wrapf(err, "probe split migration capability peer %s", address) + } + if resp == nil || !resp.GetMigrationCapable() { + return errors.New("peer does not advertise split migration capability") + } + if !slices.Contains(resp.GetCapabilities(), adapter.SplitMigrationCapabilityV2) { + return errors.Errorf("peer does not advertise %s", adapter.SplitMigrationCapabilityV2) + } + return nil +} + type runtimeConfig struct { groups []groupSpec defaultGroup uint64 @@ -1789,6 +1959,150 @@ type serversInput struct { encryptionConfChangeInterceptor internalraftadmin.MembershipChangeInterceptor } +type splitPromotionLeaderResolver func(distribution.SplitJob) (string, error) +type splitMigrationLeaderResolver func(uint64) (string, error) + +func splitPromotionTargetLeaderResolver(shardGroups map[uint64]*kv.ShardGroup) splitPromotionLeaderResolver { + return func(job distribution.SplitJob) (string, error) { + sg := shardGroups[job.TargetGroupID] + if sg == nil || sg.Engine == nil { + return "", errors.Wrapf(kv.ErrLeaderNotFound, "split promotion target group %d", job.TargetGroupID) + } + addr := strings.TrimSpace(sg.Engine.Leader().Address) + if addr == "" { + return "", errors.Wrapf(kv.ErrLeaderNotFound, "split promotion target group %d", job.TargetGroupID) + } + return addr, nil + } +} + +func splitMigrationGroupLeaderResolver(shardGroups map[uint64]*kv.ShardGroup) splitMigrationLeaderResolver { + return func(groupID uint64) (string, error) { + sg := shardGroups[groupID] + if sg == nil || sg.Engine == nil { + return "", errors.Wrapf(kv.ErrLeaderNotFound, "split migration group %d", groupID) + } + addr := strings.TrimSpace(sg.Engine.Leader().Address) + if addr == "" { + return "", errors.Wrapf(kv.ErrLeaderNotFound, "split migration group %d", groupID) + } + return addr, nil + } +} + +func splitPromotionClientFactory(resolve splitPromotionLeaderResolver, connCache *kv.GRPCConnCache) adapter.SplitPromotionClientFactory { + return func(_ context.Context, job distribution.SplitJob) (adapter.SplitPromotionClient, error) { + if resolve == nil { + return nil, errors.New("split promotion leader resolver is not configured") + } + if connCache == nil { + return nil, errors.New("split promotion gRPC connection cache is not configured") + } + addr, err := resolve(job) + if err != nil { + return nil, errors.WithStack(err) + } + conn, err := connCache.ConnFor(addr) + if err != nil { + return nil, errors.WithStack(err) + } + return pb.NewInternalClient(conn), nil + } +} + +func splitMigrationClientFactory(resolve splitMigrationLeaderResolver, connCache *kv.GRPCConnCache) adapter.SplitMigrationClientFactory { + return func(_ context.Context, job distribution.SplitJob, sourceGroupID uint64) (adapter.SplitMigrationClient, adapter.SplitMigrationClient, error) { + if resolve == nil { + return nil, nil, errors.New("split migration leader resolver is not configured") + } + if connCache == nil { + return nil, nil, errors.New("split migration gRPC connection cache is not configured") + } + sourceAddr, err := resolve(sourceGroupID) + if err != nil { + return nil, nil, err + } + targetAddr, err := resolve(job.TargetGroupID) + if err != nil { + return nil, nil, errors.WithStack(err) + } + sourceConn, err := connCache.ConnFor(sourceAddr) + if err != nil { + return nil, nil, errors.WithStack(err) + } + targetConn, err := connCache.ConnFor(targetAddr) + if err != nil { + return nil, nil, errors.WithStack(err) + } + return pb.NewInternalClient(sourceConn), pb.NewInternalClient(targetConn), nil + } +} + +func splitMigrationVoterFactory(shardGroups map[uint64]*kv.ShardGroup, connCache *kv.GRPCConnCache) adapter.SplitMigrationVoterFactory { + return func(ctx context.Context, groupID uint64) ([]adapter.SplitMigrationVoter, error) { + sg := shardGroups[groupID] + if sg == nil || sg.Engine == nil { + return nil, errors.Wrapf(kv.ErrLeaderNotFound, "split migration group %d", groupID) + } + if connCache == nil { + return nil, errors.New("split migration voter gRPC connection cache is not configured") + } + cfg, err := sg.Engine.Configuration(ctx) + if err != nil { + return nil, errors.WithStack(err) + } + voters := make([]adapter.SplitMigrationVoter, 0, len(cfg.Servers)) + for _, member := range cfg.Servers { + if member.Suffrage != raftMemberSuffrageVoter { + continue + } + voter, err := splitMigrationVoter(groupID, member, connCache) + if err != nil { + return nil, err + } + voters = append(voters, voter) + } + if len(voters) == 0 { + return nil, errors.Errorf("split migration group %d has no voters", groupID) + } + return voters, nil + } +} + +func splitMigrationVoter(groupID uint64, member raftengine.Server, connCache *kv.GRPCConnCache) (adapter.SplitMigrationVoter, error) { + addr := strings.TrimSpace(member.Address) + if member.ID == "" || addr == "" { + return adapter.SplitMigrationVoter{}, errors.Errorf("split migration group %d has voter with missing id/address", groupID) + } + conn, err := connCache.ConnFor(addr) + if err != nil { + return adapter.SplitMigrationVoter{}, errors.WithStack(err) + } + return adapter.SplitMigrationVoter{ID: member.ID, Address: addr, Client: pb.NewInternalClient(conn)}, nil +} + +func splitMigrationLocalReadinessGate(context.Context) error { + if !adapter.MigrationImportOpcodeEnabledFromEnv() { + return errors.Errorf("%s is disabled", adapter.MigrationImportOpcodeEnv) + } + if !adapter.MigrationPromoteOpcodeEnabledFromEnv() { + return errors.Errorf("%s is disabled", adapter.MigrationPromoteOpcodeEnv) + } + if !adapter.MigrationCleanupOpcodeEnabledFromEnv() { + return errors.Errorf("%s is disabled", adapter.MigrationCleanupOpcodeEnv) + } + return nil +} + +func startDistributionSplitJobRunner(ctx context.Context, eg *errgroup.Group, server *adapter.DistributionServer) { + if eg == nil || server == nil { + return + } + eg.Go(func() error { + return server.RunSplitJobRunner(ctx) + }) +} + // startServersAfterStartupRotation wires up the AdminServer, starts the // per-group Raft listeners needed for quorum traffic, waits for a fresh join // to catch up, prepares the public listeners, waits for any requested startup @@ -1932,6 +2246,7 @@ func startServersAfterStartupRotation(waitRotateOnStartup startupRotationWaiter, } startHLCLeaseRenewal(in.ctx, in.eg, in.coordinate) publicKVGate.markReady() + startDistributionSplitJobRunner(in.ctx, in.eg, in.distServer) if err := runner.startPublicServices(); err != nil { return err } @@ -2008,7 +2323,7 @@ func raftJoinRuntimesReady(ctx context.Context, runtimes []*raftGroupRuntime, lo func configurationContainsMember(configuration raftengine.Configuration, localID string) bool { for _, server := range configuration.Servers { - if server.ID == localID && (server.Suffrage == "learner" || server.Suffrage == "voter") { + if server.ID == localID && (server.Suffrage == "learner" || server.Suffrage == raftMemberSuffrageVoter) { return true } } @@ -2273,8 +2588,17 @@ func (g *startupPublicKVGate) blocked() bool { func startupRotationGatedMethod(fullMethod string) bool { switch fullMethod { case pb.Internal_Forward_FullMethodName, + pb.Internal_ApplyTargetStagedReadiness_FullMethodName, + pb.Internal_ImportRangeVersions_FullMethodName, + pb.Internal_PromoteStagedVersions_FullMethodName, + pb.Internal_CleanupMigration_FullMethodName, + pb.Internal_IssueMigrationTimestamp_FullMethodName, pb.AdminForward_Forward_FullMethodName, pb.Distribution_SplitRange_FullMethodName, + pb.Distribution_StartSplitMigration_FullMethodName, + pb.Distribution_GetSplitMigrationCapability_FullMethodName, + pb.Distribution_AbandonSplitJob_FullMethodName, + pb.Distribution_RetrySplitJob_FullMethodName, pb.RaftAdmin_AddVoter_FullMethodName, pb.RaftAdmin_AddLearner_FullMethodName, pb.RaftAdmin_PromoteLearner_FullMethodName, @@ -2508,6 +2832,7 @@ func startRaftServers( confChangeInterceptor internalraftadmin.MembershipChangeInterceptor, encWiring encryptionWriteWiring, sqsPartitionResolver kv.PartitionResolver, + readTracker *kv.ActiveTimestampTracker, ) error { forwardLogger := slog.Default().With(slog.String("component", "admin")) // extraOptsCap reserves slots for the unary + stream admin interceptor @@ -2546,6 +2871,7 @@ func startRaftServers( adapter.WithInternalStore(rt.store), adapter.WithInternalMigrationProposer(proposerForGroup(rt, shardGroups)), adapter.WithInternalRouteEngine(routeEngine), + adapter.WithInternalActiveTimestampTracker(readTracker), adapter.WithInternalMigrationExportRouting(rt.spec.id, sqsPartitionResolver), )..., )) @@ -2804,7 +3130,7 @@ func distributionCatalogStoreForGroup(runtimes []*raftGroupRuntime, groupID uint continue } if rt.spec.id == groupID { - return distribution.NewCatalogStore(rt.store) + return distribution.NewCatalogStore(rt.store, distribution.WithCatalogRouteDescriptorV2Writes(true)) } } return nil @@ -3018,6 +3344,7 @@ func (r *runtimeServerRunner) startRaftTransport() error { r.encryptionConfChangeInterceptor, r.encWiring, sqsPartitionResolver, + r.readTracker, ); err != nil { return r.startupFailure(err) } diff --git a/main_catalog_test.go b/main_catalog_test.go index 193a98219..57af6a088 100644 --- a/main_catalog_test.go +++ b/main_catalog_test.go @@ -2,11 +2,20 @@ package main import ( "context" + "errors" + "net" "testing" + "time" + "github.com/bootjp/elastickv/adapter" "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/internal/raftengine" + pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) func TestDistributionCatalogGroupID_UsesCatalogKeyRoute(t *testing.T) { @@ -68,3 +77,303 @@ func TestSetupDistributionCatalog_UsesResolvedCatalogGroup(t *testing.T) { require.NoError(t, err) require.NotNil(t, catalog) } + +func TestDistributionCatalogStoreForGroupEnablesRouteDescriptorV2Writes(t *testing.T) { + t.Parallel() + + rt := &raftGroupRuntime{ + spec: groupSpec{id: 5}, + store: store.NewMVCCStore(), + } + t.Cleanup(func() { + rt.Close() + }) + + catalog := distributionCatalogStoreForGroup([]*raftGroupRuntime{rt}, 5) + require.NotNil(t, catalog) + require.True(t, catalog.AllowsRouteDescriptorV2Writes()) +} + +func TestSplitMigrationCapabilityGateChecksAllPeers(t *testing.T) { + t.Parallel() + + peers := []splitMigrationCapabilityPeer{ + {ID: "n1", Address: "10.0.0.11:50051"}, + {ID: "n2", Address: "10.0.0.12:50051"}, + } + var probed []string + gate := newSplitMigrationCapabilityGate(staticSplitMigrationCapabilityPeerSource(peers), time.Second, func(_ context.Context, address string) error { + probed = append(probed, address) + return nil + }, nil) + + require.NoError(t, gate(context.Background())) + require.Equal(t, []string{"10.0.0.11:50051", "10.0.0.12:50051"}, probed) +} + +func TestSplitMigrationCapabilityGateFailsClosedWithoutPeers(t *testing.T) { + t.Parallel() + + gate := newSplitMigrationCapabilityGate(nil, time.Second, func(context.Context, string) error { + t.Fatal("probe must not run without peers") + return nil + }, nil) + + err := gate(context.Background()) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, "peers are not configured") +} + +func TestSplitMigrationCapabilityGateFailsClosedWhenPeerMissingCapability(t *testing.T) { + t.Parallel() + + peers := []splitMigrationCapabilityPeer{ + {ID: "n1", Address: "10.0.0.11:50051"}, + {ID: "n2", Address: "10.0.0.12:50051"}, + } + gate := newSplitMigrationCapabilityGate(staticSplitMigrationCapabilityPeerSource(peers), time.Second, func(_ context.Context, address string) error { + if address == "10.0.0.12:50051" { + return status.Error(codes.Unimplemented, "method not found") + } + return nil + }, nil) + + err := gate(context.Background()) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, "n2") + require.ErrorContains(t, err, "method not found") +} + +func TestSplitMigrationCapabilityGateFailsClosedWhenPeerSourceErrors(t *testing.T) { + t.Parallel() + + errPeerSourceUnavailable := errors.New("configuration unavailable") + gate := newSplitMigrationCapabilityGate(func(context.Context) ([]splitMigrationCapabilityPeer, error) { + return nil, errPeerSourceUnavailable + }, time.Second, func(context.Context, string) error { + t.Fatal("probe must not run when peer source fails") + return nil + }, nil) + + err := gate(context.Background()) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, "peers are not available") +} + +func TestSplitMigrationCapabilityGateFailsClosedWhenLocalReadinessFails(t *testing.T) { + t.Parallel() + + peers := []splitMigrationCapabilityPeer{ + {ID: "n1", Address: "10.0.0.11:50051"}, + } + gate := newSplitMigrationCapabilityGate(staticSplitMigrationCapabilityPeerSource(peers), time.Second, func(context.Context, string) error { + t.Fatal("probe must not run when local readiness is closed") + return nil + }, func(context.Context) error { + return status.Error(codes.FailedPrecondition, "migration opcode disabled") + }) + + err := gate(context.Background()) + require.Error(t, err) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, "local readiness") + require.ErrorContains(t, err, "migration opcode disabled") +} + +func TestSplitMigrationLocalReadinessGateRequiresMigrationOpcodes(t *testing.T) { + t.Setenv(adapter.MigrationImportOpcodeEnv, "") + t.Setenv(adapter.MigrationPromoteOpcodeEnv, "1") + t.Setenv(adapter.MigrationCleanupOpcodeEnv, "1") + err := splitMigrationLocalReadinessGate(context.Background()) + require.Error(t, err) + require.ErrorContains(t, err, adapter.MigrationImportOpcodeEnv) + + t.Setenv(adapter.MigrationImportOpcodeEnv, "1") + t.Setenv(adapter.MigrationPromoteOpcodeEnv, "") + err = splitMigrationLocalReadinessGate(context.Background()) + require.Error(t, err) + require.ErrorContains(t, err, adapter.MigrationPromoteOpcodeEnv) + + t.Setenv(adapter.MigrationPromoteOpcodeEnv, "1") + t.Setenv(adapter.MigrationCleanupOpcodeEnv, "") + err = splitMigrationLocalReadinessGate(context.Background()) + require.Error(t, err) + require.ErrorContains(t, err, adapter.MigrationCleanupOpcodeEnv) + + t.Setenv(adapter.MigrationCleanupOpcodeEnv, "1") + require.NoError(t, splitMigrationLocalReadinessGate(context.Background())) +} + +func TestSplitMigrationCapabilityPeersFromConfigurationUsesCurrentMembers(t *testing.T) { + t.Parallel() + + cfg := raftengine.Configuration{Servers: []raftengine.Server{ + {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50051"}, + {Suffrage: "learner", ID: "n2", Address: "10.0.0.12:50051"}, + {Suffrage: "", ID: "n3", Address: "10.0.0.13:50051"}, + {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50051"}, + }} + + require.Equal(t, []splitMigrationCapabilityPeer{ + {ID: "n1", Address: "10.0.0.11:50051"}, + {ID: "n2", Address: "10.0.0.12:50051"}, + {ID: "n3", Address: "10.0.0.13:50051"}, + }, splitMigrationCapabilityPeersFromConfiguration(cfg)) +} + +func TestSplitMigrationCapabilityPeerSourceForRuntimesChecksAllGroups(t *testing.T) { + t.Parallel() + + runtimes := []*raftGroupRuntime{ + { + spec: groupSpec{id: 1}, + engine: capabilityConfigEngine{cfg: raftengine.Configuration{Servers: []raftengine.Server{ + {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50051"}, + {Suffrage: "learner", ID: "n2", Address: "10.0.0.12:50051"}, + }}}, + }, + { + spec: groupSpec{id: 2}, + engine: capabilityConfigEngine{cfg: raftengine.Configuration{Servers: []raftengine.Server{ + {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50051"}, + {Suffrage: "voter", ID: "n3", Address: "10.0.0.13:50051"}, + }}}, + }, + } + + peers, err := splitMigrationCapabilityPeerSourceForRuntimes(runtimes)(context.Background()) + require.NoError(t, err) + require.Equal(t, []splitMigrationCapabilityPeer{ + {ID: "n1", Address: "10.0.0.11:50051"}, + {ID: "n2", Address: "10.0.0.12:50051"}, + {ID: "n3", Address: "10.0.0.13:50051"}, + }, peers) +} + +func TestSplitMigrationCapabilityPeerSourceForRuntimesFailsClosedOnEmptyGroup(t *testing.T) { + t.Parallel() + + runtimes := []*raftGroupRuntime{ + { + spec: groupSpec{id: 1}, + engine: capabilityConfigEngine{cfg: raftengine.Configuration{Servers: []raftengine.Server{ + {Suffrage: "voter", ID: "n1", Address: "10.0.0.11:50051"}, + }}}, + }, + { + spec: groupSpec{id: 2}, + engine: capabilityConfigEngine{cfg: raftengine.Configuration{}}, + }, + } + + peers, err := splitMigrationCapabilityPeerSourceForRuntimes(runtimes)(context.Background()) + require.Error(t, err) + require.Nil(t, peers) + require.ErrorContains(t, err, "raft group 2") + require.ErrorContains(t, err, "no split migration capability peers") +} + +func TestProbeSplitMigrationCapabilityPeerRequiresDocumentedToken(t *testing.T) { + t.Parallel() + + addr := startSplitMigrationCapabilityTestServer(t, &pb.GetSplitMigrationCapabilityResponse{ + MigrationCapable: true, + Capabilities: []string{"split_migration_v2"}, + }) + + err := probeSplitMigrationCapabilityPeer(context.Background(), addr) + require.Error(t, err) + require.ErrorContains(t, err, adapter.SplitMigrationCapabilityV2) +} + +func TestProbeSplitMigrationCapabilityPeerAcceptsDocumentedToken(t *testing.T) { + t.Parallel() + + addr := startSplitMigrationCapabilityTestServer(t, &pb.GetSplitMigrationCapabilityResponse{ + MigrationCapable: true, + Capabilities: []string{adapter.SplitMigrationCapabilityV2}, + }) + + require.NoError(t, probeSplitMigrationCapabilityPeer(context.Background(), addr)) +} + +func staticSplitMigrationCapabilityPeerSource(peers []splitMigrationCapabilityPeer) splitMigrationCapabilityPeerSource { + return func(context.Context) ([]splitMigrationCapabilityPeer, error) { + out := make([]splitMigrationCapabilityPeer, len(peers)) + copy(out, peers) + return out, nil + } +} + +type splitMigrationCapabilityTestServer struct { + pb.UnimplementedDistributionServer + resp *pb.GetSplitMigrationCapabilityResponse +} + +func (s *splitMigrationCapabilityTestServer) GetSplitMigrationCapability(context.Context, *pb.GetSplitMigrationCapabilityRequest) (*pb.GetSplitMigrationCapabilityResponse, error) { + return s.resp, nil +} + +func startSplitMigrationCapabilityTestServer(t *testing.T, resp *pb.GetSplitMigrationCapabilityResponse) string { + t.Helper() + + var listenConfig net.ListenConfig + listener, err := listenConfig.Listen(context.Background(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + grpcServer := grpc.NewServer() + pb.RegisterDistributionServer(grpcServer, &splitMigrationCapabilityTestServer{resp: resp}) + done := make(chan struct{}) + go func() { + defer close(done) + _ = grpcServer.Serve(listener) + }() + t.Cleanup(func() { + grpcServer.Stop() + <-done + }) + return listener.Addr().String() +} + +type capabilityConfigEngine struct { + cfg raftengine.Configuration + leader raftengine.LeaderInfo +} + +func (e capabilityConfigEngine) Propose(context.Context, []byte) (*raftengine.ProposalResult, error) { + return nil, nil +} + +func (e capabilityConfigEngine) ProposeAdmin(context.Context, []byte) (*raftengine.ProposalResult, error) { + return nil, nil +} + +func (e capabilityConfigEngine) State() raftengine.State { + return raftengine.StateFollower +} + +func (e capabilityConfigEngine) Leader() raftengine.LeaderInfo { + return e.leader +} + +func (e capabilityConfigEngine) VerifyLeader(context.Context) error { + return nil +} + +func (e capabilityConfigEngine) LinearizableRead(context.Context) (uint64, error) { + return 0, nil +} + +func (e capabilityConfigEngine) Status() raftengine.Status { + return raftengine.Status{} +} + +func (e capabilityConfigEngine) Configuration(context.Context) (raftengine.Configuration, error) { + return e.cfg, nil +} + +func (e capabilityConfigEngine) Close() error { + return nil +} diff --git a/main_encryption_rotate_on_startup_test.go b/main_encryption_rotate_on_startup_test.go index 5e16348ad..76b998160 100644 --- a/main_encryption_rotate_on_startup_test.go +++ b/main_encryption_rotate_on_startup_test.go @@ -419,8 +419,17 @@ func TestStartupPublicKVGate_BlocksMutatorsUntilReady(t *testing.T) { pb.RawKV_RawGet_FullMethodName, pb.TransactionalKV_Get_FullMethodName, pb.Internal_Forward_FullMethodName, + pb.Internal_ApplyTargetStagedReadiness_FullMethodName, + pb.Internal_ImportRangeVersions_FullMethodName, + pb.Internal_PromoteStagedVersions_FullMethodName, + pb.Internal_CleanupMigration_FullMethodName, + pb.Internal_IssueMigrationTimestamp_FullMethodName, pb.AdminForward_Forward_FullMethodName, pb.Distribution_SplitRange_FullMethodName, + pb.Distribution_StartSplitMigration_FullMethodName, + pb.Distribution_GetSplitMigrationCapability_FullMethodName, + pb.Distribution_AbandonSplitJob_FullMethodName, + pb.Distribution_RetrySplitJob_FullMethodName, pb.RaftAdmin_AddVoter_FullMethodName, pb.RaftAdmin_AddLearner_FullMethodName, pb.RaftAdmin_PromoteLearner_FullMethodName, diff --git a/main_leader_balance.go b/main_leader_balance.go index b7fa14751..c14013630 100644 --- a/main_leader_balance.go +++ b/main_leader_balance.go @@ -24,6 +24,7 @@ const ( defaultLeaderBalanceImbalanceThreshold = 2 defaultLeaderBalanceMaxTargetLag = 1024 defaultLeaderBalanceRPCTimeout = 5 * time.Second + raftMemberSuffrageVoter = "voter" ) type leaderBalanceConfig struct { @@ -328,7 +329,7 @@ func observeLeaderBalance(ctx context.Context, runtimes []*raftGroupRuntime, loc localLeader: status.State == raftengine.StateLeader && status.Leader.ID == localNodeID, } for _, server := range cfg.Servers { - if server.Suffrage != "voter" { + if server.Suffrage != raftMemberSuffrageVoter { continue } obs.voters[server.ID] = server diff --git a/main_proposer_for_group_test.go b/main_proposer_for_group_test.go index 017344040..f40f01ee0 100644 --- a/main_proposer_for_group_test.go +++ b/main_proposer_for_group_test.go @@ -3,8 +3,10 @@ package main import ( "testing" + "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/kv" + "github.com/cockroachdb/errors" ) // stubEngineForProposerTest satisfies raftengine.Engine at the @@ -92,3 +94,37 @@ func TestProposerForGroup_FallsBackToEngineWhenShardGroupNil(t *testing.T) { t.Errorf("proposerForGroup did not fall back to rt.engine for nil shardGroup entry; got %T, want raw engine", got) } } + +func TestSplitPromotionTargetLeaderResolverUsesTargetGroup(t *testing.T) { + t.Parallel() + + source := capabilityConfigEngine{leader: raftengine.LeaderInfo{Address: "source:50051"}} + target := capabilityConfigEngine{leader: raftengine.LeaderInfo{Address: "target:50051"}} + resolve := splitPromotionTargetLeaderResolver(map[uint64]*kv.ShardGroup{ + 1: {Engine: source}, + 2: {Engine: target}, + }) + + addr, err := resolve(distribution.SplitJob{ + SplitKey: []byte("m"), + TargetGroupID: 2, + }) + if err != nil { + t.Fatalf("resolve returned error: %v", err) + } + if addr != "target:50051" { + t.Fatalf("resolve returned %q, want target leader address", addr) + } +} + +func TestSplitPromotionTargetLeaderResolverRejectsMissingLeader(t *testing.T) { + t.Parallel() + + resolve := splitPromotionTargetLeaderResolver(map[uint64]*kv.ShardGroup{ + 2: {Engine: capabilityConfigEngine{}}, + }) + _, err := resolve(distribution.SplitJob{TargetGroupID: 2}) + if !errors.Is(err, kv.ErrLeaderNotFound) { + t.Fatalf("resolve error = %v, want ErrLeaderNotFound", err) + } +} diff --git a/proto/distribution.pb.go b/proto/distribution.pb.go index cc0513d5a..cc443c042 100644 --- a/proto/distribution.pb.go +++ b/proto/distribution.pb.go @@ -687,6 +687,7 @@ type SplitJob struct { StartedAtMs int64 `protobuf:"varint,31,opt,name=started_at_ms,json=startedAtMs,proto3" json:"started_at_ms,omitempty"` UpdatedAtMs int64 `protobuf:"varint,32,opt,name=updated_at_ms,json=updatedAtMs,proto3" json:"updated_at_ms,omitempty"` TerminalAtMs int64 `protobuf:"varint,33,opt,name=terminal_at_ms,json=terminalAtMs,proto3" json:"terminal_at_ms,omitempty"` + CapabilityRegressed bool `protobuf:"varint,34,opt,name=capability_regressed,json=capabilityRegressed,proto3" json:"capability_regressed,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -952,6 +953,13 @@ func (x *SplitJob) GetTerminalAtMs() int64 { return 0 } +func (x *SplitJob) GetCapabilityRegressed() bool { + if x != nil { + return x.CapabilityRegressed + } + return false +} + type ListRoutesRequest struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -1288,6 +1296,94 @@ func (x *StartSplitMigrationResponse) GetJobId() uint64 { return 0 } +type GetSplitMigrationCapabilityRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetSplitMigrationCapabilityRequest) Reset() { + *x = GetSplitMigrationCapabilityRequest{} + mi := &file_distribution_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetSplitMigrationCapabilityRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetSplitMigrationCapabilityRequest) ProtoMessage() {} + +func (x *GetSplitMigrationCapabilityRequest) ProtoReflect() protoreflect.Message { + mi := &file_distribution_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetSplitMigrationCapabilityRequest.ProtoReflect.Descriptor instead. +func (*GetSplitMigrationCapabilityRequest) Descriptor() ([]byte, []int) { + return file_distribution_proto_rawDescGZIP(), []int{13} +} + +type GetSplitMigrationCapabilityResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + MigrationCapable bool `protobuf:"varint,1,opt,name=migration_capable,json=migrationCapable,proto3" json:"migration_capable,omitempty"` + Capabilities []string `protobuf:"bytes,2,rep,name=capabilities,proto3" json:"capabilities,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetSplitMigrationCapabilityResponse) Reset() { + *x = GetSplitMigrationCapabilityResponse{} + mi := &file_distribution_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetSplitMigrationCapabilityResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetSplitMigrationCapabilityResponse) ProtoMessage() {} + +func (x *GetSplitMigrationCapabilityResponse) ProtoReflect() protoreflect.Message { + mi := &file_distribution_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetSplitMigrationCapabilityResponse.ProtoReflect.Descriptor instead. +func (*GetSplitMigrationCapabilityResponse) Descriptor() ([]byte, []int) { + return file_distribution_proto_rawDescGZIP(), []int{14} +} + +func (x *GetSplitMigrationCapabilityResponse) GetMigrationCapable() bool { + if x != nil { + return x.MigrationCapable + } + return false +} + +func (x *GetSplitMigrationCapabilityResponse) GetCapabilities() []string { + if x != nil { + return x.Capabilities + } + return nil +} + type GetRouteOwnershipRequest struct { state protoimpl.MessageState `protogen:"open.v1"` Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` @@ -1298,7 +1394,7 @@ type GetRouteOwnershipRequest struct { func (x *GetRouteOwnershipRequest) Reset() { *x = GetRouteOwnershipRequest{} - mi := &file_distribution_proto_msgTypes[13] + mi := &file_distribution_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1310,7 +1406,7 @@ func (x *GetRouteOwnershipRequest) String() string { func (*GetRouteOwnershipRequest) ProtoMessage() {} func (x *GetRouteOwnershipRequest) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[13] + mi := &file_distribution_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1323,7 +1419,7 @@ func (x *GetRouteOwnershipRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetRouteOwnershipRequest.ProtoReflect.Descriptor instead. func (*GetRouteOwnershipRequest) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{13} + return file_distribution_proto_rawDescGZIP(), []int{15} } func (x *GetRouteOwnershipRequest) GetKey() []byte { @@ -1351,7 +1447,7 @@ type GetRouteOwnershipResponse struct { func (x *GetRouteOwnershipResponse) Reset() { *x = GetRouteOwnershipResponse{} - mi := &file_distribution_proto_msgTypes[14] + mi := &file_distribution_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1363,7 +1459,7 @@ func (x *GetRouteOwnershipResponse) String() string { func (*GetRouteOwnershipResponse) ProtoMessage() {} func (x *GetRouteOwnershipResponse) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[14] + mi := &file_distribution_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1376,7 +1472,7 @@ func (x *GetRouteOwnershipResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetRouteOwnershipResponse.ProtoReflect.Descriptor instead. func (*GetRouteOwnershipResponse) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{14} + return file_distribution_proto_rawDescGZIP(), []int{16} } func (x *GetRouteOwnershipResponse) GetRoute() *RouteDescriptor { @@ -1411,7 +1507,7 @@ type GetIntersectingRoutesRequest struct { func (x *GetIntersectingRoutesRequest) Reset() { *x = GetIntersectingRoutesRequest{} - mi := &file_distribution_proto_msgTypes[15] + mi := &file_distribution_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1423,7 +1519,7 @@ func (x *GetIntersectingRoutesRequest) String() string { func (*GetIntersectingRoutesRequest) ProtoMessage() {} func (x *GetIntersectingRoutesRequest) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[15] + mi := &file_distribution_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1436,7 +1532,7 @@ func (x *GetIntersectingRoutesRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetIntersectingRoutesRequest.ProtoReflect.Descriptor instead. func (*GetIntersectingRoutesRequest) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{15} + return file_distribution_proto_rawDescGZIP(), []int{17} } func (x *GetIntersectingRoutesRequest) GetStart() []byte { @@ -1470,7 +1566,7 @@ type GetIntersectingRoutesResponse struct { func (x *GetIntersectingRoutesResponse) Reset() { *x = GetIntersectingRoutesResponse{} - mi := &file_distribution_proto_msgTypes[16] + mi := &file_distribution_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1482,7 +1578,7 @@ func (x *GetIntersectingRoutesResponse) String() string { func (*GetIntersectingRoutesResponse) ProtoMessage() {} func (x *GetIntersectingRoutesResponse) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[16] + mi := &file_distribution_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1495,7 +1591,7 @@ func (x *GetIntersectingRoutesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetIntersectingRoutesResponse.ProtoReflect.Descriptor instead. func (*GetIntersectingRoutesResponse) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{16} + return file_distribution_proto_rawDescGZIP(), []int{18} } func (x *GetIntersectingRoutesResponse) GetRoutes() []*RouteDescriptor { @@ -1521,7 +1617,7 @@ type GetSplitJobRequest struct { func (x *GetSplitJobRequest) Reset() { *x = GetSplitJobRequest{} - mi := &file_distribution_proto_msgTypes[17] + mi := &file_distribution_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1533,7 +1629,7 @@ func (x *GetSplitJobRequest) String() string { func (*GetSplitJobRequest) ProtoMessage() {} func (x *GetSplitJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[17] + mi := &file_distribution_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1546,7 +1642,7 @@ func (x *GetSplitJobRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetSplitJobRequest.ProtoReflect.Descriptor instead. func (*GetSplitJobRequest) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{17} + return file_distribution_proto_rawDescGZIP(), []int{19} } func (x *GetSplitJobRequest) GetJobId() uint64 { @@ -1565,7 +1661,7 @@ type GetSplitJobResponse struct { func (x *GetSplitJobResponse) Reset() { *x = GetSplitJobResponse{} - mi := &file_distribution_proto_msgTypes[18] + mi := &file_distribution_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1577,7 +1673,7 @@ func (x *GetSplitJobResponse) String() string { func (*GetSplitJobResponse) ProtoMessage() {} func (x *GetSplitJobResponse) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[18] + mi := &file_distribution_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1590,7 +1686,7 @@ func (x *GetSplitJobResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetSplitJobResponse.ProtoReflect.Descriptor instead. func (*GetSplitJobResponse) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{18} + return file_distribution_proto_rawDescGZIP(), []int{20} } func (x *GetSplitJobResponse) GetJob() *SplitJob { @@ -1611,7 +1707,7 @@ type ListSplitJobsRequest struct { func (x *ListSplitJobsRequest) Reset() { *x = ListSplitJobsRequest{} - mi := &file_distribution_proto_msgTypes[19] + mi := &file_distribution_proto_msgTypes[21] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1623,7 +1719,7 @@ func (x *ListSplitJobsRequest) String() string { func (*ListSplitJobsRequest) ProtoMessage() {} func (x *ListSplitJobsRequest) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[19] + mi := &file_distribution_proto_msgTypes[21] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1636,7 +1732,7 @@ func (x *ListSplitJobsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ListSplitJobsRequest.ProtoReflect.Descriptor instead. func (*ListSplitJobsRequest) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{19} + return file_distribution_proto_rawDescGZIP(), []int{21} } func (x *ListSplitJobsRequest) GetSinceTerminalAtMs() uint64 { @@ -1670,7 +1766,7 @@ type ListSplitJobsResponse struct { func (x *ListSplitJobsResponse) Reset() { *x = ListSplitJobsResponse{} - mi := &file_distribution_proto_msgTypes[20] + mi := &file_distribution_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1682,7 +1778,7 @@ func (x *ListSplitJobsResponse) String() string { func (*ListSplitJobsResponse) ProtoMessage() {} func (x *ListSplitJobsResponse) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[20] + mi := &file_distribution_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1695,7 +1791,7 @@ func (x *ListSplitJobsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ListSplitJobsResponse.ProtoReflect.Descriptor instead. func (*ListSplitJobsResponse) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{20} + return file_distribution_proto_rawDescGZIP(), []int{22} } func (x *ListSplitJobsResponse) GetJobs() []*SplitJob { @@ -1721,7 +1817,7 @@ type AbandonSplitJobRequest struct { func (x *AbandonSplitJobRequest) Reset() { *x = AbandonSplitJobRequest{} - mi := &file_distribution_proto_msgTypes[21] + mi := &file_distribution_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1733,7 +1829,7 @@ func (x *AbandonSplitJobRequest) String() string { func (*AbandonSplitJobRequest) ProtoMessage() {} func (x *AbandonSplitJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[21] + mi := &file_distribution_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1746,7 +1842,7 @@ func (x *AbandonSplitJobRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AbandonSplitJobRequest.ProtoReflect.Descriptor instead. func (*AbandonSplitJobRequest) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{21} + return file_distribution_proto_rawDescGZIP(), []int{23} } func (x *AbandonSplitJobRequest) GetJobId() uint64 { @@ -1764,7 +1860,7 @@ type AbandonSplitJobResponse struct { func (x *AbandonSplitJobResponse) Reset() { *x = AbandonSplitJobResponse{} - mi := &file_distribution_proto_msgTypes[22] + mi := &file_distribution_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1776,7 +1872,7 @@ func (x *AbandonSplitJobResponse) String() string { func (*AbandonSplitJobResponse) ProtoMessage() {} func (x *AbandonSplitJobResponse) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[22] + mi := &file_distribution_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1789,7 +1885,7 @@ func (x *AbandonSplitJobResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use AbandonSplitJobResponse.ProtoReflect.Descriptor instead. func (*AbandonSplitJobResponse) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{22} + return file_distribution_proto_rawDescGZIP(), []int{24} } type RetrySplitJobRequest struct { @@ -1801,7 +1897,7 @@ type RetrySplitJobRequest struct { func (x *RetrySplitJobRequest) Reset() { *x = RetrySplitJobRequest{} - mi := &file_distribution_proto_msgTypes[23] + mi := &file_distribution_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1813,7 +1909,7 @@ func (x *RetrySplitJobRequest) String() string { func (*RetrySplitJobRequest) ProtoMessage() {} func (x *RetrySplitJobRequest) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[23] + mi := &file_distribution_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1826,7 +1922,7 @@ func (x *RetrySplitJobRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RetrySplitJobRequest.ProtoReflect.Descriptor instead. func (*RetrySplitJobRequest) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{23} + return file_distribution_proto_rawDescGZIP(), []int{25} } func (x *RetrySplitJobRequest) GetJobId() uint64 { @@ -1844,7 +1940,7 @@ type RetrySplitJobResponse struct { func (x *RetrySplitJobResponse) Reset() { *x = RetrySplitJobResponse{} - mi := &file_distribution_proto_msgTypes[24] + mi := &file_distribution_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1856,7 +1952,7 @@ func (x *RetrySplitJobResponse) String() string { func (*RetrySplitJobResponse) ProtoMessage() {} func (x *RetrySplitJobResponse) ProtoReflect() protoreflect.Message { - mi := &file_distribution_proto_msgTypes[24] + mi := &file_distribution_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1869,7 +1965,7 @@ func (x *RetrySplitJobResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RetrySplitJobResponse.ProtoReflect.Descriptor instead. func (*RetrySplitJobResponse) Descriptor() ([]byte, []int) { - return file_distribution_proto_rawDescGZIP(), []int{24} + return file_distribution_proto_rawDescGZIP(), []int{26} } var File_distribution_proto protoreflect.FileDescriptor @@ -1908,7 +2004,7 @@ const file_distribution_proto_rawDesc = "" + "\x04done\x18\x05 \x01(\bR\x04done\x12#\n" + "\rscanned_bytes\x18\x06 \x01(\x04R\fscannedBytes\x12#\n" + "\raccepted_rows\x18\a \x01(\x04R\facceptedRows\x12/\n" + - "\x14last_acked_batch_seq\x18\b \x01(\x04R\x11lastAckedBatchSeq\"\xe9\f\n" + + "\x14last_acked_batch_seq\x18\b \x01(\x04R\x11lastAckedBatchSeq\"\x9c\r\n" + "\bSplitJob\x12\x15\n" + "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12&\n" + "\x0fsource_route_id\x18\x02 \x01(\x04R\rsourceRouteId\x12\x1b\n" + @@ -1947,7 +2043,8 @@ const file_distribution_proto_rawDesc = "" + "last_error\x18\x1e \x01(\tR\tlastError\x12\"\n" + "\rstarted_at_ms\x18\x1f \x01(\x03R\vstartedAtMs\x12\"\n" + "\rupdated_at_ms\x18 \x01(\x03R\vupdatedAtMs\x12$\n" + - "\x0eterminal_at_ms\x18! \x01(\x03R\fterminalAtMs\"\x13\n" + + "\x0eterminal_at_ms\x18! \x01(\x03R\fterminalAtMs\x121\n" + + "\x14capability_regressed\x18\" \x01(\bR\x13capabilityRegressed\"\x13\n" + "\x11ListRoutesRequest\"g\n" + "\x12ListRoutesResponse\x12'\n" + "\x0fcatalog_version\x18\x01 \x01(\x04R\x0ecatalogVersion\x12(\n" + @@ -1971,7 +2068,11 @@ const file_distribution_proto_rawDesc = "" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"]\n" + "\x1bStartSplitMigrationResponse\x12'\n" + "\x0fcatalog_version\x18\x01 \x01(\x04R\x0ecatalogVersion\x12\x15\n" + - "\x06job_id\x18\x02 \x01(\x04R\x05jobId\"U\n" + + "\x06job_id\x18\x02 \x01(\x04R\x05jobId\"$\n" + + "\"GetSplitMigrationCapabilityRequest\"v\n" + + "#GetSplitMigrationCapabilityResponse\x12+\n" + + "\x11migration_capable\x18\x01 \x01(\bR\x10migrationCapable\x12\"\n" + + "\fcapabilities\x18\x02 \x03(\tR\fcapabilities\"U\n" + "\x18GetRouteOwnershipRequest\x12\x10\n" + "\x03key\x18\x01 \x01(\fR\x03key\x12'\n" + "\x0fcatalog_version\x18\x02 \x01(\x04R\x0ecatalogVersion\"\x82\x01\n" + @@ -2032,7 +2133,7 @@ const file_distribution_proto_rawDesc = "" + "\x13SplitJobExportPhase\x12\x1f\n" + "\x1bSPLIT_JOB_EXPORT_PHASE_NONE\x10\x00\x12#\n" + "\x1fSPLIT_JOB_EXPORT_PHASE_BACKFILL\x10\x01\x12%\n" + - "!SPLIT_JOB_EXPORT_PHASE_DELTA_COPY\x10\x022\xf6\x05\n" + + "!SPLIT_JOB_EXPORT_PHASE_DELTA_COPY\x10\x022\xe2\x06\n" + "\fDistribution\x121\n" + "\bGetRoute\x12\x10.GetRouteRequest\x1a\x11.GetRouteResponse\"\x00\x12=\n" + "\fGetTimestamp\x12\x14.GetTimestampRequest\x1a\x15.GetTimestampResponse\"\x00\x127\n" + @@ -2040,7 +2141,8 @@ const file_distribution_proto_rawDesc = "" + "ListRoutes\x12\x12.ListRoutesRequest\x1a\x13.ListRoutesResponse\"\x00\x127\n" + "\n" + "SplitRange\x12\x12.SplitRangeRequest\x1a\x13.SplitRangeResponse\"\x00\x12R\n" + - "\x13StartSplitMigration\x12\x1b.StartSplitMigrationRequest\x1a\x1c.StartSplitMigrationResponse\"\x00\x12L\n" + + "\x13StartSplitMigration\x12\x1b.StartSplitMigrationRequest\x1a\x1c.StartSplitMigrationResponse\"\x00\x12j\n" + + "\x1bGetSplitMigrationCapability\x12#.GetSplitMigrationCapabilityRequest\x1a$.GetSplitMigrationCapabilityResponse\"\x00\x12L\n" + "\x11GetRouteOwnership\x12\x19.GetRouteOwnershipRequest\x1a\x1a.GetRouteOwnershipResponse\"\x00\x12X\n" + "\x15GetIntersectingRoutes\x12\x1d.GetIntersectingRoutesRequest\x1a\x1e.GetIntersectingRoutesResponse\"\x00\x12:\n" + "\vGetSplitJob\x12\x13.GetSplitJobRequest\x1a\x14.GetSplitJobResponse\"\x00\x12@\n" + @@ -2061,38 +2163,40 @@ func file_distribution_proto_rawDescGZIP() []byte { } var file_distribution_proto_enumTypes = make([]protoimpl.EnumInfo, 4) -var file_distribution_proto_msgTypes = make([]protoimpl.MessageInfo, 26) +var file_distribution_proto_msgTypes = make([]protoimpl.MessageInfo, 28) var file_distribution_proto_goTypes = []any{ - (RouteState)(0), // 0: RouteState - (SplitJobPhase)(0), // 1: SplitJobPhase - (SplitJobBarrierState)(0), // 2: SplitJobBarrierState - (SplitJobExportPhase)(0), // 3: SplitJobExportPhase - (*GetRouteRequest)(nil), // 4: GetRouteRequest - (*GetRouteResponse)(nil), // 5: GetRouteResponse - (*GetTimestampRequest)(nil), // 6: GetTimestampRequest - (*GetTimestampResponse)(nil), // 7: GetTimestampResponse - (*RouteDescriptor)(nil), // 8: RouteDescriptor - (*SplitJobBracketProgress)(nil), // 9: SplitJobBracketProgress - (*SplitJob)(nil), // 10: SplitJob - (*ListRoutesRequest)(nil), // 11: ListRoutesRequest - (*ListRoutesResponse)(nil), // 12: ListRoutesResponse - (*SplitRangeRequest)(nil), // 13: SplitRangeRequest - (*SplitRangeResponse)(nil), // 14: SplitRangeResponse - (*StartSplitMigrationRequest)(nil), // 15: StartSplitMigrationRequest - (*StartSplitMigrationResponse)(nil), // 16: StartSplitMigrationResponse - (*GetRouteOwnershipRequest)(nil), // 17: GetRouteOwnershipRequest - (*GetRouteOwnershipResponse)(nil), // 18: GetRouteOwnershipResponse - (*GetIntersectingRoutesRequest)(nil), // 19: GetIntersectingRoutesRequest - (*GetIntersectingRoutesResponse)(nil), // 20: GetIntersectingRoutesResponse - (*GetSplitJobRequest)(nil), // 21: GetSplitJobRequest - (*GetSplitJobResponse)(nil), // 22: GetSplitJobResponse - (*ListSplitJobsRequest)(nil), // 23: ListSplitJobsRequest - (*ListSplitJobsResponse)(nil), // 24: ListSplitJobsResponse - (*AbandonSplitJobRequest)(nil), // 25: AbandonSplitJobRequest - (*AbandonSplitJobResponse)(nil), // 26: AbandonSplitJobResponse - (*RetrySplitJobRequest)(nil), // 27: RetrySplitJobRequest - (*RetrySplitJobResponse)(nil), // 28: RetrySplitJobResponse - nil, // 29: StartSplitMigrationRequest.OptionsEntry + (RouteState)(0), // 0: RouteState + (SplitJobPhase)(0), // 1: SplitJobPhase + (SplitJobBarrierState)(0), // 2: SplitJobBarrierState + (SplitJobExportPhase)(0), // 3: SplitJobExportPhase + (*GetRouteRequest)(nil), // 4: GetRouteRequest + (*GetRouteResponse)(nil), // 5: GetRouteResponse + (*GetTimestampRequest)(nil), // 6: GetTimestampRequest + (*GetTimestampResponse)(nil), // 7: GetTimestampResponse + (*RouteDescriptor)(nil), // 8: RouteDescriptor + (*SplitJobBracketProgress)(nil), // 9: SplitJobBracketProgress + (*SplitJob)(nil), // 10: SplitJob + (*ListRoutesRequest)(nil), // 11: ListRoutesRequest + (*ListRoutesResponse)(nil), // 12: ListRoutesResponse + (*SplitRangeRequest)(nil), // 13: SplitRangeRequest + (*SplitRangeResponse)(nil), // 14: SplitRangeResponse + (*StartSplitMigrationRequest)(nil), // 15: StartSplitMigrationRequest + (*StartSplitMigrationResponse)(nil), // 16: StartSplitMigrationResponse + (*GetSplitMigrationCapabilityRequest)(nil), // 17: GetSplitMigrationCapabilityRequest + (*GetSplitMigrationCapabilityResponse)(nil), // 18: GetSplitMigrationCapabilityResponse + (*GetRouteOwnershipRequest)(nil), // 19: GetRouteOwnershipRequest + (*GetRouteOwnershipResponse)(nil), // 20: GetRouteOwnershipResponse + (*GetIntersectingRoutesRequest)(nil), // 21: GetIntersectingRoutesRequest + (*GetIntersectingRoutesResponse)(nil), // 22: GetIntersectingRoutesResponse + (*GetSplitJobRequest)(nil), // 23: GetSplitJobRequest + (*GetSplitJobResponse)(nil), // 24: GetSplitJobResponse + (*ListSplitJobsRequest)(nil), // 25: ListSplitJobsRequest + (*ListSplitJobsResponse)(nil), // 26: ListSplitJobsResponse + (*AbandonSplitJobRequest)(nil), // 27: AbandonSplitJobRequest + (*AbandonSplitJobResponse)(nil), // 28: AbandonSplitJobResponse + (*RetrySplitJobRequest)(nil), // 29: RetrySplitJobRequest + (*RetrySplitJobResponse)(nil), // 30: RetrySplitJobResponse + nil, // 31: StartSplitMigrationRequest.OptionsEntry } var file_distribution_proto_depIdxs = []int32{ 0, // 0: RouteDescriptor.state:type_name -> RouteState @@ -2106,7 +2210,7 @@ var file_distribution_proto_depIdxs = []int32{ 8, // 8: ListRoutesResponse.routes:type_name -> RouteDescriptor 8, // 9: SplitRangeResponse.left:type_name -> RouteDescriptor 8, // 10: SplitRangeResponse.right:type_name -> RouteDescriptor - 29, // 11: StartSplitMigrationRequest.options:type_name -> StartSplitMigrationRequest.OptionsEntry + 31, // 11: StartSplitMigrationRequest.options:type_name -> StartSplitMigrationRequest.OptionsEntry 8, // 12: GetRouteOwnershipResponse.route:type_name -> RouteDescriptor 8, // 13: GetIntersectingRoutesResponse.routes:type_name -> RouteDescriptor 10, // 14: GetSplitJobResponse.job:type_name -> SplitJob @@ -2116,25 +2220,27 @@ var file_distribution_proto_depIdxs = []int32{ 11, // 18: Distribution.ListRoutes:input_type -> ListRoutesRequest 13, // 19: Distribution.SplitRange:input_type -> SplitRangeRequest 15, // 20: Distribution.StartSplitMigration:input_type -> StartSplitMigrationRequest - 17, // 21: Distribution.GetRouteOwnership:input_type -> GetRouteOwnershipRequest - 19, // 22: Distribution.GetIntersectingRoutes:input_type -> GetIntersectingRoutesRequest - 21, // 23: Distribution.GetSplitJob:input_type -> GetSplitJobRequest - 23, // 24: Distribution.ListSplitJobs:input_type -> ListSplitJobsRequest - 25, // 25: Distribution.AbandonSplitJob:input_type -> AbandonSplitJobRequest - 27, // 26: Distribution.RetrySplitJob:input_type -> RetrySplitJobRequest - 5, // 27: Distribution.GetRoute:output_type -> GetRouteResponse - 7, // 28: Distribution.GetTimestamp:output_type -> GetTimestampResponse - 12, // 29: Distribution.ListRoutes:output_type -> ListRoutesResponse - 14, // 30: Distribution.SplitRange:output_type -> SplitRangeResponse - 16, // 31: Distribution.StartSplitMigration:output_type -> StartSplitMigrationResponse - 18, // 32: Distribution.GetRouteOwnership:output_type -> GetRouteOwnershipResponse - 20, // 33: Distribution.GetIntersectingRoutes:output_type -> GetIntersectingRoutesResponse - 22, // 34: Distribution.GetSplitJob:output_type -> GetSplitJobResponse - 24, // 35: Distribution.ListSplitJobs:output_type -> ListSplitJobsResponse - 26, // 36: Distribution.AbandonSplitJob:output_type -> AbandonSplitJobResponse - 28, // 37: Distribution.RetrySplitJob:output_type -> RetrySplitJobResponse - 27, // [27:38] is the sub-list for method output_type - 16, // [16:27] is the sub-list for method input_type + 17, // 21: Distribution.GetSplitMigrationCapability:input_type -> GetSplitMigrationCapabilityRequest + 19, // 22: Distribution.GetRouteOwnership:input_type -> GetRouteOwnershipRequest + 21, // 23: Distribution.GetIntersectingRoutes:input_type -> GetIntersectingRoutesRequest + 23, // 24: Distribution.GetSplitJob:input_type -> GetSplitJobRequest + 25, // 25: Distribution.ListSplitJobs:input_type -> ListSplitJobsRequest + 27, // 26: Distribution.AbandonSplitJob:input_type -> AbandonSplitJobRequest + 29, // 27: Distribution.RetrySplitJob:input_type -> RetrySplitJobRequest + 5, // 28: Distribution.GetRoute:output_type -> GetRouteResponse + 7, // 29: Distribution.GetTimestamp:output_type -> GetTimestampResponse + 12, // 30: Distribution.ListRoutes:output_type -> ListRoutesResponse + 14, // 31: Distribution.SplitRange:output_type -> SplitRangeResponse + 16, // 32: Distribution.StartSplitMigration:output_type -> StartSplitMigrationResponse + 18, // 33: Distribution.GetSplitMigrationCapability:output_type -> GetSplitMigrationCapabilityResponse + 20, // 34: Distribution.GetRouteOwnership:output_type -> GetRouteOwnershipResponse + 22, // 35: Distribution.GetIntersectingRoutes:output_type -> GetIntersectingRoutesResponse + 24, // 36: Distribution.GetSplitJob:output_type -> GetSplitJobResponse + 26, // 37: Distribution.ListSplitJobs:output_type -> ListSplitJobsResponse + 28, // 38: Distribution.AbandonSplitJob:output_type -> AbandonSplitJobResponse + 30, // 39: Distribution.RetrySplitJob:output_type -> RetrySplitJobResponse + 28, // [28:40] is the sub-list for method output_type + 16, // [16:28] is the sub-list for method input_type 16, // [16:16] is the sub-list for extension type_name 16, // [16:16] is the sub-list for extension extendee 0, // [0:16] is the sub-list for field type_name @@ -2151,7 +2257,7 @@ func file_distribution_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_distribution_proto_rawDesc), len(file_distribution_proto_rawDesc)), NumEnums: 4, - NumMessages: 26, + NumMessages: 28, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/distribution.proto b/proto/distribution.proto index 4fc05cd8f..968bfa191 100644 --- a/proto/distribution.proto +++ b/proto/distribution.proto @@ -8,6 +8,7 @@ service Distribution { rpc ListRoutes (ListRoutesRequest) returns (ListRoutesResponse) {} rpc SplitRange (SplitRangeRequest) returns (SplitRangeResponse) {} rpc StartSplitMigration (StartSplitMigrationRequest) returns (StartSplitMigrationResponse) {} + rpc GetSplitMigrationCapability (GetSplitMigrationCapabilityRequest) returns (GetSplitMigrationCapabilityResponse) {} rpc GetRouteOwnership (GetRouteOwnershipRequest) returns (GetRouteOwnershipResponse) {} rpc GetIntersectingRoutes (GetIntersectingRoutesRequest) returns (GetIntersectingRoutesResponse) {} rpc GetSplitJob (GetSplitJobRequest) returns (GetSplitJobResponse) {} @@ -127,6 +128,7 @@ message SplitJob { int64 started_at_ms = 31; int64 updated_at_ms = 32; int64 terminal_at_ms = 33; + bool capability_regressed = 34; } message ListRoutesRequest {} @@ -161,6 +163,13 @@ message StartSplitMigrationResponse { uint64 job_id = 2; } +message GetSplitMigrationCapabilityRequest {} + +message GetSplitMigrationCapabilityResponse { + bool migration_capable = 1; + repeated string capabilities = 2; +} + message GetRouteOwnershipRequest { bytes key = 1; uint64 catalog_version = 2; diff --git a/proto/distribution_grpc.pb.go b/proto/distribution_grpc.pb.go index 66ebf8511..ec02eabee 100644 --- a/proto/distribution_grpc.pb.go +++ b/proto/distribution_grpc.pb.go @@ -19,17 +19,18 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - Distribution_GetRoute_FullMethodName = "/Distribution/GetRoute" - Distribution_GetTimestamp_FullMethodName = "/Distribution/GetTimestamp" - Distribution_ListRoutes_FullMethodName = "/Distribution/ListRoutes" - Distribution_SplitRange_FullMethodName = "/Distribution/SplitRange" - Distribution_StartSplitMigration_FullMethodName = "/Distribution/StartSplitMigration" - Distribution_GetRouteOwnership_FullMethodName = "/Distribution/GetRouteOwnership" - Distribution_GetIntersectingRoutes_FullMethodName = "/Distribution/GetIntersectingRoutes" - Distribution_GetSplitJob_FullMethodName = "/Distribution/GetSplitJob" - Distribution_ListSplitJobs_FullMethodName = "/Distribution/ListSplitJobs" - Distribution_AbandonSplitJob_FullMethodName = "/Distribution/AbandonSplitJob" - Distribution_RetrySplitJob_FullMethodName = "/Distribution/RetrySplitJob" + Distribution_GetRoute_FullMethodName = "/Distribution/GetRoute" + Distribution_GetTimestamp_FullMethodName = "/Distribution/GetTimestamp" + Distribution_ListRoutes_FullMethodName = "/Distribution/ListRoutes" + Distribution_SplitRange_FullMethodName = "/Distribution/SplitRange" + Distribution_StartSplitMigration_FullMethodName = "/Distribution/StartSplitMigration" + Distribution_GetSplitMigrationCapability_FullMethodName = "/Distribution/GetSplitMigrationCapability" + Distribution_GetRouteOwnership_FullMethodName = "/Distribution/GetRouteOwnership" + Distribution_GetIntersectingRoutes_FullMethodName = "/Distribution/GetIntersectingRoutes" + Distribution_GetSplitJob_FullMethodName = "/Distribution/GetSplitJob" + Distribution_ListSplitJobs_FullMethodName = "/Distribution/ListSplitJobs" + Distribution_AbandonSplitJob_FullMethodName = "/Distribution/AbandonSplitJob" + Distribution_RetrySplitJob_FullMethodName = "/Distribution/RetrySplitJob" ) // DistributionClient is the client API for Distribution service. @@ -41,6 +42,7 @@ type DistributionClient interface { ListRoutes(ctx context.Context, in *ListRoutesRequest, opts ...grpc.CallOption) (*ListRoutesResponse, error) SplitRange(ctx context.Context, in *SplitRangeRequest, opts ...grpc.CallOption) (*SplitRangeResponse, error) StartSplitMigration(ctx context.Context, in *StartSplitMigrationRequest, opts ...grpc.CallOption) (*StartSplitMigrationResponse, error) + GetSplitMigrationCapability(ctx context.Context, in *GetSplitMigrationCapabilityRequest, opts ...grpc.CallOption) (*GetSplitMigrationCapabilityResponse, error) GetRouteOwnership(ctx context.Context, in *GetRouteOwnershipRequest, opts ...grpc.CallOption) (*GetRouteOwnershipResponse, error) GetIntersectingRoutes(ctx context.Context, in *GetIntersectingRoutesRequest, opts ...grpc.CallOption) (*GetIntersectingRoutesResponse, error) GetSplitJob(ctx context.Context, in *GetSplitJobRequest, opts ...grpc.CallOption) (*GetSplitJobResponse, error) @@ -107,6 +109,16 @@ func (c *distributionClient) StartSplitMigration(ctx context.Context, in *StartS return out, nil } +func (c *distributionClient) GetSplitMigrationCapability(ctx context.Context, in *GetSplitMigrationCapabilityRequest, opts ...grpc.CallOption) (*GetSplitMigrationCapabilityResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetSplitMigrationCapabilityResponse) + err := c.cc.Invoke(ctx, Distribution_GetSplitMigrationCapability_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *distributionClient) GetRouteOwnership(ctx context.Context, in *GetRouteOwnershipRequest, opts ...grpc.CallOption) (*GetRouteOwnershipResponse, error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GetRouteOwnershipResponse) @@ -176,6 +188,7 @@ type DistributionServer interface { ListRoutes(context.Context, *ListRoutesRequest) (*ListRoutesResponse, error) SplitRange(context.Context, *SplitRangeRequest) (*SplitRangeResponse, error) StartSplitMigration(context.Context, *StartSplitMigrationRequest) (*StartSplitMigrationResponse, error) + GetSplitMigrationCapability(context.Context, *GetSplitMigrationCapabilityRequest) (*GetSplitMigrationCapabilityResponse, error) GetRouteOwnership(context.Context, *GetRouteOwnershipRequest) (*GetRouteOwnershipResponse, error) GetIntersectingRoutes(context.Context, *GetIntersectingRoutesRequest) (*GetIntersectingRoutesResponse, error) GetSplitJob(context.Context, *GetSplitJobRequest) (*GetSplitJobResponse, error) @@ -207,6 +220,9 @@ func (UnimplementedDistributionServer) SplitRange(context.Context, *SplitRangeRe func (UnimplementedDistributionServer) StartSplitMigration(context.Context, *StartSplitMigrationRequest) (*StartSplitMigrationResponse, error) { return nil, status.Error(codes.Unimplemented, "method StartSplitMigration not implemented") } +func (UnimplementedDistributionServer) GetSplitMigrationCapability(context.Context, *GetSplitMigrationCapabilityRequest) (*GetSplitMigrationCapabilityResponse, error) { + return nil, status.Error(codes.Unimplemented, "method GetSplitMigrationCapability not implemented") +} func (UnimplementedDistributionServer) GetRouteOwnership(context.Context, *GetRouteOwnershipRequest) (*GetRouteOwnershipResponse, error) { return nil, status.Error(codes.Unimplemented, "method GetRouteOwnership not implemented") } @@ -336,6 +352,24 @@ func _Distribution_StartSplitMigration_Handler(srv interface{}, ctx context.Cont return interceptor(ctx, in, info, handler) } +func _Distribution_GetSplitMigrationCapability_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetSplitMigrationCapabilityRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(DistributionServer).GetSplitMigrationCapability(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Distribution_GetSplitMigrationCapability_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(DistributionServer).GetSplitMigrationCapability(ctx, req.(*GetSplitMigrationCapabilityRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Distribution_GetRouteOwnership_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetRouteOwnershipRequest) if err := dec(in); err != nil { @@ -471,6 +505,10 @@ var Distribution_ServiceDesc = grpc.ServiceDesc{ MethodName: "StartSplitMigration", Handler: _Distribution_StartSplitMigration_Handler, }, + { + MethodName: "GetSplitMigrationCapability", + Handler: _Distribution_GetSplitMigrationCapability_Handler, + }, { MethodName: "GetRouteOwnership", Handler: _Distribution_GetRouteOwnership_Handler, diff --git a/proto/internal.pb.go b/proto/internal.pb.go index 33d707446..90c0943da 100644 --- a/proto/internal.pb.go +++ b/proto/internal.pb.go @@ -128,6 +128,113 @@ func (Phase) EnumDescriptor() ([]byte, []int) { return file_internal_proto_rawDescGZIP(), []int{1} } +type MigrationCleanupMode int32 + +const ( + MigrationCleanupMode_MIGRATION_CLEANUP_MODE_UNSPECIFIED MigrationCleanupMode = 0 + MigrationCleanupMode_MIGRATION_CLEANUP_MODE_VERSIONS MigrationCleanupMode = 1 + MigrationCleanupMode_MIGRATION_CLEANUP_MODE_METADATA MigrationCleanupMode = 2 +) + +// Enum value maps for MigrationCleanupMode. +var ( + MigrationCleanupMode_name = map[int32]string{ + 0: "MIGRATION_CLEANUP_MODE_UNSPECIFIED", + 1: "MIGRATION_CLEANUP_MODE_VERSIONS", + 2: "MIGRATION_CLEANUP_MODE_METADATA", + } + MigrationCleanupMode_value = map[string]int32{ + "MIGRATION_CLEANUP_MODE_UNSPECIFIED": 0, + "MIGRATION_CLEANUP_MODE_VERSIONS": 1, + "MIGRATION_CLEANUP_MODE_METADATA": 2, + } +) + +func (x MigrationCleanupMode) Enum() *MigrationCleanupMode { + p := new(MigrationCleanupMode) + *p = x + return p +} + +func (x MigrationCleanupMode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (MigrationCleanupMode) Descriptor() protoreflect.EnumDescriptor { + return file_internal_proto_enumTypes[2].Descriptor() +} + +func (MigrationCleanupMode) Type() protoreflect.EnumType { + return &file_internal_proto_enumTypes[2] +} + +func (x MigrationCleanupMode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use MigrationCleanupMode.Descriptor instead. +func (MigrationCleanupMode) EnumDescriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{2} +} + +type MigrationStateProbeKind int32 + +const ( + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_UNSPECIFIED MigrationStateProbeKind = 0 + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED MigrationStateProbeKind = 1 + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED MigrationStateProbeKind = 2 + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_ROUTE_REMOVED MigrationStateProbeKind = 3 + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED MigrationStateProbeKind = 4 + MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED MigrationStateProbeKind = 5 +) + +// Enum value maps for MigrationStateProbeKind. +var ( + MigrationStateProbeKind_name = map[int32]string{ + 0: "MIGRATION_STATE_PROBE_KIND_UNSPECIFIED", + 1: "MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED", + 2: "MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED", + 3: "MIGRATION_STATE_PROBE_KIND_SOURCE_ROUTE_REMOVED", + 4: "MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED", + 5: "MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED", + } + MigrationStateProbeKind_value = map[string]int32{ + "MIGRATION_STATE_PROBE_KIND_UNSPECIFIED": 0, + "MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED": 1, + "MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED": 2, + "MIGRATION_STATE_PROBE_KIND_SOURCE_ROUTE_REMOVED": 3, + "MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED": 4, + "MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED": 5, + } +) + +func (x MigrationStateProbeKind) Enum() *MigrationStateProbeKind { + p := new(MigrationStateProbeKind) + *p = x + return p +} + +func (x MigrationStateProbeKind) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (MigrationStateProbeKind) Descriptor() protoreflect.EnumDescriptor { + return file_internal_proto_enumTypes[3].Descriptor() +} + +func (MigrationStateProbeKind) Type() protoreflect.EnumType { + return &file_internal_proto_enumTypes[3] +} + +func (x MigrationStateProbeKind) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use MigrationStateProbeKind.Descriptor instead. +func (MigrationStateProbeKind) EnumDescriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{3} +} + type Mutation struct { state protoimpl.MessageState `protogen:"open.v1"` Op Op `protobuf:"varint,1,opt,name=op,proto3,enum=Op" json:"op,omitempty"` @@ -1109,90 +1216,976 @@ func (x *PromoteStagedVersionsResponse) GetMaxPromotedTs() uint64 { return 0 } -var File_internal_proto protoreflect.FileDescriptor +type TargetStagedReadinessRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + JobId uint64 `protobuf:"varint,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` + RouteStart []byte `protobuf:"bytes,2,opt,name=route_start,json=routeStart,proto3" json:"route_start,omitempty"` + RouteEnd []byte `protobuf:"bytes,3,opt,name=route_end,json=routeEnd,proto3" json:"route_end,omitempty"` + ExpectedCutoverVersion uint64 `protobuf:"varint,4,opt,name=expected_cutover_version,json=expectedCutoverVersion,proto3" json:"expected_cutover_version,omitempty"` + MigrationJobId uint64 `protobuf:"varint,5,opt,name=migration_job_id,json=migrationJobId,proto3" json:"migration_job_id,omitempty"` + MinWriteTsExclusive uint64 `protobuf:"varint,6,opt,name=min_write_ts_exclusive,json=minWriteTsExclusive,proto3" json:"min_write_ts_exclusive,omitempty"` + Armed bool `protobuf:"varint,7,opt,name=armed,proto3" json:"armed,omitempty"` + // Source-side controls reuse the same Raft-replicated durable record. They + // are ignored by target staged-readiness checks. + SourceWriteFence bool `protobuf:"varint,8,opt,name=source_write_fence,json=sourceWriteFence,proto3" json:"source_write_fence,omitempty"` + SourceReadFence bool `protobuf:"varint,9,opt,name=source_read_fence,json=sourceReadFence,proto3" json:"source_read_fence,omitempty"` + RetentionPinTs uint64 `protobuf:"varint,10,opt,name=retention_pin_ts,json=retentionPinTs,proto3" json:"retention_pin_ts,omitempty"` + TrackWrites bool `protobuf:"varint,11,opt,name=track_writes,json=trackWrites,proto3" json:"track_writes,omitempty"` + MinAdmittedTs uint64 `protobuf:"varint,12,opt,name=min_admitted_ts,json=minAdmittedTs,proto3" json:"min_admitted_ts,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TargetStagedReadinessRequest) Reset() { + *x = TargetStagedReadinessRequest{} + mi := &file_internal_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} -const file_internal_proto_rawDesc = "" + - "\n" + - "\x0einternal.proto\"|\n" + - "\bMutation\x12\x13\n" + - "\x02op\x18\x01 \x01(\x0e2\x03.OpR\x02op\x12\x10\n" + - "\x03key\x18\x02 \x01(\fR\x03key\x12\x14\n" + - "\x05value\x18\x03 \x01(\fR\x05value\x123\n" + - "\x16commit_ts_value_offset\x18\x04 \x01(\x04R\x13commitTsValueOffset\"\x81\x02\n" + - "\aRequest\x12\x15\n" + - "\x06is_txn\x18\x01 \x01(\bR\x05isTxn\x12\x1c\n" + - "\x05phase\x18\x02 \x01(\x0e2\x06.PhaseR\x05phase\x12\x0e\n" + - "\x02ts\x18\x03 \x01(\x04R\x02ts\x12'\n" + - "\tmutations\x18\x04 \x03(\v2\t.MutationR\tmutations\x12\x1b\n" + - "\tread_keys\x18\x05 \x03(\fR\breadKeys\x124\n" + - "\x16observed_route_version\x18\x06 \x01(\x04R\x14observedRouteVersion\x125\n" + - "\x17write_fence_bypass_keys\x18\a \x03(\fR\x14writeFenceBypassKeys\"3\n" + - "\vRaftCommand\x12$\n" + - "\brequests\x18\x01 \x03(\v2\b.RequestR\brequests\"M\n" + - "\x0eForwardRequest\x12\x15\n" + - "\x06is_txn\x18\x01 \x01(\bR\x05isTxn\x12$\n" + - "\brequests\x18\x02 \x03(\v2\b.RequestR\brequests\"k\n" + - "\x0fForwardResponse\x12\x18\n" + - "\asuccess\x18\x01 \x01(\bR\asuccess\x12!\n" + - "\fcommit_index\x18\x02 \x01(\x04R\vcommitIndex\x12\x1b\n" + - "\tcommit_ts\x18\x03 \x01(\x04R\bcommitTs\"I\n" + - "\x13RelayPublishRequest\x12\x18\n" + - "\achannel\x18\x01 \x01(\fR\achannel\x12\x18\n" + - "\amessage\x18\x02 \x01(\fR\amessage\"8\n" + - "\x14RelayPublishResponse\x12 \n" + - "\vsubscribers\x18\x01 \x01(\x03R\vsubscribers\"\xc5\x03\n" + - "\x1aExportRangeVersionsRequest\x12\x1f\n" + - "\vrange_start\x18\x01 \x01(\fR\n" + - "rangeStart\x12\x1b\n" + - "\trange_end\x18\x02 \x01(\fR\brangeEnd\x12\"\n" + - "\rmax_commit_ts\x18\x03 \x01(\x04R\vmaxCommitTs\x12\"\n" + - "\rmin_commit_ts\x18\x04 \x01(\x04R\vminCommitTs\x12\x16\n" + - "\x06cursor\x18\x05 \x01(\fR\x06cursor\x12\x1f\n" + - "\vchunk_bytes\x18\x06 \x01(\rR\n" + - "chunkBytes\x12\x1f\n" + - "\vroute_start\x18\a \x01(\fR\n" + - "routeStart\x12\x1b\n" + - "\troute_end\x18\b \x01(\fR\brouteEnd\x12*\n" + - "\x11max_scanned_bytes\x18\t \x01(\x04R\x0fmaxScannedBytes\x12\x1d\n" + - "\n" + - "key_family\x18\n" + - " \x01(\rR\tkeyFamily\x124\n" + - "\x16exclude_known_internal\x18\v \x01(\bR\x14excludeKnownInternal\x12)\n" + - "\x10exclude_prefixes\x18\f \x03(\fR\x0fexcludePrefixes\"|\n" + - "\x1bExportRangeVersionsResponse\x12(\n" + - "\bversions\x18\x01 \x03(\v2\f.MVCCVersionR\bversions\x12\x1f\n" + - "\vnext_cursor\x18\x02 \x01(\fR\n" + - "nextCursor\x12\x12\n" + - "\x04done\x18\x03 \x01(\bR\x04done\"\xac\x01\n" + - "\vMVCCVersion\x12\x10\n" + - "\x03key\x18\x01 \x01(\fR\x03key\x12\x1b\n" + - "\tcommit_ts\x18\x02 \x01(\x04R\bcommitTs\x12\x1c\n" + - "\ttombstone\x18\x03 \x01(\bR\ttombstone\x12\x14\n" + - "\x05value\x18\x04 \x01(\fR\x05value\x12\x1d\n" + - "\n" + - "key_family\x18\x05 \x01(\rR\tkeyFamily\x12\x1b\n" + - "\texpire_at\x18\x06 \x01(\x04R\bexpireAt\"\xb1\x01\n" + - "\x1aImportRangeVersionsRequest\x12\x15\n" + - "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12(\n" + - "\bversions\x18\x02 \x03(\v2\f.MVCCVersionR\bversions\x12\x16\n" + - "\x06cursor\x18\x03 \x01(\fR\x06cursor\x12\x1d\n" + - "\n" + - "bracket_id\x18\x04 \x01(\x04R\tbracketId\x12\x1b\n" + - "\tbatch_seq\x18\x05 \x01(\x04R\bbatchSeq\"@\n" + - "\x1bImportRangeVersionsResponse\x12!\n" + - "\facked_cursor\x18\x01 \x01(\fR\vackedCursor\"\xb9\x01\n" + - "\x1cPromoteStagedVersionsRequest\x12\x15\n" + - "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12\x16\n" + - "\x06cursor\x18\x02 \x01(\fR\x06cursor\x12!\n" + - "\fmax_versions\x18\x03 \x01(\rR\vmaxVersions\x12\x1b\n" + - "\tmax_bytes\x18\x04 \x01(\x04R\bmaxBytes\x12*\n" + - "\x11max_scanned_bytes\x18\x05 \x01(\x04R\x0fmaxScannedBytes\"\xa1\x01\n" + - "\x1dPromoteStagedVersionsResponse\x12\x1f\n" + - "\vnext_cursor\x18\x01 \x01(\fR\n" + - "nextCursor\x12\x12\n" + - "\x04done\x18\x02 \x01(\bR\x04done\x12#\n" + - "\rpromoted_rows\x18\x03 \x01(\x04R\fpromotedRows\x12&\n" + - "\x0fmax_promoted_ts\x18\x04 \x01(\x04R\rmaxPromotedTs*&\n" + +func (x *TargetStagedReadinessRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TargetStagedReadinessRequest) ProtoMessage() {} + +func (x *TargetStagedReadinessRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TargetStagedReadinessRequest.ProtoReflect.Descriptor instead. +func (*TargetStagedReadinessRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{14} +} + +func (x *TargetStagedReadinessRequest) GetJobId() uint64 { + if x != nil { + return x.JobId + } + return 0 +} + +func (x *TargetStagedReadinessRequest) GetRouteStart() []byte { + if x != nil { + return x.RouteStart + } + return nil +} + +func (x *TargetStagedReadinessRequest) GetRouteEnd() []byte { + if x != nil { + return x.RouteEnd + } + return nil +} + +func (x *TargetStagedReadinessRequest) GetExpectedCutoverVersion() uint64 { + if x != nil { + return x.ExpectedCutoverVersion + } + return 0 +} + +func (x *TargetStagedReadinessRequest) GetMigrationJobId() uint64 { + if x != nil { + return x.MigrationJobId + } + return 0 +} + +func (x *TargetStagedReadinessRequest) GetMinWriteTsExclusive() uint64 { + if x != nil { + return x.MinWriteTsExclusive + } + return 0 +} + +func (x *TargetStagedReadinessRequest) GetArmed() bool { + if x != nil { + return x.Armed + } + return false +} + +func (x *TargetStagedReadinessRequest) GetSourceWriteFence() bool { + if x != nil { + return x.SourceWriteFence + } + return false +} + +func (x *TargetStagedReadinessRequest) GetSourceReadFence() bool { + if x != nil { + return x.SourceReadFence + } + return false +} + +func (x *TargetStagedReadinessRequest) GetRetentionPinTs() uint64 { + if x != nil { + return x.RetentionPinTs + } + return 0 +} + +func (x *TargetStagedReadinessRequest) GetTrackWrites() bool { + if x != nil { + return x.TrackWrites + } + return false +} + +func (x *TargetStagedReadinessRequest) GetMinAdmittedTs() uint64 { + if x != nil { + return x.MinAdmittedTs + } + return 0 +} + +type TargetStagedReadinessResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + MinAdmittedTs uint64 `protobuf:"varint,1,opt,name=min_admitted_ts,json=minAdmittedTs,proto3" json:"min_admitted_ts,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TargetStagedReadinessResponse) Reset() { + *x = TargetStagedReadinessResponse{} + mi := &file_internal_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TargetStagedReadinessResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TargetStagedReadinessResponse) ProtoMessage() {} + +func (x *TargetStagedReadinessResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TargetStagedReadinessResponse.ProtoReflect.Descriptor instead. +func (*TargetStagedReadinessResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{15} +} + +func (x *TargetStagedReadinessResponse) GetMinAdmittedTs() uint64 { + if x != nil { + return x.MinAdmittedTs + } + return 0 +} + +type ProbeMigrationLocksRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + RouteStart []byte `protobuf:"bytes,1,opt,name=route_start,json=routeStart,proto3" json:"route_start,omitempty"` + RouteEnd []byte `protobuf:"bytes,2,opt,name=route_end,json=routeEnd,proto3" json:"route_end,omitempty"` + Limit uint32 `protobuf:"varint,3,opt,name=limit,proto3" json:"limit,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProbeMigrationLocksRequest) Reset() { + *x = ProbeMigrationLocksRequest{} + mi := &file_internal_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProbeMigrationLocksRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProbeMigrationLocksRequest) ProtoMessage() {} + +func (x *ProbeMigrationLocksRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProbeMigrationLocksRequest.ProtoReflect.Descriptor instead. +func (*ProbeMigrationLocksRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{16} +} + +func (x *ProbeMigrationLocksRequest) GetRouteStart() []byte { + if x != nil { + return x.RouteStart + } + return nil +} + +func (x *ProbeMigrationLocksRequest) GetRouteEnd() []byte { + if x != nil { + return x.RouteEnd + } + return nil +} + +func (x *ProbeMigrationLocksRequest) GetLimit() uint32 { + if x != nil { + return x.Limit + } + return 0 +} + +type ProbeMigrationLocksResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + PendingCount uint32 `protobuf:"varint,1,opt,name=pending_count,json=pendingCount,proto3" json:"pending_count,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProbeMigrationLocksResponse) Reset() { + *x = ProbeMigrationLocksResponse{} + mi := &file_internal_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProbeMigrationLocksResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProbeMigrationLocksResponse) ProtoMessage() {} + +func (x *ProbeMigrationLocksResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProbeMigrationLocksResponse.ProtoReflect.Descriptor instead. +func (*ProbeMigrationLocksResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{17} +} + +func (x *ProbeMigrationLocksResponse) GetPendingCount() uint32 { + if x != nil { + return x.PendingCount + } + return 0 +} + +type CleanupMigrationRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + JobId uint64 `protobuf:"varint,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` + Mode MigrationCleanupMode `protobuf:"varint,2,opt,name=mode,proto3,enum=MigrationCleanupMode" json:"mode,omitempty"` + RangeStart []byte `protobuf:"bytes,3,opt,name=range_start,json=rangeStart,proto3" json:"range_start,omitempty"` + RangeEnd []byte `protobuf:"bytes,4,opt,name=range_end,json=rangeEnd,proto3" json:"range_end,omitempty"` + Cursor []byte `protobuf:"bytes,5,opt,name=cursor,proto3" json:"cursor,omitempty"` + MaxCommitTs uint64 `protobuf:"varint,6,opt,name=max_commit_ts,json=maxCommitTs,proto3" json:"max_commit_ts,omitempty"` + MaxVersions uint32 `protobuf:"varint,7,opt,name=max_versions,json=maxVersions,proto3" json:"max_versions,omitempty"` + MaxBytes uint64 `protobuf:"varint,8,opt,name=max_bytes,json=maxBytes,proto3" json:"max_bytes,omitempty"` + MaxScannedBytes uint64 `protobuf:"varint,9,opt,name=max_scanned_bytes,json=maxScannedBytes,proto3" json:"max_scanned_bytes,omitempty"` + KeyFamily uint32 `protobuf:"varint,10,opt,name=key_family,json=keyFamily,proto3" json:"key_family,omitempty"` + RouteStart []byte `protobuf:"bytes,11,opt,name=route_start,json=routeStart,proto3" json:"route_start,omitempty"` + RouteEnd []byte `protobuf:"bytes,12,opt,name=route_end,json=routeEnd,proto3" json:"route_end,omitempty"` + ExcludeKnownInternal bool `protobuf:"varint,13,opt,name=exclude_known_internal,json=excludeKnownInternal,proto3" json:"exclude_known_internal,omitempty"` + ExcludePrefixes [][]byte `protobuf:"bytes,14,rep,name=exclude_prefixes,json=excludePrefixes,proto3" json:"exclude_prefixes,omitempty"` + RequiresRouteKeyCheck bool `protobuf:"varint,15,opt,name=requires_route_key_check,json=requiresRouteKeyCheck,proto3" json:"requires_route_key_check,omitempty"` + RequiresDecodedS3 bool `protobuf:"varint,16,opt,name=requires_decoded_s3,json=requiresDecodedS3,proto3" json:"requires_decoded_s3,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CleanupMigrationRequest) Reset() { + *x = CleanupMigrationRequest{} + mi := &file_internal_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CleanupMigrationRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CleanupMigrationRequest) ProtoMessage() {} + +func (x *CleanupMigrationRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CleanupMigrationRequest.ProtoReflect.Descriptor instead. +func (*CleanupMigrationRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{18} +} + +func (x *CleanupMigrationRequest) GetJobId() uint64 { + if x != nil { + return x.JobId + } + return 0 +} + +func (x *CleanupMigrationRequest) GetMode() MigrationCleanupMode { + if x != nil { + return x.Mode + } + return MigrationCleanupMode_MIGRATION_CLEANUP_MODE_UNSPECIFIED +} + +func (x *CleanupMigrationRequest) GetRangeStart() []byte { + if x != nil { + return x.RangeStart + } + return nil +} + +func (x *CleanupMigrationRequest) GetRangeEnd() []byte { + if x != nil { + return x.RangeEnd + } + return nil +} + +func (x *CleanupMigrationRequest) GetCursor() []byte { + if x != nil { + return x.Cursor + } + return nil +} + +func (x *CleanupMigrationRequest) GetMaxCommitTs() uint64 { + if x != nil { + return x.MaxCommitTs + } + return 0 +} + +func (x *CleanupMigrationRequest) GetMaxVersions() uint32 { + if x != nil { + return x.MaxVersions + } + return 0 +} + +func (x *CleanupMigrationRequest) GetMaxBytes() uint64 { + if x != nil { + return x.MaxBytes + } + return 0 +} + +func (x *CleanupMigrationRequest) GetMaxScannedBytes() uint64 { + if x != nil { + return x.MaxScannedBytes + } + return 0 +} + +func (x *CleanupMigrationRequest) GetKeyFamily() uint32 { + if x != nil { + return x.KeyFamily + } + return 0 +} + +func (x *CleanupMigrationRequest) GetRouteStart() []byte { + if x != nil { + return x.RouteStart + } + return nil +} + +func (x *CleanupMigrationRequest) GetRouteEnd() []byte { + if x != nil { + return x.RouteEnd + } + return nil +} + +func (x *CleanupMigrationRequest) GetExcludeKnownInternal() bool { + if x != nil { + return x.ExcludeKnownInternal + } + return false +} + +func (x *CleanupMigrationRequest) GetExcludePrefixes() [][]byte { + if x != nil { + return x.ExcludePrefixes + } + return nil +} + +func (x *CleanupMigrationRequest) GetRequiresRouteKeyCheck() bool { + if x != nil { + return x.RequiresRouteKeyCheck + } + return false +} + +func (x *CleanupMigrationRequest) GetRequiresDecodedS3() bool { + if x != nil { + return x.RequiresDecodedS3 + } + return false +} + +type CleanupMigrationResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + NextCursor []byte `protobuf:"bytes,1,opt,name=next_cursor,json=nextCursor,proto3" json:"next_cursor,omitempty"` + Done bool `protobuf:"varint,2,opt,name=done,proto3" json:"done,omitempty"` + DeletedRows uint64 `protobuf:"varint,3,opt,name=deleted_rows,json=deletedRows,proto3" json:"deleted_rows,omitempty"` + DeletedBytes uint64 `protobuf:"varint,4,opt,name=deleted_bytes,json=deletedBytes,proto3" json:"deleted_bytes,omitempty"` + ScannedBytes uint64 `protobuf:"varint,5,opt,name=scanned_bytes,json=scannedBytes,proto3" json:"scanned_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CleanupMigrationResponse) Reset() { + *x = CleanupMigrationResponse{} + mi := &file_internal_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CleanupMigrationResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CleanupMigrationResponse) ProtoMessage() {} + +func (x *CleanupMigrationResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CleanupMigrationResponse.ProtoReflect.Descriptor instead. +func (*CleanupMigrationResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{19} +} + +func (x *CleanupMigrationResponse) GetNextCursor() []byte { + if x != nil { + return x.NextCursor + } + return nil +} + +func (x *CleanupMigrationResponse) GetDone() bool { + if x != nil { + return x.Done + } + return false +} + +func (x *CleanupMigrationResponse) GetDeletedRows() uint64 { + if x != nil { + return x.DeletedRows + } + return 0 +} + +func (x *CleanupMigrationResponse) GetDeletedBytes() uint64 { + if x != nil { + return x.DeletedBytes + } + return 0 +} + +func (x *CleanupMigrationResponse) GetScannedBytes() uint64 { + if x != nil { + return x.ScannedBytes + } + return 0 +} + +type ProbeMigrationStateRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + JobId uint64 `protobuf:"varint,1,opt,name=job_id,json=jobId,proto3" json:"job_id,omitempty"` + Kind MigrationStateProbeKind `protobuf:"varint,2,opt,name=kind,proto3,enum=MigrationStateProbeKind" json:"kind,omitempty"` + RouteStart []byte `protobuf:"bytes,3,opt,name=route_start,json=routeStart,proto3" json:"route_start,omitempty"` + RouteEnd []byte `protobuf:"bytes,4,opt,name=route_end,json=routeEnd,proto3" json:"route_end,omitempty"` + ExpectedCatalogVersion uint64 `protobuf:"varint,5,opt,name=expected_catalog_version,json=expectedCatalogVersion,proto3" json:"expected_catalog_version,omitempty"` + ExpectedGroupId uint64 `protobuf:"varint,6,opt,name=expected_group_id,json=expectedGroupId,proto3" json:"expected_group_id,omitempty"` + MigrationJobId uint64 `protobuf:"varint,7,opt,name=migration_job_id,json=migrationJobId,proto3" json:"migration_job_id,omitempty"` + MinWriteTsExclusive uint64 `protobuf:"varint,8,opt,name=min_write_ts_exclusive,json=minWriteTsExclusive,proto3" json:"min_write_ts_exclusive,omitempty"` + SourceWriteFence bool `protobuf:"varint,9,opt,name=source_write_fence,json=sourceWriteFence,proto3" json:"source_write_fence,omitempty"` + SourceReadFence bool `protobuf:"varint,10,opt,name=source_read_fence,json=sourceReadFence,proto3" json:"source_read_fence,omitempty"` + TrackWrites bool `protobuf:"varint,11,opt,name=track_writes,json=trackWrites,proto3" json:"track_writes,omitempty"` + RetentionPinTs uint64 `protobuf:"varint,12,opt,name=retention_pin_ts,json=retentionPinTs,proto3" json:"retention_pin_ts,omitempty"` + ReadDrainNotBeforeMs int64 `protobuf:"varint,13,opt,name=read_drain_not_before_ms,json=readDrainNotBeforeMs,proto3" json:"read_drain_not_before_ms,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProbeMigrationStateRequest) Reset() { + *x = ProbeMigrationStateRequest{} + mi := &file_internal_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProbeMigrationStateRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProbeMigrationStateRequest) ProtoMessage() {} + +func (x *ProbeMigrationStateRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProbeMigrationStateRequest.ProtoReflect.Descriptor instead. +func (*ProbeMigrationStateRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{20} +} + +func (x *ProbeMigrationStateRequest) GetJobId() uint64 { + if x != nil { + return x.JobId + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetKind() MigrationStateProbeKind { + if x != nil { + return x.Kind + } + return MigrationStateProbeKind_MIGRATION_STATE_PROBE_KIND_UNSPECIFIED +} + +func (x *ProbeMigrationStateRequest) GetRouteStart() []byte { + if x != nil { + return x.RouteStart + } + return nil +} + +func (x *ProbeMigrationStateRequest) GetRouteEnd() []byte { + if x != nil { + return x.RouteEnd + } + return nil +} + +func (x *ProbeMigrationStateRequest) GetExpectedCatalogVersion() uint64 { + if x != nil { + return x.ExpectedCatalogVersion + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetExpectedGroupId() uint64 { + if x != nil { + return x.ExpectedGroupId + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetMigrationJobId() uint64 { + if x != nil { + return x.MigrationJobId + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetMinWriteTsExclusive() uint64 { + if x != nil { + return x.MinWriteTsExclusive + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetSourceWriteFence() bool { + if x != nil { + return x.SourceWriteFence + } + return false +} + +func (x *ProbeMigrationStateRequest) GetSourceReadFence() bool { + if x != nil { + return x.SourceReadFence + } + return false +} + +func (x *ProbeMigrationStateRequest) GetTrackWrites() bool { + if x != nil { + return x.TrackWrites + } + return false +} + +func (x *ProbeMigrationStateRequest) GetRetentionPinTs() uint64 { + if x != nil { + return x.RetentionPinTs + } + return 0 +} + +func (x *ProbeMigrationStateRequest) GetReadDrainNotBeforeMs() int64 { + if x != nil { + return x.ReadDrainNotBeforeMs + } + return 0 +} + +type ProbeMigrationStateResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Ready bool `protobuf:"varint,1,opt,name=ready,proto3" json:"ready,omitempty"` + CatalogVersion uint64 `protobuf:"varint,2,opt,name=catalog_version,json=catalogVersion,proto3" json:"catalog_version,omitempty"` + MinAdmittedTs uint64 `protobuf:"varint,3,opt,name=min_admitted_ts,json=minAdmittedTs,proto3" json:"min_admitted_ts,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ProbeMigrationStateResponse) Reset() { + *x = ProbeMigrationStateResponse{} + mi := &file_internal_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ProbeMigrationStateResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ProbeMigrationStateResponse) ProtoMessage() {} + +func (x *ProbeMigrationStateResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ProbeMigrationStateResponse.ProtoReflect.Descriptor instead. +func (*ProbeMigrationStateResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{21} +} + +func (x *ProbeMigrationStateResponse) GetReady() bool { + if x != nil { + return x.Ready + } + return false +} + +func (x *ProbeMigrationStateResponse) GetCatalogVersion() uint64 { + if x != nil { + return x.CatalogVersion + } + return 0 +} + +func (x *ProbeMigrationStateResponse) GetMinAdmittedTs() uint64 { + if x != nil { + return x.MinAdmittedTs + } + return 0 +} + +type IssueMigrationTimestampRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *IssueMigrationTimestampRequest) Reset() { + *x = IssueMigrationTimestampRequest{} + mi := &file_internal_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *IssueMigrationTimestampRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IssueMigrationTimestampRequest) ProtoMessage() {} + +func (x *IssueMigrationTimestampRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[22] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IssueMigrationTimestampRequest.ProtoReflect.Descriptor instead. +func (*IssueMigrationTimestampRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{22} +} + +type IssueMigrationTimestampResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Timestamp uint64 `protobuf:"varint,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + LastCommitTs uint64 `protobuf:"varint,2,opt,name=last_commit_ts,json=lastCommitTs,proto3" json:"last_commit_ts,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *IssueMigrationTimestampResponse) Reset() { + *x = IssueMigrationTimestampResponse{} + mi := &file_internal_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *IssueMigrationTimestampResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*IssueMigrationTimestampResponse) ProtoMessage() {} + +func (x *IssueMigrationTimestampResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[23] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use IssueMigrationTimestampResponse.ProtoReflect.Descriptor instead. +func (*IssueMigrationTimestampResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{23} +} + +func (x *IssueMigrationTimestampResponse) GetTimestamp() uint64 { + if x != nil { + return x.Timestamp + } + return 0 +} + +func (x *IssueMigrationTimestampResponse) GetLastCommitTs() uint64 { + if x != nil { + return x.LastCommitTs + } + return 0 +} + +var File_internal_proto protoreflect.FileDescriptor + +const file_internal_proto_rawDesc = "" + + "\n" + + "\x0einternal.proto\"|\n" + + "\bMutation\x12\x13\n" + + "\x02op\x18\x01 \x01(\x0e2\x03.OpR\x02op\x12\x10\n" + + "\x03key\x18\x02 \x01(\fR\x03key\x12\x14\n" + + "\x05value\x18\x03 \x01(\fR\x05value\x123\n" + + "\x16commit_ts_value_offset\x18\x04 \x01(\x04R\x13commitTsValueOffset\"\x81\x02\n" + + "\aRequest\x12\x15\n" + + "\x06is_txn\x18\x01 \x01(\bR\x05isTxn\x12\x1c\n" + + "\x05phase\x18\x02 \x01(\x0e2\x06.PhaseR\x05phase\x12\x0e\n" + + "\x02ts\x18\x03 \x01(\x04R\x02ts\x12'\n" + + "\tmutations\x18\x04 \x03(\v2\t.MutationR\tmutations\x12\x1b\n" + + "\tread_keys\x18\x05 \x03(\fR\breadKeys\x124\n" + + "\x16observed_route_version\x18\x06 \x01(\x04R\x14observedRouteVersion\x125\n" + + "\x17write_fence_bypass_keys\x18\a \x03(\fR\x14writeFenceBypassKeys\"3\n" + + "\vRaftCommand\x12$\n" + + "\brequests\x18\x01 \x03(\v2\b.RequestR\brequests\"M\n" + + "\x0eForwardRequest\x12\x15\n" + + "\x06is_txn\x18\x01 \x01(\bR\x05isTxn\x12$\n" + + "\brequests\x18\x02 \x03(\v2\b.RequestR\brequests\"k\n" + + "\x0fForwardResponse\x12\x18\n" + + "\asuccess\x18\x01 \x01(\bR\asuccess\x12!\n" + + "\fcommit_index\x18\x02 \x01(\x04R\vcommitIndex\x12\x1b\n" + + "\tcommit_ts\x18\x03 \x01(\x04R\bcommitTs\"I\n" + + "\x13RelayPublishRequest\x12\x18\n" + + "\achannel\x18\x01 \x01(\fR\achannel\x12\x18\n" + + "\amessage\x18\x02 \x01(\fR\amessage\"8\n" + + "\x14RelayPublishResponse\x12 \n" + + "\vsubscribers\x18\x01 \x01(\x03R\vsubscribers\"\xc5\x03\n" + + "\x1aExportRangeVersionsRequest\x12\x1f\n" + + "\vrange_start\x18\x01 \x01(\fR\n" + + "rangeStart\x12\x1b\n" + + "\trange_end\x18\x02 \x01(\fR\brangeEnd\x12\"\n" + + "\rmax_commit_ts\x18\x03 \x01(\x04R\vmaxCommitTs\x12\"\n" + + "\rmin_commit_ts\x18\x04 \x01(\x04R\vminCommitTs\x12\x16\n" + + "\x06cursor\x18\x05 \x01(\fR\x06cursor\x12\x1f\n" + + "\vchunk_bytes\x18\x06 \x01(\rR\n" + + "chunkBytes\x12\x1f\n" + + "\vroute_start\x18\a \x01(\fR\n" + + "routeStart\x12\x1b\n" + + "\troute_end\x18\b \x01(\fR\brouteEnd\x12*\n" + + "\x11max_scanned_bytes\x18\t \x01(\x04R\x0fmaxScannedBytes\x12\x1d\n" + + "\n" + + "key_family\x18\n" + + " \x01(\rR\tkeyFamily\x124\n" + + "\x16exclude_known_internal\x18\v \x01(\bR\x14excludeKnownInternal\x12)\n" + + "\x10exclude_prefixes\x18\f \x03(\fR\x0fexcludePrefixes\"|\n" + + "\x1bExportRangeVersionsResponse\x12(\n" + + "\bversions\x18\x01 \x03(\v2\f.MVCCVersionR\bversions\x12\x1f\n" + + "\vnext_cursor\x18\x02 \x01(\fR\n" + + "nextCursor\x12\x12\n" + + "\x04done\x18\x03 \x01(\bR\x04done\"\xac\x01\n" + + "\vMVCCVersion\x12\x10\n" + + "\x03key\x18\x01 \x01(\fR\x03key\x12\x1b\n" + + "\tcommit_ts\x18\x02 \x01(\x04R\bcommitTs\x12\x1c\n" + + "\ttombstone\x18\x03 \x01(\bR\ttombstone\x12\x14\n" + + "\x05value\x18\x04 \x01(\fR\x05value\x12\x1d\n" + + "\n" + + "key_family\x18\x05 \x01(\rR\tkeyFamily\x12\x1b\n" + + "\texpire_at\x18\x06 \x01(\x04R\bexpireAt\"\xb1\x01\n" + + "\x1aImportRangeVersionsRequest\x12\x15\n" + + "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12(\n" + + "\bversions\x18\x02 \x03(\v2\f.MVCCVersionR\bversions\x12\x16\n" + + "\x06cursor\x18\x03 \x01(\fR\x06cursor\x12\x1d\n" + + "\n" + + "bracket_id\x18\x04 \x01(\x04R\tbracketId\x12\x1b\n" + + "\tbatch_seq\x18\x05 \x01(\x04R\bbatchSeq\"@\n" + + "\x1bImportRangeVersionsResponse\x12!\n" + + "\facked_cursor\x18\x01 \x01(\fR\vackedCursor\"\xb9\x01\n" + + "\x1cPromoteStagedVersionsRequest\x12\x15\n" + + "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12\x16\n" + + "\x06cursor\x18\x02 \x01(\fR\x06cursor\x12!\n" + + "\fmax_versions\x18\x03 \x01(\rR\vmaxVersions\x12\x1b\n" + + "\tmax_bytes\x18\x04 \x01(\x04R\bmaxBytes\x12*\n" + + "\x11max_scanned_bytes\x18\x05 \x01(\x04R\x0fmaxScannedBytes\"\xa1\x01\n" + + "\x1dPromoteStagedVersionsResponse\x12\x1f\n" + + "\vnext_cursor\x18\x01 \x01(\fR\n" + + "nextCursor\x12\x12\n" + + "\x04done\x18\x02 \x01(\bR\x04done\x12#\n" + + "\rpromoted_rows\x18\x03 \x01(\x04R\fpromotedRows\x12&\n" + + "\x0fmax_promoted_ts\x18\x04 \x01(\x04R\rmaxPromotedTs\"\xf1\x03\n" + + "\x1cTargetStagedReadinessRequest\x12\x15\n" + + "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12\x1f\n" + + "\vroute_start\x18\x02 \x01(\fR\n" + + "routeStart\x12\x1b\n" + + "\troute_end\x18\x03 \x01(\fR\brouteEnd\x128\n" + + "\x18expected_cutover_version\x18\x04 \x01(\x04R\x16expectedCutoverVersion\x12(\n" + + "\x10migration_job_id\x18\x05 \x01(\x04R\x0emigrationJobId\x123\n" + + "\x16min_write_ts_exclusive\x18\x06 \x01(\x04R\x13minWriteTsExclusive\x12\x14\n" + + "\x05armed\x18\a \x01(\bR\x05armed\x12,\n" + + "\x12source_write_fence\x18\b \x01(\bR\x10sourceWriteFence\x12*\n" + + "\x11source_read_fence\x18\t \x01(\bR\x0fsourceReadFence\x12(\n" + + "\x10retention_pin_ts\x18\n" + + " \x01(\x04R\x0eretentionPinTs\x12!\n" + + "\ftrack_writes\x18\v \x01(\bR\vtrackWrites\x12&\n" + + "\x0fmin_admitted_ts\x18\f \x01(\x04R\rminAdmittedTs\"G\n" + + "\x1dTargetStagedReadinessResponse\x12&\n" + + "\x0fmin_admitted_ts\x18\x01 \x01(\x04R\rminAdmittedTs\"p\n" + + "\x1aProbeMigrationLocksRequest\x12\x1f\n" + + "\vroute_start\x18\x01 \x01(\fR\n" + + "routeStart\x12\x1b\n" + + "\troute_end\x18\x02 \x01(\fR\brouteEnd\x12\x14\n" + + "\x05limit\x18\x03 \x01(\rR\x05limit\"B\n" + + "\x1bProbeMigrationLocksResponse\x12#\n" + + "\rpending_count\x18\x01 \x01(\rR\fpendingCount\"\xe8\x04\n" + + "\x17CleanupMigrationRequest\x12\x15\n" + + "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12)\n" + + "\x04mode\x18\x02 \x01(\x0e2\x15.MigrationCleanupModeR\x04mode\x12\x1f\n" + + "\vrange_start\x18\x03 \x01(\fR\n" + + "rangeStart\x12\x1b\n" + + "\trange_end\x18\x04 \x01(\fR\brangeEnd\x12\x16\n" + + "\x06cursor\x18\x05 \x01(\fR\x06cursor\x12\"\n" + + "\rmax_commit_ts\x18\x06 \x01(\x04R\vmaxCommitTs\x12!\n" + + "\fmax_versions\x18\a \x01(\rR\vmaxVersions\x12\x1b\n" + + "\tmax_bytes\x18\b \x01(\x04R\bmaxBytes\x12*\n" + + "\x11max_scanned_bytes\x18\t \x01(\x04R\x0fmaxScannedBytes\x12\x1d\n" + + "\n" + + "key_family\x18\n" + + " \x01(\rR\tkeyFamily\x12\x1f\n" + + "\vroute_start\x18\v \x01(\fR\n" + + "routeStart\x12\x1b\n" + + "\troute_end\x18\f \x01(\fR\brouteEnd\x124\n" + + "\x16exclude_known_internal\x18\r \x01(\bR\x14excludeKnownInternal\x12)\n" + + "\x10exclude_prefixes\x18\x0e \x03(\fR\x0fexcludePrefixes\x127\n" + + "\x18requires_route_key_check\x18\x0f \x01(\bR\x15requiresRouteKeyCheck\x12.\n" + + "\x13requires_decoded_s3\x18\x10 \x01(\bR\x11requiresDecodedS3\"\xbc\x01\n" + + "\x18CleanupMigrationResponse\x12\x1f\n" + + "\vnext_cursor\x18\x01 \x01(\fR\n" + + "nextCursor\x12\x12\n" + + "\x04done\x18\x02 \x01(\bR\x04done\x12!\n" + + "\fdeleted_rows\x18\x03 \x01(\x04R\vdeletedRows\x12#\n" + + "\rdeleted_bytes\x18\x04 \x01(\x04R\fdeletedBytes\x12#\n" + + "\rscanned_bytes\x18\x05 \x01(\x04R\fscannedBytes\"\xc3\x04\n" + + "\x1aProbeMigrationStateRequest\x12\x15\n" + + "\x06job_id\x18\x01 \x01(\x04R\x05jobId\x12,\n" + + "\x04kind\x18\x02 \x01(\x0e2\x18.MigrationStateProbeKindR\x04kind\x12\x1f\n" + + "\vroute_start\x18\x03 \x01(\fR\n" + + "routeStart\x12\x1b\n" + + "\troute_end\x18\x04 \x01(\fR\brouteEnd\x128\n" + + "\x18expected_catalog_version\x18\x05 \x01(\x04R\x16expectedCatalogVersion\x12*\n" + + "\x11expected_group_id\x18\x06 \x01(\x04R\x0fexpectedGroupId\x12(\n" + + "\x10migration_job_id\x18\a \x01(\x04R\x0emigrationJobId\x123\n" + + "\x16min_write_ts_exclusive\x18\b \x01(\x04R\x13minWriteTsExclusive\x12,\n" + + "\x12source_write_fence\x18\t \x01(\bR\x10sourceWriteFence\x12*\n" + + "\x11source_read_fence\x18\n" + + " \x01(\bR\x0fsourceReadFence\x12!\n" + + "\ftrack_writes\x18\v \x01(\bR\vtrackWrites\x12(\n" + + "\x10retention_pin_ts\x18\f \x01(\x04R\x0eretentionPinTs\x126\n" + + "\x18read_drain_not_before_ms\x18\r \x01(\x03R\x14readDrainNotBeforeMs\"\x84\x01\n" + + "\x1bProbeMigrationStateResponse\x12\x14\n" + + "\x05ready\x18\x01 \x01(\bR\x05ready\x12'\n" + + "\x0fcatalog_version\x18\x02 \x01(\x04R\x0ecatalogVersion\x12&\n" + + "\x0fmin_admitted_ts\x18\x03 \x01(\x04R\rminAdmittedTs\" \n" + + "\x1eIssueMigrationTimestampRequest\"e\n" + + "\x1fIssueMigrationTimestampResponse\x12\x1c\n" + + "\ttimestamp\x18\x01 \x01(\x04R\ttimestamp\x12$\n" + + "\x0elast_commit_ts\x18\x02 \x01(\x04R\flastCommitTs*&\n" + "\x02Op\x12\a\n" + "\x03PUT\x10\x00\x12\a\n" + "\x03DEL\x10\x01\x12\x0e\n" + @@ -1203,13 +2196,29 @@ const file_internal_proto_rawDesc = "" + "\aPREPARE\x10\x01\x12\n" + "\n" + "\x06COMMIT\x10\x02\x12\t\n" + - "\x05ABORT\x10\x032\xfd\x02\n" + + "\x05ABORT\x10\x03*\x88\x01\n" + + "\x14MigrationCleanupMode\x12&\n" + + "\"MIGRATION_CLEANUP_MODE_UNSPECIFIED\x10\x00\x12#\n" + + "\x1fMIGRATION_CLEANUP_MODE_VERSIONS\x10\x01\x12#\n" + + "\x1fMIGRATION_CLEANUP_MODE_METADATA\x10\x02*\xc9\x02\n" + + "\x17MigrationStateProbeKind\x12*\n" + + "&MIGRATION_STATE_PROBE_KIND_UNSPECIFIED\x10\x00\x12.\n" + + "*MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED\x10\x01\x128\n" + + "4MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED\x10\x02\x123\n" + + "/MIGRATION_STATE_PROBE_KIND_SOURCE_ROUTE_REMOVED\x10\x03\x122\n" + + ".MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED\x10\x04\x12/\n" + + "+MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED\x10\x052\xaf\x06\n" + "\bInternal\x12.\n" + "\aForward\x12\x0f.ForwardRequest\x1a\x10.ForwardResponse\"\x00\x12=\n" + "\fRelayPublish\x12\x14.RelayPublishRequest\x1a\x15.RelayPublishResponse\"\x00\x12T\n" + "\x13ExportRangeVersions\x12\x1b.ExportRangeVersionsRequest\x1a\x1c.ExportRangeVersionsResponse\"\x000\x01\x12R\n" + "\x13ImportRangeVersions\x12\x1b.ImportRangeVersionsRequest\x1a\x1c.ImportRangeVersionsResponse\"\x00\x12X\n" + - "\x15PromoteStagedVersions\x12\x1d.PromoteStagedVersionsRequest\x1a\x1e.PromoteStagedVersionsResponse\"\x00B#Z!github.com/bootjp/elastickv/protob\x06proto3" + "\x15PromoteStagedVersions\x12\x1d.PromoteStagedVersionsRequest\x1a\x1e.PromoteStagedVersionsResponse\"\x00\x12]\n" + + "\x1aApplyTargetStagedReadiness\x12\x1d.TargetStagedReadinessRequest\x1a\x1e.TargetStagedReadinessResponse\"\x00\x12R\n" + + "\x13ProbeMigrationLocks\x12\x1b.ProbeMigrationLocksRequest\x1a\x1c.ProbeMigrationLocksResponse\"\x00\x12I\n" + + "\x10CleanupMigration\x12\x18.CleanupMigrationRequest\x1a\x19.CleanupMigrationResponse\"\x00\x12R\n" + + "\x13ProbeMigrationState\x12\x1b.ProbeMigrationStateRequest\x1a\x1c.ProbeMigrationStateResponse\"\x00\x12^\n" + + "\x17IssueMigrationTimestamp\x12\x1f.IssueMigrationTimestampRequest\x1a .IssueMigrationTimestampResponse\"\x00B#Z!github.com/bootjp/elastickv/protob\x06proto3" var ( file_internal_proto_rawDescOnce sync.Once @@ -1223,49 +2232,73 @@ func file_internal_proto_rawDescGZIP() []byte { return file_internal_proto_rawDescData } -var file_internal_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_internal_proto_enumTypes = make([]protoimpl.EnumInfo, 4) +var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 24) var file_internal_proto_goTypes = []any{ - (Op)(0), // 0: Op - (Phase)(0), // 1: Phase - (*Mutation)(nil), // 2: Mutation - (*Request)(nil), // 3: Request - (*RaftCommand)(nil), // 4: RaftCommand - (*ForwardRequest)(nil), // 5: ForwardRequest - (*ForwardResponse)(nil), // 6: ForwardResponse - (*RelayPublishRequest)(nil), // 7: RelayPublishRequest - (*RelayPublishResponse)(nil), // 8: RelayPublishResponse - (*ExportRangeVersionsRequest)(nil), // 9: ExportRangeVersionsRequest - (*ExportRangeVersionsResponse)(nil), // 10: ExportRangeVersionsResponse - (*MVCCVersion)(nil), // 11: MVCCVersion - (*ImportRangeVersionsRequest)(nil), // 12: ImportRangeVersionsRequest - (*ImportRangeVersionsResponse)(nil), // 13: ImportRangeVersionsResponse - (*PromoteStagedVersionsRequest)(nil), // 14: PromoteStagedVersionsRequest - (*PromoteStagedVersionsResponse)(nil), // 15: PromoteStagedVersionsResponse + (Op)(0), // 0: Op + (Phase)(0), // 1: Phase + (MigrationCleanupMode)(0), // 2: MigrationCleanupMode + (MigrationStateProbeKind)(0), // 3: MigrationStateProbeKind + (*Mutation)(nil), // 4: Mutation + (*Request)(nil), // 5: Request + (*RaftCommand)(nil), // 6: RaftCommand + (*ForwardRequest)(nil), // 7: ForwardRequest + (*ForwardResponse)(nil), // 8: ForwardResponse + (*RelayPublishRequest)(nil), // 9: RelayPublishRequest + (*RelayPublishResponse)(nil), // 10: RelayPublishResponse + (*ExportRangeVersionsRequest)(nil), // 11: ExportRangeVersionsRequest + (*ExportRangeVersionsResponse)(nil), // 12: ExportRangeVersionsResponse + (*MVCCVersion)(nil), // 13: MVCCVersion + (*ImportRangeVersionsRequest)(nil), // 14: ImportRangeVersionsRequest + (*ImportRangeVersionsResponse)(nil), // 15: ImportRangeVersionsResponse + (*PromoteStagedVersionsRequest)(nil), // 16: PromoteStagedVersionsRequest + (*PromoteStagedVersionsResponse)(nil), // 17: PromoteStagedVersionsResponse + (*TargetStagedReadinessRequest)(nil), // 18: TargetStagedReadinessRequest + (*TargetStagedReadinessResponse)(nil), // 19: TargetStagedReadinessResponse + (*ProbeMigrationLocksRequest)(nil), // 20: ProbeMigrationLocksRequest + (*ProbeMigrationLocksResponse)(nil), // 21: ProbeMigrationLocksResponse + (*CleanupMigrationRequest)(nil), // 22: CleanupMigrationRequest + (*CleanupMigrationResponse)(nil), // 23: CleanupMigrationResponse + (*ProbeMigrationStateRequest)(nil), // 24: ProbeMigrationStateRequest + (*ProbeMigrationStateResponse)(nil), // 25: ProbeMigrationStateResponse + (*IssueMigrationTimestampRequest)(nil), // 26: IssueMigrationTimestampRequest + (*IssueMigrationTimestampResponse)(nil), // 27: IssueMigrationTimestampResponse } var file_internal_proto_depIdxs = []int32{ 0, // 0: Mutation.op:type_name -> Op 1, // 1: Request.phase:type_name -> Phase - 2, // 2: Request.mutations:type_name -> Mutation - 3, // 3: RaftCommand.requests:type_name -> Request - 3, // 4: ForwardRequest.requests:type_name -> Request - 11, // 5: ExportRangeVersionsResponse.versions:type_name -> MVCCVersion - 11, // 6: ImportRangeVersionsRequest.versions:type_name -> MVCCVersion - 5, // 7: Internal.Forward:input_type -> ForwardRequest - 7, // 8: Internal.RelayPublish:input_type -> RelayPublishRequest - 9, // 9: Internal.ExportRangeVersions:input_type -> ExportRangeVersionsRequest - 12, // 10: Internal.ImportRangeVersions:input_type -> ImportRangeVersionsRequest - 14, // 11: Internal.PromoteStagedVersions:input_type -> PromoteStagedVersionsRequest - 6, // 12: Internal.Forward:output_type -> ForwardResponse - 8, // 13: Internal.RelayPublish:output_type -> RelayPublishResponse - 10, // 14: Internal.ExportRangeVersions:output_type -> ExportRangeVersionsResponse - 13, // 15: Internal.ImportRangeVersions:output_type -> ImportRangeVersionsResponse - 15, // 16: Internal.PromoteStagedVersions:output_type -> PromoteStagedVersionsResponse - 12, // [12:17] is the sub-list for method output_type - 7, // [7:12] is the sub-list for method input_type - 7, // [7:7] is the sub-list for extension type_name - 7, // [7:7] is the sub-list for extension extendee - 0, // [0:7] is the sub-list for field type_name + 4, // 2: Request.mutations:type_name -> Mutation + 5, // 3: RaftCommand.requests:type_name -> Request + 5, // 4: ForwardRequest.requests:type_name -> Request + 13, // 5: ExportRangeVersionsResponse.versions:type_name -> MVCCVersion + 13, // 6: ImportRangeVersionsRequest.versions:type_name -> MVCCVersion + 2, // 7: CleanupMigrationRequest.mode:type_name -> MigrationCleanupMode + 3, // 8: ProbeMigrationStateRequest.kind:type_name -> MigrationStateProbeKind + 7, // 9: Internal.Forward:input_type -> ForwardRequest + 9, // 10: Internal.RelayPublish:input_type -> RelayPublishRequest + 11, // 11: Internal.ExportRangeVersions:input_type -> ExportRangeVersionsRequest + 14, // 12: Internal.ImportRangeVersions:input_type -> ImportRangeVersionsRequest + 16, // 13: Internal.PromoteStagedVersions:input_type -> PromoteStagedVersionsRequest + 18, // 14: Internal.ApplyTargetStagedReadiness:input_type -> TargetStagedReadinessRequest + 20, // 15: Internal.ProbeMigrationLocks:input_type -> ProbeMigrationLocksRequest + 22, // 16: Internal.CleanupMigration:input_type -> CleanupMigrationRequest + 24, // 17: Internal.ProbeMigrationState:input_type -> ProbeMigrationStateRequest + 26, // 18: Internal.IssueMigrationTimestamp:input_type -> IssueMigrationTimestampRequest + 8, // 19: Internal.Forward:output_type -> ForwardResponse + 10, // 20: Internal.RelayPublish:output_type -> RelayPublishResponse + 12, // 21: Internal.ExportRangeVersions:output_type -> ExportRangeVersionsResponse + 15, // 22: Internal.ImportRangeVersions:output_type -> ImportRangeVersionsResponse + 17, // 23: Internal.PromoteStagedVersions:output_type -> PromoteStagedVersionsResponse + 19, // 24: Internal.ApplyTargetStagedReadiness:output_type -> TargetStagedReadinessResponse + 21, // 25: Internal.ProbeMigrationLocks:output_type -> ProbeMigrationLocksResponse + 23, // 26: Internal.CleanupMigration:output_type -> CleanupMigrationResponse + 25, // 27: Internal.ProbeMigrationState:output_type -> ProbeMigrationStateResponse + 27, // 28: Internal.IssueMigrationTimestamp:output_type -> IssueMigrationTimestampResponse + 19, // [19:29] is the sub-list for method output_type + 9, // [9:19] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name } func init() { file_internal_proto_init() } @@ -1278,8 +2311,8 @@ func file_internal_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_internal_proto_rawDesc), len(file_internal_proto_rawDesc)), - NumEnums: 2, - NumMessages: 14, + NumEnums: 4, + NumMessages: 24, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/internal.proto b/proto/internal.proto index 7432916b8..07662a02d 100644 --- a/proto/internal.proto +++ b/proto/internal.proto @@ -10,6 +10,11 @@ service Internal { rpc ExportRangeVersions(ExportRangeVersionsRequest) returns (stream ExportRangeVersionsResponse) {} rpc ImportRangeVersions(ImportRangeVersionsRequest) returns (ImportRangeVersionsResponse) {} rpc PromoteStagedVersions(PromoteStagedVersionsRequest) returns (PromoteStagedVersionsResponse) {} + rpc ApplyTargetStagedReadiness(TargetStagedReadinessRequest) returns (TargetStagedReadinessResponse) {} + rpc ProbeMigrationLocks(ProbeMigrationLocksRequest) returns (ProbeMigrationLocksResponse) {} + rpc CleanupMigration(CleanupMigrationRequest) returns (CleanupMigrationResponse) {} + rpc ProbeMigrationState(ProbeMigrationStateRequest) returns (ProbeMigrationStateResponse) {} + rpc IssueMigrationTimestamp(IssueMigrationTimestampRequest) returns (IssueMigrationTimestampResponse) {} } // internal.proto is node to node communication message in raft replication. @@ -157,3 +162,105 @@ message PromoteStagedVersionsResponse { uint64 promoted_rows = 3; uint64 max_promoted_ts = 4; } + +message TargetStagedReadinessRequest { + uint64 job_id = 1; + bytes route_start = 2; + bytes route_end = 3; + uint64 expected_cutover_version = 4; + uint64 migration_job_id = 5; + uint64 min_write_ts_exclusive = 6; + bool armed = 7; + // Source-side controls reuse the same Raft-replicated durable record. They + // are ignored by target staged-readiness checks. + bool source_write_fence = 8; + bool source_read_fence = 9; + uint64 retention_pin_ts = 10; + bool track_writes = 11; + uint64 min_admitted_ts = 12; +} + +message TargetStagedReadinessResponse { + uint64 min_admitted_ts = 1; +} + +message ProbeMigrationLocksRequest { + bytes route_start = 1; + bytes route_end = 2; + uint32 limit = 3; +} + +message ProbeMigrationLocksResponse { + uint32 pending_count = 1; +} + +enum MigrationCleanupMode { + MIGRATION_CLEANUP_MODE_UNSPECIFIED = 0; + MIGRATION_CLEANUP_MODE_VERSIONS = 1; + MIGRATION_CLEANUP_MODE_METADATA = 2; +} + +message CleanupMigrationRequest { + uint64 job_id = 1; + MigrationCleanupMode mode = 2; + bytes range_start = 3; + bytes range_end = 4; + bytes cursor = 5; + uint64 max_commit_ts = 6; + uint32 max_versions = 7; + uint64 max_bytes = 8; + uint64 max_scanned_bytes = 9; + uint32 key_family = 10; + bytes route_start = 11; + bytes route_end = 12; + bool exclude_known_internal = 13; + repeated bytes exclude_prefixes = 14; + bool requires_route_key_check = 15; + bool requires_decoded_s3 = 16; +} + +message CleanupMigrationResponse { + bytes next_cursor = 1; + bool done = 2; + uint64 deleted_rows = 3; + uint64 deleted_bytes = 4; + uint64 scanned_bytes = 5; +} + +enum MigrationStateProbeKind { + MIGRATION_STATE_PROBE_KIND_UNSPECIFIED = 0; + MIGRATION_STATE_PROBE_KIND_CONTROL_APPLIED = 1; + MIGRATION_STATE_PROBE_KIND_TARGET_DESCRIPTOR_CLEARED = 2; + MIGRATION_STATE_PROBE_KIND_SOURCE_ROUTE_REMOVED = 3; + MIGRATION_STATE_PROBE_KIND_SOURCE_READ_DRAINED = 4; + MIGRATION_STATE_PROBE_KIND_METADATA_CLEARED = 5; +} + +message ProbeMigrationStateRequest { + uint64 job_id = 1; + MigrationStateProbeKind kind = 2; + bytes route_start = 3; + bytes route_end = 4; + uint64 expected_catalog_version = 5; + uint64 expected_group_id = 6; + uint64 migration_job_id = 7; + uint64 min_write_ts_exclusive = 8; + bool source_write_fence = 9; + bool source_read_fence = 10; + bool track_writes = 11; + uint64 retention_pin_ts = 12; + int64 read_drain_not_before_ms = 13; +} + +message ProbeMigrationStateResponse { + bool ready = 1; + uint64 catalog_version = 2; + uint64 min_admitted_ts = 3; +} + +message IssueMigrationTimestampRequest {} + +message IssueMigrationTimestampResponse { + uint64 timestamp = 1; + uint64 last_commit_ts = 2; +} diff --git a/proto/internal_grpc.pb.go b/proto/internal_grpc.pb.go index ba679ed80..18ed111d8 100644 --- a/proto/internal_grpc.pb.go +++ b/proto/internal_grpc.pb.go @@ -19,11 +19,16 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - Internal_Forward_FullMethodName = "/Internal/Forward" - Internal_RelayPublish_FullMethodName = "/Internal/RelayPublish" - Internal_ExportRangeVersions_FullMethodName = "/Internal/ExportRangeVersions" - Internal_ImportRangeVersions_FullMethodName = "/Internal/ImportRangeVersions" - Internal_PromoteStagedVersions_FullMethodName = "/Internal/PromoteStagedVersions" + Internal_Forward_FullMethodName = "/Internal/Forward" + Internal_RelayPublish_FullMethodName = "/Internal/RelayPublish" + Internal_ExportRangeVersions_FullMethodName = "/Internal/ExportRangeVersions" + Internal_ImportRangeVersions_FullMethodName = "/Internal/ImportRangeVersions" + Internal_PromoteStagedVersions_FullMethodName = "/Internal/PromoteStagedVersions" + Internal_ApplyTargetStagedReadiness_FullMethodName = "/Internal/ApplyTargetStagedReadiness" + Internal_ProbeMigrationLocks_FullMethodName = "/Internal/ProbeMigrationLocks" + Internal_CleanupMigration_FullMethodName = "/Internal/CleanupMigration" + Internal_ProbeMigrationState_FullMethodName = "/Internal/ProbeMigrationState" + Internal_IssueMigrationTimestamp_FullMethodName = "/Internal/IssueMigrationTimestamp" ) // InternalClient is the client API for Internal service. @@ -36,6 +41,11 @@ type InternalClient interface { ExportRangeVersions(ctx context.Context, in *ExportRangeVersionsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[ExportRangeVersionsResponse], error) ImportRangeVersions(ctx context.Context, in *ImportRangeVersionsRequest, opts ...grpc.CallOption) (*ImportRangeVersionsResponse, error) PromoteStagedVersions(ctx context.Context, in *PromoteStagedVersionsRequest, opts ...grpc.CallOption) (*PromoteStagedVersionsResponse, error) + ApplyTargetStagedReadiness(ctx context.Context, in *TargetStagedReadinessRequest, opts ...grpc.CallOption) (*TargetStagedReadinessResponse, error) + ProbeMigrationLocks(ctx context.Context, in *ProbeMigrationLocksRequest, opts ...grpc.CallOption) (*ProbeMigrationLocksResponse, error) + CleanupMigration(ctx context.Context, in *CleanupMigrationRequest, opts ...grpc.CallOption) (*CleanupMigrationResponse, error) + ProbeMigrationState(ctx context.Context, in *ProbeMigrationStateRequest, opts ...grpc.CallOption) (*ProbeMigrationStateResponse, error) + IssueMigrationTimestamp(ctx context.Context, in *IssueMigrationTimestampRequest, opts ...grpc.CallOption) (*IssueMigrationTimestampResponse, error) } type internalClient struct { @@ -105,6 +115,56 @@ func (c *internalClient) PromoteStagedVersions(ctx context.Context, in *PromoteS return out, nil } +func (c *internalClient) ApplyTargetStagedReadiness(ctx context.Context, in *TargetStagedReadinessRequest, opts ...grpc.CallOption) (*TargetStagedReadinessResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(TargetStagedReadinessResponse) + err := c.cc.Invoke(ctx, Internal_ApplyTargetStagedReadiness_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *internalClient) ProbeMigrationLocks(ctx context.Context, in *ProbeMigrationLocksRequest, opts ...grpc.CallOption) (*ProbeMigrationLocksResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ProbeMigrationLocksResponse) + err := c.cc.Invoke(ctx, Internal_ProbeMigrationLocks_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *internalClient) CleanupMigration(ctx context.Context, in *CleanupMigrationRequest, opts ...grpc.CallOption) (*CleanupMigrationResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(CleanupMigrationResponse) + err := c.cc.Invoke(ctx, Internal_CleanupMigration_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *internalClient) ProbeMigrationState(ctx context.Context, in *ProbeMigrationStateRequest, opts ...grpc.CallOption) (*ProbeMigrationStateResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ProbeMigrationStateResponse) + err := c.cc.Invoke(ctx, Internal_ProbeMigrationState_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *internalClient) IssueMigrationTimestamp(ctx context.Context, in *IssueMigrationTimestampRequest, opts ...grpc.CallOption) (*IssueMigrationTimestampResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(IssueMigrationTimestampResponse) + err := c.cc.Invoke(ctx, Internal_IssueMigrationTimestamp_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // InternalServer is the server API for Internal service. // All implementations must embed UnimplementedInternalServer // for forward compatibility. @@ -115,6 +175,11 @@ type InternalServer interface { ExportRangeVersions(*ExportRangeVersionsRequest, grpc.ServerStreamingServer[ExportRangeVersionsResponse]) error ImportRangeVersions(context.Context, *ImportRangeVersionsRequest) (*ImportRangeVersionsResponse, error) PromoteStagedVersions(context.Context, *PromoteStagedVersionsRequest) (*PromoteStagedVersionsResponse, error) + ApplyTargetStagedReadiness(context.Context, *TargetStagedReadinessRequest) (*TargetStagedReadinessResponse, error) + ProbeMigrationLocks(context.Context, *ProbeMigrationLocksRequest) (*ProbeMigrationLocksResponse, error) + CleanupMigration(context.Context, *CleanupMigrationRequest) (*CleanupMigrationResponse, error) + ProbeMigrationState(context.Context, *ProbeMigrationStateRequest) (*ProbeMigrationStateResponse, error) + IssueMigrationTimestamp(context.Context, *IssueMigrationTimestampRequest) (*IssueMigrationTimestampResponse, error) mustEmbedUnimplementedInternalServer() } @@ -140,6 +205,21 @@ func (UnimplementedInternalServer) ImportRangeVersions(context.Context, *ImportR func (UnimplementedInternalServer) PromoteStagedVersions(context.Context, *PromoteStagedVersionsRequest) (*PromoteStagedVersionsResponse, error) { return nil, status.Error(codes.Unimplemented, "method PromoteStagedVersions not implemented") } +func (UnimplementedInternalServer) ApplyTargetStagedReadiness(context.Context, *TargetStagedReadinessRequest) (*TargetStagedReadinessResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ApplyTargetStagedReadiness not implemented") +} +func (UnimplementedInternalServer) ProbeMigrationLocks(context.Context, *ProbeMigrationLocksRequest) (*ProbeMigrationLocksResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ProbeMigrationLocks not implemented") +} +func (UnimplementedInternalServer) CleanupMigration(context.Context, *CleanupMigrationRequest) (*CleanupMigrationResponse, error) { + return nil, status.Error(codes.Unimplemented, "method CleanupMigration not implemented") +} +func (UnimplementedInternalServer) ProbeMigrationState(context.Context, *ProbeMigrationStateRequest) (*ProbeMigrationStateResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ProbeMigrationState not implemented") +} +func (UnimplementedInternalServer) IssueMigrationTimestamp(context.Context, *IssueMigrationTimestampRequest) (*IssueMigrationTimestampResponse, error) { + return nil, status.Error(codes.Unimplemented, "method IssueMigrationTimestamp not implemented") +} func (UnimplementedInternalServer) mustEmbedUnimplementedInternalServer() {} func (UnimplementedInternalServer) testEmbeddedByValue() {} @@ -244,6 +324,96 @@ func _Internal_PromoteStagedVersions_Handler(srv interface{}, ctx context.Contex return interceptor(ctx, in, info, handler) } +func _Internal_ApplyTargetStagedReadiness_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(TargetStagedReadinessRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InternalServer).ApplyTargetStagedReadiness(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Internal_ApplyTargetStagedReadiness_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InternalServer).ApplyTargetStagedReadiness(ctx, req.(*TargetStagedReadinessRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Internal_ProbeMigrationLocks_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ProbeMigrationLocksRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InternalServer).ProbeMigrationLocks(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Internal_ProbeMigrationLocks_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InternalServer).ProbeMigrationLocks(ctx, req.(*ProbeMigrationLocksRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Internal_CleanupMigration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CleanupMigrationRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InternalServer).CleanupMigration(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Internal_CleanupMigration_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InternalServer).CleanupMigration(ctx, req.(*CleanupMigrationRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Internal_ProbeMigrationState_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ProbeMigrationStateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InternalServer).ProbeMigrationState(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Internal_ProbeMigrationState_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InternalServer).ProbeMigrationState(ctx, req.(*ProbeMigrationStateRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Internal_IssueMigrationTimestamp_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(IssueMigrationTimestampRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InternalServer).IssueMigrationTimestamp(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Internal_IssueMigrationTimestamp_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InternalServer).IssueMigrationTimestamp(ctx, req.(*IssueMigrationTimestampRequest)) + } + return interceptor(ctx, in, info, handler) +} + // Internal_ServiceDesc is the grpc.ServiceDesc for Internal service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -267,6 +437,26 @@ var Internal_ServiceDesc = grpc.ServiceDesc{ MethodName: "PromoteStagedVersions", Handler: _Internal_PromoteStagedVersions_Handler, }, + { + MethodName: "ApplyTargetStagedReadiness", + Handler: _Internal_ApplyTargetStagedReadiness_Handler, + }, + { + MethodName: "ProbeMigrationLocks", + Handler: _Internal_ProbeMigrationLocks_Handler, + }, + { + MethodName: "CleanupMigration", + Handler: _Internal_CleanupMigration_Handler, + }, + { + MethodName: "ProbeMigrationState", + Handler: _Internal_ProbeMigrationState_Handler, + }, + { + MethodName: "IssueMigrationTimestamp", + Handler: _Internal_IssueMigrationTimestamp_Handler, + }, }, Streams: []grpc.StreamDesc{ { diff --git a/proto/service.pb.go b/proto/service.pb.go index 80a0a6820..df9370abd 100644 --- a/proto/service.pb.go +++ b/proto/service.pb.go @@ -2271,6 +2271,214 @@ func (x *TransferTarget) GetTargetAddress() string { return "" } +type FetchChunkBlobRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + ContentSha256 []byte `protobuf:"bytes,1,opt,name=content_sha256,json=contentSha256,proto3" json:"content_sha256,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FetchChunkBlobRequest) Reset() { + *x = FetchChunkBlobRequest{} + mi := &file_service_proto_msgTypes[39] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FetchChunkBlobRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FetchChunkBlobRequest) ProtoMessage() {} + +func (x *FetchChunkBlobRequest) ProtoReflect() protoreflect.Message { + mi := &file_service_proto_msgTypes[39] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FetchChunkBlobRequest.ProtoReflect.Descriptor instead. +func (*FetchChunkBlobRequest) Descriptor() ([]byte, []int) { + return file_service_proto_rawDescGZIP(), []int{39} +} + +func (x *FetchChunkBlobRequest) GetContentSha256() []byte { + if x != nil { + return x.ContentSha256 + } + return nil +} + +type FetchChunkBlobResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Payload []byte `protobuf:"bytes,1,opt,name=payload,proto3" json:"payload,omitempty"` + Eof bool `protobuf:"varint,2,opt,name=eof,proto3" json:"eof,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FetchChunkBlobResponse) Reset() { + *x = FetchChunkBlobResponse{} + mi := &file_service_proto_msgTypes[40] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FetchChunkBlobResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FetchChunkBlobResponse) ProtoMessage() {} + +func (x *FetchChunkBlobResponse) ProtoReflect() protoreflect.Message { + mi := &file_service_proto_msgTypes[40] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FetchChunkBlobResponse.ProtoReflect.Descriptor instead. +func (*FetchChunkBlobResponse) Descriptor() ([]byte, []int) { + return file_service_proto_rawDescGZIP(), []int{40} +} + +func (x *FetchChunkBlobResponse) GetPayload() []byte { + if x != nil { + return x.Payload + } + return nil +} + +func (x *FetchChunkBlobResponse) GetEof() bool { + if x != nil { + return x.Eof + } + return false +} + +type PushChunkBlobRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + ContentSha256 []byte `protobuf:"bytes,1,opt,name=content_sha256,json=contentSha256,proto3" json:"content_sha256,omitempty"` + Payload []byte `protobuf:"bytes,2,opt,name=payload,proto3" json:"payload,omitempty"` + Eof bool `protobuf:"varint,3,opt,name=eof,proto3" json:"eof,omitempty"` + CommitTs uint64 `protobuf:"varint,4,opt,name=commit_ts,json=commitTs,proto3" json:"commit_ts,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PushChunkBlobRequest) Reset() { + *x = PushChunkBlobRequest{} + mi := &file_service_proto_msgTypes[41] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PushChunkBlobRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PushChunkBlobRequest) ProtoMessage() {} + +func (x *PushChunkBlobRequest) ProtoReflect() protoreflect.Message { + mi := &file_service_proto_msgTypes[41] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PushChunkBlobRequest.ProtoReflect.Descriptor instead. +func (*PushChunkBlobRequest) Descriptor() ([]byte, []int) { + return file_service_proto_rawDescGZIP(), []int{41} +} + +func (x *PushChunkBlobRequest) GetContentSha256() []byte { + if x != nil { + return x.ContentSha256 + } + return nil +} + +func (x *PushChunkBlobRequest) GetPayload() []byte { + if x != nil { + return x.Payload + } + return nil +} + +func (x *PushChunkBlobRequest) GetEof() bool { + if x != nil { + return x.Eof + } + return false +} + +func (x *PushChunkBlobRequest) GetCommitTs() uint64 { + if x != nil { + return x.CommitTs + } + return 0 +} + +type PushChunkBlobResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Durable bool `protobuf:"varint,1,opt,name=durable,proto3" json:"durable,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PushChunkBlobResponse) Reset() { + *x = PushChunkBlobResponse{} + mi := &file_service_proto_msgTypes[42] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PushChunkBlobResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PushChunkBlobResponse) ProtoMessage() {} + +func (x *PushChunkBlobResponse) ProtoReflect() protoreflect.Message { + mi := &file_service_proto_msgTypes[42] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PushChunkBlobResponse.ProtoReflect.Descriptor instead. +func (*PushChunkBlobResponse) Descriptor() ([]byte, []int) { + return file_service_proto_rawDescGZIP(), []int{42} +} + +func (x *PushChunkBlobResponse) GetDurable() bool { + if x != nil { + return x.Durable + } + return false +} + type RaftAdminTransferLeadershipResponse struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -2279,7 +2487,7 @@ type RaftAdminTransferLeadershipResponse struct { func (x *RaftAdminTransferLeadershipResponse) Reset() { *x = RaftAdminTransferLeadershipResponse{} - mi := &file_service_proto_msgTypes[39] + mi := &file_service_proto_msgTypes[43] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2291,7 +2499,7 @@ func (x *RaftAdminTransferLeadershipResponse) String() string { func (*RaftAdminTransferLeadershipResponse) ProtoMessage() {} func (x *RaftAdminTransferLeadershipResponse) ProtoReflect() protoreflect.Message { - mi := &file_service_proto_msgTypes[39] + mi := &file_service_proto_msgTypes[43] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2304,7 +2512,7 @@ func (x *RaftAdminTransferLeadershipResponse) ProtoReflect() protoreflect.Messag // Deprecated: Use RaftAdminTransferLeadershipResponse.ProtoReflect.Descriptor instead. func (*RaftAdminTransferLeadershipResponse) Descriptor() ([]byte, []int) { - return file_service_proto_rawDescGZIP(), []int{39} + return file_service_proto_rawDescGZIP(), []int{43} } var File_service_proto protoreflect.FileDescriptor @@ -2406,7 +2614,7 @@ const file_service_proto_rawDesc = "" + "\bstart_ts\x18\x01 \x01(\x04R\astartTs\",\n" + "\x10RollbackResponse\x12\x18\n" + "\asuccess\x18\x01 \x01(\bR\asuccess\"\x18\n" + - "\x16RaftAdminStatusRequest\"\xa2\x03\n" + + "\x16RaftAdminStatusRequest\"\xd8\x03\n" + "\x17RaftAdminStatusResponse\x12%\n" + "\x05state\x18\x01 \x01(\x0e2\x0f.RaftAdminStateR\x05state\x12\x1b\n" + "\tleader_id\x18\x02 \x01(\tR\bleaderId\x12%\n" + @@ -2420,7 +2628,7 @@ const file_service_proto_rawDesc = "" + "fsmPending\x12\x1b\n" + "\tnum_peers\x18\n" + " \x01(\x04R\bnumPeers\x12,\n" + - "\x12last_contact_nanos\x18\v \x01(\x03R\x10lastContactNanos\"\x1f\n" + + "\x12last_contact_nanos\x18\v \x01(\x03R\x10lastContactNanosJ\x04\b\f\x10\rJ\x04\b\r\x10\x0eR\x13configuration_indexR\x13pending_conf_change\"\x1f\n" + "\x1dRaftAdminConfigurationRequest\"W\n" + "\x0fRaftAdminMember\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x18\n" + @@ -2454,7 +2662,19 @@ const file_service_proto_rawDesc = "" + "\x11target_candidates\x18\x05 \x03(\v2\x0f.TransferTargetR\x10targetCandidates\"T\n" + "\x0eTransferTarget\x12\x1b\n" + "\ttarget_id\x18\x01 \x01(\tR\btargetId\x12%\n" + - "\x0etarget_address\x18\x02 \x01(\tR\rtargetAddress\"%\n" + + "\x0etarget_address\x18\x02 \x01(\tR\rtargetAddress\">\n" + + "\x15FetchChunkBlobRequest\x12%\n" + + "\x0econtent_sha256\x18\x01 \x01(\fR\rcontentSha256\"D\n" + + "\x16FetchChunkBlobResponse\x12\x18\n" + + "\apayload\x18\x01 \x01(\fR\apayload\x12\x10\n" + + "\x03eof\x18\x02 \x01(\bR\x03eof\"\x86\x01\n" + + "\x14PushChunkBlobRequest\x12%\n" + + "\x0econtent_sha256\x18\x01 \x01(\fR\rcontentSha256\x12\x18\n" + + "\apayload\x18\x02 \x01(\fR\apayload\x12\x10\n" + + "\x03eof\x18\x03 \x01(\bR\x03eof\x12\x1b\n" + + "\tcommit_ts\x18\x04 \x01(\x04R\bcommitTs\"1\n" + + "\x15PushChunkBlobResponse\x12\x18\n" + + "\adurable\x18\x01 \x01(\bR\adurable\"%\n" + "#RaftAdminTransferLeadershipResponse*\xa9\x01\n" + "\x0eRaftAdminState\x12\x1c\n" + "\x18RAFT_ADMIN_STATE_UNKNOWN\x10\x00\x12\x1d\n" + @@ -2484,7 +2704,10 @@ const file_service_proto_rawDesc = "" + "AddLearner\x12\x1b.RaftAdminAddLearnerRequest\x1a%.RaftAdminConfigurationChangeResponse\"\x00\x12Z\n" + "\x0ePromoteLearner\x12\x1f.RaftAdminPromoteLearnerRequest\x1a%.RaftAdminConfigurationChangeResponse\"\x00\x12V\n" + "\fRemoveServer\x12\x1d.RaftAdminRemoveServerRequest\x1a%.RaftAdminConfigurationChangeResponse\"\x00\x12a\n" + - "\x12TransferLeadership\x12#.RaftAdminTransferLeadershipRequest\x1a$.RaftAdminTransferLeadershipResponse\"\x00B#Z!github.com/bootjp/elastickv/protob\x06proto3" + "\x12TransferLeadership\x12#.RaftAdminTransferLeadershipRequest\x1a$.RaftAdminTransferLeadershipResponse\"\x002\x98\x01\n" + + "\vS3BlobFetch\x12E\n" + + "\x0eFetchChunkBlob\x12\x16.FetchChunkBlobRequest\x1a\x17.FetchChunkBlobResponse\"\x000\x01\x12B\n" + + "\rPushChunkBlob\x12\x15.PushChunkBlobRequest\x1a\x16.PushChunkBlobResponse\"\x00(\x01B#Z!github.com/bootjp/elastickv/protob\x06proto3" var ( file_service_proto_rawDescOnce sync.Once @@ -2499,7 +2722,7 @@ func file_service_proto_rawDescGZIP() []byte { } var file_service_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_service_proto_msgTypes = make([]protoimpl.MessageInfo, 40) +var file_service_proto_msgTypes = make([]protoimpl.MessageInfo, 44) var file_service_proto_goTypes = []any{ (RaftAdminState)(0), // 0: RaftAdminState (*RawPutRequest)(nil), // 1: RawPutRequest @@ -2541,7 +2764,11 @@ var file_service_proto_goTypes = []any{ (*RaftAdminConfigurationChangeResponse)(nil), // 37: RaftAdminConfigurationChangeResponse (*RaftAdminTransferLeadershipRequest)(nil), // 38: RaftAdminTransferLeadershipRequest (*TransferTarget)(nil), // 39: TransferTarget - (*RaftAdminTransferLeadershipResponse)(nil), // 40: RaftAdminTransferLeadershipResponse + (*FetchChunkBlobRequest)(nil), // 40: FetchChunkBlobRequest + (*FetchChunkBlobResponse)(nil), // 41: FetchChunkBlobResponse + (*PushChunkBlobRequest)(nil), // 42: PushChunkBlobRequest + (*PushChunkBlobResponse)(nil), // 43: PushChunkBlobResponse + (*RaftAdminTransferLeadershipResponse)(nil), // 44: RaftAdminTransferLeadershipResponse } var file_service_proto_depIdxs = []int32{ 10, // 0: RawScanAtResponse.kv:type_name -> RawKVPair @@ -2572,27 +2799,31 @@ var file_service_proto_depIdxs = []int32{ 35, // 25: RaftAdmin.PromoteLearner:input_type -> RaftAdminPromoteLearnerRequest 36, // 26: RaftAdmin.RemoveServer:input_type -> RaftAdminRemoveServerRequest 38, // 27: RaftAdmin.TransferLeadership:input_type -> RaftAdminTransferLeadershipRequest - 2, // 28: RawKV.RawPut:output_type -> RawPutResponse - 4, // 29: RawKV.RawGet:output_type -> RawGetResponse - 6, // 30: RawKV.RawDelete:output_type -> RawDeleteResponse - 8, // 31: RawKV.RawLatestCommitTS:output_type -> RawLatestCommitTSResponse - 11, // 32: RawKV.RawScanAt:output_type -> RawScanAtResponse - 13, // 33: TransactionalKV.Put:output_type -> PutResponse - 17, // 34: TransactionalKV.Get:output_type -> GetResponse - 15, // 35: TransactionalKV.Delete:output_type -> DeleteResponse - 21, // 36: TransactionalKV.Scan:output_type -> ScanResponse - 23, // 37: TransactionalKV.PreWrite:output_type -> PreCommitResponse - 25, // 38: TransactionalKV.Commit:output_type -> CommitResponse - 27, // 39: TransactionalKV.Rollback:output_type -> RollbackResponse - 29, // 40: RaftAdmin.Status:output_type -> RaftAdminStatusResponse - 32, // 41: RaftAdmin.Configuration:output_type -> RaftAdminConfigurationResponse - 37, // 42: RaftAdmin.AddVoter:output_type -> RaftAdminConfigurationChangeResponse - 37, // 43: RaftAdmin.AddLearner:output_type -> RaftAdminConfigurationChangeResponse - 37, // 44: RaftAdmin.PromoteLearner:output_type -> RaftAdminConfigurationChangeResponse - 37, // 45: RaftAdmin.RemoveServer:output_type -> RaftAdminConfigurationChangeResponse - 40, // 46: RaftAdmin.TransferLeadership:output_type -> RaftAdminTransferLeadershipResponse - 28, // [28:47] is the sub-list for method output_type - 9, // [9:28] is the sub-list for method input_type + 40, // 28: S3BlobFetch.FetchChunkBlob:input_type -> FetchChunkBlobRequest + 42, // 29: S3BlobFetch.PushChunkBlob:input_type -> PushChunkBlobRequest + 2, // 30: RawKV.RawPut:output_type -> RawPutResponse + 4, // 31: RawKV.RawGet:output_type -> RawGetResponse + 6, // 32: RawKV.RawDelete:output_type -> RawDeleteResponse + 8, // 33: RawKV.RawLatestCommitTS:output_type -> RawLatestCommitTSResponse + 11, // 34: RawKV.RawScanAt:output_type -> RawScanAtResponse + 13, // 35: TransactionalKV.Put:output_type -> PutResponse + 17, // 36: TransactionalKV.Get:output_type -> GetResponse + 15, // 37: TransactionalKV.Delete:output_type -> DeleteResponse + 21, // 38: TransactionalKV.Scan:output_type -> ScanResponse + 23, // 39: TransactionalKV.PreWrite:output_type -> PreCommitResponse + 25, // 40: TransactionalKV.Commit:output_type -> CommitResponse + 27, // 41: TransactionalKV.Rollback:output_type -> RollbackResponse + 29, // 42: RaftAdmin.Status:output_type -> RaftAdminStatusResponse + 32, // 43: RaftAdmin.Configuration:output_type -> RaftAdminConfigurationResponse + 37, // 44: RaftAdmin.AddVoter:output_type -> RaftAdminConfigurationChangeResponse + 37, // 45: RaftAdmin.AddLearner:output_type -> RaftAdminConfigurationChangeResponse + 37, // 46: RaftAdmin.PromoteLearner:output_type -> RaftAdminConfigurationChangeResponse + 37, // 47: RaftAdmin.RemoveServer:output_type -> RaftAdminConfigurationChangeResponse + 44, // 48: RaftAdmin.TransferLeadership:output_type -> RaftAdminTransferLeadershipResponse + 41, // 49: S3BlobFetch.FetchChunkBlob:output_type -> FetchChunkBlobResponse + 43, // 50: S3BlobFetch.PushChunkBlob:output_type -> PushChunkBlobResponse + 30, // [30:51] is the sub-list for method output_type + 9, // [9:30] is the sub-list for method input_type 9, // [9:9] is the sub-list for extension type_name 9, // [9:9] is the sub-list for extension extendee 0, // [0:9] is the sub-list for field type_name @@ -2609,9 +2840,9 @@ func file_service_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_service_proto_rawDesc), len(file_service_proto_rawDesc)), NumEnums: 1, - NumMessages: 40, + NumMessages: 44, NumExtensions: 0, - NumServices: 3, + NumServices: 4, }, GoTypes: file_service_proto_goTypes, DependencyIndexes: file_service_proto_depIdxs, diff --git a/proto/service.proto b/proto/service.proto index 1e6937364..d1983d093 100644 --- a/proto/service.proto +++ b/proto/service.proto @@ -31,6 +31,11 @@ service RaftAdmin { rpc TransferLeadership(RaftAdminTransferLeadershipRequest) returns (RaftAdminTransferLeadershipResponse) {} } +service S3BlobFetch { + rpc FetchChunkBlob(FetchChunkBlobRequest) returns (stream FetchChunkBlobResponse) {} + rpc PushChunkBlob(stream PushChunkBlobRequest) returns (PushChunkBlobResponse) {} +} + message RawPutRequest { bytes key = 1; bytes value = 2; @@ -190,6 +195,9 @@ enum RaftAdminState { message RaftAdminStatusRequest {} message RaftAdminStatusResponse { + reserved 12, 13; + reserved "configuration_index", "pending_conf_change"; + RaftAdminState state = 1; string leader_id = 2; string leader_address = 3; @@ -267,4 +275,24 @@ message TransferTarget { string target_address = 2; } +message FetchChunkBlobRequest { + bytes content_sha256 = 1; +} + +message FetchChunkBlobResponse { + bytes payload = 1; + bool eof = 2; +} + +message PushChunkBlobRequest { + bytes content_sha256 = 1; + bytes payload = 2; + bool eof = 3; + uint64 commit_ts = 4; +} + +message PushChunkBlobResponse { + bool durable = 1; +} + message RaftAdminTransferLeadershipResponse {} diff --git a/proto/service_grpc.pb.go b/proto/service_grpc.pb.go index 484d04c64..1254bcbf9 100644 --- a/proto/service_grpc.pb.go +++ b/proto/service_grpc.pb.go @@ -931,3 +931,139 @@ var RaftAdmin_ServiceDesc = grpc.ServiceDesc{ Streams: []grpc.StreamDesc{}, Metadata: "service.proto", } + +const ( + S3BlobFetch_FetchChunkBlob_FullMethodName = "/S3BlobFetch/FetchChunkBlob" + S3BlobFetch_PushChunkBlob_FullMethodName = "/S3BlobFetch/PushChunkBlob" +) + +// S3BlobFetchClient is the client API for S3BlobFetch service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +type S3BlobFetchClient interface { + FetchChunkBlob(ctx context.Context, in *FetchChunkBlobRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[FetchChunkBlobResponse], error) + PushChunkBlob(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStreamingClient[PushChunkBlobRequest, PushChunkBlobResponse], error) +} + +type s3BlobFetchClient struct { + cc grpc.ClientConnInterface +} + +func NewS3BlobFetchClient(cc grpc.ClientConnInterface) S3BlobFetchClient { + return &s3BlobFetchClient{cc} +} + +func (c *s3BlobFetchClient) FetchChunkBlob(ctx context.Context, in *FetchChunkBlobRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[FetchChunkBlobResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &S3BlobFetch_ServiceDesc.Streams[0], S3BlobFetch_FetchChunkBlob_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[FetchChunkBlobRequest, FetchChunkBlobResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type S3BlobFetch_FetchChunkBlobClient = grpc.ServerStreamingClient[FetchChunkBlobResponse] + +func (c *s3BlobFetchClient) PushChunkBlob(ctx context.Context, opts ...grpc.CallOption) (grpc.ClientStreamingClient[PushChunkBlobRequest, PushChunkBlobResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &S3BlobFetch_ServiceDesc.Streams[1], S3BlobFetch_PushChunkBlob_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[PushChunkBlobRequest, PushChunkBlobResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type S3BlobFetch_PushChunkBlobClient = grpc.ClientStreamingClient[PushChunkBlobRequest, PushChunkBlobResponse] + +// S3BlobFetchServer is the server API for S3BlobFetch service. +// All implementations must embed UnimplementedS3BlobFetchServer +// for forward compatibility. +type S3BlobFetchServer interface { + FetchChunkBlob(*FetchChunkBlobRequest, grpc.ServerStreamingServer[FetchChunkBlobResponse]) error + PushChunkBlob(grpc.ClientStreamingServer[PushChunkBlobRequest, PushChunkBlobResponse]) error + mustEmbedUnimplementedS3BlobFetchServer() +} + +// UnimplementedS3BlobFetchServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedS3BlobFetchServer struct{} + +func (UnimplementedS3BlobFetchServer) FetchChunkBlob(*FetchChunkBlobRequest, grpc.ServerStreamingServer[FetchChunkBlobResponse]) error { + return status.Error(codes.Unimplemented, "method FetchChunkBlob not implemented") +} +func (UnimplementedS3BlobFetchServer) PushChunkBlob(grpc.ClientStreamingServer[PushChunkBlobRequest, PushChunkBlobResponse]) error { + return status.Error(codes.Unimplemented, "method PushChunkBlob not implemented") +} +func (UnimplementedS3BlobFetchServer) mustEmbedUnimplementedS3BlobFetchServer() {} +func (UnimplementedS3BlobFetchServer) testEmbeddedByValue() {} + +// UnsafeS3BlobFetchServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to S3BlobFetchServer will +// result in compilation errors. +type UnsafeS3BlobFetchServer interface { + mustEmbedUnimplementedS3BlobFetchServer() +} + +func RegisterS3BlobFetchServer(s grpc.ServiceRegistrar, srv S3BlobFetchServer) { + // If the following call panics, it indicates UnimplementedS3BlobFetchServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&S3BlobFetch_ServiceDesc, srv) +} + +func _S3BlobFetch_FetchChunkBlob_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(FetchChunkBlobRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(S3BlobFetchServer).FetchChunkBlob(m, &grpc.GenericServerStream[FetchChunkBlobRequest, FetchChunkBlobResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type S3BlobFetch_FetchChunkBlobServer = grpc.ServerStreamingServer[FetchChunkBlobResponse] + +func _S3BlobFetch_PushChunkBlob_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(S3BlobFetchServer).PushChunkBlob(&grpc.GenericServerStream[PushChunkBlobRequest, PushChunkBlobResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type S3BlobFetch_PushChunkBlobServer = grpc.ClientStreamingServer[PushChunkBlobRequest, PushChunkBlobResponse] + +// S3BlobFetch_ServiceDesc is the grpc.ServiceDesc for S3BlobFetch service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var S3BlobFetch_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "S3BlobFetch", + HandlerType: (*S3BlobFetchServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "FetchChunkBlob", + Handler: _S3BlobFetch_FetchChunkBlob_Handler, + ServerStreams: true, + }, + { + StreamName: "PushChunkBlob", + Handler: _S3BlobFetch_PushChunkBlob_Handler, + ClientStreams: true, + }, + }, + Metadata: "service.proto", +} diff --git a/store/lsm_migration.go b/store/lsm_migration.go index 247b13205..d6cd48cb4 100644 --- a/store/lsm_migration.go +++ b/store/lsm_migration.go @@ -25,6 +25,9 @@ func (s *pebbleStore) exportVersionsLocked(ctx context.Context, opts ExportVersi if opts.MaxVersions <= 0 { return ExportVersionsResult{Done: true}, nil } + if readTSCompacted(exportRetentionReadTS(opts), s.effectiveMinRetainedTS()) { + return ExportVersionsResult{}, ErrReadTSCompacted + } iter, err := s.db.NewIter(pebbleExportIterOptions(opts)) if err != nil { @@ -552,3 +555,25 @@ func (s *pebbleStore) MigrationHLCFloor(_ context.Context, jobID uint64) (uint64 } return floor, nil } + +func (s *pebbleStore) MigrationImportMetadataPresent(_ context.Context, jobID uint64) (bool, error) { + s.dbMu.RLock() + defer s.dbMu.RUnlock() + floors, err := s.readMigrationHLCFloors() + if err != nil { + return false, err + } + if floors[jobID] != 0 { + return true, nil + } + acks, err := s.readMigrationImportAcks() + if err != nil { + return false, err + } + for id := range acks { + if id.jobID == jobID { + return true, nil + } + } + return false, nil +} diff --git a/store/lsm_store.go b/store/lsm_store.go index 4f6f65299..e1f5f1931 100644 --- a/store/lsm_store.go +++ b/store/lsm_store.go @@ -195,19 +195,20 @@ var metaAppliedIndexBytes = []byte(metaAppliedIndex) // (PR #915 round-3) so the snapshot-persist checkpoint cannot rewind // metaAppliedIndex below a concurrent per-Apply value. // 4. mtx – guards the in-memory metadata fields -// (lastCommitTS, minRetainedTS, pendingMinRetainedTS). +// (lastCommitTS, minRetainedTS, pendingMinRetainedTS, migrationReadinessCache). type pebbleStore struct { - db *pebble.DB - cache *pebble.Cache // owns one ref; Unref on Close / reopen. - dbMu sync.RWMutex // guards s.db pointer – see lock ordering above - log *slog.Logger - lastCommitTS uint64 - minRetainedTS uint64 - pendingMinRetainedTS uint64 - mtx sync.RWMutex - applyMu sync.Mutex // serializes ApplyMutations: conflict check → commit - maintenanceMu sync.Mutex - dir string + db *pebble.DB + cache *pebble.Cache // owns one ref; Unref on Close / reopen. + dbMu sync.RWMutex // guards s.db pointer – see lock ordering above + log *slog.Logger + lastCommitTS uint64 + minRetainedTS uint64 + pendingMinRetainedTS uint64 + migrationReadinessCache []TargetStagedReadinessState + mtx sync.RWMutex + applyMu sync.Mutex // serializes ApplyMutations: conflict check → commit + maintenanceMu sync.Mutex + dir string // writeConflicts tracks per-(kind, key_prefix) OCC conflict counts // detected inside ApplyMutations. Polled by the monitoring // WriteConflictCollector; not part of the authoritative OCC path. @@ -354,9 +355,15 @@ func NewPebbleStore(dir string, opts ...PebbleStoreOption) (MVCCStore, error) { cleanupOnInitFail() return nil, err } + readinessStates, err := s.loadPebbleTargetReadinessStatesFromDB() + if err != nil { + cleanupOnInitFail() + return nil, err + } s.lastCommitTS = maxTS s.minRetainedTS = minRetainedTS s.pendingMinRetainedTS = pendingMinRetainedTS + s.migrationReadinessCache = readinessStates return s, nil } @@ -3039,6 +3046,7 @@ func (s *pebbleStore) handleRestorePeekError(err error, header []byte) error { s.lastCommitTS = 0 s.minRetainedTS = 0 s.pendingMinRetainedTS = 0 + s.migrationReadinessCache = nil if setErr := writePebbleUint64(s.db, metaLastCommitTSBytes, 0, pebble.NoSync); setErr != nil { return errors.WithStack(setErr) } @@ -3209,22 +3217,43 @@ func writeNativeSnapshotToTempDir(r io.Reader, tmpDir string, ts uint64) error { // Entries are written to a temporary Pebble directory and only swapped into // place after the CRC32 checksum is verified, preserving the existing store // on failure. -func readStreamingMVCCRestoreHeader(r io.Reader) (io.Reader, hash.Hash32, uint32, uint64, uint64, error) { - expectedChecksum, err := readMVCCSnapshotHeader(r) +func readStreamingMVCCRestoreHeader( + r io.Reader, +) (io.Reader, hash.Hash32, uint32, uint64, uint64, []TargetStagedReadinessState, map[migrationAckID]migrationImportAck, map[uint64]uint64, error) { + expectedChecksum, version, err := readMVCCSnapshotHeader(r) if err != nil { - return nil, nil, 0, 0, 0, err + return nil, nil, 0, 0, 0, nil, nil, nil, err } hash := crc32.NewIEEE() body := io.TeeReader(r, hash) lastCommitTS, minRetainedTS, err := readMVCCSnapshotMetadata(body) if err != nil { - return nil, nil, 0, 0, 0, err + return nil, nil, 0, 0, 0, nil, nil, nil, err } - return body, hash, expectedChecksum, lastCommitTS, minRetainedTS, nil + readinessStates, err := readMVCCSnapshotReadinessStates(body, version) + if err != nil { + return nil, nil, 0, 0, 0, nil, nil, nil, err + } + bufferedBody := bufio.NewReader(body) + importAcks, hlcFloors, err := readMVCCSnapshotMigrationMetadata(bufferedBody, version) + if err != nil { + return nil, nil, 0, 0, 0, nil, nil, nil, err + } + return bufferedBody, hash, expectedChecksum, lastCommitTS, minRetainedTS, readinessStates, importAcks, hlcFloors, nil } -func writeStreamingMVCCRestoreTempDB(dir string, body io.Reader, hash hash.Hash32, expectedChecksum uint32, lastCommitTS uint64, minRetainedTS uint64) (string, error) { +func writeStreamingMVCCRestoreTempDB( + dir string, + body io.Reader, + hash hash.Hash32, + expectedChecksum uint32, + lastCommitTS uint64, + minRetainedTS uint64, + readinessStates []TargetStagedReadinessState, + importAcks map[migrationAckID]migrationImportAck, + hlcFloors map[uint64]uint64, +) (string, error) { tmpDir := filepath.Clean(dir) + ".restore-tmp" if err := os.RemoveAll(tmpDir); err != nil { return "", errors.WithStack(err) @@ -3244,6 +3273,14 @@ func writeStreamingMVCCRestoreTempDB(dir string, body io.Reader, hash hash.Hash3 _ = os.RemoveAll(tmpDir) } + if err := writePebbleTargetReadinessStates(tmpDB, readinessStates); err != nil { + cleanupTmp() + return "", err + } + if err := writePebbleMigrationImportMetadata(tmpDB, importAcks, hlcFloors); err != nil { + cleanupTmp() + return "", err + } if err := writeMVCCEntriesToDB(body, tmpDB); err != nil { cleanupTmp() return "", err @@ -3263,13 +3300,50 @@ func writeStreamingMVCCRestoreTempDB(dir string, body io.Reader, hash hash.Hash3 return tmpDir, nil } +func writePebbleTargetReadinessStates(db *pebble.DB, states []TargetStagedReadinessState) error { + for _, state := range states { + if err := db.Set(migrationReadyKey(state.JobID), encodeTargetStagedReadinessState(state), pebble.NoSync); err != nil { + return errors.WithStack(err) + } + } + return nil +} + +func writePebbleMigrationImportMetadata( + db *pebble.DB, + acks map[migrationAckID]migrationImportAck, + floors map[uint64]uint64, +) error { + if len(acks) != 0 { + if err := db.Set(migrationAckMetaKeyBytes, encodeMigrationImportAcks(acks), pebble.NoSync); err != nil { + return errors.WithStack(err) + } + } + if len(floors) != 0 { + if err := db.Set(migrationHLCFloorMetaKeyBytes, encodeMigrationHLCFloors(floors), pebble.NoSync); err != nil { + return errors.WithStack(err) + } + } + return nil +} + func (s *pebbleStore) restoreFromStreamingMVCC(r io.Reader) error { - body, hash, expectedChecksum, lastCommitTS, minRetainedTS, err := readStreamingMVCCRestoreHeader(r) + body, hash, expectedChecksum, lastCommitTS, minRetainedTS, readinessStates, importAcks, hlcFloors, err := readStreamingMVCCRestoreHeader(r) if err != nil { return err } - tmpDir, err := writeStreamingMVCCRestoreTempDB(s.dir, body, hash, expectedChecksum, lastCommitTS, minRetainedTS) + tmpDir, err := writeStreamingMVCCRestoreTempDB( + s.dir, + body, + hash, + expectedChecksum, + lastCommitTS, + minRetainedTS, + readinessStates, + importAcks, + hlcFloors, + ) if err != nil { return err } @@ -3356,9 +3430,15 @@ func (s *pebbleStore) swapInTempDB(tmpDir string) error { cleanupOnSwapFail() return err } + readinessStates, err := s.loadPebbleTargetReadinessStatesFromDB() + if err != nil { + cleanupOnSwapFail() + return err + } s.lastCommitTS = lastCommitTS s.minRetainedTS = minRetainedTS s.pendingMinRetainedTS = pendingMinRetainedTS + s.migrationReadinessCache = readinessStates return nil } diff --git a/store/lsm_store_test.go b/store/lsm_store_test.go index ecea4526a..c8aec471c 100644 --- a/store/lsm_store_test.go +++ b/store/lsm_store_test.go @@ -890,6 +890,18 @@ func TestPebbleStore_RestoreFromStreamingMVCC(t *testing.T) { require.NoError(t, src.PutAt(ctx, []byte("key1"), []byte("val1-updated"), 20, 0)) require.NoError(t, src.DeleteAt(ctx, []byte("key2"), 15)) require.NoError(t, src.PutWithTTLAt(ctx, []byte("key3"), []byte("val3"), 30, 9999)) + readyState := TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 12, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + } + readyWriter, ok := src.(MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, readyWriter.ApplyTargetStagedReadiness(ctx, readyState)) snap, err := src.Snapshot() require.NoError(t, err) @@ -927,6 +939,55 @@ func TestPebbleStore_RestoreFromStreamingMVCC(t *testing.T) { assert.Equal(t, []byte("val3"), val) assert.Equal(t, src.LastCommitTS(), dst.LastCommitTS()) + readyReader, ok := dst.(MigrationTargetReadinessReader) + require.True(t, ok) + states, err := readyReader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []TargetStagedReadinessState{readyState}, states) +} + +func TestPebbleStore_RestoreFromStreamingMVCCPreservesMigrationImportMetadata(t *testing.T) { + ctx := context.Background() + src := NewMVCCStore() + _, err := src.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 7, + BracketID: 3, + BatchSeq: 1, + Cursor: []byte("c1"), + Versions: []MVCCVersion{{Key: []byte("snapshotted"), CommitTS: 50, Value: []byte("v50")}}, + }) + require.NoError(t, err) + + snap, err := src.Snapshot() + require.NoError(t, err) + defer snap.Close() + + dir, err := os.MkdirTemp("", "pebble-migrate-import-metadata-*") + require.NoError(t, err) + defer os.RemoveAll(dir) + dst, err := NewPebbleStore(dir) + require.NoError(t, err) + defer dst.Close() + require.NoError(t, dst.Restore(bytes.NewReader(snapshotBytes(t, snap)))) + + val, err := dst.GetAt(ctx, []byte("snapshotted"), 50) + require.NoError(t, err) + require.Equal(t, []byte("v50"), val) + floor, err := dst.MigrationHLCFloor(ctx, 7) + require.NoError(t, err) + require.Equal(t, uint64(50), floor) + res, err := dst.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 7, + BracketID: 3, + BatchSeq: 1, + Cursor: []byte("fresh"), + Versions: []MVCCVersion{{Key: []byte("fresh"), CommitTS: 60, Value: []byte("v60")}}, + }) + require.NoError(t, err) + require.True(t, res.Duplicate) + require.Equal(t, []byte("c1"), res.AckedCursor) + _, err = dst.GetAt(ctx, []byte("fresh"), 60) + require.ErrorIs(t, err, ErrKeyNotFound) } func TestPebbleStore_RestoreFromStreamingMVCCPreservesMinRetainedTS(t *testing.T) { @@ -975,6 +1036,17 @@ func TestPebbleStore_Restore_EmptySnapshot(t *testing.T) { // Pre-populate so we can verify the restore clears the data. require.NoError(t, s.PutAt(ctx, []byte("k"), []byte("v"), 42, 0)) + readyWriter, ok := s.(MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, readyWriter.ApplyTargetStagedReadiness(ctx, TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 12, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + })) requirePebbleRetentionController(t, s).SetMinRetainedTS(21) require.Equal(t, uint64(42), s.LastCommitTS()) @@ -986,6 +1058,11 @@ func TestPebbleStore_Restore_EmptySnapshot(t *testing.T) { assert.ErrorIs(t, err, ErrKeyNotFound) assert.Equal(t, uint64(0), s.LastCommitTS()) assert.Equal(t, uint64(0), requirePebbleRetentionController(t, s).MinRetainedTS()) + readyReader, ok := s.(MigrationTargetReadinessReader) + require.True(t, ok) + states, err := readyReader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + assert.Empty(t, states) } // TestPebbleStore_Restore_TruncatedHeader verifies that a partial magic diff --git a/store/migration_cleanup.go b/store/migration_cleanup.go new file mode 100644 index 000000000..78198224c --- /dev/null +++ b/store/migration_cleanup.go @@ -0,0 +1,205 @@ +package store + +import ( + "bytes" + "context" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/v2" +) + +func cleanupExportOptions(opts CleanupVersionsOptions) ExportVersionsOptions { + return ExportVersionsOptions{ + StartKey: opts.StartKey, + EndKey: opts.EndKey, + Cursor: opts.Cursor, + MaxCommitTSInclusive: opts.MaxCommitTS, + MaxVersions: opts.MaxVersions, + MaxBytes: opts.MaxBytes, + MaxScannedBytes: opts.MaxScannedBytes, + KeyFamily: opts.KeyFamily, + AcceptVersion: opts.AcceptVersion, + } +} + +func cleanupResult(exported ExportVersionsResult) CleanupVersionsResult { + var deletedBytes uint64 + for _, version := range exported.Versions { + deletedBytes += versionExportSize(version.Key, len(version.Value)) + } + return CleanupVersionsResult{ + NextCursor: bytes.Clone(exported.NextCursor), + Done: exported.Done, + DeletedRows: uint64(len(exported.Versions)), //nolint:gosec // bounded by MaxVersions. + DeletedBytes: deletedBytes, + ScannedBytes: exported.ScannedBytes, + } +} + +func (s *mvccStore) CleanupVersions(ctx context.Context, opts CleanupVersionsOptions) (CleanupVersionsResult, error) { + if opts.MaxVersions <= 0 { + return CleanupVersionsResult{Done: true}, nil + } + s.mtx.Lock() + defer s.mtx.Unlock() + + exportOpts := normalizeExportVersionsOptions(cleanupExportOptions(opts)) + pos, err := decodeExportCursorForOptions(exportOpts) + if err != nil { + return CleanupVersionsResult{}, err + } + exported, err := s.exportVersionsLocked(ctx, exportOpts, pos) + if err != nil { + return CleanupVersionsResult{}, err + } + for _, version := range exported.Versions { + s.removeVersionLocked(version.Key, version.CommitTS) + } + return cleanupResult(exported), nil +} + +func (s *mvccStore) ClearMigrationState(_ context.Context, jobID, _ uint64) error { + if jobID == 0 { + return errors.New("migration cleanup job id is required") + } + s.mtx.Lock() + defer s.mtx.Unlock() + + for id := range s.migrationAcks { + if id.jobID == jobID { + delete(s.migrationAcks, id) + } + } + delete(s.migrationHLCFloors, jobID) + delete(s.migrationPromotions, jobID) + delete(s.migrationReadiness, jobID) + s.migrationReadinessCache = upsertReadinessCacheWithoutJob(s.migrationReadinessCache, jobID) + return nil +} + +func upsertReadinessCacheWithoutJob(states []TargetStagedReadinessState, jobID uint64) []TargetStagedReadinessState { + out := states[:0] + for _, state := range states { + if state.JobID != jobID { + out = append(out, state) + } + } + return out +} + +func (s *pebbleStore) CleanupVersions(ctx context.Context, opts CleanupVersionsOptions) (CleanupVersionsResult, error) { + if opts.MaxVersions <= 0 { + return CleanupVersionsResult{Done: true}, nil + } + s.dbMu.RLock() + defer s.dbMu.RUnlock() + s.applyMu.Lock() + defer s.applyMu.Unlock() + + exported, err := s.exportVersionsLocked(ctx, cleanupExportOptions(opts)) + if err != nil { + return CleanupVersionsResult{}, err + } + batch := s.db.NewBatch() + defer batch.Close() + for _, version := range exported.Versions { + if err := batch.Delete(encodeKey(version.Key, version.CommitTS), nil); err != nil { + return CleanupVersionsResult{}, errors.WithStack(err) + } + } + if opts.AppliedIndex > 0 { + if err := setPebbleUint64InBatch(batch, metaAppliedIndexBytes, opts.AppliedIndex); err != nil { + return CleanupVersionsResult{}, err + } + } + if err := batch.Commit(s.cleanupWriteOpts(opts.AppliedIndex)); err != nil { + return CleanupVersionsResult{}, errors.WithStack(err) + } + return cleanupResult(exported), nil +} + +func (s *pebbleStore) ClearMigrationState(_ context.Context, jobID, appliedIndex uint64) error { + if jobID == 0 { + return errors.New("migration cleanup job id is required") + } + s.dbMu.RLock() + defer s.dbMu.RUnlock() + s.applyMu.Lock() + defer s.applyMu.Unlock() + + metadata, err := s.migrationMetadataWithoutJob(jobID) + if err != nil { + return err + } + + batch := s.db.NewBatch() + defer batch.Close() + if err := stageMigrationMetadataCleanup(batch, jobID, appliedIndex, metadata); err != nil { + return err + } + if err := batch.Commit(s.cleanupWriteOpts(appliedIndex)); err != nil { + return errors.WithStack(err) + } + s.mtx.Lock() + s.migrationReadinessCache = upsertReadinessCacheWithoutJob(s.migrationReadinessCache, jobID) + s.mtx.Unlock() + return nil +} + +type migrationMetadataState struct { + acks map[migrationAckID]migrationImportAck + floors map[uint64]uint64 + promotions map[uint64]PromotionState +} + +func (s *pebbleStore) migrationMetadataWithoutJob(jobID uint64) (migrationMetadataState, error) { + acks, err := s.readMigrationImportAcks() + if err != nil { + return migrationMetadataState{}, err + } + for id := range acks { + if id.jobID == jobID { + delete(acks, id) + } + } + floors, err := s.readMigrationHLCFloors() + if err != nil { + return migrationMetadataState{}, err + } + delete(floors, jobID) + promotions, err := s.readPebblePromotionStates() + if err != nil { + return migrationMetadataState{}, err + } + delete(promotions, jobID) + return migrationMetadataState{acks: acks, floors: floors, promotions: promotions}, nil +} + +func stageMigrationMetadataCleanup(batch *pebble.Batch, jobID, appliedIndex uint64, metadata migrationMetadataState) error { + if err := batch.Set(migrationAckMetaKeyBytes, encodeMigrationImportAcks(metadata.acks), nil); err != nil { + return errors.WithStack(err) + } + if err := batch.Set(migrationHLCFloorMetaKeyBytes, encodeMigrationHLCFloors(metadata.floors), nil); err != nil { + return errors.WithStack(err) + } + if err := batch.Set(migrationPromoteMetaKeyBytes, encodeMigrationPromotionStates(metadata.promotions), nil); err != nil { + return errors.WithStack(err) + } + if err := batch.Delete(migrationReadyKey(jobID), nil); err != nil { + return errors.WithStack(err) + } + if appliedIndex == 0 { + return nil + } + return setPebbleUint64InBatch(batch, metaAppliedIndexBytes, appliedIndex) +} + +func (s *pebbleStore) cleanupWriteOpts(appliedIndex uint64) *pebble.WriteOptions { + if appliedIndex > 0 { + return s.raftApplyWriteOpts() + } + return s.directApplyWriteOpts() +} + +var _ MigrationCleaner = (*mvccStore)(nil) +var _ MigrationCleaner = (*pebbleStore)(nil) diff --git a/store/migration_promote.go b/store/migration_promote.go index d8a8bc27b..c79bac7e5 100644 --- a/store/migration_promote.go +++ b/store/migration_promote.go @@ -104,7 +104,7 @@ func (s *mvccStore) planMemoryPromotionLocked( if err != nil { return ExportVersionsResult{}, nil, PromoteVersionsResult{}, err } - exported, err := s.exportMemoryVersionsLocked(ctx, exportOpts, pos) + exported, err := s.exportVersionsLocked(ctx, exportOpts, pos) if err != nil { return ExportVersionsResult{}, nil, PromoteVersionsResult{}, err } @@ -142,55 +142,24 @@ func (s *mvccStore) MigrationPromotionState(_ context.Context, jobID uint64) (Pr return clonePromotionState(state), ok, nil } -func (s *mvccStore) exportMemoryVersionsLocked(ctx context.Context, opts ExportVersionsOptions, pos exportCursorPosition) (ExportVersionsResult, error) { - result := newExportVersionsResult(opts.MaxVersions) - it := s.tree.Iterator() - if !s.seekMemoryExportStart(&it, opts.StartKey, pos) { - result.Done = true - return result, nil - } - for ok := true; ok; ok = it.Next() { - key, keyOK := it.Key().([]byte) - if err := checkExportKey(ctx, key, keyOK, opts.EndKey); err != nil { - if errors.Is(err, errExportReachedEnd) { - result.Done = true - result.NextCursor = nil - return result, nil - } - return ExportVersionsResult{}, err - } - if !keyOK { - continue - } - done, err := exportMemoryIteratorKey(ctx, opts, pos, key, it.Value(), &result) - if err != nil || !done { - return result, err - } - } - result.Done = true - result.NextCursor = nil - return result, nil -} - -func (s *mvccStore) removeVersionLocked(key []byte, commitTS uint64) bool { +func (s *mvccStore) removeVersionLocked(key []byte, commitTS uint64) { existing, ok := s.tree.Get(key) if !ok { - return false + return } versions, _ := existing.([]VersionedValue) idx := findVersionIndex(versions, commitTS) if idx < 0 { - return false + return } next := make([]VersionedValue, len(versions)-1) copy(next, versions[:idx]) copy(next[idx:], versions[idx+1:]) if len(next) == 0 { s.tree.Remove(key) - return true + return } s.tree.Put(bytes.Clone(key), next) - return true } func findVersionIndex(versions []VersionedValue, commitTS uint64) int { diff --git a/store/migration_readiness.go b/store/migration_readiness.go new file mode 100644 index 000000000..60cc4b224 --- /dev/null +++ b/store/migration_readiness.go @@ -0,0 +1,358 @@ +package store + +import ( + "bytes" + "context" + "encoding/binary" + "sort" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/v2" +) + +const ( + targetReadinessCodecVersionV1 byte = 1 + targetReadinessCodecVersion byte = 2 + targetReadinessArmedFlag byte = 1 + targetReadinessSourceWrite byte = 1 + targetReadinessSourceRead byte = 2 + targetReadinessTrackWrites byte = 4 +) + +func validateTargetStagedReadinessState(state TargetStagedReadinessState) error { + switch { + case state.JobID == 0: + return errors.New("target staged readiness job_id is required") + case state.MigrationJobID == 0: + return errors.New("target staged readiness migration_job_id is required") + case state.Armed && state.MinWriteTSExclusive == 0: + return errors.New("target staged readiness min_write_ts_exclusive is required when armed") + case state.RouteEnd != nil && bytes.Compare(state.RouteStart, state.RouteEnd) >= 0: + return errors.New("target staged readiness route range is invalid") + } + return validateSourceMigrationControlState(state) +} + +func validateSourceMigrationControlState(state TargetStagedReadinessState) error { + if state.SourceReadFence && !state.SourceWriteFence { + return errors.New("source read fence requires source write fence") + } + if (state.SourceWriteFence || state.SourceReadFence) && state.RetentionPinTS == 0 { + return errors.New("source migration control retention pin is required") + } + if state.TrackWrites && state.RetentionPinTS == 0 { + return errors.New("migration write tracker retention pin is required") + } + return nil +} + +func (s *mvccStore) ApplyTargetStagedReadiness(_ context.Context, state TargetStagedReadinessState) error { + if err := validateTargetStagedReadinessState(state); err != nil { + return err + } + s.mtx.Lock() + defer s.mtx.Unlock() + cloned := cloneTargetStagedReadinessState(state) + s.migrationReadiness[state.JobID] = cloned + s.migrationReadinessCache = upsertTargetStagedReadinessStateCache(s.migrationReadinessCache, cloned) + return nil +} + +func (s *mvccStore) ApplyTargetStagedReadinessAt(ctx context.Context, state TargetStagedReadinessState, _ uint64) error { + return s.ApplyTargetStagedReadiness(ctx, state) +} + +func (s *mvccStore) MigrationTargetReadinessStates(_ context.Context) ([]TargetStagedReadinessState, error) { + s.mtx.RLock() + defer s.mtx.RUnlock() + return cloneTargetStagedReadinessStates(s.migrationReadinessCache), nil +} + +func (s *pebbleStore) ApplyTargetStagedReadiness(_ context.Context, state TargetStagedReadinessState) error { + if err := validateTargetStagedReadinessState(state); err != nil { + return err + } + s.dbMu.RLock() + defer s.dbMu.RUnlock() + s.mtx.Lock() + defer s.mtx.Unlock() + if err := s.db.Set(migrationReadyKey(state.JobID), encodeTargetStagedReadinessState(state), s.directApplyWriteOpts()); err != nil { + return errors.WithStack(err) + } + s.migrationReadinessCache = upsertTargetStagedReadinessStateCache(s.migrationReadinessCache, state) + return nil +} + +func (s *pebbleStore) ApplyTargetStagedReadinessAt(_ context.Context, state TargetStagedReadinessState, appliedIndex uint64) error { + if err := validateTargetStagedReadinessState(state); err != nil { + return err + } + s.dbMu.RLock() + defer s.dbMu.RUnlock() + s.applyMu.Lock() + defer s.applyMu.Unlock() + + writeAppliedIndex, err := s.shouldWriteAppliedIndexLocked(appliedIndex) + if err != nil { + return err + } + + batch := s.db.NewBatch() + defer func() { _ = batch.Close() }() + if err := batch.Set(migrationReadyKey(state.JobID), encodeTargetStagedReadinessState(state), nil); err != nil { + return errors.WithStack(err) + } + if writeAppliedIndex { + if err := setPebbleUint64InBatch(batch, metaAppliedIndexBytes, appliedIndex); err != nil { + return err + } + } + + s.mtx.Lock() + defer s.mtx.Unlock() + if err := batch.Commit(s.raftApplyWriteOpts()); err != nil { + return errors.WithStack(err) + } + s.migrationReadinessCache = upsertTargetStagedReadinessStateCache(s.migrationReadinessCache, state) + return nil +} + +func (s *pebbleStore) MigrationTargetReadinessStates(_ context.Context) ([]TargetStagedReadinessState, error) { + s.mtx.RLock() + defer s.mtx.RUnlock() + return cloneTargetStagedReadinessStates(s.migrationReadinessCache), nil +} + +func (s *pebbleStore) loadPebbleTargetReadinessStatesFromDB() ([]TargetStagedReadinessState, error) { + iter, err := s.db.NewIter(&pebble.IterOptions{ + LowerBound: []byte(migrationReadyPrefix), + UpperBound: PrefixScanEnd([]byte(migrationReadyPrefix)), + }) + if err != nil { + return nil, errors.WithStack(err) + } + defer iter.Close() + + var out []TargetStagedReadinessState + for valid := iter.First(); valid; valid = iter.Next() { + if !isTargetReadinessRecordKey(iter.Key()) { + continue + } + state, ok := decodeTargetStagedReadinessState(iter.Value()) + if !ok { + return nil, errors.New("corrupt target staged readiness state") + } + out = append(out, state) + } + if err := iter.Error(); err != nil { + return nil, errors.WithStack(err) + } + sortTargetStagedReadinessStates(out) + return out, nil +} + +func isTargetReadinessRecordKey(rawKey []byte) bool { + return len(rawKey) == len(migrationReadyPrefix)+migrationUint64Bytes && + bytes.HasPrefix(rawKey, []byte(migrationReadyPrefix)) +} + +func encodeTargetStagedReadinessState(state TargetStagedReadinessState) []byte { + buf := make([]byte, 0, targetReadinessEncodedSize(state)) + buf = append(buf, targetReadinessCodecVersion) + if state.Armed { + buf = append(buf, targetReadinessArmedFlag) + } else { + buf = append(buf, 0) + } + var sourceFlags byte + if state.SourceWriteFence { + sourceFlags |= targetReadinessSourceWrite + } + if state.SourceReadFence { + sourceFlags |= targetReadinessSourceRead + } + if state.TrackWrites { + sourceFlags |= targetReadinessTrackWrites + } + buf = append(buf, sourceFlags) + buf = binary.BigEndian.AppendUint64(buf, state.JobID) + buf = binary.BigEndian.AppendUint64(buf, state.ExpectedCutoverVersion) + buf = binary.BigEndian.AppendUint64(buf, state.MigrationJobID) + buf = binary.BigEndian.AppendUint64(buf, state.MinWriteTSExclusive) + buf = binary.BigEndian.AppendUint64(buf, state.RetentionPinTS) + buf = binary.BigEndian.AppendUint64(buf, state.MinAdmittedTS) + buf = appendVarBytes(buf, state.RouteStart) + if state.RouteEnd == nil { + buf = append(buf, 0) + return buf + } + buf = append(buf, 1) + return appendVarBytes(buf, state.RouteEnd) +} + +func decodeTargetStagedReadinessState(data []byte) (TargetStagedReadinessState, bool) { + state, rest, ok := decodeTargetReadinessHeader(data) + if !ok { + return TargetStagedReadinessState{}, false + } + if !decodeTargetReadinessRange(&state, rest) { + return TargetStagedReadinessState{}, false + } + if err := validateTargetStagedReadinessState(state); err != nil { + return TargetStagedReadinessState{}, false + } + return state, true +} + +func decodeTargetReadinessHeader(data []byte) (TargetStagedReadinessState, []byte, bool) { + if len(data) == 0 { + return TargetStagedReadinessState{}, nil, false + } + if data[0] == targetReadinessCodecVersionV1 { + return decodeTargetReadinessHeaderV1(data) + } + if len(data) < 3+6*migrationUint64Bytes || data[0] != targetReadinessCodecVersion { + return TargetStagedReadinessState{}, nil, false + } + if data[1] != 0 && data[1] != targetReadinessArmedFlag { + return TargetStagedReadinessState{}, nil, false + } + sourceFlags := data[2] + if sourceFlags & ^(targetReadinessSourceWrite|targetReadinessSourceRead|targetReadinessTrackWrites) != 0 { + return TargetStagedReadinessState{}, nil, false + } + state := TargetStagedReadinessState{ + Armed: data[1] == targetReadinessArmedFlag, + SourceWriteFence: sourceFlags&targetReadinessSourceWrite != 0, + SourceReadFence: sourceFlags&targetReadinessSourceRead != 0, + TrackWrites: sourceFlags&targetReadinessTrackWrites != 0, + } + rest := data[3:] + state.JobID = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.ExpectedCutoverVersion = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.MigrationJobID = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.MinWriteTSExclusive = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.RetentionPinTS = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.MinAdmittedTS = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + return state, rest, true +} + +func decodeTargetReadinessHeaderV1(data []byte) (TargetStagedReadinessState, []byte, bool) { + if len(data) < 2+4*migrationUint64Bytes || data[1] != 0 && data[1] != targetReadinessArmedFlag { + return TargetStagedReadinessState{}, nil, false + } + state := TargetStagedReadinessState{Armed: data[1] == targetReadinessArmedFlag} + rest := data[2:] + state.JobID = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.ExpectedCutoverVersion = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.MigrationJobID = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + state.MinWriteTSExclusive = binary.BigEndian.Uint64(rest[:migrationUint64Bytes]) + rest = rest[migrationUint64Bytes:] + return state, rest, true +} + +func decodeTargetReadinessRange(state *TargetStagedReadinessState, data []byte) bool { + routeStart, rest, ok := readVarBytes(data) + if !ok || len(rest) == 0 { + return false + } + state.RouteStart = routeStart + endPresent := rest[0] + rest = rest[1:] + if endPresent == 0 { + return len(rest) == 0 + } + if endPresent != 1 { + return false + } + state.RouteEnd, rest, ok = readVarBytes(rest) + return ok && len(rest) == 0 +} + +func targetReadinessEncodedSize(state TargetStagedReadinessState) int { + size := 3 + 6*migrationUint64Bytes + binary.MaxVarintLen64 + len(state.RouteStart) + 1 + if state.RouteEnd != nil { + size += binary.MaxVarintLen64 + len(state.RouteEnd) + } + return size +} + +func appendVarBytes(dst []byte, value []byte) []byte { + dst = binary.AppendUvarint(dst, lenAsUint64(len(value))) + return append(dst, value...) +} + +func readVarBytes(data []byte) ([]byte, []byte, bool) { + l, n := binary.Uvarint(data) + if n <= 0 { + return nil, nil, false + } + rest := data[n:] + if l > lenAsUint64(len(rest)) { + return nil, nil, false + } + return bytes.Clone(rest[:l]), rest[l:], true +} + +func cloneTargetStagedReadinessState(state TargetStagedReadinessState) TargetStagedReadinessState { + return TargetStagedReadinessState{ + JobID: state.JobID, + RouteStart: bytes.Clone(state.RouteStart), + RouteEnd: bytes.Clone(state.RouteEnd), + ExpectedCutoverVersion: state.ExpectedCutoverVersion, + MigrationJobID: state.MigrationJobID, + MinWriteTSExclusive: state.MinWriteTSExclusive, + Armed: state.Armed, + SourceWriteFence: state.SourceWriteFence, + SourceReadFence: state.SourceReadFence, + RetentionPinTS: state.RetentionPinTS, + TrackWrites: state.TrackWrites, + MinAdmittedTS: state.MinAdmittedTS, + } +} + +func cloneTargetStagedReadinessStates(states []TargetStagedReadinessState) []TargetStagedReadinessState { + if len(states) == 0 { + return nil + } + out := make([]TargetStagedReadinessState, 0, len(states)) + for _, state := range states { + out = append(out, cloneTargetStagedReadinessState(state)) + } + return out +} + +func upsertTargetStagedReadinessStateCache(cache []TargetStagedReadinessState, state TargetStagedReadinessState) []TargetStagedReadinessState { + cloned := cloneTargetStagedReadinessState(state) + for i := range cache { + if cache[i].JobID == cloned.JobID { + cache[i] = cloned + return cache + } + } + cache = append(cache, cloned) + sortTargetStagedReadinessStates(cache) + return cache +} + +func sortTargetStagedReadinessStates(states []TargetStagedReadinessState) { + sort.Slice(states, func(i, j int) bool { + return states[i].JobID < states[j].JobID + }) +} + +var _ MigrationTargetReadinessReader = (*mvccStore)(nil) +var _ MigrationTargetReadinessReader = (*pebbleStore)(nil) +var _ MigrationTargetReadinessWriter = (*mvccStore)(nil) +var _ MigrationTargetReadinessWriter = (*pebbleStore)(nil) +var _ MigrationTargetReadinessRaftWriter = (*mvccStore)(nil) +var _ MigrationTargetReadinessRaftWriter = (*pebbleStore)(nil) diff --git a/store/migration_readiness_test.go b/store/migration_readiness_test.go new file mode 100644 index 000000000..5b3db499b --- /dev/null +++ b/store/migration_readiness_test.go @@ -0,0 +1,79 @@ +package store + +import ( + "encoding/binary" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestTargetStagedReadinessCodecRoundTripSourceControls(t *testing.T) { + t.Parallel() + + want := TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 11, + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + SourceWriteFence: true, + SourceReadFence: true, + RetentionPinTS: 80, + TrackWrites: true, + MinAdmittedTS: 70, + } + + got, ok := decodeTargetStagedReadinessState(encodeTargetStagedReadinessState(want)) + require.True(t, ok) + require.Equal(t, want, got) +} + +func TestTargetStagedReadinessCodecDecodesV1(t *testing.T) { + t.Parallel() + + data := []byte{targetReadinessCodecVersionV1, targetReadinessArmedFlag} + for _, value := range []uint64{9, 11, 7, 100} { + data = binary.BigEndian.AppendUint64(data, value) + } + data = appendVarBytes(data, []byte("m")) + data = append(data, 1) + data = appendVarBytes(data, []byte("z")) + + got, ok := decodeTargetStagedReadinessState(data) + require.True(t, ok) + require.Equal(t, TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("m"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 11, + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + }, got) +} + +func TestValidateTargetStagedReadinessSourceControls(t *testing.T) { + t.Parallel() + + base := TargetStagedReadinessState{ + JobID: 9, + MigrationJobID: 7, + MinWriteTSExclusive: 100, + Armed: true, + } + + readOnly := base + readOnly.SourceReadFence = true + readOnly.RetentionPinTS = 80 + require.ErrorContains(t, validateTargetStagedReadinessState(readOnly), "requires source write fence") + + withoutPin := base + withoutPin.SourceWriteFence = true + require.ErrorContains(t, validateTargetStagedReadinessState(withoutPin), "retention pin is required") + + trackerWithoutPin := base + trackerWithoutPin.TrackWrites = true + require.ErrorContains(t, validateTargetStagedReadinessState(trackerWithoutPin), "tracker retention pin is required") +} diff --git a/store/migration_versions.go b/store/migration_versions.go index 8008d971c..32afdd2e6 100644 --- a/store/migration_versions.go +++ b/store/migration_versions.go @@ -22,7 +22,9 @@ const ( migrationMetadataVersion = 1 migrationAckPrefix = "!migstage|ack|" + migrationReadyPrefix = "!migstage|ready|" migrationUint64Bytes = 8 + migrationAckKeyIDBytes = 2 * migrationUint64Bytes exportVersionSizeOverhead = 24 defaultSparseExportMaxScannedBytes = 1 << 20 ) @@ -203,10 +205,18 @@ func exportUsesSparseScanBudget(opts ExportVersionsOptions) bool { opts.EndKey != nil } +func migrationReadyKey(jobID uint64) []byte { + key := make([]byte, len(migrationReadyPrefix)+migrationUint64Bytes) + copy(key, migrationReadyPrefix) + binary.BigEndian.PutUint64(key[len(migrationReadyPrefix):], jobID) + return key +} + func isMigrationMetadataKey(rawKey []byte) bool { return bytes.Equal(rawKey, migrationAckMetaKeyBytes) || bytes.Equal(rawKey, migrationHLCFloorMetaKeyBytes) || - bytes.Equal(rawKey, migrationPromoteMetaKeyBytes) + bytes.Equal(rawKey, migrationPromoteMetaKeyBytes) || + (len(rawKey) == len(migrationReadyPrefix)+migrationUint64Bytes && bytes.HasPrefix(rawKey, []byte(migrationReadyPrefix))) } func encodeMigrationImportAcks(acks map[migrationAckID]migrationImportAck) []byte { @@ -431,7 +441,14 @@ func (s *mvccStore) ExportVersions(ctx context.Context, opts ExportVersionsOptio s.mtx.RLock() defer s.mtx.RUnlock() + if err := s.checkExportReadTSLocked(opts); err != nil { + return ExportVersionsResult{}, err + } + + return s.exportVersionsLocked(ctx, opts, pos) +} +func (s *mvccStore) exportVersionsLocked(ctx context.Context, opts ExportVersionsOptions, pos exportCursorPosition) (ExportVersionsResult, error) { result := newExportVersionsResult(opts.MaxVersions) it := s.tree.Iterator() if !s.seekMemoryExportStart(&it, opts.StartKey, pos) { @@ -462,6 +479,20 @@ func (s *mvccStore) ExportVersions(ctx context.Context, opts ExportVersionsOptio return result, nil } +func (s *mvccStore) checkExportReadTSLocked(opts ExportVersionsOptions) error { + if readTSCompacted(exportRetentionReadTS(opts), s.minRetainedTS) { + return ErrReadTSCompacted + } + return nil +} + +func exportRetentionReadTS(opts ExportVersionsOptions) uint64 { + if opts.ReadTS != 0 { + return opts.ReadTS + } + return opts.MaxCommitTSInclusive +} + var errExportReachedEnd = errors.New("export reached end") var errExportChunkFull = errors.New("export chunk full") @@ -646,3 +677,17 @@ func (s *mvccStore) MigrationHLCFloor(_ context.Context, jobID uint64) (uint64, defer s.mtx.RUnlock() return s.migrationHLCFloors[jobID], nil } + +func (s *mvccStore) MigrationImportMetadataPresent(_ context.Context, jobID uint64) (bool, error) { + s.mtx.RLock() + defer s.mtx.RUnlock() + if s.migrationHLCFloors[jobID] != 0 { + return true, nil + } + for id := range s.migrationAcks { + if id.jobID == jobID { + return true, nil + } + } + return false, nil +} diff --git a/store/migration_versions_test.go b/store/migration_versions_test.go index 2f93a78d2..0dcaac18a 100644 --- a/store/migration_versions_test.go +++ b/store/migration_versions_test.go @@ -57,6 +57,42 @@ func TestExportVersionsPreservesRawVersionMetadata(t *testing.T) { }) } +func TestExportVersionsReadTSPreservesCompactionErrors(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + require.NoError(t, st.PutAt(ctx, []byte("k"), []byte("v10"), 10, 0)) + retention, ok := st.(RetentionController) + require.True(t, ok) + retention.SetMinRetainedTS(20) + + _, err := st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: []byte("k"), + EndKey: []byte("l"), + MaxCommitTSInclusive: 15, + ReadTS: 15, + MaxVersions: 10, + }) + require.ErrorIs(t, err, ErrReadTSCompacted) + + _, err = st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: []byte("k"), + EndKey: []byte("l"), + MaxCommitTSInclusive: 15, + MaxVersions: 10, + }) + require.ErrorIs(t, err, ErrReadTSCompacted) + + result, err := st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: []byte("k"), + EndKey: []byte("l"), + MaxCommitTSInclusive: 25, + MaxVersions: 10, + }) + require.NoError(t, err) + require.Len(t, result.Versions, 1) + }) +} + func TestExportVersionsExcludesTxnLocks(t *testing.T) { runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { ctx := context.Background() @@ -921,6 +957,38 @@ func TestImportVersionsIdempotencyAndMetadata(t *testing.T) { }) } +func TestMigrationImportMetadataPresentClearsWithMigrationState(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + reader, ok := st.(MigrationImportMetadataReader) + require.True(t, ok) + cleaner, ok := st.(MigrationCleaner) + require.True(t, ok) + + present, err := reader.MigrationImportMetadataPresent(ctx, 1) + require.NoError(t, err) + require.False(t, present) + _, err = st.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 1, + BracketID: 2, + BatchSeq: 1, + Cursor: []byte("ack-only"), + }) + require.NoError(t, err) + present, err = reader.MigrationImportMetadataPresent(ctx, 1) + require.NoError(t, err) + require.True(t, present) + present, err = reader.MigrationImportMetadataPresent(ctx, 2) + require.NoError(t, err) + require.False(t, present) + + require.NoError(t, cleaner.ClearMigrationState(ctx, 1, 0)) + present, err = reader.MigrationImportMetadataPresent(ctx, 1) + require.NoError(t, err) + require.False(t, present) + }) +} + func TestPromoteVersionsMovesStagedVersionsAndDeletesStagedRows(t *testing.T) { runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { ctx := context.Background() @@ -1033,6 +1101,138 @@ func TestPromoteVersionsMovesStagedVersionsAndDeletesStagedRows(t *testing.T) { }) } +func TestTargetStagedReadinessStatePersistsAndIsCloned(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + writer, ok := st.(MigrationTargetReadinessWriter) + require.True(t, ok) + reader, ok := st.(MigrationTargetReadinessReader) + require.True(t, ok) + + state := TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 12, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + } + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, state)) + + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []TargetStagedReadinessState{state}, states) + + states[0].RouteStart[0] = 'x' + states, err = reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []byte("a"), states[0].RouteStart) + + updated := state + updated.MinWriteTSExclusive = 101 + updated.RouteStart = []byte("b") + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, updated)) + + states, err = reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []TargetStagedReadinessState{updated}, states) + + exported, err := st.ExportVersions(ctx, ExportVersionsOptions{ + StartKey: []byte("!"), + EndKey: []byte("~"), + MaxVersions: 100, + }) + require.NoError(t, err) + require.Empty(t, exported.Versions) + }) +} + +func TestTargetStagedReadinessRejectsArmedStateWithoutWriteFloor(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + writer, ok := st.(MigrationTargetReadinessWriter) + require.True(t, ok) + + err := writer.ApplyTargetStagedReadiness(context.Background(), TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 12, + MigrationJobID: 9, + Armed: true, + }) + require.ErrorContains(t, err, "min_write_ts_exclusive") + }) +} + +func TestPebbleTargetStagedReadinessPersistsAcrossReopen(t *testing.T) { + ctx := context.Background() + dir, err := os.MkdirTemp("", "migration-ready-persist-*") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + + st, err := NewPebbleStore(dir) + require.NoError(t, err) + state := TargetStagedReadinessState{ + JobID: 11, + RouteStart: []byte("m"), + RouteEnd: nil, + ExpectedCutoverVersion: 22, + MigrationJobID: 11, + MinWriteTSExclusive: 333, + Armed: true, + } + writer, ok := st.(MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, state)) + require.NoError(t, st.Close()) + + reopened, err := NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, reopened.Close()) }) + reader, ok := reopened.(MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []TargetStagedReadinessState{state}, states) +} + +func TestPebbleTargetStagedReadinessIgnoresNonRecordPrefixKeys(t *testing.T) { + ctx := context.Background() + dir, err := os.MkdirTemp("", "migration-ready-prefix-*") + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, os.RemoveAll(dir)) }) + + st, err := NewPebbleStore(dir) + require.NoError(t, err) + state := TargetStagedReadinessState{ + JobID: 11, + RouteStart: []byte("m"), + RouteEnd: nil, + ExpectedCutoverVersion: 22, + MigrationJobID: 11, + MinWriteTSExclusive: 333, + Armed: true, + } + writer, ok := st.(MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, state)) + pebbleStore, ok := st.(*pebbleStore) + require.True(t, ok) + junkKey := append([]byte(migrationReadyPrefix), []byte("side-record")...) + require.NoError(t, pebbleStore.db.Set(junkKey, []byte("not-readiness-state"), pebbleStore.directApplyWriteOpts())) + require.NoError(t, st.Close()) + + reopened, err := NewPebbleStore(dir) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, reopened.Close()) }) + reader, ok := reopened.(MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []TargetStagedReadinessState{state}, states) +} + func TestPromoteVersionsIgnoresClientCursorWhenStateMissing(t *testing.T) { runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { ctx := context.Background() @@ -1279,3 +1479,76 @@ func TestPebbleSnapshotPreservesMigrationMetadata(t *testing.T) { _, err = dst.GetAt(ctx, []byte("fresh"), 60) require.ErrorIs(t, err, ErrKeyNotFound) } + +func TestCleanupVersionsRemovesOnlySelectedCommittedVersions(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + require.NoError(t, st.PutAt(ctx, []byte("a"), []byte("v1"), 10, 0)) + require.NoError(t, st.PutAt(ctx, []byte("a"), []byte("v2"), 20, 0)) + require.NoError(t, st.PutAt(ctx, []byte("b"), []byte("keep"), 10, 0)) + + cleaner, ok := st.(MigrationCleaner) + require.True(t, ok) + first, err := cleaner.CleanupVersions(ctx, CleanupVersionsOptions{ + JobID: 1, + StartKey: []byte("a"), + EndKey: []byte("b"), + MaxCommitTS: 10, + MaxVersions: 1, + MaxBytes: 1 << 20, + }) + require.NoError(t, err) + require.Equal(t, uint64(1), first.DeletedRows) + + value, err := st.GetAt(ctx, []byte("a"), 20) + require.NoError(t, err) + require.Equal(t, []byte("v2"), value) + _, err = st.GetAt(ctx, []byte("a"), 10) + require.ErrorIs(t, err, ErrKeyNotFound) + value, err = st.GetAt(ctx, []byte("b"), 20) + require.NoError(t, err) + require.Equal(t, []byte("keep"), value) + }) +} + +func TestClearMigrationStateRemovesPerJobProofs(t *testing.T) { + runMigrationStoreSuite(t, func(t *testing.T, st MVCCStore) { + ctx := context.Background() + _, err := st.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 7, + BracketID: 1, + BatchSeq: 1, + Versions: []MVCCVersion{{ + Key: []byte("!dist|migstage|row"), + CommitTS: 50, + Value: []byte("v"), + }}, + }) + require.NoError(t, err) + writer, ok := st.(MigrationTargetReadinessWriter) + require.True(t, ok) + require.NoError(t, writer.ApplyTargetStagedReadiness(ctx, TargetStagedReadinessState{ + JobID: 7, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 7, + MinWriteTSExclusive: 50, + Armed: true, + })) + + cleaner, ok := st.(MigrationCleaner) + require.True(t, ok) + require.NoError(t, cleaner.ClearMigrationState(ctx, 7, 0)) + floor, err := st.MigrationHLCFloor(ctx, 7) + require.NoError(t, err) + require.Zero(t, floor) + reader, ok := st.(MigrationTargetReadinessReader) + require.True(t, ok) + states, err := reader.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + for _, state := range states { + require.NotEqual(t, uint64(7), state.JobID) + } + }) +} diff --git a/store/mvcc_store.go b/store/mvcc_store.go index 4d2b61dfa..98dffed88 100644 --- a/store/mvcc_store.go +++ b/store/mvcc_store.go @@ -25,9 +25,13 @@ type VersionedValue struct { } const ( - mvccSnapshotVersion = uint32(1) - maxSnapshotKeySize = 1 << 20 // 1 MiB per key - maxSnapshotVersionCount = 1 << 20 // 1M versions per key + mvccSnapshotVersion = uint32(2) + mvccSnapshotLegacyVersion = uint32(1) + maxSnapshotKeySize = 1 << 20 // 1 MiB per key + maxSnapshotVersionCount = 1 << 20 // 1M versions per key + maxSnapshotReadinessStateCount = 1 << 20 + maxSnapshotReadinessEncodedBytes = 1 << 20 + maxSnapshotMigrationMetadataBytes = 1 << 20 ) // maxSnapshotValueSize caps the allowed size of a single value during streaming @@ -37,6 +41,7 @@ const ( var maxSnapshotValueSize = 256 << 20 // 256 MiB var mvccSnapshotMagic = [8]byte{'E', 'K', 'V', 'M', 'V', 'C', 'C', '2'} +var mvccSnapshotMigrationMetadataMarker = [8]byte{'E', 'K', 'V', 'M', 'I', 'G', '2', '!'} type compactEntry struct { key []byte @@ -60,14 +65,16 @@ func byteSliceComparator(a, b any) int { // mvccStore is an in-memory MVCC implementation backed by a treemap for // deterministic iteration order and range scans. type mvccStore struct { - tree *treemap.Map // key []byte -> []VersionedValue - mtx sync.RWMutex - log *slog.Logger - lastCommitTS uint64 - minRetainedTS uint64 - migrationAcks map[migrationAckID]migrationImportAck - migrationHLCFloors map[uint64]uint64 - migrationPromotions map[uint64]PromotionState + tree *treemap.Map // key []byte -> []VersionedValue + mtx sync.RWMutex + log *slog.Logger + lastCommitTS uint64 + minRetainedTS uint64 + migrationAcks map[migrationAckID]migrationImportAck + migrationHLCFloors map[uint64]uint64 + migrationPromotions map[uint64]PromotionState + migrationReadiness map[uint64]TargetStagedReadinessState + migrationReadinessCache []TargetStagedReadinessState // writeConflicts mirrors the per-(kind, key_prefix) counter from // the pebble-backed store so the in-memory implementation shows up // in the same Prometheus series (even if the counts are usually @@ -117,6 +124,7 @@ func NewMVCCStore(opts ...MVCCStoreOption) MVCCStore { migrationAcks: make(map[migrationAckID]migrationImportAck), migrationHLCFloors: make(map[uint64]uint64), migrationPromotions: make(map[uint64]PromotionState), + migrationReadiness: make(map[uint64]TargetStagedReadinessState), writeConflicts: newWriteConflictCounter(), } for _, opt := range opts { @@ -845,12 +853,19 @@ func isStreamingMVCCSnapshot(r *bufio.Reader) (bool, error) { } func (s *mvccStore) writeSnapshotFile(f *os.File) error { - checksumOffset, err := writeMVCCSnapshotHeader(f) + s.mtx.RLock() + defer s.mtx.RUnlock() + + version := mvccSnapshotLegacyVersion + if s.hasSnapshotMigrationMetadataLocked() { + version = mvccSnapshotVersion + } + checksumOffset, err := writeMVCCSnapshotHeader(f, version) if err != nil { return err } - sum, err := s.writeSnapshotBody(f) + sum, err := s.writeSnapshotBodyLocked(f, version) if err != nil { return err } @@ -858,11 +873,11 @@ func (s *mvccStore) writeSnapshotFile(f *os.File) error { return finalizeMVCCSnapshotFile(f, checksumOffset, sum) } -func writeMVCCSnapshotHeader(f *os.File) (int64, error) { +func writeMVCCSnapshotHeader(f *os.File, version uint32) (int64, error) { if _, err := f.Write(mvccSnapshotMagic[:]); err != nil { return 0, errors.WithStack(err) } - if err := binary.Write(f, binary.LittleEndian, mvccSnapshotVersion); err != nil { + if err := binary.Write(f, binary.LittleEndian, version); err != nil { return 0, errors.WithStack(err) } checksumOffset, err := f.Seek(0, io.SeekCurrent) @@ -875,20 +890,20 @@ func writeMVCCSnapshotHeader(f *os.File) (int64, error) { return checksumOffset, nil } -func (s *mvccStore) writeSnapshotBody(f *os.File) (uint32, error) { +func (s *mvccStore) writeSnapshotBodyLocked(f *os.File, version uint32) (uint32, error) { hash := crc32.NewIEEE() bw := bufio.NewWriter(f) w := io.MultiWriter(bw, hash) - s.mtx.RLock() - defer s.mtx.RUnlock() - if err := binary.Write(w, binary.LittleEndian, s.lastCommitTS); err != nil { return 0, errors.WithStack(err) } if err := binary.Write(w, binary.LittleEndian, s.minRetainedTS); err != nil { return 0, errors.WithStack(err) } + if err := s.writeSnapshotMigrationMetadataLocked(w, version); err != nil { + return 0, err + } iter := s.tree.Iterator() for iter.Next() { key, ok := iter.Key().([]byte) @@ -909,6 +924,25 @@ func (s *mvccStore) writeSnapshotBody(f *os.File) (uint32, error) { return hash.Sum32(), nil } +func (s *mvccStore) writeSnapshotMigrationMetadataLocked(w io.Writer, version uint32) error { + if version < mvccSnapshotVersion { + return nil + } + if err := writeMVCCSnapshotReadinessStates(w, s.migrationReadinessCache); err != nil { + return err + } + if !hasExtendedMVCCSnapshotMigrationMetadata(s.migrationAcks, s.migrationHLCFloors) { + return nil + } + return writeMVCCSnapshotMigrationMetadata(w, s.migrationAcks, s.migrationHLCFloors) +} + +func (s *mvccStore) hasSnapshotMigrationMetadataLocked() bool { + return len(s.migrationReadinessCache) != 0 || + len(s.migrationAcks) != 0 || + len(s.migrationHLCFloors) != 0 +} + func finalizeMVCCSnapshotFile(f *os.File, checksumOffset int64, sum uint32) error { if _, err := f.Seek(checksumOffset, io.SeekStart); err != nil { return errors.WithStack(err) @@ -959,6 +993,49 @@ func writeMVCCSnapshotVersion(w io.Writer, version VersionedValue) error { return nil } +func writeMVCCSnapshotReadinessStates(w io.Writer, states []TargetStagedReadinessState) error { + if err := binary.Write(w, binary.LittleEndian, uint64(len(states))); err != nil { + return errors.WithStack(err) + } + for _, state := range states { + encoded := encodeTargetStagedReadinessState(state) + if err := binary.Write(w, binary.LittleEndian, uint64(len(encoded))); err != nil { + return errors.WithStack(err) + } + if _, err := w.Write(encoded); err != nil { + return errors.WithStack(err) + } + } + return nil +} + +func hasExtendedMVCCSnapshotMigrationMetadata(acks map[migrationAckID]migrationImportAck, floors map[uint64]uint64) bool { + return len(acks) != 0 || len(floors) != 0 +} + +func writeMVCCSnapshotMigrationMetadata(w io.Writer, acks map[migrationAckID]migrationImportAck, floors map[uint64]uint64) error { + if _, err := w.Write(mvccSnapshotMigrationMetadataMarker[:]); err != nil { + return errors.WithStack(err) + } + if err := writeMVCCSnapshotEncodedMetadata(w, "migration import acks", encodeMigrationImportAcks(acks)); err != nil { + return err + } + return writeMVCCSnapshotEncodedMetadata(w, "migration HLC floors", encodeMigrationHLCFloors(floors)) +} + +func writeMVCCSnapshotEncodedMetadata(w io.Writer, label string, encoded []byte) error { + if len(encoded) > maxSnapshotMigrationMetadataBytes { + return errors.Wrapf(ErrValueTooLarge, "%s %d > %d", label, len(encoded), maxSnapshotMigrationMetadataBytes) + } + if err := binary.Write(w, binary.LittleEndian, uint64(len(encoded))); err != nil { + return errors.WithStack(err) + } + if _, err := w.Write(encoded); err != nil { + return errors.WithStack(err) + } + return nil +} + func mvccSnapshotTombstoneByte(tombstone bool) byte { if tombstone { return 1 @@ -967,12 +1044,12 @@ func mvccSnapshotTombstoneByte(tombstone bool) byte { } func (s *mvccStore) restoreStreamingSnapshot(r io.Reader) error { - expected, err := readMVCCSnapshotHeader(r) + expected, version, err := readMVCCSnapshotHeader(r) if err != nil { return err } - tree, lastCommitTS, minRetainedTS, actual, err := restoreStreamingMVCCSnapshotBody(r) + tree, lastCommitTS, minRetainedTS, readinessStates, importAcks, hlcFloors, actual, err := restoreStreamingMVCCSnapshotBody(r, version) if err != nil { return err } @@ -985,51 +1062,65 @@ func (s *mvccStore) restoreStreamingSnapshot(r io.Reader) error { s.tree = tree s.lastCommitTS = lastCommitTS s.minRetainedTS = minRetainedTS - s.migrationAcks = make(map[migrationAckID]migrationImportAck) - s.migrationHLCFloors = make(map[uint64]uint64) + s.migrationAcks = importAcks + s.migrationHLCFloors = hlcFloors s.migrationPromotions = make(map[uint64]PromotionState) + s.migrationReadiness = targetReadinessStateMap(readinessStates) + s.migrationReadinessCache = cloneTargetStagedReadinessStates(readinessStates) return nil } -func readMVCCSnapshotHeader(r io.Reader) (uint32, error) { +func readMVCCSnapshotHeader(r io.Reader) (uint32, uint32, error) { var magic [8]byte if _, err := io.ReadFull(r, magic[:]); err != nil { - return 0, errors.WithStack(err) + return 0, 0, errors.WithStack(err) } if magic != mvccSnapshotMagic { - return 0, errors.WithStack(ErrInvalidChecksum) + return 0, 0, errors.WithStack(ErrInvalidChecksum) } var version uint32 if err := binary.Read(r, binary.LittleEndian, &version); err != nil { - return 0, errors.WithStack(err) + return 0, 0, errors.WithStack(err) } - if version != mvccSnapshotVersion { - return 0, errors.WithStack(errors.Newf("unsupported mvcc snapshot version %d", version)) + if version != mvccSnapshotLegacyVersion && version != mvccSnapshotVersion { + return 0, 0, errors.WithStack(errors.Newf("unsupported mvcc snapshot version %d", version)) } var expected uint32 if err := binary.Read(r, binary.LittleEndian, &expected); err != nil { - return 0, errors.WithStack(err) + return 0, 0, errors.WithStack(err) } - return expected, nil + return expected, version, nil } -func restoreStreamingMVCCSnapshotBody(r io.Reader) (*treemap.Map, uint64, uint64, uint32, error) { +func restoreStreamingMVCCSnapshotBody( + r io.Reader, + version uint32, +) (*treemap.Map, uint64, uint64, []TargetStagedReadinessState, map[migrationAckID]migrationImportAck, map[uint64]uint64, uint32, error) { hash := crc32.NewIEEE() body := io.TeeReader(r, hash) lastCommitTS, minRetainedTS, err := readMVCCSnapshotMetadata(body) if err != nil { - return nil, 0, 0, 0, err + return nil, 0, 0, nil, nil, nil, 0, err + } + readinessStates, err := readMVCCSnapshotReadinessStates(body, version) + if err != nil { + return nil, 0, 0, nil, nil, nil, 0, err + } + bufferedBody := bufio.NewReader(body) + importAcks, hlcFloors, err := readMVCCSnapshotMigrationMetadata(bufferedBody, version) + if err != nil { + return nil, 0, 0, nil, nil, nil, 0, err } - tree, err := readMVCCSnapshotTree(body) + tree, err := readMVCCSnapshotTree(bufferedBody) if err != nil { - return nil, 0, 0, 0, err + return nil, 0, 0, nil, nil, nil, 0, err } - return tree, lastCommitTS, minRetainedTS, hash.Sum32(), nil + return tree, lastCommitTS, minRetainedTS, readinessStates, importAcks, hlcFloors, hash.Sum32(), nil } func readMVCCSnapshotMetadata(r io.Reader) (uint64, uint64, error) { @@ -1044,6 +1135,107 @@ func readMVCCSnapshotMetadata(r io.Reader) (uint64, uint64, error) { return lastCommitTS, minRetainedTS, nil } +func readMVCCSnapshotReadinessStates(r io.Reader, version uint32) ([]TargetStagedReadinessState, error) { + if version < mvccSnapshotVersion { + return nil, nil + } + var count uint64 + if err := binary.Read(r, binary.LittleEndian, &count); err != nil { + return nil, errors.WithStack(err) + } + if count > maxSnapshotReadinessStateCount { + return nil, errors.Wrapf(ErrSnapshotVersionCountTooLarge, "readiness states %d > %d", count, maxSnapshotReadinessStateCount) + } + states := make([]TargetStagedReadinessState, 0, count) + for range count { + var encodedLen uint64 + if err := binary.Read(r, binary.LittleEndian, &encodedLen); err != nil { + return nil, errors.WithStack(err) + } + if encodedLen > maxSnapshotReadinessEncodedBytes { + return nil, errors.Wrapf(ErrValueTooLarge, "readiness state %d > %d", encodedLen, maxSnapshotReadinessEncodedBytes) + } + encoded := make([]byte, encodedLen) + if _, err := io.ReadFull(r, encoded); err != nil { + return nil, errors.WithStack(err) + } + state, ok := decodeTargetStagedReadinessState(encoded) + if !ok { + return nil, errors.New("corrupt target staged readiness state") + } + states = append(states, state) + } + sortTargetStagedReadinessStates(states) + return states, nil +} + +func readMVCCSnapshotMigrationMetadata( + r *bufio.Reader, + version uint32, +) (map[migrationAckID]migrationImportAck, map[uint64]uint64, error) { + emptyAcks := make(map[migrationAckID]migrationImportAck) + emptyFloors := make(map[uint64]uint64) + if version < mvccSnapshotVersion { + return emptyAcks, emptyFloors, nil + } + marker, err := r.Peek(len(mvccSnapshotMigrationMetadataMarker)) + if err != nil { + if errors.Is(err, io.EOF) { + return emptyAcks, emptyFloors, nil + } + return nil, nil, errors.WithStack(err) + } + if !bytes.Equal(marker, mvccSnapshotMigrationMetadataMarker[:]) { + return emptyAcks, emptyFloors, nil + } + if _, err := r.Discard(len(mvccSnapshotMigrationMetadataMarker)); err != nil { + return nil, nil, errors.WithStack(err) + } + + encodedAcks, err := readMVCCSnapshotEncodedMetadata(r, "migration import acks") + if err != nil { + return nil, nil, err + } + importAcks, ok := decodeMigrationImportAcks(encodedAcks) + if !ok { + return nil, nil, errors.New("corrupt migration import ack snapshot metadata") + } + encodedFloors, err := readMVCCSnapshotEncodedMetadata(r, "migration HLC floors") + if err != nil { + return nil, nil, err + } + hlcFloors, ok := decodeMigrationHLCFloors(encodedFloors) + if !ok { + return nil, nil, errors.New("corrupt migration HLC floor snapshot metadata") + } + return importAcks, hlcFloors, nil +} + +func readMVCCSnapshotEncodedMetadata(r io.Reader, label string) ([]byte, error) { + var encodedLen uint64 + if err := binary.Read(r, binary.LittleEndian, &encodedLen); err != nil { + return nil, errors.WithStack(err) + } + decodedLen, err := restoreFieldLenInt(encodedLen, label, maxSnapshotMigrationMetadataBytes) + if err != nil { + return nil, err + } + encoded := make([]byte, decodedLen) + if _, err := io.ReadFull(r, encoded); err != nil { + return nil, errors.WithStack(err) + } + return encoded, nil +} + +func targetReadinessStateMap(states []TargetStagedReadinessState) map[uint64]TargetStagedReadinessState { + out := make(map[uint64]TargetStagedReadinessState, len(states)) + for _, state := range states { + cloned := cloneTargetStagedReadinessState(state) + out[cloned.JobID] = cloned + } + return out +} + func readMVCCSnapshotTree(r io.Reader) (*treemap.Map, error) { tree := treemap.NewWith(byteSliceComparator) for { diff --git a/store/mvcc_store_snapshot_test.go b/store/mvcc_store_snapshot_test.go index 3032a1cc3..1fc81d7d1 100644 --- a/store/mvcc_store_snapshot_test.go +++ b/store/mvcc_store_snapshot_test.go @@ -3,6 +3,7 @@ package store import ( "bytes" "context" + "encoding/binary" "testing" "github.com/stretchr/testify/require" @@ -21,6 +22,7 @@ func TestMVCCStore_SnapshotRestoreRoundTrip(t *testing.T) { defer snap.Close() raw := snapshotBytes(t, snap) + require.Equal(t, mvccSnapshotLegacyVersion, binary.LittleEndian.Uint32(raw[len(mvccSnapshotMagic):])) // Mutate source after snapshot so restore must reflect snapshot point-in-time. require.NoError(t, src.PutAt(ctx, []byte("k1"), []byte("v3"), 30, 0)) @@ -38,6 +40,123 @@ func TestMVCCStore_SnapshotRestoreRoundTrip(t *testing.T) { require.Equal(t, []byte("v2"), v) } +func TestMVCCStore_SnapshotRestorePreservesTargetReadiness(t *testing.T) { + t.Parallel() + + ctx := context.Background() + src := newTestMVCCStore(t) + + stale := TargetStagedReadinessState{ + JobID: 8, + RouteStart: []byte("old"), + RouteEnd: []byte("oldz"), + ExpectedCutoverVersion: 1, + MigrationJobID: 8, + MinWriteTSExclusive: 80, + Armed: true, + } + require.NoError(t, src.ApplyTargetStagedReadiness(ctx, stale)) + + snapState := TargetStagedReadinessState{ + JobID: 9, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + ExpectedCutoverVersion: 12, + MigrationJobID: 9, + MinWriteTSExclusive: 100, + Armed: true, + } + require.NoError(t, src.ApplyTargetStagedReadiness(ctx, snapState)) + + snap, err := src.Snapshot() + require.NoError(t, err) + defer snap.Close() + raw := snapshotBytes(t, snap) + require.Equal(t, mvccSnapshotVersion, binary.LittleEndian.Uint32(raw[len(mvccSnapshotMagic):])) + + dst := newTestMVCCStore(t) + require.NoError(t, dst.ApplyTargetStagedReadiness(ctx, TargetStagedReadinessState{ + JobID: 77, + RouteStart: []byte("dst-only"), + RouteEnd: []byte("dst-z"), + ExpectedCutoverVersion: 2, + MigrationJobID: 77, + MinWriteTSExclusive: 700, + Armed: true, + })) + + require.NoError(t, dst.Restore(bytes.NewReader(raw))) + states, err := dst.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []TargetStagedReadinessState{stale, snapState}, states) + + states[0].RouteStart[0] = 'x' + states, err = dst.MigrationTargetReadinessStates(ctx) + require.NoError(t, err) + require.Equal(t, []byte("old"), states[0].RouteStart) +} + +func TestMVCCStore_SnapshotRestorePreservesMigrationImportMetadata(t *testing.T) { + t.Parallel() + + ctx := context.Background() + src := newTestMVCCStore(t) + res, err := src.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 7, + BracketID: 3, + BatchSeq: 1, + Cursor: []byte("c1"), + Versions: []MVCCVersion{{Key: []byte("snapshotted"), CommitTS: 50, Value: []byte("v50")}}, + }) + require.NoError(t, err) + require.False(t, res.Duplicate) + + snap, err := src.Snapshot() + require.NoError(t, err) + defer snap.Close() + raw := snapshotBytes(t, snap) + require.Equal(t, mvccSnapshotVersion, binary.LittleEndian.Uint32(raw[len(mvccSnapshotMagic):])) + + dst := newTestMVCCStore(t) + require.NoError(t, dst.Restore(bytes.NewReader(raw))) + val, err := dst.GetAt(ctx, []byte("snapshotted"), 50) + require.NoError(t, err) + require.Equal(t, []byte("v50"), val) + floor, err := dst.MigrationHLCFloor(ctx, 7) + require.NoError(t, err) + require.Equal(t, uint64(50), floor) + present, err := dst.MigrationImportMetadataPresent(ctx, 7) + require.NoError(t, err) + require.True(t, present) + + res, err = dst.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 7, + BracketID: 3, + BatchSeq: 1, + Cursor: []byte("fresh"), + Versions: []MVCCVersion{{Key: []byte("fresh"), CommitTS: 60, Value: []byte("v60")}}, + }) + require.NoError(t, err) + require.True(t, res.Duplicate) + require.Equal(t, []byte("c1"), res.AckedCursor) + _, err = dst.GetAt(ctx, []byte("fresh"), 60) + require.ErrorIs(t, err, ErrKeyNotFound) + + res, err = dst.ImportVersions(ctx, ImportVersionsOptions{ + JobID: 7, + BracketID: 3, + BatchSeq: 2, + Cursor: []byte("c2"), + Versions: []MVCCVersion{{Key: []byte("fresh"), CommitTS: 60, Value: []byte("v60")}}, + }) + require.NoError(t, err) + require.False(t, res.Duplicate) + require.Equal(t, []byte("c2"), res.AckedCursor) + val, err = dst.GetAt(ctx, []byte("fresh"), 60) + require.NoError(t, err) + require.Equal(t, []byte("v60"), val) +} + func TestMVCCStore_RestoreRejectsInvalidChecksum(t *testing.T) { t.Parallel() diff --git a/store/store.go b/store/store.go index 5a746d8c3..b23c522d3 100644 --- a/store/store.go +++ b/store/store.go @@ -87,13 +87,15 @@ type ExportVersionsOptions struct { EndKey []byte MinCommitTSExclusive uint64 MaxCommitTSInclusive uint64 - Cursor []byte - MaxVersions int - MaxBytes uint64 - MaxScannedBytes uint64 - KeyFamily uint32 - AcceptKey func([]byte) bool - AcceptVersion func(key []byte, value []byte) bool + // ReadTS asks export to enforce the same retention watermark as GetAt/ScanAt. + ReadTS uint64 + Cursor []byte + MaxVersions int + MaxBytes uint64 + MaxScannedBytes uint64 + KeyFamily uint32 + AcceptKey func([]byte) bool + AcceptVersion func(key []byte, value []byte) bool } // ExportVersionsResult is one resumable chunk of raw MVCC versions. @@ -161,17 +163,91 @@ type PromotionState struct { LastError string } +// CleanupVersionsOptions physically removes exact committed versions selected +// by the migration bracket filter. It is cursor-resumable and is used only +// after ownership has moved or while abandoning an unpublished migration. +type CleanupVersionsOptions struct { + JobID uint64 + AppliedIndex uint64 + StartKey []byte + EndKey []byte + Cursor []byte + MaxCommitTS uint64 + MaxVersions int + MaxBytes uint64 + MaxScannedBytes uint64 + KeyFamily uint32 + AcceptVersion func(key, value []byte) bool +} + +// CleanupVersionsResult reports one bounded physical-delete chunk. +type CleanupVersionsResult struct { + NextCursor []byte + Done bool + DeletedRows uint64 + DeletedBytes uint64 + ScannedBytes uint64 +} + +// TargetStagedReadinessState is a target-local guard that makes a target voter +// fail closed for a moving route until it has either the staged descriptor or +// the cleared descriptor with the retained write timestamp floor. +type TargetStagedReadinessState struct { + JobID uint64 + RouteStart []byte + RouteEnd []byte + ExpectedCutoverVersion uint64 + MigrationJobID uint64 + MinWriteTSExclusive uint64 + Armed bool + SourceWriteFence bool + SourceReadFence bool + RetentionPinTS uint64 + TrackWrites bool + MinAdmittedTS uint64 +} + // MigrationPromoter is implemented by stores that can promote staged range // migration data into the live keyspace. type MigrationPromoter interface { PromoteVersions(ctx context.Context, opts PromoteVersionsOptions) (PromoteVersionsResult, error) } +// MigrationCleaner removes source or abandoned target versions and the +// per-job migration proof records through the Raft apply path. +type MigrationCleaner interface { + CleanupVersions(ctx context.Context, opts CleanupVersionsOptions) (CleanupVersionsResult, error) + ClearMigrationState(ctx context.Context, jobID, appliedIndex uint64) error +} + // MigrationPromotionStateReader reads target-local staged promotion state. type MigrationPromotionStateReader interface { MigrationPromotionState(ctx context.Context, jobID uint64) (PromotionState, bool, error) } +// MigrationTargetReadinessWriter persists a target-local staged-readiness +// guard through the target Raft group. +type MigrationTargetReadinessWriter interface { + ApplyTargetStagedReadiness(ctx context.Context, state TargetStagedReadinessState) error +} + +// MigrationTargetReadinessRaftWriter persists a target-local readiness guard +// from the Raft apply loop while bundling the applied index with that update. +type MigrationTargetReadinessRaftWriter interface { + ApplyTargetStagedReadinessAt(ctx context.Context, state TargetStagedReadinessState, appliedIndex uint64) error +} + +// MigrationTargetReadinessReader reads retained target-local readiness guards. +type MigrationTargetReadinessReader interface { + MigrationTargetReadinessStates(ctx context.Context) ([]TargetStagedReadinessState, error) +} + +// MigrationImportMetadataReader reports whether target-local import proof +// metadata remains for a migration job. +type MigrationImportMetadataReader interface { + MigrationImportMetadataPresent(ctx context.Context, jobID uint64) (bool, error) +} + // OpType describes a mutation kind. type OpType int