diff --git a/adapter/admin_grpc.go b/adapter/admin_grpc.go index 247b37c3e..50d16a735 100644 --- a/adapter/admin_grpc.go +++ b/adapter/admin_grpc.go @@ -33,6 +33,13 @@ type KeyVizSampler interface { Snapshot(from, to time.Time) []keyviz.MatrixColumn } +// AutoSplitRuntime is the atomic runtime control exposed by the authenticated +// Admin service. +type AutoSplitRuntime interface { + Enabled() bool + SetEnabled(enabled bool) +} + // AdminGroup exposes per-Raft-group state to the Admin service. It is a narrow // subset of raftengine.Engine so tests can supply an in-memory fake without // standing up a real Raft cluster. Configuration is polled on each @@ -77,7 +84,8 @@ type AdminServer struct { // Nil means keyviz is disabled — the RPC returns Unavailable. // Guarded by groupsMu (same lock as groups/now) so RegisterSampler // pairs atomically with concurrent RPC reads. - sampler KeyVizSampler + sampler KeyVizSampler + autoSplitRuntime AutoSplitRuntime pb.UnimplementedAdminServer } @@ -145,6 +153,32 @@ func (s *AdminServer) SetCapability(name string, enabled bool) { s.groupsMu.Unlock() } +// RegisterAutoSplitRuntime wires the process-local automatic split switch. +func (s *AdminServer) RegisterAutoSplitRuntime(runtime AutoSplitRuntime) { + s.groupsMu.Lock() + s.autoSplitRuntime = runtime + s.groupsMu.Unlock() +} + +// SetAutoSplitEnabled atomically changes whether the scheduler may issue new +// SplitRange calls. Detector observation continues while disabled. +func (s *AdminServer) SetAutoSplitEnabled( + _ context.Context, + req *pb.SetAutoSplitEnabledRequest, +) (*pb.SetAutoSplitEnabledResponse, error) { + if req == nil { + return nil, grpcStatusError(codes.InvalidArgument, "request is required") + } + s.groupsMu.RLock() + runtime := s.autoSplitRuntime + s.groupsMu.RUnlock() + if runtime == nil { + return nil, grpcStatusError(codes.Unavailable, "automatic splitting is not configured") + } + runtime.SetEnabled(req.GetEnabled()) + return &pb.SetAutoSplitEnabledResponse{Enabled: runtime.Enabled()}, nil +} + // GetClusterOverview returns the local node identity, the current member // list, and per-group leader identity collected from the engines registered // via RegisterGroup. The member list is the union of (a) the bootstrap seed diff --git a/adapter/admin_grpc_autosplit_test.go b/adapter/admin_grpc_autosplit_test.go new file mode 100644 index 000000000..1c4906394 --- /dev/null +++ b/adapter/admin_grpc_autosplit_test.go @@ -0,0 +1,60 @@ +package adapter + +import ( + "context" + "sync" + "testing" + + "github.com/bootjp/elastickv/distribution/autosplit" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestAdminSetAutoSplitEnabledControlsRuntimeSwitch(t *testing.T) { + t.Parallel() + server := NewAdminServer(NodeIdentity{NodeID: "node-a"}, nil) + + _, err := server.SetAutoSplitEnabled(context.Background(), &pb.SetAutoSplitEnabledRequest{Enabled: false}) + require.Equal(t, codes.Unavailable, status.Code(err)) + + runtime := autosplit.NewRuntimeSwitch(true) + server.RegisterAutoSplitRuntime(runtime) + disabled, err := server.SetAutoSplitEnabled(context.Background(), &pb.SetAutoSplitEnabledRequest{Enabled: false}) + require.NoError(t, err) + require.False(t, disabled.GetEnabled()) + require.True(t, runtime.KillSwitch(context.Background())) + + enabled, err := server.SetAutoSplitEnabled(context.Background(), &pb.SetAutoSplitEnabledRequest{Enabled: true}) + require.NoError(t, err) + require.True(t, enabled.GetEnabled()) + require.False(t, runtime.KillSwitch(context.Background())) +} + +func TestAdminSetAutoSplitEnabledIsConcurrentSafe(t *testing.T) { + t.Parallel() + server := NewAdminServer(NodeIdentity{NodeID: "node-a"}, nil) + runtime := autosplit.NewRuntimeSwitch(true) + server.RegisterAutoSplitRuntime(runtime) + + var wg sync.WaitGroup + errs := make(chan error, 100) + for i := range 100 { + wg.Add(1) + go func() { + defer wg.Done() + _, err := server.SetAutoSplitEnabled(context.Background(), &pb.SetAutoSplitEnabledRequest{Enabled: i%2 == 0}) + errs <- err + }() + } + wg.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } + + _, err := server.SetAutoSplitEnabled(context.Background(), &pb.SetAutoSplitEnabledRequest{Enabled: true}) + require.NoError(t, err) + require.True(t, runtime.Enabled()) +} diff --git a/adapter/distribution_server.go b/adapter/distribution_server.go index 072294d78..ae9d6f5ff 100644 --- a/adapter/distribution_server.go +++ b/adapter/distribution_server.go @@ -649,6 +649,9 @@ func (s *DistributionServer) applyEngineSnapshot(snapshot distribution.CatalogSn return grpcStatusError(codes.FailedPrecondition, errDistributionEngineNotConfigured.Error()) } if err := s.engine.ApplySnapshot(snapshot); err != nil { + if errors.Is(err, distribution.ErrEngineSnapshotVersionStale) { + return nil + } return grpcStatusErrorf(codes.Internal, "apply engine snapshot: %v", err) } return nil diff --git a/adapter/distribution_server_test.go b/adapter/distribution_server_test.go index ce268cddc..5f95d1c72 100644 --- a/adapter/distribution_server_test.go +++ b/adapter/distribution_server_test.go @@ -653,6 +653,29 @@ func TestDistributionServerSplitRange_ReturnsExactCommittedSplitVersion(t *testi require.Equal(t, uint64(3), latest.Version) } +func TestDistributionServerApplyEngineSnapshotAcceptsNewerEngine(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + recent := distribution.CatalogSnapshot{ + Version: 3, + Routes: []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte(""), + End: nil, + GroupID: 1, + State: distribution.RouteStateActive, + }}, + } + require.NoError(t, engine.ApplySnapshot(recent)) + + s := NewDistributionServer(engine, nil) + stale := recent + stale.Version = 2 + require.NoError(t, s.applyEngineSnapshot(stale)) + require.Equal(t, uint64(3), engine.Version()) +} + func TestDistributionServerSplitRange_RetriesCatalogReloadUntilVisible(t *testing.T) { t.Parallel() diff --git a/adapter/internal.go b/adapter/internal.go index bf4279e31..cef2561fc 100644 --- a/adapter/internal.go +++ b/adapter/internal.go @@ -18,6 +18,17 @@ func WithInternalTimestampAllocator(alloc kv.TimestampAllocator) InternalOption } } +// ForwardWriteObserver observes successfully committed leader-side forwarded +// writes. It lets the autosplit sampler account for writes that entered through +// followers without coupling adapter.Internal to keyviz. +type ForwardWriteObserver func([]*pb.Request) + +func WithInternalForwardWriteObserver(observer ForwardWriteObserver) InternalOption { + return func(i *Internal) { + i.forwardWriteObserver = observer + } +} + func NewInternalWithEngine(txm kv.Transactional, leader raftengine.LeaderView, clock *kv.HLC, relay *RedisPubSubRelay, opts ...InternalOption) *Internal { i := &Internal{ leader: leader, @@ -32,11 +43,12 @@ func NewInternalWithEngine(txm kv.Transactional, leader raftengine.LeaderView, c } type Internal struct { - leader raftengine.LeaderView - transactionManager kv.Transactional - clock *kv.HLC - tsAllocator kv.TimestampAllocator - relay *RedisPubSubRelay + leader raftengine.LeaderView + transactionManager kv.Transactional + clock *kv.HLC + tsAllocator kv.TimestampAllocator + relay *RedisPubSubRelay + forwardWriteObserver ForwardWriteObserver pb.UnimplementedInternalServer } @@ -70,6 +82,9 @@ func (i *Internal) Forward(ctx context.Context, req *pb.ForwardRequest) (*pb.For CommitIndex: 0, }, errors.WithStack(err) } + if i.forwardWriteObserver != nil { + i.forwardWriteObserver(req.GetRequests()) + } return &pb.ForwardResponse{ Success: true, diff --git a/adapter/internal_test.go b/adapter/internal_test.go index fcabc5356..eced05a42 100644 --- a/adapter/internal_test.go +++ b/adapter/internal_test.go @@ -5,11 +5,40 @@ import ( "encoding/binary" "testing" + "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/kv" pb "github.com/bootjp/elastickv/proto" "github.com/stretchr/testify/require" ) +func TestInternalForwardObservesCommittedWrites(t *testing.T) { + t.Parallel() + + reqs := []*pb.Request{{ + Mutations: []*pb.Mutation{{ + Op: pb.Op_PUT, + Key: []byte("hot"), + Value: []byte("value"), + }}, + }} + txn := &forwardObserverTxn{} + var observed []*pb.Request + internal := NewInternalWithEngine(txn, forwardObserverLeader{}, nil, nil, WithInternalForwardWriteObserver(func(reqs []*pb.Request) { + observed = reqs + })) + + resp, err := internal.Forward(context.Background(), &pb.ForwardRequest{Requests: reqs}) + + require.NoError(t, err) + require.True(t, resp.Success) + require.Equal(t, uint64(9), resp.CommitIndex) + require.Len(t, observed, 1) + require.Same(t, reqs[0], observed[0]) + require.Len(t, txn.reqs, 1) + require.Same(t, reqs[0], txn.reqs[0]) + require.Equal(t, uint64(1), reqs[0].Ts) +} + func TestStampTxnTimestamps_RejectsMaxStartTS(t *testing.T) { t.Parallel() @@ -241,3 +270,25 @@ func TestStampTxnTimestamps_UsesSingleTxnStartTS(t *testing.T) { require.Greater(t, meta.CommitTS, uint64(9)) require.Equal(t, meta.CommitTS, commitTS) } + +type forwardObserverLeader struct{} + +func (forwardObserverLeader) State() raftengine.State { return raftengine.StateLeader } +func (forwardObserverLeader) Leader() raftengine.LeaderInfo { + return raftengine.LeaderInfo{ID: "self", Address: "127.0.0.1:0"} +} +func (forwardObserverLeader) VerifyLeader(context.Context) error { return nil } +func (forwardObserverLeader) LinearizableRead(context.Context) (uint64, error) { return 0, nil } + +type forwardObserverTxn struct { + reqs []*pb.Request +} + +func (t *forwardObserverTxn) Commit(_ context.Context, reqs []*pb.Request) (*kv.TransactionResponse, error) { + t.reqs = reqs + return &kv.TransactionResponse{CommitIndex: 9}, nil +} + +func (t *forwardObserverTxn) Abort(context.Context, []*pb.Request) (*kv.TransactionResponse, error) { + return &kv.TransactionResponse{}, nil +} diff --git a/cmd/server/demo.go b/cmd/server/demo.go index e231e9ec1..7581ae7cc 100644 --- a/cmd/server/demo.go +++ b/cmd/server/demo.go @@ -6,6 +6,7 @@ import ( "flag" "io" "log/slog" + "math" "net" "net/http" "os" @@ -15,10 +16,12 @@ import ( "github.com/bootjp/elastickv/adapter" "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/distribution/autosplit" internalutil "github.com/bootjp/elastickv/internal" internalraftadmin "github.com/bootjp/elastickv/internal/raftadmin" "github.com/bootjp/elastickv/internal/raftengine" etcdraftengine "github.com/bootjp/elastickv/internal/raftengine/etcd" + "github.com/bootjp/elastickv/keyviz" "github.com/bootjp/elastickv/kv" "github.com/bootjp/elastickv/monitoring" pb "github.com/bootjp/elastickv/proto" @@ -46,6 +49,35 @@ var ( raftRedisMap = flag.String("raftRedisMap", "", "Map of Raft address to Redis address (raftAddr=redisAddr,...)") raftS3Map = flag.String("raftS3Map", "", "Map of Raft address to S3 address (raftAddr=s3Addr,...)") raftDynamoMap = flag.String("raftDynamoMap", "", "Map of Raft address to DynamoDB address (raftAddr=dynamoAddr,...)") + + keyvizEnabled = flag.Bool("keyvizEnabled", false, "Enable the in-memory key visualizer sampler") + keyvizStep = flag.Duration("keyvizStep", keyviz.DefaultStep, "Flush interval / matrix-column resolution for the keyviz sampler") + keyvizMaxTrackedRoutes = flag.Int("keyvizMaxTrackedRoutes", keyviz.DefaultMaxTrackedRoutes, "Maximum routes tracked individually before excess routes coarsen into virtual buckets") + keyvizMaxMemberRoutesPerSlot = flag.Int("keyvizMaxMemberRoutesPerSlot", keyviz.DefaultMaxMemberRoutesPerSlot, "Maximum members listed on a virtual bucket") + keyvizHistoryColumns = flag.Int("keyvizHistoryColumns", keyviz.DefaultHistoryColumns, "Maximum matrix columns retained in the keyviz ring buffer") + keyvizKeyBucketsPerRoute = flag.Int("keyvizKeyBucketsPerRoute", keyviz.DefaultKeyBucketsPerRoute, "Order-preserving sub-range buckets per individual route") + keyvizHotKeysEnabled = flag.Bool("keyvizHotKeysEnabled", false, "Enable per-route Top-K hot-key sampling") + keyvizHotKeysPerRoute = flag.Int("keyvizHotKeysPerRoute", keyviz.DefaultHotKeysPerRoute, "Space-Saving sketch capacity per route") + keyvizHotKeysSampleRate = flag.Int("keyvizHotKeysSampleRate", keyviz.DefaultHotKeysSampleRate, "Hot-key observation sample rate") + keyvizHotKeysQueueSize = flag.Int("keyvizHotKeysQueueSize", keyviz.DefaultHotKeysQueueSize, "Hot-key aggregator queue capacity") + keyvizHotKeysMaxKeyLen = flag.Int("keyvizHotKeysMaxKeyLen", keyviz.DefaultHotKeysMaxKeyLen, "Maximum key length retained by hot-key sampling") + + autoSplit = flag.Bool("autoSplit", false, "Enable automatic same-group hotspot range splits on the catalog leader") + autoSplitEvalInterval = flag.Duration("autoSplitEvalInterval", autosplit.DefaultEvalInterval, "Interval between automatic hotspot split evaluations") + autoSplitSplitCooldown = flag.Duration("autoSplitCooldown", autosplit.DefaultSplitCooldown, "Minimum cooldown before a route created by SplitRange can be auto-split again") + autoSplitSplitTimeout = flag.Duration("autoSplitSplitTimeout", autosplit.DefaultSplitTimeout, "Timeout for each automatic SplitRange RPC") + autoSplitKillSwitchFile = flag.String("autoSplitKillSwitchFile", "", "If non-empty and the file exists, the auto-split scheduler observes but skips new splits") + autoSplitDefaultBuckets = flag.Int("autoSplitDefaultBuckets", autosplit.DefaultSamplerBuckets, "KeyViz buckets per route implied by --autoSplit when --keyvizKeyBucketsPerRoute was not explicitly set") + autoSplitWriteWeight = flag.Float64("autoSplitWriteWeight", autosplit.DefaultConfig().WriteWeight, "Weighted ops/min multiplier for writes in automatic hotspot detection") + autoSplitReadWeight = flag.Float64("autoSplitReadWeight", autosplit.DefaultConfig().ReadWeight, "Weighted ops/min multiplier for reads in automatic hotspot detection") + autoSplitThresholdOpsMin = flag.Float64("autoSplitThreshold", autosplit.DefaultConfig().ThresholdOpsMin, "Weighted ops/min threshold per committed KeyViz column for automatic hotspot detection") + autoSplitCandidateWindows = flag.Int("autoSplitCandidateWindows", autosplit.DefaultConfig().CandidateWindows, "Consecutive committed KeyViz columns over threshold required before an automatic split") + autoSplitMaxRoutes = flag.Int("autoSplitMaxRoutes", autosplit.DefaultConfig().MaxRoutes, "Maximum live routes allowed after automatic splits") + autoSplitMaxSplitsPerCycle = flag.Int("autoSplitMaxPerCycle", autosplit.DefaultConfig().MaxSplitsPerCycle, "Maximum automatic split decisions scheduled per evaluation cycle") + autoSplitTopKeyShare = flag.Float64("autoSplitTopKeyShare", autosplit.DefaultConfig().TopKeyShare, "Minimum lower-bound share of route writes required for hot-key isolation") + autoSplitTopKeyAbsoluteFloor = flag.Float64("autoSplitTopKeyAbsoluteFloor", 0, "Minimum lower-bound hot-key weighted ops/min; zero uses half --autoSplitThreshold") + + keyvizKeyBucketsPerRouteExplicit bool ) const ( @@ -92,6 +124,7 @@ type config struct { func main() { flag.Parse() + recordExplicitDemoFlags(flag.CommandLine) eg, runCtx := errgroup.WithContext(context.Background()) @@ -233,6 +266,344 @@ func effectiveDemoMetricsToken(token string) string { return "demo-metrics-token" } +func recordExplicitDemoFlags(fs *flag.FlagSet) { + if fs == nil { + return + } + fs.Visit(func(f *flag.Flag) { + if f.Name == "keyvizKeyBucketsPerRoute" { + keyvizKeyBucketsPerRouteExplicit = true + } + }) +} + +func buildDemoKeyVizSampler() *keyviz.MemSampler { + if !*keyvizEnabled && !*autoSplit { + return nil + } + keyBucketsPerRoute := *keyvizKeyBucketsPerRoute + if *autoSplit && !keyvizKeyBucketsPerRouteExplicit && keyBucketsPerRoute <= keyviz.DefaultKeyBucketsPerRoute { + keyBucketsPerRoute = *autoSplitDefaultBuckets + } + return keyviz.NewMemSampler(keyviz.MemSamplerOptions{ + Step: *keyvizStep, + HistoryColumns: *keyvizHistoryColumns, + MaxTrackedRoutes: *keyvizMaxTrackedRoutes, + MaxMemberRoutesPerSlot: *keyvizMaxMemberRoutesPerSlot, + KeyBucketsPerRoute: keyBucketsPerRoute, + HotKeysEnabled: *keyvizHotKeysEnabled, + HotKeysPerRoute: *keyvizHotKeysPerRoute, + HotKeysSampleRate: *keyvizHotKeysSampleRate, + HotKeysQueueSize: *keyvizHotKeysQueueSize, + HotKeysMaxKeyLen: *keyvizHotKeysMaxKeyLen, + }) +} + +func seedDemoKeyVizRoutes(s *keyviz.MemSampler, engine *distribution.Engine) { + if s == nil || engine == nil { + return + } + for _, route := range engine.Stats() { + s.RegisterRoute(route.RouteID, route.Start, route.End, route.GroupID) + } +} + +func startDemoKeyVizFlusher(ctx context.Context, eg *errgroup.Group, s *keyviz.MemSampler) { + if eg == nil || s == nil { + return + } + eg.Go(func() error { + keyviz.RunFlusher(ctx, s, s.Step()) + s.Flush() + return nil + }) + eg.Go(func() error { + keyviz.RunHotKeysAggregator(ctx, s) + return nil + }) +} + +func demoAutoSplitConfigFromFlags(coordinator *kv.Coordinate) (autosplit.SchedulerConfig, error) { + cfg := autosplit.SchedulerConfig{ + Enabled: *autoSplit, + EvalInterval: *autoSplitEvalInterval, + SplitCooldown: *autoSplitSplitCooldown, + SplitTimeout: *autoSplitSplitTimeout, + KillSwitchFile: strings.TrimSpace(*autoSplitKillSwitchFile), + Logger: slog.Default(), + Detector: autosplit.Config{ + WriteWeight: *autoSplitWriteWeight, + ReadWeight: *autoSplitReadWeight, + ThresholdOpsMin: *autoSplitThresholdOpsMin, + CandidateWindows: *autoSplitCandidateWindows, + MaxRoutes: *autoSplitMaxRoutes, + MaxSplitsPerCycle: *autoSplitMaxSplitsPerCycle, + TopKeyShare: *autoSplitTopKeyShare, + TopKeyAbsoluteFloor: *autoSplitTopKeyAbsoluteFloor, + }, + } + if coordinator != nil { + cfg.IsLeader = demoAutoSplitLeaderGate(coordinator) + cfg.Leadership = func() (bool, uint64) { + return coordinator.LeadershipForKey(distribution.CatalogVersionKey()) + } + cfg.GroupLeadership = coordinator.GroupLeadership + } + if err := validateDemoAutoSplitConfig(cfg); err != nil { + return autosplit.SchedulerConfig{}, err + } + return cfg, nil +} + +func validateDemoAutoSplitConfig(cfg autosplit.SchedulerConfig) error { + if !cfg.Enabled { + return nil + } + if err := validateDemoAutoSplitSamplerConfig(cfg.Detector); err != nil { + return err + } + if err := validateDemoAutoSplitDetectorConfig(cfg.Detector); err != nil { + return err + } + return validateDemoAutoSplitSchedulerConfig(cfg) +} + +func validateDemoAutoSplitSamplerConfig(cfg autosplit.Config) error { + if demoAutoSplitUsesDefaultBuckets() { + if *autoSplitDefaultBuckets <= keyviz.DefaultKeyBucketsPerRoute { + return errors.New("--autoSplitDefaultBuckets must be greater than 1") + } + if *autoSplitDefaultBuckets > keyviz.MaxKeyBucketsPerRoute { + return errors.Errorf("--autoSplitDefaultBuckets (%d) must be <= %d", *autoSplitDefaultBuckets, keyviz.MaxKeyBucketsPerRoute) + } + } + if cfg.MaxRoutes > *keyvizMaxTrackedRoutes { + return errors.Errorf("--autoSplitMaxRoutes (%d) must be <= --keyvizMaxTrackedRoutes (%d); raise keyviz capacity or lower the auto-split route cap", cfg.MaxRoutes, *keyvizMaxTrackedRoutes) + } + return nil +} + +func demoAutoSplitUsesDefaultBuckets() bool { + return *autoSplit && + !keyvizKeyBucketsPerRouteExplicit && + *keyvizKeyBucketsPerRoute <= keyviz.DefaultKeyBucketsPerRoute +} + +func validateDemoAutoSplitDetectorConfig(cfg autosplit.Config) error { + if err := validateDemoAutoSplitWeights(cfg.WriteWeight, cfg.ReadWeight); err != nil { + return err + } + if err := validateDemoAutoSplitDetectorLimits(cfg); err != nil { + return err + } + return validateDemoAutoSplitHotKeyThresholds(cfg) +} + +func validateDemoAutoSplitDetectorLimits(cfg autosplit.Config) error { + if !isFiniteDemoAutoSplitFloat(cfg.ThresholdOpsMin) || cfg.ThresholdOpsMin <= 0 { + return errors.New("--autoSplitThreshold must be positive") + } + if cfg.CandidateWindows <= 0 { + return errors.New("--autoSplitCandidateWindows must be positive") + } + if cfg.MaxRoutes <= 0 { + return errors.New("--autoSplitMaxRoutes must be positive") + } + if cfg.MaxSplitsPerCycle <= 0 { + return errors.New("--autoSplitMaxPerCycle must be positive") + } + return nil +} + +func validateDemoAutoSplitHotKeyThresholds(cfg autosplit.Config) error { + if !isFiniteDemoAutoSplitFloat(cfg.TopKeyShare) || cfg.TopKeyShare <= 0 || cfg.TopKeyShare > 1 { + return errors.New("--autoSplitTopKeyShare must be in (0, 1]") + } + if !isFiniteDemoAutoSplitFloat(cfg.TopKeyAbsoluteFloor) || cfg.TopKeyAbsoluteFloor < 0 { + return errors.New("--autoSplitTopKeyAbsoluteFloor must be non-negative") + } + return nil +} + +func validateDemoAutoSplitWeights(writeWeight, readWeight float64) error { + if !isFiniteDemoAutoSplitFloat(writeWeight) || !isFiniteDemoAutoSplitFloat(readWeight) || + writeWeight < 0 || readWeight < 0 || (writeWeight == 0 && readWeight == 0) { + return errors.New("--autoSplitWriteWeight and --autoSplitReadWeight must be non-negative and at least one must be positive") + } + return nil +} + +func isFiniteDemoAutoSplitFloat(v float64) bool { + return !math.IsNaN(v) && !math.IsInf(v, 0) +} + +func validateDemoAutoSplitSchedulerConfig(cfg autosplit.SchedulerConfig) error { + if cfg.EvalInterval <= 0 { + return errors.New("--autoSplitEvalInterval must be positive") + } + if cfg.SplitCooldown <= 0 { + return errors.New("--autoSplitCooldown must be positive") + } + if cfg.SplitTimeout <= 0 { + return errors.New("--autoSplitSplitTimeout must be positive") + } + return nil +} + +func startDemoAutoSplitScheduler( + ctx context.Context, + eg *errgroup.Group, + catalog *distribution.CatalogStore, + distServer *adapter.DistributionServer, + coordinator *kv.Coordinate, + sampler *keyviz.MemSampler, + cfg autosplit.SchedulerConfig, +) { + if eg == nil || !cfg.Enabled { + return + } + if sampler == nil { + cfg.Logger.Warn("autosplit: sampler not configured; scheduler disabled") + return + } + if coordinator != nil { + cfg.IsLeader = demoAutoSplitLeaderGate(coordinator) + cfg.Leadership = func() (bool, uint64) { + return coordinator.LeadershipForKey(distribution.CatalogVersionKey()) + } + cfg.GroupLeadership = coordinator.GroupLeadership + } + scheduler := autosplit.NewScheduler( + cfg, + catalog, + demoAutoSplitDistributionSplitter{server: distServer}, + sampler, + sampler, + ) + eg.Go(func() error { + return scheduler.Run(ctx) + }) +} + +type demoAutoSplitDistributionSplitter struct { + server *adapter.DistributionServer +} + +func demoAutoSplitLeaderGate(coordinator *kv.Coordinate) func() bool { + if coordinator == nil { + return nil + } + return func() bool { + return coordinator.IsLeaderForKey(distribution.CatalogVersionKey()) + } +} + +func (s demoAutoSplitDistributionSplitter) SplitRange(ctx context.Context, req autosplit.SplitRequest) (autosplit.SplitResult, error) { + if s.server == nil { + return autosplit.SplitResult{}, errors.New("autosplit: distribution server is not configured") + } + if req.TargetGroupID != 0 { + return autosplit.SplitResult{}, errors.New("autosplit: cross-group target selection is not enabled") + } + resp, err := s.server.SplitRange(ctx, &pb.SplitRangeRequest{ + ExpectedCatalogVersion: req.ExpectedCatalogVersion, + RouteId: req.RouteID, + SplitKey: distribution.CloneBytes(req.SplitKey), + }) + if err != nil { + return autosplit.SplitResult{}, errors.Wrap(err, "autosplit: split range") + } + return autosplit.SplitResult{ + CatalogVersion: resp.GetCatalogVersion(), + Left: demoAutoSplitRouteFromProto(resp.GetLeft()), + Right: demoAutoSplitRouteFromProto(resp.GetRight()), + }, nil +} + +func demoAutoSplitRouteFromProto(route *pb.RouteDescriptor) distribution.RouteDescriptor { + if route == nil { + return distribution.RouteDescriptor{} + } + state := distribution.RouteStateWriteFenced + switch route.GetState() { + case pb.RouteState_ROUTE_STATE_UNSPECIFIED, + pb.RouteState_ROUTE_STATE_WRITE_FENCED: + state = distribution.RouteStateWriteFenced + case pb.RouteState_ROUTE_STATE_ACTIVE: + state = distribution.RouteStateActive + case pb.RouteState_ROUTE_STATE_MIGRATING_SOURCE: + state = distribution.RouteStateMigratingSource + case pb.RouteState_ROUTE_STATE_MIGRATING_TARGET: + state = distribution.RouteStateMigratingTarget + } + return distribution.RouteDescriptor{ + RouteID: route.GetRouteId(), + Start: distribution.CloneBytes(route.GetStart()), + End: distribution.CloneBytes(route.GetEnd()), + GroupID: route.GetRaftGroupId(), + State: state, + ParentRouteID: route.GetParentRouteId(), + SplitAtHLC: route.GetSplitAtHlc(), + } +} + +type demoDistributionRuntime struct { + engine *distribution.Engine + catalog *distribution.CatalogStore + server *adapter.DistributionServer + autoSplitCfg autosplit.SchedulerConfig + reconciler *autosplit.RouteReconciler + sampler *keyviz.MemSampler +} + +func setupDemoDistributionRuntime( + ctx context.Context, + st store.MVCCStore, + coordinator *kv.Coordinate, + readTracker *kv.ActiveTimestampTracker, + metricsRegistry *monitoring.Registry, +) (demoDistributionRuntime, error) { + runtime := demoDistributionRuntime{ + engine: distribution.NewEngineWithDefaultRoute(), + catalog: distribution.NewCatalogStore(st), + } + if _, err := distribution.EnsureCatalogSnapshot(ctx, runtime.catalog, runtime.engine); err != nil { + return demoDistributionRuntime{}, errors.WithStack(err) + } + runtime.server = adapter.NewDistributionServer( + runtime.engine, + runtime.catalog, + adapter.WithDistributionCoordinator(coordinator), + adapter.WithDistributionActiveTimestampTracker(readTracker), + ) + var err error + runtime.autoSplitCfg, err = demoAutoSplitConfigFromFlags(coordinator) + if err != nil { + return demoDistributionRuntime{}, err + } + runtime.autoSplitCfg.Observer = autosplit.NewPrometheusObserver(metricsRegistry.Registerer()) + runtime.sampler = buildDemoKeyVizSampler() + seedDemoKeyVizRoutes(runtime.sampler, runtime.engine) + if runtime.sampler != nil { + runtime.reconciler = autosplit.NewRouteReconciler(runtime.sampler) + initialSnapshot, snapshotErr := runtime.catalog.Snapshot(ctx) + if snapshotErr != nil { + return demoDistributionRuntime{}, errors.Wrap(snapshotErr, "load initial catalog for keyviz reconciliation") + } + runtime.reconciler.Reconcile(initialSnapshot.Routes) + runtime.autoSplitCfg.Reconciler = runtime.reconciler + } + coordinator.WithSamplerRouteResolver(runtime.sampler, demoSamplerRouteResolver(runtime.engine)) + return runtime, nil +} + +func demoSamplerRouteResolver(engine *distribution.Engine) func([]byte) (uint64, bool) { + return func(key []byte) (uint64, bool) { + route, ok := engine.GetRoute(kv.RouteKey(key)) + return route.RouteID, ok && route.State == distribution.RouteStateActive + } +} + // setupFSMStore creates and returns the MVCCStore for the Raft FSM. // When raftDataDir is non-empty the store is persisted under that directory; // otherwise a temporary directory is used and registered for cleanup on exit. @@ -472,17 +843,16 @@ func run(ctx context.Context, eg *errgroup.Group, cfg config) error { // the closure here matches the symmetric construction order. _ = coordinator.Close() }() - distEngine := distribution.NewEngineWithDefaultRoute() - distCatalog := distribution.NewCatalogStore(st) - if _, err := distribution.EnsureCatalogSnapshot(ctx, distCatalog, distEngine); err != nil { - return errors.WithStack(err) + distributionRuntime, err := setupDemoDistributionRuntime(ctx, st, coordinator, readTracker, metricsRegistry) + if err != nil { + return err } - distServer := adapter.NewDistributionServer( - distEngine, - distCatalog, - adapter.WithDistributionCoordinator(coordinator), - adapter.WithDistributionActiveTimestampTracker(readTracker), - ) + distEngine := distributionRuntime.engine + distCatalog := distributionRuntime.catalog + distServer := distributionRuntime.server + autoSplitCfg := distributionRuntime.autoSplitCfg + autoSplitReconciler := distributionRuntime.reconciler + sampler := distributionRuntime.sampler metricsRegistry.RaftObserver().Start(ctx, []monitoring.RaftRuntime{{ GroupID: 1, StatusReader: engine, @@ -531,7 +901,11 @@ func run(ctx context.Context, eg *errgroup.Group, cfg config) error { } eg.Go(func() error { coordinator.RunHLCLeaseRenewal(ctx); return nil }) - eg.Go(catalogWatcherTask(ctx, distCatalog, distEngine)) + eg.Go(catalogWatcherTask(ctx, distCatalog, distEngine, func(snapshot distribution.CatalogSnapshot) { + autoSplitReconciler.Reconcile(snapshot.Routes) + })) + startDemoKeyVizFlusher(ctx, eg, sampler) + startDemoAutoSplitScheduler(ctx, eg, distCatalog, distServer, coordinator, sampler, autoSplitCfg) eg.Go(func() error { return compactor.Run(ctx) }) eg.Go(func() error { return deltaCompactor.Run(ctx) }) eg.Go(grpcShutdownTask(ctx, s, grpcSock, cfg.address, grpcSvc)) @@ -607,9 +981,24 @@ func setupPprofHTTPServer(ctx context.Context, lc net.ListenConfig, pprofAddress return pprofL, ps, nil } -func catalogWatcherTask(ctx context.Context, distCatalog *distribution.CatalogStore, distEngine *distribution.Engine) func() error { +func catalogWatcherTask( + ctx context.Context, + distCatalog *distribution.CatalogStore, + distEngine *distribution.Engine, + observers ...distribution.CatalogSnapshotObserver, +) func() error { return func() error { - if err := distribution.RunCatalogWatcher(ctx, distCatalog, distEngine, slog.Default()); err != nil { + var opts []distribution.CatalogWatcherOption + if len(observers) > 0 { + opts = append(opts, distribution.WithCatalogWatcherSnapshotObserver(func(snapshot distribution.CatalogSnapshot) { + for _, observer := range observers { + if observer != nil { + observer(snapshot) + } + } + })) + } + if err := distribution.RunCatalogWatcher(ctx, distCatalog, distEngine, slog.Default(), opts...); err != nil { return errors.Wrapf(err, "catalog watcher failed") } return nil diff --git a/cmd/server/demo_test.go b/cmd/server/demo_test.go index bb31b34da..e6ad3bcfa 100644 --- a/cmd/server/demo_test.go +++ b/cmd/server/demo_test.go @@ -2,16 +2,82 @@ package main import ( "context" + "math" "net" "net/http" "os" "path/filepath" "testing" + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/distribution/autosplit" + "github.com/bootjp/elastickv/keyviz" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" ) +func TestValidateDemoAutoSplitDetectorConfigRejectsNonFiniteValues(t *testing.T) { + t.Parallel() + tests := []struct { + name string + mutate func(*autosplit.Config) + }{ + {name: "threshold nan", mutate: func(cfg *autosplit.Config) { cfg.ThresholdOpsMin = math.NaN() }}, + {name: "write weight infinity", mutate: func(cfg *autosplit.Config) { cfg.WriteWeight = math.Inf(1) }}, + {name: "read weight nan", mutate: func(cfg *autosplit.Config) { cfg.ReadWeight = math.NaN() }}, + {name: "top key share nan", mutate: func(cfg *autosplit.Config) { cfg.TopKeyShare = math.NaN() }}, + {name: "top key floor infinity", mutate: func(cfg *autosplit.Config) { cfg.TopKeyAbsoluteFloor = math.Inf(1) }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + cfg := autosplit.DefaultConfig() + test.mutate(&cfg) + require.Error(t, validateDemoAutoSplitDetectorConfig(cfg)) + }) + } +} + +func TestValidateDemoAutoSplitSamplerConfigAllowsExplicitBucketsWithUnusedInvalidDefault(t *testing.T) { + oldAutoSplit := *autoSplit + oldExplicit := keyvizKeyBucketsPerRouteExplicit + oldKeyBuckets := *keyvizKeyBucketsPerRoute + oldDefaultBuckets := *autoSplitDefaultBuckets + t.Cleanup(func() { + *autoSplit = oldAutoSplit + keyvizKeyBucketsPerRouteExplicit = oldExplicit + *keyvizKeyBucketsPerRoute = oldKeyBuckets + *autoSplitDefaultBuckets = oldDefaultBuckets + }) + + *autoSplit = true + *keyvizKeyBucketsPerRoute = 8 + *autoSplitDefaultBuckets = keyviz.DefaultKeyBucketsPerRoute + keyvizKeyBucketsPerRouteExplicit = true + require.NoError(t, validateDemoAutoSplitSamplerConfig(autosplit.DefaultConfig())) + + keyvizKeyBucketsPerRouteExplicit = false + *keyvizKeyBucketsPerRoute = keyviz.DefaultKeyBucketsPerRoute + require.ErrorContains(t, validateDemoAutoSplitSamplerConfig(autosplit.DefaultConfig()), "--autoSplitDefaultBuckets") +} + +func TestDemoSamplerRouteResolverNormalizesInternalKeys(t *testing.T) { + t.Parallel() + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 1, Start: []byte("a"), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 2, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateActive}, + }, + })) + + routeID, ok := demoSamplerRouteResolver(engine)([]byte("!redis|meta|z")) + + require.True(t, ok) + require.Equal(t, uint64(2), routeID) +} + func TestEffectiveDemoMetricsToken(t *testing.T) { require.Equal(t, "custom-token", effectiveDemoMetricsToken(" custom-token ")) require.Equal(t, "demo-metrics-token", effectiveDemoMetricsToken("")) diff --git a/distribution/autosplit/detector.go b/distribution/autosplit/detector.go index 32af77991..5d341bbd1 100644 --- a/distribution/autosplit/detector.go +++ b/distribution/autosplit/detector.go @@ -2,6 +2,7 @@ package autosplit import ( "bytes" + "math" "sort" "time" @@ -10,56 +11,78 @@ import ( ) const ( - defaultWriteWeight = 4 - defaultReadWeight = 1 - defaultThresholdOpsMin = 50_000 - defaultCandidateWindows = 3 - defaultMaxRoutes = 1024 - defaultMaxSplitsPerCycle = 1 - opsPerMinute = 60 + defaultWriteWeight = 4 + defaultReadWeight = 1 + defaultThresholdOpsMin = 50_000 + defaultCandidateWindows = 3 + defaultMaxRoutes = 1024 + defaultMaxSplitsPerCycle = 1 + defaultTopKeyShare = 0.8 + defaultTopKeyFloorDivisor = 2 + opsPerMinute = 60 ) // SplitOrigin describes how an automatic split key was selected. type SplitOrigin string const ( - SplitOriginP50Mid SplitOrigin = "p50_mid" - SplitOriginP50LastBucketLo SplitOrigin = "p50_last_bucket_lo" - SplitOriginP50FirstBucketHi SplitOrigin = "p50_first_bucket_hi" + SplitOriginP50Mid SplitOrigin = "p50_mid" + SplitOriginP50LastBucketLo SplitOrigin = "p50_last_bucket_lo" + SplitOriginP50FirstBucketHi SplitOrigin = "p50_first_bucket_hi" + SplitOriginIsolationCompound SplitOrigin = "isolation_compound" + SplitOriginIsolationSingleLowerEdge SplitOrigin = "isolation_single_lower_edge" + SplitOriginIsolationSingleUpperEdge SplitOrigin = "isolation_single_upper_edge" +) + +// IsolationDeclineReason explains why aligned Top-K evidence fell through to +// the sub-range p50 selector. +type IsolationDeclineReason string + +const ( + IsolationDeclineAbsoluteFloor IsolationDeclineReason = "absolute_floor" + IsolationDeclineTopKDegraded IsolationDeclineReason = "topk_degraded" + IsolationDeclineTopKInsufficient IsolationDeclineReason = "topk_insufficient" + IsolationDeclineTopKErrorBound IsolationDeclineReason = "topk_error_bound" ) // SkipReason explains why a route did not produce a split decision. type SkipReason string const ( - SkipReasonNoSplitKey SkipReason = "no_split_key" - SkipReasonRouteCap SkipReason = "route_cap" - SkipReasonBudgetExhausted SkipReason = "budget_exhausted" - SkipReasonNonActiveState SkipReason = "non_active_state" - SkipReasonAggregateRow SkipReason = "aggregate_row" - SkipReasonCooldown SkipReason = "cooldown" - SkipReasonInvalidWindow SkipReason = "invalid_window" + SkipReasonNoSplitKey SkipReason = "no_split_key" + SkipReasonRouteCap SkipReason = "route_cap" + SkipReasonBudgetExhausted SkipReason = "budget_exhausted" + SkipReasonNonActiveState SkipReason = "non_active_state" + SkipReasonAggregateRow SkipReason = "aggregate_row" + SkipReasonCooldown SkipReason = "cooldown" + SkipReasonInvalidWindow SkipReason = "invalid_window" + SkipReasonLeadershipFence SkipReason = "leadership_fence" + SkipReasonUnsplittableHotKey SkipReason = "unsplittable_hot_key" ) // Config controls the pure detector and scheduler admission checks. type Config struct { - WriteWeight float64 - ReadWeight float64 - ThresholdOpsMin float64 - CandidateWindows int - MaxRoutes int - MaxSplitsPerCycle int + WriteWeight float64 + ReadWeight float64 + ThresholdOpsMin float64 + CandidateWindows int + MaxRoutes int + MaxSplitsPerCycle int + TopKeyShare float64 + TopKeyAbsoluteFloor float64 } // DefaultConfig returns the M3 detector defaults from the design doc. func DefaultConfig() Config { return Config{ - WriteWeight: defaultWriteWeight, - ReadWeight: defaultReadWeight, - ThresholdOpsMin: defaultThresholdOpsMin, - CandidateWindows: defaultCandidateWindows, - MaxRoutes: defaultMaxRoutes, - MaxSplitsPerCycle: defaultMaxSplitsPerCycle, + WriteWeight: defaultWriteWeight, + ReadWeight: defaultReadWeight, + ThresholdOpsMin: defaultThresholdOpsMin, + CandidateWindows: defaultCandidateWindows, + MaxRoutes: defaultMaxRoutes, + MaxSplitsPerCycle: defaultMaxSplitsPerCycle, + TopKeyShare: defaultTopKeyShare, + TopKeyAbsoluteFloor: defaultThresholdOpsMin / defaultTopKeyFloorDivisor, } } @@ -82,7 +105,8 @@ type RouteLoad struct { // Runtime integration passes only committed windows with a proven duration. // keyviz.MatrixColumn.WindowStart is authoritative when present; legacy // in-memory rows may be accepted only when the previous contiguous column proves -// the lower boundary. +// the lower boundary. MatrixColumn carries the exact committed (WindowStart, +// At] boundary used to align route load and Top-K evidence. type ColumnWindow struct { Column keyviz.MatrixColumn Duration time.Duration @@ -90,36 +114,52 @@ type ColumnWindow struct { // Input is one pure detector evaluation. type Input struct { - Routes []distribution.RouteDescriptor - Windows []ColumnWindow - Now time.Time + Routes []distribution.RouteDescriptor + Windows []ColumnWindow + EvidenceFences map[uint64]EvidenceFence + Now time.Time + LiveRouteCount int +} + +// EvidenceFence excludes sampler history that was not collected wholly while +// this node held the relevant default-group and shard-group leadership. +type EvidenceFence struct { + ProcessedThrough time.Time + WindowStartNotBefore time.Time } // Decision is a scheduler-ready automatic split decision. type Decision struct { - RouteID uint64 - SplitKey []byte - SplitOrigin SplitOrigin - TargetGroupID uint64 - RouteDelta int - - ScoreOpsMin float64 - ConsecutiveOver int - LeftLoad float64 - RightLoad float64 + RouteID uint64 + SplitKey []byte + SecondSplitKey []byte + SplitOrigin SplitOrigin + TargetGroupID uint64 + RouteDelta int + RouteStart []byte + RouteEnd []byte + RouteGroupID uint64 + + ScoreOpsMin float64 + PerColumnScoreOpsMin float64 + ConsecutiveOver int + LeftLoad float64 + RightLoad float64 } // Event records a deterministic skip or reset reason from an evaluation. type Event struct { - RouteID uint64 - Reason SkipReason - At time.Time + RouteID uint64 + Reason SkipReason + IsolationReason IsolationDeclineReason + At time.Time } // Result is the complete output of one detector evaluation. type Result struct { Decisions []Decision Events []Event + Promoted int } // RouteStatus is the observable detector state for one route. @@ -131,12 +171,16 @@ type RouteStatus struct { // DetectorState carries leader-local confidence and cooldown state. type DetectorState struct { - routes map[uint64]RouteStatus + routes map[uint64]RouteStatus + scoreHistory map[uint64][]scoreSample } // NewDetectorState creates empty detector state. func NewDetectorState() *DetectorState { - return &DetectorState{routes: map[uint64]RouteStatus{}} + return &DetectorState{ + routes: map[uint64]RouteStatus{}, + scoreHistory: map[uint64][]scoreSample{}, + } } // RouteStatus returns the current detector state for routeID. @@ -156,6 +200,7 @@ func (s *DetectorState) ResetConfidence(routeID uint64) { status := s.routes[routeID] status.ConsecutiveOver = 0 s.routes[routeID] = status + delete(s.scoreHistory, routeID) } // ApplyRouteState applies a catalog state transition to the detector state. @@ -178,6 +223,7 @@ func (s *DetectorState) SetCooldown(routeID uint64, until time.Time) { status.CooldownUntil = until status.ConsecutiveOver = 0 s.routes[routeID] = status + delete(s.scoreHistory, routeID) } // AggregateColumnRows groups all non-aggregate rows by (RouteID, RaftGroupID). @@ -216,9 +262,13 @@ func Evaluate(cfg Config, state *DetectorState, in Input) Result { state.gc(live) for _, window := range windows { - processWindow(cfg, state, active, window, in.Now, latestHot, &result) + processWindow(cfg, state, active, window, in.EvidenceFences, in.Now, latestHot, &result) + } + liveRouteCount := in.LiveRouteCount + if liveRouteCount <= 0 || liveRouteCount < len(live) { + liveRouteCount = len(live) } - selectDecisions(cfg, state, live, latestHot, &result) + selectDecisions(cfg, state, liveRouteCount, latestHot, &result) return result } @@ -255,6 +305,7 @@ func processWindow( state *DetectorState, active []distribution.RouteDescriptor, window ColumnWindow, + fences map[uint64]EvidenceFence, now time.Time, latestHot map[uint64]candidate, result *Result, @@ -274,7 +325,7 @@ func processWindow( } for _, route := range active { - processRouteWindow(cfg, state, route, window, now, aggregated, latestHot, result) + processRouteWindow(cfg, state, route, window, fences[route.RouteID], now, aggregated, latestHot, result) } } @@ -283,6 +334,7 @@ func processRouteWindow( state *DetectorState, route distribution.RouteDescriptor, window ColumnWindow, + fence EvidenceFence, now time.Time, aggregated columnAggregation, latestHot map[uint64]candidate, @@ -292,6 +344,22 @@ func processRouteWindow( if !window.Column.At.After(status.LastProcessedAt) { return } + windowStart := window.Column.WindowStart + if windowStart.IsZero() { + windowStart = window.Column.At.Add(-window.Duration) + } + if !fence.ProcessedThrough.IsZero() && !window.Column.At.After(fence.ProcessedThrough) { + state.resetConfidenceThrough(route.RouteID, window.Column.At) + delete(latestHot, route.RouteID) + result.Events = append(result.Events, Event{RouteID: route.RouteID, Reason: SkipReasonLeadershipFence, At: window.Column.At}) + return + } + if !fence.WindowStartNotBefore.IsZero() && windowStart.Before(fence.WindowStartNotBefore) { + state.resetConfidenceThrough(route.RouteID, window.Column.At) + delete(latestHot, route.RouteID) + result.Events = append(result.Events, Event{RouteID: route.RouteID, Reason: SkipReasonLeadershipFence, At: window.Column.At}) + return + } if now.Before(status.CooldownUntil) || windowOverlapsCooldown(window, status.CooldownUntil) { status.ConsecutiveOver = 0 status.LastProcessedAt = window.Column.At @@ -304,6 +372,8 @@ func processRouteWindow( key := RouteKey{RouteID: route.RouteID, RaftGroupID: route.GroupID} load := aggregated.loads[key] score := scoreOpsPerMinute(load, window.Duration, cfg) + state.recordScore(route.RouteID, load, window.Duration, cfg.CandidateWindows) + smoothedScore := state.smoothedScore(route.RouteID, cfg) if score < cfg.ThresholdOpsMin { status.ConsecutiveOver = 0 status.LastProcessedAt = window.Column.At @@ -316,17 +386,21 @@ func processRouteWindow( status.LastProcessedAt = window.Column.At state.routes[route.RouteID] = status latestHot[route.RouteID] = candidate{ - route: route, - rows: aggregated.rows[key], - scoreOpsMin: score, - consecutiveOver: status.ConsecutiveOver, + route: route, + rows: aggregated.rows[key], + load: load, + duration: window.Duration, + hotKeys: alignedHotKeys(window.Column, route.RouteID), + perColumnScoreOpsMin: score, + smoothedScoreOpsMin: smoothedScore, + consecutiveOver: status.ConsecutiveOver, } } func selectDecisions( cfg Config, state *DetectorState, - live map[uint64]distribution.RouteDescriptor, + liveRouteCount int, latestHot map[uint64]candidate, result *Result, ) { @@ -337,25 +411,21 @@ func selectDecisions( } } sort.Slice(ordered, func(i, j int) bool { - if ordered[i].scoreOpsMin == ordered[j].scoreOpsMin { + if ordered[i].perColumnScoreOpsMin == ordered[j].perColumnScoreOpsMin { return ordered[i].route.RouteID < ordered[j].route.RouteID } - return ordered[i].scoreOpsMin > ordered[j].scoreOpsMin + return ordered[i].perColumnScoreOpsMin > ordered[j].perColumnScoreOpsMin }) reservedDelta := 0 + result.Promoted = len(ordered) for _, candidate := range ordered { if len(result.Decisions) >= cfg.MaxSplitsPerCycle { result.Events = append(result.Events, Event{RouteID: candidate.route.RouteID, Reason: SkipReasonBudgetExhausted}) continue } - decision, ok := selectP50Decision(cfg, candidate) + decision, ok := admitCandidate(cfg, liveRouteCount, reservedDelta, candidate, result) if !ok { - result.Events = append(result.Events, Event{RouteID: candidate.route.RouteID, Reason: SkipReasonNoSplitKey}) - continue - } - if cfg.MaxRoutes > 0 && len(live)+reservedDelta+decision.RouteDelta > cfg.MaxRoutes { - result.Events = append(result.Events, Event{RouteID: candidate.route.RouteID, Reason: SkipReasonRouteCap}) continue } reservedDelta += decision.RouteDelta @@ -363,6 +433,35 @@ func selectDecisions( } } +func admitCandidate( + cfg Config, + liveRoutes int, + reservedDelta int, + candidate candidate, + result *Result, +) (Decision, bool) { + decision, isolationReason, terminalReason, ok := selectDecision(cfg, candidate) + if isolationReason != "" { + result.Events = append(result.Events, Event{ + RouteID: candidate.route.RouteID, + IsolationReason: isolationReason, + }) + } + if terminalReason != "" { + result.Events = append(result.Events, Event{RouteID: candidate.route.RouteID, Reason: terminalReason}) + return Decision{}, false + } + if !ok { + result.Events = append(result.Events, Event{RouteID: candidate.route.RouteID, Reason: SkipReasonNoSplitKey}) + return Decision{}, false + } + if cfg.MaxRoutes > 0 && liveRoutes+reservedDelta+decision.RouteDelta > cfg.MaxRoutes { + result.Events = append(result.Events, Event{RouteID: candidate.route.RouteID, Reason: SkipReasonRouteCap}) + return Decision{}, false + } + return decision, true +} + func windowOverlapsCooldown(window ColumnWindow, cooldownUntil time.Time) bool { if cooldownUntil.IsZero() { return false @@ -371,10 +470,19 @@ func windowOverlapsCooldown(window ColumnWindow, cooldownUntil time.Time) bool { } type candidate struct { - route distribution.RouteDescriptor - rows []keyviz.MatrixRow - scoreOpsMin float64 - consecutiveOver int + route distribution.RouteDescriptor + rows []keyviz.MatrixRow + load RouteLoad + duration time.Duration + hotKeys []keyviz.KeyvizHotKeysSnapshot + perColumnScoreOpsMin float64 + smoothedScoreOpsMin float64 + consecutiveOver int +} + +type scoreSample struct { + load RouteLoad + duration time.Duration } type columnAggregation struct { @@ -387,6 +495,9 @@ func (s *DetectorState) ensure() { if s.routes == nil { s.routes = map[uint64]RouteStatus{} } + if s.scoreHistory == nil { + s.scoreHistory = map[uint64][]scoreSample{} + } } func (s *DetectorState) resetConfidenceThrough(routeID uint64, through time.Time) { @@ -400,6 +511,7 @@ func (s *DetectorState) resetConfidenceThrough(routeID uint64, through time.Time status.LastProcessedAt = through } s.routes[routeID] = status + delete(s.scoreHistory, routeID) } func (s *DetectorState) resetConfidenceAt(routeID uint64, at time.Time) bool { @@ -414,6 +526,7 @@ func (s *DetectorState) resetConfidenceAt(routeID uint64, at time.Time) bool { status.ConsecutiveOver = 0 status.LastProcessedAt = at s.routes[routeID] = status + delete(s.scoreHistory, routeID) return true } @@ -421,10 +534,36 @@ func (s *DetectorState) gc(live map[uint64]distribution.RouteDescriptor) { for routeID := range s.routes { if _, ok := live[routeID]; !ok { delete(s.routes, routeID) + delete(s.scoreHistory, routeID) } } } +func (s *DetectorState) recordScore(routeID uint64, load RouteLoad, duration time.Duration, limit int) { + if duration <= 0 || limit <= 0 { + return + } + history := s.scoreHistory[routeID] + history = append(history, scoreSample{load: load, duration: duration}) + if len(history) > limit { + history = history[len(history)-limit:] + } + s.scoreHistory[routeID] = history +} + +func (s *DetectorState) smoothedScore(routeID uint64, cfg Config) float64 { + var load RouteLoad + var duration time.Duration + for _, sample := range s.scoreHistory[routeID] { + load.Reads += sample.load.Reads + load.Writes += sample.load.Writes + load.ReadBytes += sample.load.ReadBytes + load.WriteBytes += sample.load.WriteBytes + duration += sample.duration + } + return scoreOpsPerMinute(load, duration, cfg) +} + func (cfg Config) withDefaults() Config { defaults := DefaultConfig() if cfg.WriteWeight == 0 && cfg.ReadWeight == 0 { @@ -443,6 +582,12 @@ func (cfg Config) withDefaults() Config { if cfg.MaxSplitsPerCycle <= 0 { cfg.MaxSplitsPerCycle = defaults.MaxSplitsPerCycle } + if cfg.TopKeyShare <= 0 || cfg.TopKeyShare > 1 { + cfg.TopKeyShare = defaults.TopKeyShare + } + if cfg.TopKeyAbsoluteFloor <= 0 { + cfg.TopKeyAbsoluteFloor = cfg.ThresholdOpsMin / defaultTopKeyFloorDivisor + } return cfg } @@ -478,22 +623,193 @@ func scoreOpsPerMinute(load RouteLoad, duration time.Duration, cfg Config) float return writeRate*cfg.WriteWeight + readRate*cfg.ReadWeight } +func alignedHotKeys(col keyviz.MatrixColumn, routeID uint64) []keyviz.KeyvizHotKeysSnapshot { + out := make([]keyviz.KeyvizHotKeysSnapshot, 0, len(col.HotKeys)) + for _, snapshot := range col.HotKeys { + if snapshot.RouteID != routeID || + !snapshot.WindowStart.Equal(col.WindowStart) || + !snapshot.WindowEnd.Equal(col.At) { + continue + } + out = append(out, snapshot) + } + return out +} + +func selectDecision(cfg Config, candidate candidate) (Decision, IsolationDeclineReason, SkipReason, bool) { + decision, reason, terminalReason, considered, ok := selectTopKeyDecision(cfg, candidate) + if ok { + return decision, "", "", true + } + if terminalReason != "" { + return Decision{}, reason, terminalReason, false + } + p50, p50OK := selectP50Decision(cfg, candidate) + if considered { + return p50, reason, "", p50OK + } + return p50, "", "", p50OK +} + +type hotKeyEstimate struct { + key []byte + lower float64 + upper float64 +} + +func selectTopKeyDecision(cfg Config, candidate candidate) (Decision, IsolationDeclineReason, SkipReason, bool, bool) { + if len(candidate.hotKeys) == 0 || candidate.load.Writes == 0 || candidate.duration <= 0 { + return Decision{}, "", "", false, false + } + + estimates, reason := buildHotKeyEstimates(candidate.hotKeys) + if reason != "" { + return Decision{}, reason, "", true, false + } + if len(estimates) == 0 { + return Decision{}, IsolationDeclineTopKInsufficient, "", true, false + } + + seconds := candidate.duration.Seconds() + writes := float64(candidate.load.Writes) + for _, estimate := range sortedHotKeyEstimates(estimates) { + decision, isolationReason, terminalReason, matched, ok := evaluateHotKeyEstimate( + cfg, candidate, estimate, writes, seconds, + ) + if matched { + return decision, isolationReason, terminalReason, true, ok + } + } + return Decision{}, "", "", true, false +} + +func buildHotKeyEstimates( + snapshots []keyviz.KeyvizHotKeysSnapshot, +) (map[string]hotKeyEstimate, IsolationDeclineReason) { + estimates := make(map[string]hotKeyEstimate) + for _, snapshot := range snapshots { + if snapshot.DroppedSamples > 0 || snapshot.SkippedLongKeys > 0 { + return nil, IsolationDeclineTopKDegraded + } + if snapshot.Capacity <= 0 || snapshot.SampledN == 0 || snapshot.SampleRate <= 0 { + return nil, IsolationDeclineTopKInsufficient + } + errorBound := uint64(math.Ceil(float64(snapshot.SampledN) / float64(snapshot.Capacity))) + for _, entry := range snapshot.Entries { + lowerCount := uint64(0) + if entry.Count > errorBound { + lowerCount = entry.Count - errorBound + } + key := string(entry.Key) + estimate := estimates[key] + if estimate.key == nil { + estimate.key = distribution.CloneBytes(entry.Key) + } + estimate.lower += float64(lowerCount) * float64(snapshot.SampleRate) + estimate.upper += float64(entry.Count) * float64(snapshot.SampleRate) + estimates[key] = estimate + } + } + return estimates, "" +} + +func sortedHotKeyEstimates(estimates map[string]hotKeyEstimate) []hotKeyEstimate { + ordered := make([]hotKeyEstimate, 0, len(estimates)) + for _, estimate := range estimates { + ordered = append(ordered, estimate) + } + sort.Slice(ordered, func(i, j int) bool { + if ordered[i].lower == ordered[j].lower { + return bytes.Compare(ordered[i].key, ordered[j].key) < 0 + } + return ordered[i].lower > ordered[j].lower + }) + return ordered +} + +func evaluateHotKeyEstimate( + cfg Config, + candidate candidate, + estimate hotKeyEstimate, + writes float64, + seconds float64, +) (Decision, IsolationDeclineReason, SkipReason, bool, bool) { + lowerShare := math.Min(estimate.lower, writes) / writes + upperShare := math.Min(estimate.upper, writes) / writes + lowerContribution := estimate.lower / seconds * opsPerMinute * cfg.WriteWeight + upperContribution := estimate.upper / seconds * opsPerMinute * cfg.WriteWeight + if lowerShare < cfg.TopKeyShare { + if upperShare >= cfg.TopKeyShare && upperContribution >= cfg.TopKeyAbsoluteFloor { + return Decision{}, IsolationDeclineTopKErrorBound, "", true, false + } + return Decision{}, "", "", false, false + } + if lowerContribution < cfg.TopKeyAbsoluteFloor { + return Decision{}, IsolationDeclineAbsoluteFloor, "", true, false + } + decision, ok := isolationDecision(candidate, estimate.key) + if !ok { + return Decision{}, "", SkipReasonUnsplittableHotKey, true, false + } + return decision, "", "", true, true +} + +func isolationDecision(candidate candidate, hotKey []byte) (Decision, bool) { + route := candidate.route + successor := append(distribution.CloneBytes(hotKey), 0) + decision := baseDecision(candidate) + + switch { + case bytes.Equal(hotKey, route.Start): + if route.End != nil && bytes.Compare(successor, route.End) >= 0 { + return Decision{}, false + } + decision.SplitKey = successor + decision.SplitOrigin = SplitOriginIsolationSingleLowerEdge + decision.RouteDelta = 1 + case !splitKeyInsideRoute(route, hotKey): + return Decision{}, false + case route.End != nil && bytes.Compare(successor, route.End) >= 0: + decision.SplitKey = distribution.CloneBytes(hotKey) + decision.SplitOrigin = SplitOriginIsolationSingleUpperEdge + decision.RouteDelta = 1 + default: + decision.SplitKey = distribution.CloneBytes(hotKey) + decision.SecondSplitKey = successor + decision.SplitOrigin = SplitOriginIsolationCompound + decision.RouteDelta = 2 + } + if !splitKeyInsideRoute(route, decision.SplitKey) { + return Decision{}, false + } + return decision, true +} + +func baseDecision(candidate candidate) Decision { + return Decision{ + RouteID: candidate.route.RouteID, + RouteStart: distribution.CloneBytes(candidate.route.Start), + RouteEnd: distribution.CloneBytes(candidate.route.End), + RouteGroupID: candidate.route.GroupID, + TargetGroupID: 0, + ScoreOpsMin: candidate.smoothedScoreOpsMin, + PerColumnScoreOpsMin: candidate.perColumnScoreOpsMin, + ConsecutiveOver: candidate.consecutiveOver, + } +} + func selectP50Decision(cfg Config, candidate candidate) (Decision, bool) { key, origin, leftLoad, rightLoad, ok := selectP50SplitKey(cfg, candidate.route, candidate.rows) if !ok { return Decision{}, false } - return Decision{ - RouteID: candidate.route.RouteID, - SplitKey: key, - SplitOrigin: origin, - TargetGroupID: 0, - RouteDelta: 1, - ScoreOpsMin: candidate.scoreOpsMin, - ConsecutiveOver: candidate.consecutiveOver, - LeftLoad: leftLoad, - RightLoad: rightLoad, - }, true + decision := baseDecision(candidate) + decision.SplitKey = key + decision.SplitOrigin = origin + decision.RouteDelta = 1 + decision.LeftLoad = leftLoad + decision.RightLoad = rightLoad + return decision, true } func selectP50SplitKey(cfg Config, route distribution.RouteDescriptor, rows []keyviz.MatrixRow) ([]byte, SplitOrigin, float64, float64, bool) { diff --git a/distribution/autosplit/detector_rapid_test.go b/distribution/autosplit/detector_rapid_test.go new file mode 100644 index 000000000..d014b4e93 --- /dev/null +++ b/distribution/autosplit/detector_rapid_test.go @@ -0,0 +1,88 @@ +package autosplit + +import ( + "bytes" + "testing" + "time" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/keyviz" + "github.com/stretchr/testify/require" + "pgregory.net/rapid" +) + +func TestRapidConsecutivePromotionAndSplitBounds(t *testing.T) { + rapid.Check(t, func(rt *rapid.T) { + candidateWindows := rapid.IntRange(1, 5).Draw(rt, "candidate_windows") + hot := rapid.SliceOfN(rapid.Bool(), 1, 24).Draw(rt, "hot_columns") + base := time.Unix(2_000, 0) + windows := make([]ColumnWindow, 0, len(hot)) + finalRun := 0 + for i, over := range hot { + writes := uint64(10) + if over { + writes = 100 + finalRun++ + } else { + finalRun = 0 + } + at := base.Add(time.Duration(i+1) * time.Minute) + windows = append(windows, testWindow(at, time.Minute, + testRow(1, 1, 0, 2, "a", "m", writes/2, 0), + testRow(1, 1, 1, 2, "m", "z", writes-writes/2, 0), + )) + } + cfg := testConfig() + cfg.CandidateWindows = candidateWindows + state := NewDetectorState() + result := Evaluate(cfg, state, Input{ + Routes: []distribution.RouteDescriptor{testRoute(1, 1, "a", "z")}, + Windows: windows, + Now: windows[len(windows)-1].Column.At, + }) + + require.Equal(rt, finalRun, state.RouteStatus(1).ConsecutiveOver) + require.Equal(rt, finalRun >= candidateWindows, len(result.Decisions) == 1) + for _, decision := range result.Decisions { + require.Greater(rt, bytes.Compare(decision.SplitKey, []byte("a")), 0) + require.Less(rt, bytes.Compare(decision.SplitKey, []byte("z")), 0) + } + }) +} + +func TestRapidCooldownAndCycleBudget(t *testing.T) { + rapid.Check(t, func(rt *rapid.T) { + routeCount := rapid.IntRange(1, 12).Draw(rt, "route_count") + budget := rapid.IntRange(1, 5).Draw(rt, "budget") + cooling := rapid.Bool().Draw(rt, "cooling") + at := time.Unix(3_000, 0) + routes := make([]distribution.RouteDescriptor, 0, routeCount) + rows := make([]keyviz.MatrixRow, 0, routeCount*2) + state := NewDetectorState() + routeID := uint64(0) + for range routeCount { + routeID++ + routes = append(routes, testRoute(routeID, 1, "a", "z")) + rows = append(rows, + testRow(routeID, 1, 0, 2, "a", "m", 60, 0), + testRow(routeID, 1, 1, 2, "m", "z", 60, 0), + ) + if cooling { + state.SetCooldown(routeID, at.Add(time.Minute)) + } + } + cfg := testConfig() + cfg.MaxRoutes = routeCount + budget + 10 + cfg.MaxSplitsPerCycle = budget + result := Evaluate(cfg, state, Input{ + Routes: routes, + Windows: []ColumnWindow{testWindow(at, time.Minute, rows...)}, + Now: at, + }) + + require.LessOrEqual(rt, len(result.Decisions), budget) + if cooling { + require.Empty(rt, result.Decisions) + } + }) +} diff --git a/distribution/autosplit/detector_test.go b/distribution/autosplit/detector_test.go index 493358c68..2ec1a935c 100644 --- a/distribution/autosplit/detector_test.go +++ b/distribution/autosplit/detector_test.go @@ -61,6 +61,34 @@ func TestReadWeightContributesToScore(t *testing.T) { require.InDelta(t, 40, result.Decisions[0].ScoreOpsMin, 0.0001) } +func TestDecisionReportsSmoothedScoreWithoutChangingPerColumnPromotion(t *testing.T) { + t.Parallel() + start := time.Unix(160, 0) + route := testRoute(2, 3, "a", "z") + windows := []ColumnWindow{ + testWindow(start.Add(time.Minute), time.Minute, + testRow(2, 3, 0, 2, "a", "m", 50, 0), + testRow(2, 3, 1, 2, "m", "z", 50, 0), + ), + testWindow(start.Add(3*time.Minute), 2*time.Minute, + testRow(2, 3, 0, 2, "a", "m", 200, 0), + testRow(2, 3, 1, 2, "m", "z", 200, 0), + ), + } + + result := Evaluate(Config{ + WriteWeight: 1, + ReadWeight: 0, + ThresholdOpsMin: 50, + CandidateWindows: 2, + MaxSplitsPerCycle: 1, + }, NewDetectorState(), Input{Routes: []distribution.RouteDescriptor{route}, Windows: windows, Now: start.Add(3 * time.Minute)}) + + require.Len(t, result.Decisions, 1) + require.InDelta(t, 200, result.Decisions[0].PerColumnScoreOpsMin, 0.0001) + require.InDelta(t, 500.0/3.0, result.Decisions[0].ScoreOpsMin, 0.0001) +} + func TestInvalidWindowIsSkipped(t *testing.T) { t.Parallel() at := time.Unix(175, 0) @@ -467,6 +495,24 @@ func TestRouteCapUsesCycleLocalReservation(t *testing.T) { requireEvent(t, result.Events, 2, SkipReasonRouteCap) } +func TestRouteCapUsesFullLiveRouteCountWithLocalSubset(t *testing.T) { + t.Parallel() + at := time.Unix(525, 0) + route := testRoute(1, 1, "a", "z") + cfg := testConfig() + cfg.MaxRoutes = 2 + + result := Evaluate(cfg, NewDetectorState(), Input{ + Routes: []distribution.RouteDescriptor{route}, + Windows: []ColumnWindow{hotWindow(at)}, + Now: at, + LiveRouteCount: 2, + }) + + require.Empty(t, result.Decisions) + requireEvent(t, result.Events, 1, SkipReasonRouteCap) +} + func TestDefaultConfigSetsRouteCap(t *testing.T) { t.Parallel() @@ -625,6 +671,214 @@ func TestP50CoalescesDuplicateSubBucketRows(t *testing.T) { require.InDelta(t, 100, result.Decisions[0].RightLoad, 0.0001) } +func TestTopKeyIsolationUsesAlignedLowerBoundBeforeP50(t *testing.T) { + t.Parallel() + at := time.Unix(900, 0) + route := testRoute(1, 1, "a", "z") + + for _, sample := range []struct { + rate int + writes uint64 + }{{rate: 1, writes: 100}, {rate: 16, writes: 1600}} { + t.Run("sample_rate_"+time.Duration(sample.rate).String(), func(t *testing.T) { + t.Parallel() + window := topKeyWindow(at, time.Minute, route, sample.writes, keyviz.KeyvizHotKeysSnapshot{ + SampledN: 100, + SampleRate: sample.rate, + Capacity: 100, + Entries: []keyviz.KeyvizHotKeyEntry{ + {Key: []byte("m"), Count: 91}, + }, + }) + cfg := testConfig() + cfg.TopKeyAbsoluteFloor = 50 * float64(sample.rate) + + result := Evaluate(cfg, NewDetectorState(), Input{ + Routes: []distribution.RouteDescriptor{route}, Windows: []ColumnWindow{window}, Now: at, + }) + + require.Len(t, result.Decisions, 1) + require.Equal(t, SplitOriginIsolationCompound, result.Decisions[0].SplitOrigin) + require.Equal(t, []byte("m"), result.Decisions[0].SplitKey) + require.Equal(t, []byte{'m', 0}, result.Decisions[0].SecondSplitKey) + require.Equal(t, 2, result.Decisions[0].RouteDelta) + require.Empty(t, result.Events) + }) + } +} + +func TestTopKeyIsolationDeclinesConservativelyAndFallsBackToP50(t *testing.T) { + t.Parallel() + at := time.Unix(1000, 0) + route := testRoute(1, 1, "a", "z") + + tests := []struct { + name string + snapshot keyviz.KeyvizHotKeysSnapshot + floor float64 + wantReason IsolationDeclineReason + }{ + { + name: "absolute floor", + snapshot: keyviz.KeyvizHotKeysSnapshot{SampledN: 100, SampleRate: 1, Capacity: 100, + Entries: []keyviz.KeyvizHotKeyEntry{{Key: []byte("m"), Count: 91}}}, + floor: 100, + wantReason: IsolationDeclineAbsoluteFloor, + }, + { + name: "space saving uncertainty", + snapshot: keyviz.KeyvizHotKeysSnapshot{SampledN: 100, SampleRate: 1, Capacity: 10, + Entries: []keyviz.KeyvizHotKeyEntry{{Key: []byte("m"), Count: 85}}}, + floor: 50, + wantReason: IsolationDeclineTopKErrorBound, + }, + { + name: "degraded queue", + snapshot: keyviz.KeyvizHotKeysSnapshot{SampledN: 100, SampleRate: 1, Capacity: 100, DroppedSamples: 1, + Entries: []keyviz.KeyvizHotKeyEntry{{Key: []byte("m"), Count: 100}}}, + floor: 50, + wantReason: IsolationDeclineTopKDegraded, + }, + { + name: "empty sketch", + snapshot: keyviz.KeyvizHotKeysSnapshot{SampleRate: 1, Capacity: 100}, + floor: 50, + wantReason: IsolationDeclineTopKInsufficient, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + window := topKeyWindow(at, time.Minute, route, 100, tc.snapshot) + cfg := testConfig() + cfg.TopKeyAbsoluteFloor = tc.floor + + result := Evaluate(cfg, NewDetectorState(), Input{ + Routes: []distribution.RouteDescriptor{route}, Windows: []ColumnWindow{window}, Now: at, + }) + + require.Len(t, result.Decisions, 1) + require.Equal(t, SplitOriginP50FirstBucketHi, result.Decisions[0].SplitOrigin) + requireIsolationEvent(t, result.Events, 1, tc.wantReason) + }) + } +} + +func TestTopKeySnapshotRequiresExactCommittedWindow(t *testing.T) { + t.Parallel() + at := time.Unix(1100, 0) + route := testRoute(1, 1, "a", "z") + window := topKeyWindow(at, time.Minute, route, 100, keyviz.KeyvizHotKeysSnapshot{ + SampledN: 100, SampleRate: 1, Capacity: 100, + Entries: []keyviz.KeyvizHotKeyEntry{{Key: []byte("m"), Count: 91}}, + }) + window.Column.HotKeys[0].WindowStart = window.Column.WindowStart.Add(time.Second) + + result := Evaluate(testConfig(), NewDetectorState(), Input{ + Routes: []distribution.RouteDescriptor{route}, Windows: []ColumnWindow{window}, Now: at, + }) + + require.Len(t, result.Decisions, 1) + require.Equal(t, SplitOriginP50FirstBucketHi, result.Decisions[0].SplitOrigin) +} + +func TestTopKeyIsolationBoundaryForms(t *testing.T) { + t.Parallel() + at := time.Unix(1200, 0) + tests := []struct { + name string + route distribution.RouteDescriptor + hotKey []byte + wantOrigin SplitOrigin + wantKey []byte + wantSecond []byte + wantDelta int + wantNone bool + }{ + {name: "compound", route: testRoute(1, 1, "a", "z"), hotKey: []byte("m"), + wantOrigin: SplitOriginIsolationCompound, wantKey: []byte("m"), wantSecond: []byte{'m', 0}, wantDelta: 2}, + {name: "lower edge", route: testRoute(1, 1, "m", "z"), hotKey: []byte("m"), + wantOrigin: SplitOriginIsolationSingleLowerEdge, wantKey: []byte{'m', 0}, wantDelta: 1}, + {name: "lower edge unbounded", route: testRouteNilEnd(1, 1, "m"), hotKey: []byte("m"), + wantOrigin: SplitOriginIsolationSingleLowerEdge, wantKey: []byte{'m', 0}, wantDelta: 1}, + {name: "upper edge", route: testRoute(1, 1, "a", "m\x00"), hotKey: []byte("m"), + wantOrigin: SplitOriginIsolationSingleUpperEdge, wantKey: []byte("m"), wantDelta: 1}, + {name: "already isolated", route: testRoute(1, 1, "m", "m\x00"), hotKey: []byte("m"), wantNone: true}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + window := topKeyOnlyWindow(at, time.Minute, tc.route, 100, tc.hotKey) + result := Evaluate(testConfig(), NewDetectorState(), Input{ + Routes: []distribution.RouteDescriptor{tc.route}, Windows: []ColumnWindow{window}, Now: at, + }) + if tc.wantNone { + require.Empty(t, result.Decisions) + return + } + require.Len(t, result.Decisions, 1) + require.Equal(t, tc.wantOrigin, result.Decisions[0].SplitOrigin) + require.Equal(t, tc.wantKey, result.Decisions[0].SplitKey) + require.Equal(t, tc.wantSecond, result.Decisions[0].SecondSplitKey) + require.Equal(t, tc.wantDelta, result.Decisions[0].RouteDelta) + }) + } +} + +func TestCompoundIsolationHonorsActualRouteDelta(t *testing.T) { + t.Parallel() + at := time.Unix(1300, 0) + routes := []distribution.RouteDescriptor{ + testRoute(1, 1, "a", "m"), + testRoute(2, 1, "m", "z"), + } + window := topKeyOnlyWindow(at, time.Minute, routes[0], 100, []byte("g")) + cfg := testConfig() + cfg.MaxRoutes = 3 + + result := Evaluate(cfg, NewDetectorState(), Input{Routes: routes, Windows: []ColumnWindow{window}, Now: at}) + + require.Empty(t, result.Decisions) + requireEvent(t, result.Events, 1, SkipReasonRouteCap) +} + +func TestEvidenceFenceRejectsPreLeadershipAndStraddlingWindows(t *testing.T) { + t.Parallel() + base := time.Unix(1400, 0) + route := testRoute(1, 1, "a", "z") + cfg := testConfig() + cfg.CandidateWindows = 2 + fence := EvidenceFence{ + ProcessedThrough: base, + WindowStartNotBefore: base.Add(30 * time.Second), + } + windows := []ColumnWindow{ + hotWindowWithStart(base, base.Add(-time.Minute)), + hotWindowWithStart(base.Add(time.Minute), base), + hotWindowWithStart(base.Add(2*time.Minute), base.Add(time.Minute)), + } + state := NewDetectorState() + + first := Evaluate(cfg, state, Input{ + Routes: []distribution.RouteDescriptor{route}, Windows: windows, + EvidenceFences: map[uint64]EvidenceFence{1: fence}, Now: base.Add(2 * time.Minute), + }) + + require.Empty(t, first.Decisions) + require.Equal(t, 1, state.RouteStatus(1).ConsecutiveOver) + requireEvent(t, first.Events, 1, SkipReasonLeadershipFence) + + final := Evaluate(cfg, state, Input{ + Routes: []distribution.RouteDescriptor{route}, + Windows: []ColumnWindow{hotWindowWithStart(base.Add(3*time.Minute), base.Add(2*time.Minute))}, + EvidenceFences: map[uint64]EvidenceFence{1: fence}, Now: base.Add(3 * time.Minute), + }) + + require.Len(t, final.Decisions, 1) +} + func testConfig() Config { return Config{ WriteWeight: 1, @@ -664,6 +918,45 @@ func testWindow(at time.Time, duration time.Duration, rows ...keyviz.MatrixRow) } } +func topKeyWindow( + at time.Time, + duration time.Duration, + route distribution.RouteDescriptor, + writes uint64, + snapshot keyviz.KeyvizHotKeysSnapshot, +) ColumnWindow { + window := testWindow(at, duration, + testRow(route.RouteID, route.GroupID, 0, 2, string(route.Start), "m", writes/2, 0), + testRow(route.RouteID, route.GroupID, 1, 2, "m", string(route.End), writes-writes/2, 0), + ) + window.Column.WindowStart = at.Add(-duration) + snapshot.RouteID = route.RouteID + snapshot.WindowStart = window.Column.WindowStart + snapshot.WindowEnd = at + window.Column.HotKeys = []keyviz.KeyvizHotKeysSnapshot{snapshot} + return window +} + +func topKeyOnlyWindow( + at time.Time, + duration time.Duration, + route distribution.RouteDescriptor, + writes uint64, + hotKey []byte, +) ColumnWindow { + window := testWindow(at, duration, keyviz.MatrixRow{ + RouteID: route.RouteID, RaftGroupID: route.GroupID, Start: route.Start, End: route.End, + SubBucketCount: 1, Writes: writes, + }) + window.Column.WindowStart = at.Add(-duration) + window.Column.HotKeys = []keyviz.KeyvizHotKeysSnapshot{{ + RouteID: route.RouteID, WindowStart: window.Column.WindowStart, WindowEnd: at, + SampledN: 100, SampleRate: 1, Capacity: 100, + Entries: []keyviz.KeyvizHotKeyEntry{{Key: hotKey, Count: 91}}, + }} + return window +} + func hotWindow(at time.Time) ColumnWindow { return testWindow(at, time.Minute, testRow(1, 1, 0, 2, "a", "m", 60, 0), @@ -671,6 +964,13 @@ func hotWindow(at time.Time) ColumnWindow { ) } +func hotWindowWithStart(at, start time.Time) ColumnWindow { + window := hotWindow(at) + window.Column.WindowStart = start + window.Duration = at.Sub(start) + return window +} + func coldWindow(at time.Time) ColumnWindow { return testWindow(at, time.Minute, testRow(1, 1, 0, 2, "a", "m", 10, 0), @@ -712,3 +1012,13 @@ func requireEvent(t *testing.T, events []Event, routeID uint64, reason SkipReaso } t.Fatalf("missing event route_id=%d reason=%s in %#v", routeID, reason, events) } + +func requireIsolationEvent(t *testing.T, events []Event, routeID uint64, reason IsolationDeclineReason) { + t.Helper() + for _, event := range events { + if event.RouteID == routeID && event.IsolationReason == reason { + return + } + } + t.Fatalf("missing isolation event route_id=%d reason=%s in %#v", routeID, reason, events) +} diff --git a/distribution/autosplit/metrics.go b/distribution/autosplit/metrics.go new file mode 100644 index 000000000..9020613ae --- /dev/null +++ b/distribution/autosplit/metrics.go @@ -0,0 +1,178 @@ +package autosplit + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +// Observer receives bounded-cardinality scheduler outcomes. Implementations +// must not attach route IDs or key bytes as metric labels. +type Observer interface { + ObserveCandidatesPromoted(count int) + ObserveSplitScheduled() + ObserveSplitFailed(reason string) + ObserveSkipped(reason SkipReason) + ObserveIsolationDeclined(reason IsolationDeclineReason) + ObserveCompoundPartial() + ObserveState(enabled bool, trackedRoutes, cooldownActive int, evalDuration time.Duration) +} + +type prometheusObserver struct { + candidatesPromoted prometheus.Counter + splitsScheduled prometheus.Counter + splitsFailed *prometheus.CounterVec + skipped *prometheus.CounterVec + isolationDeclined *prometheus.CounterVec + compoundPartial prometheus.Counter + enabled prometheus.Gauge + trackedRoutes prometheus.Gauge + cooldownActive prometheus.Gauge + evalDuration prometheus.Gauge +} + +// NewPrometheusObserver registers the standalone auto-split metric families. +func NewPrometheusObserver(registerer prometheus.Registerer) Observer { + if registerer == nil { + return nil + } + o := &prometheusObserver{ + candidatesPromoted: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "autosplit_candidates_promoted_total", + Help: "Hot routes promoted after consecutive committed windows.", + }), + splitsScheduled: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "autosplit_splits_scheduled_total", + Help: "Automatic SplitRange RPCs issued.", + }), + splitsFailed: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "autosplit_splits_failed_total", + Help: "Automatic SplitRange RPC failures by bounded reason.", + }, []string{"reason"}), + skipped: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "autosplit_skipped_total", + Help: "Automatic split decisions skipped before an RPC by bounded reason.", + }, []string{"reason"}), + isolationDeclined: prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "autosplit_isolation_declined_total", + Help: "Top-K isolation attempts declined by bounded reason.", + }, []string{"reason"}), + compoundPartial: prometheus.NewCounter(prometheus.CounterOpts{ + Name: "autosplit_compound_partial_total", + Help: "Compound isolations whose first split committed but second split did not.", + }), + enabled: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "autosplit_enabled", + Help: "Whether automatic splitting is enabled on this node.", + }), + trackedRoutes: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "autosplit_tracked_routes", + Help: "Routes in the leader-local detector state map.", + }), + cooldownActive: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "autosplit_cooldown_active", + Help: "Tracked routes currently in split cooldown.", + }), + evalDuration: prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "autosplit_eval_duration_seconds", + Help: "Wall time of the latest automatic split evaluation cycle.", + }), + } + registerer.MustRegister( + o.candidatesPromoted, + o.splitsScheduled, + o.splitsFailed, + o.skipped, + o.isolationDeclined, + o.compoundPartial, + o.enabled, + o.trackedRoutes, + o.cooldownActive, + o.evalDuration, + ) + return o +} + +func (o *prometheusObserver) ObserveCandidatesPromoted(count int) { + if o != nil && count > 0 { + o.candidatesPromoted.Add(float64(count)) + } +} + +func (o *prometheusObserver) ObserveSplitScheduled() { + if o != nil { + o.splitsScheduled.Inc() + } +} + +func (o *prometheusObserver) ObserveSplitFailed(reason string) { + if o != nil { + o.splitsFailed.WithLabelValues(normalizeSplitFailureReason(reason)).Inc() + } +} + +func (o *prometheusObserver) ObserveSkipped(reason SkipReason) { + if o == nil || !metricSkipReason(reason) { + return + } + o.skipped.WithLabelValues(string(reason)).Inc() +} + +func (o *prometheusObserver) ObserveIsolationDeclined(reason IsolationDeclineReason) { + if o == nil || reason == "" { + return + } + o.isolationDeclined.WithLabelValues(string(reason)).Inc() +} + +func (o *prometheusObserver) ObserveCompoundPartial() { + if o != nil { + o.compoundPartial.Inc() + } +} + +func (o *prometheusObserver) ObserveState( + enabled bool, + trackedRoutes int, + cooldownActive int, + evalDuration time.Duration, +) { + if o == nil { + return + } + if enabled { + o.enabled.Set(1) + } else { + o.enabled.Set(0) + } + o.trackedRoutes.Set(float64(trackedRoutes)) + o.cooldownActive.Set(float64(cooldownActive)) + o.evalDuration.Set(evalDuration.Seconds()) +} + +func metricSkipReason(reason SkipReason) bool { + switch reason { + case SkipReasonNoSplitKey, + SkipReasonRouteCap, + SkipReasonBudgetExhausted, + SkipReasonNonActiveState, + SkipReasonAggregateRow, + SkipReasonUnsplittableHotKey: + return true + case SkipReasonCooldown, + SkipReasonInvalidWindow, + SkipReasonLeadershipFence: + return false + default: + return false + } +} + +func normalizeSplitFailureReason(reason string) string { + switch reason { + case "cas_conflict", "target_unavailable": + return reason + default: + return "rpc_error" + } +} diff --git a/distribution/autosplit/metrics_test.go b/distribution/autosplit/metrics_test.go new file mode 100644 index 000000000..391d900e3 --- /dev/null +++ b/distribution/autosplit/metrics_test.go @@ -0,0 +1,42 @@ +package autosplit + +import ( + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" +) + +func TestPrometheusObserverRegistersBoundedMetrics(t *testing.T) { + t.Parallel() + registry := prometheus.NewRegistry() + observer, ok := NewPrometheusObserver(registry).(*prometheusObserver) + require.True(t, ok) + + observer.ObserveCandidatesPromoted(2) + observer.ObserveSplitScheduled() + observer.ObserveSplitFailed("unexpected_detail") + observer.ObserveSkipped(SkipReasonRouteCap) + observer.ObserveSkipped(SkipReasonCooldown) + observer.ObserveIsolationDeclined(IsolationDeclineTopKErrorBound) + observer.ObserveCompoundPartial() + observer.ObserveState(true, 7, 2, 250*time.Millisecond) + + require.Equal(t, float64(2), testutil.ToFloat64(observer.candidatesPromoted)) + require.Equal(t, float64(1), testutil.ToFloat64(observer.splitsScheduled)) + require.Equal(t, float64(1), testutil.ToFloat64(observer.splitsFailed.WithLabelValues("rpc_error"))) + require.Equal(t, float64(1), testutil.ToFloat64(observer.skipped.WithLabelValues(string(SkipReasonRouteCap)))) + require.Equal(t, float64(0), testutil.ToFloat64(observer.skipped.WithLabelValues(string(SkipReasonCooldown)))) + require.Equal(t, float64(1), testutil.ToFloat64(observer.isolationDeclined.WithLabelValues(string(IsolationDeclineTopKErrorBound)))) + require.Equal(t, float64(1), testutil.ToFloat64(observer.compoundPartial)) + require.Equal(t, float64(1), testutil.ToFloat64(observer.enabled)) + require.Equal(t, float64(7), testutil.ToFloat64(observer.trackedRoutes)) + require.Equal(t, float64(2), testutil.ToFloat64(observer.cooldownActive)) + require.InDelta(t, 0.25, testutil.ToFloat64(observer.evalDuration), 0.0001) + + families, err := registry.Gather() + require.NoError(t, err) + require.Len(t, families, 10) +} diff --git a/distribution/autosplit/runtime_switch.go b/distribution/autosplit/runtime_switch.go new file mode 100644 index 000000000..d974bd29f --- /dev/null +++ b/distribution/autosplit/runtime_switch.go @@ -0,0 +1,37 @@ +package autosplit + +import ( + "context" + "sync/atomic" +) + +// RuntimeSwitch is the process-local operator switch for automatic splitting. +// It is independent from the startup Enabled flag: disabling it keeps detector +// observation active while blocking all new SplitRange calls. +type RuntimeSwitch struct { + enabled atomic.Bool +} + +// NewRuntimeSwitch creates a runtime switch with the requested initial state. +func NewRuntimeSwitch(enabled bool) *RuntimeSwitch { + s := &RuntimeSwitch{} + s.enabled.Store(enabled) + return s +} + +// Enabled returns whether new automatic splits are allowed. +func (s *RuntimeSwitch) Enabled() bool { + return s != nil && s.enabled.Load() +} + +// SetEnabled atomically changes whether new automatic splits are allowed. +func (s *RuntimeSwitch) SetEnabled(enabled bool) { + if s != nil { + s.enabled.Store(enabled) + } +} + +// KillSwitch adapts RuntimeSwitch to SchedulerConfig.KillSwitch. +func (s *RuntimeSwitch) KillSwitch(context.Context) bool { + return !s.Enabled() +} diff --git a/distribution/autosplit/runtime_switch_test.go b/distribution/autosplit/runtime_switch_test.go new file mode 100644 index 000000000..82a4e3973 --- /dev/null +++ b/distribution/autosplit/runtime_switch_test.go @@ -0,0 +1,19 @@ +package autosplit + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRuntimeSwitchAdaptsToKillSwitch(t *testing.T) { + t.Parallel() + runtime := NewRuntimeSwitch(true) + require.True(t, runtime.Enabled()) + require.False(t, runtime.KillSwitch(context.Background())) + + runtime.SetEnabled(false) + require.False(t, runtime.Enabled()) + require.True(t, runtime.KillSwitch(context.Background())) +} diff --git a/distribution/autosplit/scheduler.go b/distribution/autosplit/scheduler.go new file mode 100644 index 000000000..288588917 --- /dev/null +++ b/distribution/autosplit/scheduler.go @@ -0,0 +1,1035 @@ +package autosplit + +import ( + "bytes" + "context" + "encoding/hex" + "log/slog" + "math" + "os" + "sort" + "sync" + "time" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/keyviz" + "github.com/bootjp/elastickv/kv" + "github.com/cockroachdb/errors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const ( + DefaultEvalInterval = keyviz.DefaultStep + DefaultSplitCooldown = 10 * time.Minute + DefaultSplitTimeout = 5 * time.Second + DefaultSamplerBuckets = 16 + defaultSnapshotLookbackCols = 6 + snapshotLookbackPaddingCols = 2 + maxLoggedSplitKeyBytes = 64 +) + +// CatalogSnapshotSource supplies the latest route catalog snapshot used for one +// detector/scheduler cycle. +type CatalogSnapshotSource interface { + Snapshot(ctx context.Context) (distribution.CatalogSnapshot, error) +} + +// Splitter executes catalog mutations. Production wiring calls +// proto.Distribution.SplitRange through the local DistributionServer. +type Splitter interface { + SplitRange(ctx context.Context, req SplitRequest) (SplitResult, error) +} + +// SplitRequest is the scheduler's stable request surface. TargetGroupID is +// carried for the post-M2 hook; M3 standalone always sends zero. +type SplitRequest struct { + ExpectedCatalogVersion uint64 + RouteID uint64 + SplitKey []byte + TargetGroupID uint64 + ParentStart []byte + ParentEnd []byte + ParentGroupID uint64 +} + +// SplitResult contains the committed catalog version and children when +// SplitRange succeeds. +type SplitResult struct { + CatalogVersion uint64 + Left distribution.RouteDescriptor + Right distribution.RouteDescriptor +} + +// MatrixSampler is the keyviz surface the scheduler consumes. +type MatrixSampler interface { + Snapshot(from, to time.Time) []keyviz.MatrixColumn + Step() time.Duration +} + +type historyColumnser interface { + HistoryColumns() int +} + +// RouteRegistrar mirrors keyviz.MemSampler's route-membership methods. +type RouteRegistrar interface { + RegisterRoute(routeID uint64, start, end []byte, groupID uint64) bool + RemoveRoute(routeID uint64) +} + +// KillSwitch reports whether the current cycle must observe only and skip new +// SplitRange calls. +type KillSwitch func(ctx context.Context) bool + +// LeadershipSnapshot reports whether this node owns the catalog key and the +// current Raft term for that group. +type LeadershipSnapshot func() (bool, uint64) + +// GroupLeadershipSnapshot reports local leadership and term for one shard +// group. +type GroupLeadershipSnapshot func(groupID uint64) (bool, uint64) + +// SchedulerConfig controls background auto-split execution. +type SchedulerConfig struct { + Enabled bool + Detector Config + EvalInterval time.Duration + SplitCooldown time.Duration + SplitTimeout time.Duration + KillSwitchFile string + KillSwitch KillSwitch + IsLeader func() bool + Leadership LeadershipSnapshot + GroupLeadership GroupLeadershipSnapshot + Logger *slog.Logger + Reconciler *RouteReconciler + Observer Observer +} + +// SchedulerResult describes one scheduler tick. +type SchedulerResult struct { + CatalogVersion uint64 + Detector Result + Scheduled int + Failed int + KillSwitch bool + Leader bool +} + +// Scheduler runs the pure detector and commits accepted decisions through +// SplitRange. +type Scheduler struct { + cfg SchedulerConfig + source CatalogSnapshotSource + splitter Splitter + sampler MatrixSampler + reconciler *RouteReconciler + state *DetectorState + wasLeader bool + leaderTerm uint64 + leaderStartedAt time.Time + leaderWatermark time.Time + catalogVersion uint64 + pendingCompounds map[uint64]pendingCompound + groupLeadership map[uint64]groupLeadershipState +} + +type groupLeadershipState struct { + leader bool + term uint64 + startedAt time.Time + watermark time.Time +} + +type pendingCompound struct { + parentRouteID uint64 + intermediate distribution.RouteDescriptor + splitKey []byte +} + +type registeredRoute struct { + start []byte + end []byte + groupID uint64 +} + +// RouteReconciler keeps a sampler's registered route descriptors synchronized +// with catalog snapshots. It is safe for the catalog watcher and scheduler to +// share across goroutines. +type RouteReconciler struct { + mu sync.Mutex + registrar RouteRegistrar + registered map[uint64]registeredRoute +} + +// NewRouteReconciler creates a catalog-to-sampler membership reconciler. +func NewRouteReconciler(registrar RouteRegistrar) *RouteReconciler { + return &RouteReconciler{ + registrar: registrar, + registered: make(map[uint64]registeredRoute), + } +} + +// NewScheduler builds a leader-local scheduler. It is inert when cfg.Enabled is +// false; callers may still construct it in tests. +func NewScheduler(cfg SchedulerConfig, source CatalogSnapshotSource, splitter Splitter, sampler MatrixSampler, registrar RouteRegistrar) *Scheduler { + cfg = cfg.withDefaults() + reconciler := cfg.Reconciler + if reconciler == nil { + reconciler = NewRouteReconciler(registrar) + } + scheduler := &Scheduler{ + cfg: cfg, + source: source, + splitter: splitter, + sampler: sampler, + reconciler: reconciler, + state: NewDetectorState(), + pendingCompounds: make(map[uint64]pendingCompound), + groupLeadership: make(map[uint64]groupLeadershipState), + } + if cfg.Observer != nil { + cfg.Observer.ObserveState(cfg.Enabled, 0, 0, 0) + } + return scheduler +} + +// Run ticks until ctx is canceled. Per-cycle errors are logged and retried; a +// detector cycle is best-effort and must not tear down the data plane. +func (s *Scheduler) Run(ctx context.Context) error { + if s == nil || !s.cfg.Enabled { + return nil + } + ticker := time.NewTicker(s.cfg.EvalInterval) + defer ticker.Stop() + s.tickAndLog(ctx, time.Now()) + for { + select { + case <-ctx.Done(): + return nil + case now := <-ticker.C: + s.tickAndLog(ctx, now) + } + } +} + +// Tick executes one scheduler cycle. Tests call this directly. +func (s *Scheduler) Tick(ctx context.Context, now time.Time) (SchedulerResult, error) { + cycleStarted := time.Now() + runtimeEnabled := false + defer func() { + if s != nil && s.cfg.Observer != nil { + tracked, cooldown := s.stateCounts(now) + s.cfg.Observer.ObserveState(runtimeEnabled, tracked, cooldown, time.Since(cycleStarted)) + } + }() + out := SchedulerResult{} + if s == nil || !s.cfg.Enabled { + return out, nil + } + if ctx == nil { + ctx = context.Background() + } + runtimeEnabled = !s.killSwitchActive(ctx) + if !s.ensureLeadership(now) { + return out, nil + } + prep, err := s.prepareTick(ctx, now, runtimeEnabled) + if err != nil { + return out, err + } + prep.result.Detector = s.evaluateTick(now, prep) + return s.finishTick(ctx, runtimeEnabled, prep), nil +} + +type tickPreparation struct { + snapshot distribution.CatalogSnapshot + routes []distribution.RouteDescriptor + fences map[uint64]EvidenceFence + usedBudget int + catalogGap bool + stopAfterPrepare bool + result SchedulerResult +} + +func (s *Scheduler) ensureLeadership(now time.Time) bool { + leader, leaderTerm := s.leadership() + if !leader { + s.wasLeader = false + return false + } + if !s.wasLeader || (leaderTerm != 0 && leaderTerm != s.leaderTerm) { + s.resetForLeadership(now) + s.wasLeader = true + } + s.leaderTerm = leaderTerm + return true +} + +func (s *Scheduler) prepareTick( + ctx context.Context, + now time.Time, + runtimeEnabled bool, +) (tickPreparation, error) { + snapshot, err := s.source.Snapshot(ctx) + if err != nil { + return tickPreparation{}, errors.Wrap(err, "autosplit: load catalog snapshot") + } + prep := tickPreparation{ + snapshot: snapshot, + catalogGap: s.catalogVersion != 0 && snapshot.Version != s.catalogVersion, + result: SchedulerResult{ + CatalogVersion: snapshot.Version, + Leader: true, + }, + } + if runtimeEnabled { + var pendingScheduled, pendingFailed int + prep.snapshot, pendingScheduled, pendingFailed, prep.usedBudget, prep.stopAfterPrepare = s.finalizePendingCompounds(ctx, snapshot) + prep.result.CatalogVersion = prep.snapshot.Version + prep.result.Scheduled += pendingScheduled + prep.result.Failed += pendingFailed + if prep.stopAfterPrepare { + s.catalogVersion = prep.snapshot.Version + return prep, nil + } + } + prep.routes, prep.fences = s.syncCatalogSnapshot(prep.snapshot, now) + s.catalogVersion = prep.snapshot.Version + return prep, nil +} + +func (s *Scheduler) syncCatalogSnapshot( + snapshot distribution.CatalogSnapshot, + now time.Time, +) ([]distribution.RouteDescriptor, map[uint64]EvidenceFence) { + s.reconcileSamplerRoutes(snapshot.Routes) + SeedCooldownsFromRoutes(s.state, snapshot.Routes, s.cfg.SplitCooldown, now) + return s.routesLedLocally(snapshot.Routes, now) +} + +func (s *Scheduler) evaluateTick(now time.Time, prep tickPreparation) Result { + if prep.stopAfterPrepare { + return Result{} + } + windows := s.committedWindows(prep.routes, now) + s.resetConfidenceForHistoryGaps(prep.routes, windows) + if prep.catalogGap { + if prep.fences == nil { + prep.fences = make(map[uint64]EvidenceFence, len(prep.routes)) + } + s.fenceUnprovenCatalogGap(prep.routes, prep.fences, newestWindowAt(windows)) + } + result := Evaluate(s.cfg.Detector, s.state, Input{ + Routes: prep.routes, + Windows: windows, + EvidenceFences: prep.fences, + Now: now, + LiveRouteCount: len(prep.snapshot.Routes), + }) + s.observeDetectorResult(result) + return result +} + +func (s *Scheduler) observeDetectorResult(result Result) { + for _, event := range result.Events { + s.logEvent(event) + if s.cfg.Observer != nil { + s.cfg.Observer.ObserveSkipped(event.Reason) + s.cfg.Observer.ObserveIsolationDeclined(event.IsolationReason) + } + } + if s.cfg.Observer != nil { + s.cfg.Observer.ObserveCandidatesPromoted(result.Promoted) + } +} + +func (s *Scheduler) finishTick( + ctx context.Context, + runtimeEnabled bool, + prep tickPreparation, +) SchedulerResult { + if prep.stopAfterPrepare { + return prep.result + } + if !runtimeEnabled { + prep.result.KillSwitch = true + if len(prep.result.Detector.Decisions) > 0 { + s.cfg.Logger.InfoContext(ctx, "autosplit: kill switch active; skipping splits", + slog.Int("decisions", len(prep.result.Detector.Decisions))) + } + return prep.result + } + + remainingBudget := s.cfg.Detector.MaxSplitsPerCycle - prep.usedBudget + if remainingBudget <= 0 { + return prep.result + } + decisions := prep.result.Detector.Decisions + if len(decisions) > remainingBudget { + decisions = decisions[:remainingBudget] + } + scheduled, failed, catalogVersion := s.executeDecisions(ctx, prep.snapshot.Version, decisions) + prep.result.Scheduled += scheduled + prep.result.Failed += failed + prep.result.CatalogVersion = catalogVersion + s.catalogVersion = catalogVersion + return prep.result +} + +func (s *Scheduler) resetConfidenceForHistoryGaps( + routes []distribution.RouteDescriptor, + windows []ColumnWindow, +) { + for _, route := range routes { + processedThrough := s.state.RouteStatus(route.RouteID).LastProcessedAt + if processedThrough.IsZero() { + continue + } + for _, window := range windows { + if !window.Column.At.After(processedThrough) { + continue + } + windowStart := window.Column.WindowStart + if windowStart.IsZero() { + windowStart = window.Column.At.Add(-window.Duration) + } + if windowStart.After(processedThrough) { + s.state.ResetConfidence(route.RouteID) + } + break + } + } +} + +func (s *Scheduler) fenceUnprovenCatalogGap( + routes []distribution.RouteDescriptor, + fences map[uint64]EvidenceFence, + processedThrough time.Time, +) { + for _, route := range routes { + s.state.ResetConfidence(route.RouteID) + fence := fences[route.RouteID] + if processedThrough.After(fence.ProcessedThrough) { + fence.ProcessedThrough = processedThrough + } + fences[route.RouteID] = fence + } +} + +func (s *Scheduler) tickAndLog(ctx context.Context, now time.Time) { + if _, err := s.Tick(ctx, now); err != nil && !errors.Is(err, context.Canceled) { + s.cfg.Logger.WarnContext(ctx, "autosplit: cycle failed", slog.Any("err", err)) + } +} + +func (s *Scheduler) executeSplit( + ctx context.Context, + catalogVersion uint64, + routeID uint64, + splitKey []byte, + targetGroupID uint64, + parentStart []byte, + parentEnd []byte, + parentGroupID uint64, + splitOrigin SplitOrigin, + score float64, + consecutiveOver int, +) (SplitResult, error) { + execCtx, cancel := context.WithTimeout(ctx, s.cfg.SplitTimeout) + defer cancel() + + s.cfg.Logger.InfoContext(ctx, "autosplit: scheduling split", + slog.Uint64("route_id", routeID), + slog.Uint64("catalog_version", catalogVersion), + slog.String("split_key", loggedSplitKey(splitKey)), + slog.String("split_origin", string(splitOrigin)), + slog.Float64("score", score), + slog.Int("consecutive_over", consecutiveOver), + slog.Uint64("target_group_id", targetGroupID)) + + result, err := s.splitter.SplitRange(execCtx, SplitRequest{ + ExpectedCatalogVersion: catalogVersion, + RouteID: routeID, + SplitKey: distribution.CloneBytes(splitKey), + TargetGroupID: targetGroupID, + ParentStart: distribution.CloneBytes(parentStart), + ParentEnd: distribution.CloneBytes(parentEnd), + ParentGroupID: parentGroupID, + }) + if s.cfg.Observer != nil { + s.cfg.Observer.ObserveSplitScheduled() + } + if err != nil { + if s.cfg.Observer != nil { + s.cfg.Observer.ObserveSplitFailed(splitFailureReason(err, targetGroupID)) + } + return SplitResult{}, errors.Wrap(err, "autosplit: split range") + } + s.cfg.Logger.InfoContext(ctx, "autosplit: split committed", + slog.Uint64("route_id", routeID), + slog.Uint64("catalog_version", result.CatalogVersion), + slog.String("split_key", loggedSplitKey(splitKey)), + slog.String("split_origin", string(splitOrigin))) + return result, nil +} + +func (s *Scheduler) executeDecisions(ctx context.Context, catalogVersion uint64, decisions []Decision) (int, int, uint64) { + scheduled := 0 + failed := 0 + for _, decision := range decisions { + nextCatalogVersion, err := s.executeDecision(ctx, catalogVersion, decision) + if err != nil { + failed++ + s.cfg.Logger.WarnContext(ctx, "autosplit: split failed", + slog.Uint64("route_id", decision.RouteID), + slog.Uint64("catalog_version", catalogVersion), + slog.String("split_key", loggedSplitKey(decision.SplitKey)), + slog.String("split_origin", string(decision.SplitOrigin)), + slog.Uint64("target_group_id", decision.TargetGroupID), + slog.Any("err", err)) + if nextCatalogVersion > catalogVersion { + catalogVersion = nextCatalogVersion + } + continue + } + catalogVersion = nextCatalogVersion + scheduled++ + } + return scheduled, failed, catalogVersion +} + +func (s *Scheduler) executeDecision(ctx context.Context, catalogVersion uint64, decision Decision) (uint64, error) { + first, err := s.executeSplit( + ctx, + catalogVersion, + decision.RouteID, + decision.SplitKey, + decision.TargetGroupID, + decision.RouteStart, + decision.RouteEnd, + decision.RouteGroupID, + decision.SplitOrigin, + decision.ScoreOpsMin, + decision.ConsecutiveOver, + ) + if err != nil { + return catalogVersion, err + } + if decision.SplitOrigin != SplitOriginIsolationCompound { + return first.CatalogVersion, nil + } + if !validCompoundIntermediate(first, decision) { + return first.CatalogVersion, errors.New("autosplit: split response did not contain the expected compound intermediate child") + } + + pending := pendingCompound{ + parentRouteID: decision.RouteID, + intermediate: distribution.CloneRouteDescriptor(first.Right), + splitKey: distribution.CloneBytes(decision.SecondSplitKey), + } + s.pendingCompounds[decision.RouteID] = pending + second, err := s.executePendingCompound(ctx, first.CatalogVersion, pending) + if err != nil { + if s.cfg.Observer != nil { + s.cfg.Observer.ObserveCompoundPartial() + } + return first.CatalogVersion, err + } + delete(s.pendingCompounds, decision.RouteID) + return second.CatalogVersion, nil +} + +func validCompoundIntermediate(result SplitResult, decision Decision) bool { + left, right := result.Left, result.Right + if !validCompoundChildTopology(left, right, decision) { + return false + } + parent := distribution.RouteDescriptor{Start: left.Start, End: right.End} + return splitKeyInsideRoute(parent, right.Start) && + splitKeyInsideRoute(right, decision.SecondSplitKey) +} + +func validCompoundChildTopology(left, right distribution.RouteDescriptor, decision Decision) bool { + return left.RouteID != 0 && right.RouteID != 0 && left.RouteID != right.RouteID && + left.State == distribution.RouteStateActive && right.State == distribution.RouteStateActive && + left.GroupID == decision.RouteGroupID && right.GroupID == decision.RouteGroupID && + bytes.Equal(left.Start, decision.RouteStart) && + bytes.Equal(left.End, right.Start) && + bytes.Equal(right.End, decision.RouteEnd) +} + +func (s *Scheduler) executePendingCompound( + ctx context.Context, + catalogVersion uint64, + pending pendingCompound, +) (SplitResult, error) { + return s.executeSplit( + ctx, + catalogVersion, + pending.intermediate.RouteID, + pending.splitKey, + 0, + pending.intermediate.Start, + pending.intermediate.End, + pending.intermediate.GroupID, + SplitOriginIsolationCompound, + 0, + 0, + ) +} + +func (s *Scheduler) finalizePendingCompounds( + ctx context.Context, + snapshot distribution.CatalogSnapshot, +) (distribution.CatalogSnapshot, int, int, int, bool) { + if len(s.pendingCompounds) == 0 { + return snapshot, 0, 0, 0, false + } + parentIDs := make([]uint64, 0, len(s.pendingCompounds)) + for parentID := range s.pendingCompounds { + parentIDs = append(parentIDs, parentID) + } + sort.Slice(parentIDs, func(i, j int) bool { return parentIDs[i] < parentIDs[j] }) + + scheduled := 0 + failed := 0 + attempted := 0 + for _, parentID := range parentIDs { + if attempted >= s.cfg.Detector.MaxSplitsPerCycle { + break + } + pending := s.pendingCompounds[parentID] + intermediate, ok := findActiveRoute(snapshot.Routes, pending.intermediate.RouteID) + if !ok || !samePendingIntermediate(intermediate, pending) || + len(snapshot.Routes)+1 > s.cfg.Detector.MaxRoutes { + delete(s.pendingCompounds, parentID) + s.cfg.Logger.WarnContext(ctx, "autosplit: dropping invalid compound finalization", + slog.Uint64("parent_route_id", parentID), + slog.Uint64("intermediate_route_id", pending.intermediate.RouteID)) + continue + } + pending.intermediate = intermediate + s.pendingCompounds[parentID] = pending + attempted++ + result, err := s.executePendingCompound(ctx, snapshot.Version, pending) + if err != nil { + failed++ + s.cfg.Logger.WarnContext(ctx, "autosplit: compound finalization failed", + slog.Uint64("parent_route_id", parentID), + slog.Uint64("intermediate_route_id", pending.intermediate.RouteID), + slog.Any("err", err)) + continue + } + delete(s.pendingCompounds, parentID) + scheduled++ + snapshot.Version = result.CatalogVersion + refreshed, refreshErr := s.source.Snapshot(ctx) + if refreshErr != nil { + s.cfg.Logger.WarnContext(ctx, "autosplit: refresh after compound finalization failed", + slog.Any("err", refreshErr)) + return snapshot, scheduled, failed, attempted, true + } + snapshot = refreshed + } + return snapshot, scheduled, failed, attempted, false +} + +func findActiveRoute(routes []distribution.RouteDescriptor, routeID uint64) (distribution.RouteDescriptor, bool) { + for _, route := range routes { + if route.RouteID == routeID && route.State == distribution.RouteStateActive { + return route, true + } + } + return distribution.RouteDescriptor{}, false +} + +func samePendingIntermediate(route distribution.RouteDescriptor, pending pendingCompound) bool { + return route.GroupID == pending.intermediate.GroupID && + bytes.Equal(route.Start, pending.intermediate.Start) && + bytes.Equal(route.End, pending.intermediate.End) && + splitKeyInsideRoute(route, pending.splitKey) +} + +func (s *Scheduler) committedWindows(routes []distribution.RouteDescriptor, now time.Time) []ColumnWindow { + if s.sampler == nil { + return nil + } + step := s.sampler.Step() + if step <= 0 { + step = keyviz.DefaultStep + } + from := s.snapshotStart(routes, now, step) + cols := s.sampler.Snapshot(from, now.Add(time.Nanosecond)) + windows, _, _ := CommittedWindowsFromColumns(cols, s.oldestProcessedAt(routes)) + if s.leaderWatermark.IsZero() && s.leaderStartedAt.IsZero() { + return windows + } + filtered := windows[:0] + for _, window := range windows { + if !s.leaderWatermark.IsZero() && !window.Column.At.After(s.leaderWatermark) { + continue + } + windowStart := window.Column.At.Add(-window.Duration) + if !s.leaderStartedAt.IsZero() && windowStart.Before(s.leaderStartedAt) { + continue + } + filtered = append(filtered, window) + } + return filtered +} + +func (s *Scheduler) oldestProcessedAt(routes []distribution.RouteDescriptor) time.Time { + oldest := time.Time{} + for _, route := range routes { + status := s.state.RouteStatus(route.RouteID) + if status.LastProcessedAt.IsZero() { + continue + } + if oldest.IsZero() || status.LastProcessedAt.Before(oldest) { + oldest = status.LastProcessedAt + } + } + return oldest +} + +func (s *Scheduler) snapshotStart(routes []distribution.RouteDescriptor, now time.Time, step time.Duration) time.Time { + oldest := s.oldestProcessedAt(routes) + if !oldest.IsZero() { + return oldest.Add(-step) + } + cols := defaultSnapshotLookbackCols + if s.cfg.Detector.CandidateWindows > 0 { + cols = s.cfg.Detector.CandidateWindows + snapshotLookbackPaddingCols + } + if h, ok := s.sampler.(historyColumnser); ok && h.HistoryColumns() > cols { + cols = h.HistoryColumns() + } + return now.Add(-time.Duration(cols+1) * step) +} + +func (s *Scheduler) reconcileSamplerRoutes(routes []distribution.RouteDescriptor) { + if s.reconciler == nil { + return + } + s.reconciler.Reconcile(routes) +} + +// Reconcile applies one complete live catalog snapshot to sampler membership. +func (r *RouteReconciler) Reconcile(routes []distribution.RouteDescriptor) { + if r == nil || r.registrar == nil { + return + } + r.mu.Lock() + defer r.mu.Unlock() + + live := make(map[uint64]struct{}, len(routes)) + for _, route := range routes { + live[route.RouteID] = struct{}{} + registered, ok := r.registered[route.RouteID] + next := registeredRouteFromDescriptor(route) + if ok && registered.equal(next) { + continue + } + if ok { + r.registrar.RemoveRoute(route.RouteID) + } + r.registrar.RegisterRoute(route.RouteID, route.Start, route.End, route.GroupID) + r.registered[route.RouteID] = next + } + for routeID := range r.registered { + if _, ok := live[routeID]; ok { + continue + } + r.registrar.RemoveRoute(routeID) + delete(r.registered, routeID) + } +} + +func registeredRouteFromDescriptor(route distribution.RouteDescriptor) registeredRoute { + return registeredRoute{ + start: distribution.CloneBytes(route.Start), + end: distribution.CloneBytes(route.End), + groupID: route.GroupID, + } +} + +func (r registeredRoute) equal(other registeredRoute) bool { + return r.groupID == other.groupID && + bytes.Equal(r.start, other.start) && + bytes.Equal(r.end, other.end) +} + +func (s *Scheduler) routesLedLocally( + routes []distribution.RouteDescriptor, + now time.Time, +) ([]distribution.RouteDescriptor, map[uint64]EvidenceFence) { + if s.cfg.GroupLeadership == nil { + return routes, nil + } + groupRoutes := make(map[uint64][]distribution.RouteDescriptor) + for _, route := range routes { + groupRoutes[route.GroupID] = append(groupRoutes[route.GroupID], route) + } + eligible := make([]distribution.RouteDescriptor, 0, len(routes)) + fences := make(map[uint64]EvidenceFence, len(routes)) + watermark := time.Time{} + watermarkLoaded := false + for groupID, ownedRoutes := range groupRoutes { + state, ledLocally := s.locallyLedGroupState(groupID, now, &watermark, &watermarkLoaded) + if !ledLocally { + continue + } + for _, route := range ownedRoutes { + eligible = append(eligible, route) + fences[route.RouteID] = EvidenceFence{ + ProcessedThrough: state.watermark, + WindowStartNotBefore: state.startedAt, + } + } + } + for groupID := range s.groupLeadership { + if _, ok := groupRoutes[groupID]; !ok { + delete(s.groupLeadership, groupID) + s.dropPendingCompoundsForGroup(groupID) + } + } + return eligible, fences +} + +func (s *Scheduler) locallyLedGroupState( + groupID uint64, + now time.Time, + watermark *time.Time, + watermarkLoaded *bool, +) (groupLeadershipState, bool) { + leader, term := s.cfg.GroupLeadership(groupID) + previous, known := s.groupLeadership[groupID] + if !leader { + previous.leader = false + previous.term = term + s.groupLeadership[groupID] = previous + s.dropPendingCompoundsForGroup(groupID) + return previous, false + } + transition := !known || !previous.leader || (term != 0 && term != previous.term) + if transition { + if !*watermarkLoaded { + *watermark = s.freshestColumnAt(now) + *watermarkLoaded = true + } + previous = groupLeadershipState{ + leader: true, + term: term, + startedAt: now, + watermark: *watermark, + } + s.dropPendingCompoundsForGroup(groupID) + } else { + previous.leader = true + previous.term = term + } + s.groupLeadership[groupID] = previous + return previous, true +} + +func (s *Scheduler) dropPendingCompoundsForGroup(groupID uint64) { + for parentID, pending := range s.pendingCompounds { + if pending.intermediate.GroupID == groupID { + delete(s.pendingCompounds, parentID) + } + } +} + +func (s *Scheduler) resetForLeadership(now time.Time) { + s.state = NewDetectorState() + s.pendingCompounds = make(map[uint64]pendingCompound) + s.catalogVersion = 0 + s.leaderStartedAt = now + s.leaderWatermark = s.freshestColumnAt(now) + s.cfg.Logger.Info("autosplit: leadership acquired; detector state reset", + slog.Time("leader_started_at", s.leaderStartedAt), + slog.Time("processed_watermark", s.leaderWatermark)) +} + +func (s *Scheduler) freshestColumnAt(now time.Time) time.Time { + if s.sampler == nil { + return time.Time{} + } + step := s.sampler.Step() + if step <= 0 { + step = keyviz.DefaultStep + } + from := now.Add(-time.Duration(defaultSnapshotLookbackCols) * step) + if h, ok := s.sampler.(historyColumnser); ok && h.HistoryColumns() > 0 { + from = now.Add(-time.Duration(h.HistoryColumns()+1) * step) + } + cols := s.sampler.Snapshot(from, now.Add(time.Nanosecond)) + if len(cols) == 0 { + return time.Time{} + } + sort.SliceStable(cols, func(i, j int) bool { + return cols[i].At.Before(cols[j].At) + }) + return cols[len(cols)-1].At +} + +func (s *Scheduler) leadership() (bool, uint64) { + if s.cfg.Leadership != nil { + return s.cfg.Leadership() + } + if s.cfg.IsLeader == nil { + return true, 0 + } + return s.cfg.IsLeader(), 0 +} + +func (s *Scheduler) stateCounts(now time.Time) (int, int) { + if s == nil || s.state == nil { + return 0, 0 + } + tracked := len(s.state.routes) + cooldown := 0 + for _, route := range s.state.routes { + if now.Before(route.CooldownUntil) { + cooldown++ + } + } + return tracked, cooldown +} + +func splitFailureReason(err error, targetGroupID uint64) string { + if status.Code(err) == codes.Aborted { + return "cas_conflict" + } + code := status.Code(err) + if targetGroupID != 0 && + (code == codes.FailedPrecondition || code == codes.NotFound || code == codes.Unavailable) { + return "target_unavailable" + } + return "rpc_error" +} + +func (s *Scheduler) killSwitchActive(ctx context.Context) bool { + if s.cfg.KillSwitch != nil && s.cfg.KillSwitch(ctx) { + return true + } + if s.cfg.KillSwitchFile == "" { + return false + } + _, err := os.Stat(s.cfg.KillSwitchFile) + if err == nil { + return true + } + if errors.Is(err, os.ErrNotExist) { + return false + } + s.cfg.Logger.WarnContext(ctx, "autosplit: kill switch stat failed; skipping splits", + slog.String("path", s.cfg.KillSwitchFile), + slog.Any("err", err)) + return true +} + +func (s *Scheduler) logEvent(event Event) { + if event.IsolationReason != "" { + args := []any{slog.String("reason", string(event.IsolationReason))} + if event.RouteID != 0 { + args = append(args, slog.Uint64("route_id", event.RouteID)) + } + s.cfg.Logger.Debug("autosplit: isolation declined", args...) + } + if event.Reason == "" { + return + } + args := []any{ + slog.String("reason", string(event.Reason)), + } + if event.RouteID != 0 { + args = append(args, slog.Uint64("route_id", event.RouteID)) + } + if !event.At.IsZero() { + args = append(args, slog.Time("at", event.At)) + } + s.cfg.Logger.Debug("autosplit: detector skip", args...) +} + +func (cfg SchedulerConfig) withDefaults() SchedulerConfig { + cfg.Detector = cfg.Detector.withDefaults() + if cfg.EvalInterval <= 0 { + cfg.EvalInterval = DefaultEvalInterval + } + if cfg.SplitCooldown <= 0 { + cfg.SplitCooldown = DefaultSplitCooldown + } + if cfg.SplitTimeout <= 0 { + cfg.SplitTimeout = DefaultSplitTimeout + } + if cfg.Logger == nil { + cfg.Logger = slog.Default() + } + return cfg +} + +// SeedCooldownsFromRoutes rebuilds leader-local cooldown state from durable +// child-route lineage. It uses only the HLC physical millis component. +func SeedCooldownsFromRoutes(state *DetectorState, routes []distribution.RouteDescriptor, cooldown time.Duration, now time.Time) { + if state == nil || cooldown <= 0 { + return + } + for _, route := range routes { + until := CooldownUntilFromSplitAtHLC(route.SplitAtHLC, cooldown, now) + if until.IsZero() { + continue + } + status := state.RouteStatus(route.RouteID) + if until.After(status.CooldownUntil) { + state.SetCooldown(route.RouteID, until) + } + } +} + +// CooldownUntilFromSplitAtHLC returns a monotonic enforcement deadline derived +// from SplitAtHLC's physical millis. A zero result means no remaining cooldown. +func CooldownUntilFromSplitAtHLC(splitAtHLC uint64, cooldown time.Duration, now time.Time) time.Time { + if splitAtHLC == 0 || cooldown <= 0 { + return time.Time{} + } + remaining := RemainingCooldownFromSplitAtHLC(splitAtHLC, cooldown, now) + if remaining <= 0 { + return time.Time{} + } + return now.Add(remaining) +} + +// RemainingCooldownFromSplitAtHLC exposes the arithmetic for regression tests. +func RemainingCooldownFromSplitAtHLC(splitAtHLC uint64, cooldown time.Duration, now time.Time) time.Duration { + if splitAtHLC == 0 || cooldown <= 0 { + return 0 + } + splitMs := HLCPhysicalMillis(splitAtHLC) + elapsedMs := now.UnixMilli() - splitMs + if elapsedMs < 0 { + elapsedMs = 0 + } + remaining := cooldown - time.Duration(elapsedMs)*time.Millisecond + if remaining < 0 { + return 0 + } + return remaining +} + +// HLCPhysicalMillis extracts the physical Unix-ms half of a packed HLC value. +func HLCPhysicalMillis(v uint64) int64 { + physical := v >> kv.HLCLogicalBits + if physical > math.MaxInt64 { + return math.MaxInt64 + } + return int64(physical) +} + +func loggedSplitKey(key []byte) string { + if len(key) <= maxLoggedSplitKeyBytes { + return hex.EncodeToString(key) + } + return hex.EncodeToString(key[:maxLoggedSplitKeyBytes]) + "..." +} diff --git a/distribution/autosplit/scheduler_test.go b/distribution/autosplit/scheduler_test.go new file mode 100644 index 000000000..6d0313e90 --- /dev/null +++ b/distribution/autosplit/scheduler_test.go @@ -0,0 +1,812 @@ +package autosplit + +import ( + "bytes" + "context" + "testing" + "time" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/keyviz" + "github.com/bootjp/elastickv/kv" + "github.com/cockroachdb/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestCommittedWindowsFromColumnsDerivesDurations(t *testing.T) { + t.Parallel() + base := time.Unix(1_700_000_000, 0) + cols := []keyviz.MatrixColumn{ + {At: base.Add(2 * time.Minute)}, + {At: base}, + {At: base.Add(time.Minute)}, + } + + windows, _, skipped := CommittedWindowsFromColumns(cols, time.Time{}) + + require.Equal(t, 1, skipped) + require.Len(t, windows, 3) + require.Equal(t, base, windows[0].Column.At) + require.Zero(t, windows[0].Duration) + require.Equal(t, base.Add(time.Minute), windows[1].Column.At) + require.Equal(t, time.Minute, windows[1].Duration) + require.Equal(t, base.Add(2*time.Minute), windows[2].Column.At) + require.Equal(t, time.Minute, windows[2].Duration) +} + +func TestCommittedWindowsUsesStoredWindowStart(t *testing.T) { + t.Parallel() + base := time.Unix(1_700_000_000, 0) + cols := []keyviz.MatrixColumn{{ + WindowStart: base, + At: base.Add(90 * time.Second), + }} + + windows, _, skipped := CommittedWindowsFromColumns(cols, time.Time{}) + + require.Zero(t, skipped) + require.Len(t, windows, 1) + require.Equal(t, 90*time.Second, windows[0].Duration) +} + +func TestSchedulerTickSchedulesSplitRange(t *testing.T) { + t.Parallel() + now := time.Unix(1_700_000_000, 0) + source := &fakeCatalogSnapshotSource{snapshot: distribution.CatalogSnapshot{ + Version: 7, + Routes: []distribution.RouteDescriptor{testRoute(1, 1, "a", "z")}, + }} + splitter := &fakeSplitter{} + scheduler := newTestScheduler(source, splitter, hotColumns(now, 2), nil) + scheduler.wasLeader = true + + result, err := scheduler.Tick(context.Background(), now.Add(2*time.Minute)) + + require.NoError(t, err) + require.Equal(t, 1, result.Scheduled) + require.Len(t, splitter.requests, 1) + require.Equal(t, uint64(7), splitter.requests[0].ExpectedCatalogVersion) + require.Equal(t, uint64(1), splitter.requests[0].RouteID) + require.Equal(t, []byte("m"), splitter.requests[0].SplitKey) + require.Equal(t, uint64(0), splitter.requests[0].TargetGroupID) +} + +func TestSchedulerUsesCommittedCatalogVersionBetweenSplits(t *testing.T) { + t.Parallel() + now := time.Unix(1_700_000_000, 0) + source := &fakeCatalogSnapshotSource{snapshot: distribution.CatalogSnapshot{ + Version: 7, + Routes: []distribution.RouteDescriptor{ + testRoute(1, 1, "a", "m"), + testRoute(2, 1, "m", "z"), + }, + }} + splitter := &fakeSplitter{} + scheduler := newTestScheduler(source, splitter, dualRouteHotColumns(now, 2), nil) + scheduler.wasLeader = true + scheduler.cfg.Detector.MaxSplitsPerCycle = 2 + + result, err := scheduler.Tick(context.Background(), now.Add(2*time.Minute)) + + require.NoError(t, err) + require.Equal(t, 2, result.Scheduled) + require.Len(t, splitter.requests, 2) + require.Equal(t, uint64(7), splitter.requests[0].ExpectedCatalogVersion) + require.Equal(t, uint64(8), splitter.requests[1].ExpectedCatalogVersion) + require.Equal(t, uint64(9), result.CatalogVersion) +} + +func TestSchedulerKillSwitchSkipsSplitRange(t *testing.T) { + t.Parallel() + now := time.Unix(1_700_000_000, 0) + source := &fakeCatalogSnapshotSource{snapshot: distribution.CatalogSnapshot{ + Version: 7, + Routes: []distribution.RouteDescriptor{testRoute(1, 1, "a", "z")}, + }} + splitter := &fakeSplitter{} + scheduler := newTestScheduler(source, splitter, hotColumns(now, 2), func(context.Context) bool { + return true + }) + registry := prometheus.NewRegistry() + observer, ok := NewPrometheusObserver(registry).(*prometheusObserver) + require.True(t, ok) + scheduler.cfg.Observer = observer + scheduler.wasLeader = true + + result, err := scheduler.Tick(context.Background(), now.Add(2*time.Minute)) + + require.NoError(t, err) + require.True(t, result.KillSwitch) + require.Equal(t, 0, result.Scheduled) + require.Equal(t, float64(0), testutil.ToFloat64(observer.enabled)) + require.Empty(t, splitter.requests) +} + +func TestSchedulerCatalogVersionGapConsumesBufferedEvidence(t *testing.T) { + t.Parallel() + now := time.Unix(1_700_000_000, 0) + source := &fakeCatalogSnapshotSource{snapshot: distribution.CatalogSnapshot{ + Version: 7, + Routes: []distribution.RouteDescriptor{testRoute(1, 1, "a", "z")}, + }} + scheduler := newTestScheduler(source, &fakeSplitter{}, hotColumns(now, 1), nil) + scheduler.wasLeader = true + + first, err := scheduler.Tick(context.Background(), now.Add(time.Minute)) + require.NoError(t, err) + require.Empty(t, first.Detector.Decisions) + require.Equal(t, 1, scheduler.state.RouteStatus(1).ConsecutiveOver) + + source.snapshot.Version = 8 + sampler := requireFakeSampler(t, scheduler) + sampler.cols = append(sampler.cols, hotColumn(now.Add(2*time.Minute))) + gap, err := scheduler.Tick(context.Background(), now.Add(2*time.Minute)) + require.NoError(t, err) + require.Empty(t, gap.Detector.Decisions) + require.Equal(t, 0, scheduler.state.RouteStatus(1).ConsecutiveOver) + require.Equal(t, now.Add(2*time.Minute), scheduler.state.RouteStatus(1).LastProcessedAt) + + sampler.cols = append(sampler.cols, hotColumn(now.Add(3*time.Minute))) + after, err := scheduler.Tick(context.Background(), now.Add(3*time.Minute)) + require.NoError(t, err) + require.Empty(t, after.Detector.Decisions) + require.Equal(t, 1, scheduler.state.RouteStatus(1).ConsecutiveOver) +} + +func TestSchedulerHistoryGapClearsPriorConfidence(t *testing.T) { + t.Parallel() + now := time.Unix(1_700_000_000, 0) + source := &fakeCatalogSnapshotSource{snapshot: distribution.CatalogSnapshot{ + Version: 7, + Routes: []distribution.RouteDescriptor{testRoute(1, 1, "a", "z")}, + }} + scheduler := newTestScheduler(source, &fakeSplitter{}, hotColumns(now, 1), nil) + scheduler.wasLeader = true + + first, err := scheduler.Tick(context.Background(), now.Add(time.Minute)) + require.NoError(t, err) + require.Empty(t, first.Detector.Decisions) + require.Equal(t, 1, scheduler.state.RouteStatus(1).ConsecutiveOver) + + sampler := requireFakeSampler(t, scheduler) + sampler.cols = []keyviz.MatrixColumn{ + hotMatrixColumn(now.Add(5*time.Minute), now.Add(4*time.Minute)), + } + afterGap, err := scheduler.Tick(context.Background(), now.Add(5*time.Minute)) + require.NoError(t, err) + require.Empty(t, afterGap.Detector.Decisions) + require.Equal(t, 0, scheduler.state.RouteStatus(1).ConsecutiveOver) + require.Equal(t, now.Add(5*time.Minute), scheduler.state.RouteStatus(1).LastProcessedAt) +} + +func TestSplitFailureReasonUsesBoundedLabels(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + err error + targetGroupID uint64 + want string + }{ + {name: "catalog CAS", err: status.Error(codes.Aborted, "version"), want: "cas_conflict"}, + {name: "wrapped catalog CAS", err: errors.Wrap(status.Error(codes.Aborted, "version"), "split"), want: "cas_conflict"}, + {name: "target unavailable", err: status.Error(codes.FailedPrecondition, "target"), targetGroupID: 2, want: "target_unavailable"}, + {name: "same-group precondition", err: status.Error(codes.FailedPrecondition, "route"), want: "rpc_error"}, + {name: "other RPC", err: status.Error(codes.Internal, "failed"), targetGroupID: 2, want: "rpc_error"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + require.Equal(t, tt.want, splitFailureReason(tt.err, tt.targetGroupID)) + }) + } +} + +func TestSchedulerFailedSplitDoesNotSeedCooldown(t *testing.T) { + t.Parallel() + now := time.Unix(1_700_000_000, 0) + source := &fakeCatalogSnapshotSource{snapshot: distribution.CatalogSnapshot{ + Version: 7, + Routes: []distribution.RouteDescriptor{testRoute(1, 1, "a", "z")}, + }} + splitter := &fakeSplitter{err: errors.New("cas conflict")} + scheduler := newTestScheduler(source, splitter, hotColumns(now, 2), nil) + scheduler.wasLeader = true + + result, err := scheduler.Tick(context.Background(), now.Add(2*time.Minute)) + require.NoError(t, err) + require.Equal(t, 1, result.Failed) + require.Len(t, splitter.requests, 1) + require.True(t, scheduler.state.RouteStatus(1).CooldownUntil.IsZero()) + + sampler := requireFakeSampler(t, scheduler) + sampler.cols = append(sampler.cols, hotColumn(now.Add(3*time.Minute))) + result, err = scheduler.Tick(context.Background(), now.Add(3*time.Minute)) + + require.NoError(t, err) + require.Equal(t, 1, result.Failed) + require.Len(t, splitter.requests, 2) + require.True(t, scheduler.state.RouteStatus(1).CooldownUntil.IsZero()) +} + +func TestSchedulerSeedsCooldownFromSplitAtHLC(t *testing.T) { + t.Parallel() + now := time.UnixMilli(1_700_000_000_000) + route := testRoute(2, 1, "a", "z") + route.SplitAtHLC = packTestHLC(now.Add(-time.Minute)) + source := &fakeCatalogSnapshotSource{snapshot: distribution.CatalogSnapshot{ + Version: 8, + Routes: []distribution.RouteDescriptor{route}, + }} + splitter := &fakeSplitter{} + scheduler := newTestScheduler(source, splitter, hotColumns(now.Add(-3*time.Minute), 3), nil) + scheduler.wasLeader = true + + result, err := scheduler.Tick(context.Background(), now) + + require.NoError(t, err) + require.Equal(t, 0, result.Scheduled) + require.Empty(t, splitter.requests) + require.WithinDuration(t, now.Add(9*time.Minute), scheduler.state.RouteStatus(2).CooldownUntil, time.Millisecond) + requireEvent(t, result.Detector.Events, 2, SkipReasonCooldown) +} + +func TestRemainingCooldownUsesHLCPhysicalMillis(t *testing.T) { + t.Parallel() + now := time.UnixMilli(1_700_000_000_000) + splitAt := packTestHLC(now.Add(-time.Minute)) | 42 + + remaining := RemainingCooldownFromSplitAtHLC(splitAt, 10*time.Minute, now) + + require.Equal(t, 9*time.Minute, remaining) + rawElapsedMs := packTestHLC(now) - splitAt + require.Greater(t, rawElapsedMs, uint64(600_000)) + + future := RemainingCooldownFromSplitAtHLC(packTestHLC(now.Add(time.Minute)), 10*time.Minute, now) + require.Equal(t, 10*time.Minute, future) +} + +func TestSchedulerLeadershipResetIgnoresPreStartHistory(t *testing.T) { + t.Parallel() + start := time.Unix(1_700_000_000, 0) + source := &fakeCatalogSnapshotSource{snapshot: distribution.CatalogSnapshot{ + Version: 7, + Routes: []distribution.RouteDescriptor{testRoute(1, 1, "a", "z")}, + }} + splitter := &fakeSplitter{} + scheduler := newTestScheduler(source, splitter, hotColumns(start, 3), nil) + + result, err := scheduler.Tick(context.Background(), start.Add(3*time.Minute)) + require.NoError(t, err) + require.Equal(t, 0, result.Scheduled) + require.Empty(t, splitter.requests) + + sampler := requireFakeSampler(t, scheduler) + sampler.cols = append(sampler.cols, + hotColumn(start.Add(4*time.Minute)), + hotColumn(start.Add(5*time.Minute)), + hotColumn(start.Add(6*time.Minute)), + ) + result, err = scheduler.Tick(context.Background(), start.Add(6*time.Minute)) + + require.NoError(t, err) + require.Equal(t, 1, result.Scheduled) + require.Len(t, splitter.requests, 1) + require.Equal(t, []byte("m"), splitter.requests[0].SplitKey) +} + +func TestSchedulerReconcilesSamplerRoutes(t *testing.T) { + t.Parallel() + now := time.Unix(1_700_000_000, 0) + source := &fakeCatalogSnapshotSource{snapshot: distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{testRoute(1, 1, "a", "m")}, + }} + registrar := &fakeRegistrar{} + scheduler := newTestScheduler(source, &fakeSplitter{}, nil, nil) + scheduler.reconciler = NewRouteReconciler(registrar) + scheduler.wasLeader = true + + _, err := scheduler.Tick(context.Background(), now) + require.NoError(t, err) + require.Equal(t, []uint64{1}, registrar.registered) + + source.snapshot = distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{testRoute(2, 1, "m", "z")}, + } + _, err = scheduler.Tick(context.Background(), now.Add(time.Minute)) + + require.NoError(t, err) + require.Equal(t, []uint64{1, 2}, registrar.registered) + require.Equal(t, []uint64{1}, registrar.removed) +} + +func TestSchedulerReregistersChangedSamplerRoute(t *testing.T) { + t.Parallel() + now := time.Unix(1_700_000_000, 0) + source := &fakeCatalogSnapshotSource{snapshot: distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{testRoute(1, 1, "a", "m")}, + }} + registrar := &fakeRegistrar{} + scheduler := newTestScheduler(source, &fakeSplitter{}, nil, nil) + scheduler.reconciler = NewRouteReconciler(registrar) + scheduler.wasLeader = true + + _, err := scheduler.Tick(context.Background(), now) + require.NoError(t, err) + + source.snapshot = distribution.CatalogSnapshot{ + Version: 2, + Routes: []distribution.RouteDescriptor{testRoute(1, 2, "m", "z")}, + } + _, err = scheduler.Tick(context.Background(), now.Add(time.Minute)) + + require.NoError(t, err) + require.Equal(t, []uint64{1, 1}, registrar.registered) + require.Equal(t, []uint64{1}, registrar.removed) + require.Len(t, registrar.registrations, 2) + require.Equal(t, uint64(2), registrar.registrations[1].groupID) + require.Equal(t, []byte("m"), registrar.registrations[1].start) + require.Equal(t, []byte("z"), registrar.registrations[1].end) +} + +func TestSchedulerExecutesCompoundIsolationBackToBack(t *testing.T) { + t.Parallel() + source := &fakeCatalogSnapshotSource{snapshot: distribution.CatalogSnapshot{ + Version: 7, + Routes: []distribution.RouteDescriptor{testRoute(1, 1, "a", "z")}, + }} + splitter := &fakeSplitter{} + splitter.onSuccess = func(req SplitRequest, result SplitResult) { + applyFakeSplit(source, req, result) + } + scheduler := newTestScheduler(source, splitter, nil, nil) + + version, err := scheduler.executeDecision(context.Background(), 7, testCompoundDecision()) + + require.NoError(t, err) + require.Equal(t, uint64(9), version) + require.Len(t, splitter.requests, 2) + require.Equal(t, uint64(7), splitter.requests[0].ExpectedCatalogVersion) + require.Equal(t, uint64(8), splitter.requests[1].ExpectedCatalogVersion) + require.Equal(t, uint64(1), splitter.requests[0].RouteID) + require.Equal(t, splitter.requests[0].SplitKey, splitter.requests[1].ParentStart) + require.Equal(t, []byte("m"), splitter.requests[0].SplitKey) + require.Equal(t, []byte{'m', 0}, splitter.requests[1].SplitKey) + require.Zero(t, splitter.requests[0].TargetGroupID) + require.Zero(t, splitter.requests[1].TargetGroupID) + require.Empty(t, scheduler.pendingCompounds) + require.Equal(t, uint64(9), source.snapshot.Version) + require.Len(t, source.snapshot.Routes, 3) +} + +func TestSchedulerCompoundUsesNormalizedCommittedBoundary(t *testing.T) { + t.Parallel() + source := &fakeCatalogSnapshotSource{snapshot: distribution.CatalogSnapshot{ + Version: 7, + Routes: []distribution.RouteDescriptor{testRoute(1, 1, "a", "z")}, + }} + splitter := &fakeSplitter{ + normalizeSplitKey: func(key []byte) []byte { + if bytes.Equal(key, []byte("m")) { + return []byte("l") + } + return key + }, + } + splitter.onSuccess = func(req SplitRequest, result SplitResult) { + applyFakeSplit(source, req, result) + } + scheduler := newTestScheduler(source, splitter, nil, nil) + + version, err := scheduler.executeDecision(context.Background(), 7, testCompoundDecision()) + + require.NoError(t, err) + require.Equal(t, uint64(9), version) + require.Len(t, splitter.requests, 2) + require.Equal(t, []byte("m"), splitter.requests[0].SplitKey) + require.Equal(t, []byte("l"), splitter.requests[1].ParentStart) + require.Equal(t, []byte{'m', 0}, splitter.requests[1].SplitKey) + require.Empty(t, scheduler.pendingCompounds) +} + +func TestSchedulerRetriesPendingCompoundWithoutNewSamplerColumn(t *testing.T) { + t.Parallel() + now := time.Unix(1_700_000_000, 0) + source := &fakeCatalogSnapshotSource{snapshot: distribution.CatalogSnapshot{ + Version: 7, + Routes: []distribution.RouteDescriptor{testRoute(1, 1, "a", "z")}, + }} + killed := true + splitter := &fakeSplitter{failAt: map[int]error{2: errors.New("temporary")}} + splitter.onSuccess = func(req SplitRequest, result SplitResult) { + applyFakeSplit(source, req, result) + } + scheduler := newTestScheduler(source, splitter, nil, func(context.Context) bool { return killed }) + scheduler.wasLeader = true + + version, err := scheduler.executeDecision(context.Background(), 7, testCompoundDecision()) + require.Error(t, err) + require.Equal(t, uint64(8), version) + require.Len(t, scheduler.pendingCompounds, 1) + require.Len(t, splitter.requests, 2) + require.Equal(t, uint64(8), source.snapshot.Version) + require.Len(t, source.snapshot.Routes, 2) + + blocked, err := scheduler.Tick(context.Background(), now) + require.NoError(t, err) + require.True(t, blocked.KillSwitch) + require.Len(t, splitter.requests, 2) + require.Len(t, scheduler.pendingCompounds, 1) + + killed = false + delete(splitter.failAt, 2) + result, err := scheduler.Tick(context.Background(), now.Add(time.Second)) + + require.NoError(t, err) + require.Equal(t, 1, result.Scheduled) + require.Zero(t, result.Failed) + require.Len(t, splitter.requests, 3) + require.Equal(t, uint64(8), splitter.requests[2].ExpectedCatalogVersion) + require.Equal(t, []byte{'m', 0}, splitter.requests[2].SplitKey) + require.Empty(t, scheduler.pendingCompounds) + require.Equal(t, uint64(9), source.snapshot.Version) + require.Len(t, source.snapshot.Routes, 3) +} + +func TestSchedulerStopsTickAfterCompoundRefreshFailure(t *testing.T) { + t.Parallel() + + now := time.Unix(1_700_000_000, 0) + source := &fakeCatalogSnapshotSource{ + snapshot: distribution.CatalogSnapshot{ + Version: 8, + Routes: []distribution.RouteDescriptor{ + testRoute(10, 1, "m", "z"), + testRoute(11, 1, "a", "m"), + }, + }, + failAt: map[int]error{2: errors.New("refresh failed")}, + } + splitter := &fakeSplitter{} + splitter.onSuccess = func(req SplitRequest, result SplitResult) { + applyFakeSplit(source, req, result) + } + scheduler := newTestScheduler(source, splitter, []keyviz.MatrixColumn{ + { + At: now.Add(-2 * time.Minute), + WindowStart: now.Add(-3 * time.Minute), + Rows: []keyviz.MatrixRow{ + testRow(10, 1, 0, 2, "m", "n", 60, 0), + testRow(10, 1, 1, 2, "n", "z", 60, 0), + }, + }, + { + At: now.Add(-time.Minute), + WindowStart: now.Add(-2 * time.Minute), + Rows: []keyviz.MatrixRow{ + testRow(10, 1, 0, 2, "m", "n", 60, 0), + testRow(10, 1, 1, 2, "n", "z", 60, 0), + }, + }, + }, nil) + scheduler.wasLeader = true + scheduler.cfg.Detector.MaxSplitsPerCycle = 2 + registrar := &fakeRegistrar{} + scheduler.reconciler = NewRouteReconciler(registrar) + scheduler.pendingCompounds[1] = pendingCompound{ + parentRouteID: 1, + intermediate: testRoute(10, 1, "m", "z"), + splitKey: []byte{'m', 0}, + } + + result, err := scheduler.Tick(context.Background(), now) + + require.NoError(t, err) + require.Equal(t, 1, result.Scheduled) + require.Zero(t, result.Failed) + require.Empty(t, result.Detector.Decisions) + require.Equal(t, uint64(9), result.CatalogVersion) + require.Equal(t, uint64(9), scheduler.catalogVersion) + require.Len(t, splitter.requests, 1) + require.Empty(t, registrar.registered, "must not reconcile the pre-refresh route set after catalog version advances") +} + +func TestSchedulerCatalogTermChangeClearsLeaderLocalState(t *testing.T) { + t.Parallel() + now := time.Unix(1_700_000_000, 0) + term := uint64(2) + source := &fakeCatalogSnapshotSource{snapshot: distribution.CatalogSnapshot{ + Version: 7, Routes: []distribution.RouteDescriptor{testRoute(1, 1, "a", "z")}, + }} + scheduler := newTestScheduler(source, &fakeSplitter{}, nil, nil) + scheduler.cfg.Leadership = func() (bool, uint64) { return true, term } + scheduler.wasLeader = true + scheduler.leaderTerm = 1 + scheduler.state.routes[1] = RouteStatus{ConsecutiveOver: 2, LastProcessedAt: now.Add(-time.Minute)} + scheduler.pendingCompounds[1] = pendingCompound{ + parentRouteID: 1, + intermediate: testRoute(10, 1, "m", "z"), + splitKey: []byte{'m', 0}, + } + + result, err := scheduler.Tick(context.Background(), now) + + require.NoError(t, err) + require.True(t, result.Leader) + require.Equal(t, term, scheduler.leaderTerm) + require.Zero(t, scheduler.state.RouteStatus(1).ConsecutiveOver) + require.Empty(t, scheduler.pendingCompounds) +} + +func TestSchedulerShardTermChangeReearnsPostTransferWindows(t *testing.T) { + t.Parallel() + base := time.Unix(1_700_000_000, 0) + groupTerm := uint64(10) + source := &fakeCatalogSnapshotSource{snapshot: distribution.CatalogSnapshot{ + Version: 7, Routes: []distribution.RouteDescriptor{testRoute(1, 1, "a", "z")}, + }} + splitter := &fakeSplitter{} + scheduler := newTestScheduler(source, splitter, []keyviz.MatrixColumn{ + hotMatrixColumn(base, base.Add(-time.Minute)), + }, func(context.Context) bool { return true }) + scheduler.wasLeader = true + scheduler.cfg.GroupLeadership = func(groupID uint64) (bool, uint64) { + return groupID == 1, groupTerm + } + + first, err := scheduler.Tick(context.Background(), base) + require.NoError(t, err) + require.Empty(t, first.Detector.Decisions) + + sampler := requireFakeSampler(t, scheduler) + sampler.cols = append(sampler.cols, + hotMatrixColumn(base.Add(time.Minute), base), + hotMatrixColumn(base.Add(2*time.Minute), base.Add(time.Minute)), + hotMatrixColumn(base.Add(3*time.Minute), base.Add(2*time.Minute)), + ) + promoted, err := scheduler.Tick(context.Background(), base.Add(3*time.Minute)) + require.NoError(t, err) + require.Len(t, promoted.Detector.Decisions, 1) + + groupTerm++ + _, err = scheduler.Tick(context.Background(), base.Add(3*time.Minute+30*time.Second)) + require.NoError(t, err) + sampler.cols = append(sampler.cols, + hotMatrixColumn(base.Add(4*time.Minute), base.Add(3*time.Minute)), + hotMatrixColumn(base.Add(5*time.Minute), base.Add(4*time.Minute)), + ) + notYet, err := scheduler.Tick(context.Background(), base.Add(5*time.Minute)) + require.NoError(t, err) + require.Empty(t, notYet.Detector.Decisions) + require.Equal(t, 1, scheduler.state.RouteStatus(1).ConsecutiveOver) + + sampler.cols = append(sampler.cols, + hotMatrixColumn(base.Add(6*time.Minute), base.Add(5*time.Minute)), + ) + reearned, err := scheduler.Tick(context.Background(), base.Add(6*time.Minute)) + require.NoError(t, err) + require.Equal(t, 2, scheduler.state.RouteStatus(1).ConsecutiveOver) + require.Len(t, reearned.Detector.Decisions, 1) +} + +func testCompoundDecision() Decision { + return Decision{ + RouteID: 1, + SplitKey: []byte("m"), + SecondSplitKey: []byte{'m', 0}, + SplitOrigin: SplitOriginIsolationCompound, + RouteDelta: 2, + RouteStart: []byte("a"), + RouteEnd: []byte("z"), + RouteGroupID: 1, + } +} + +func applyFakeSplit(source *fakeCatalogSnapshotSource, req SplitRequest, result SplitResult) { + routes := make([]distribution.RouteDescriptor, 0, len(source.snapshot.Routes)+1) + for _, route := range source.snapshot.Routes { + if route.RouteID == req.RouteID { + continue + } + routes = append(routes, route) + } + routes = append(routes, result.Left, result.Right) + source.snapshot = distribution.CatalogSnapshot{Version: result.CatalogVersion, Routes: routes} +} + +func newTestScheduler(source *fakeCatalogSnapshotSource, splitter *fakeSplitter, cols []keyviz.MatrixColumn, killSwitch KillSwitch) *Scheduler { + cfg := SchedulerConfig{ + Enabled: true, + Detector: Config{ + WriteWeight: 1, + ReadWeight: 0, + ThresholdOpsMin: 50, + CandidateWindows: 2, + MaxRoutes: 8, + MaxSplitsPerCycle: 1, + }, + EvalInterval: time.Minute, + SplitCooldown: 10 * time.Minute, + KillSwitch: killSwitch, + } + return NewScheduler(cfg, source, splitter, &fakeSampler{step: time.Minute, history: 16, cols: cols}, nil) +} + +func hotColumns(start time.Time, hotCount int) []keyviz.MatrixColumn { + cols := make([]keyviz.MatrixColumn, 0, hotCount+1) + cols = append(cols, keyviz.MatrixColumn{At: start}) + for i := 1; i <= hotCount; i++ { + cols = append(cols, hotColumn(start.Add(time.Duration(i)*time.Minute))) + } + return cols +} + +func dualRouteHotColumns(start time.Time, hotCount int) []keyviz.MatrixColumn { + cols := make([]keyviz.MatrixColumn, 0, hotCount+1) + cols = append(cols, keyviz.MatrixColumn{At: start}) + for i := 1; i <= hotCount; i++ { + cols = append(cols, dualRouteHotColumn(start.Add(time.Duration(i)*time.Minute))) + } + return cols +} + +func hotColumn(at time.Time) keyviz.MatrixColumn { + return keyviz.MatrixColumn{ + At: at, + Rows: []keyviz.MatrixRow{ + testRow(1, 1, 0, 2, "a", "m", 60, 0), + testRow(1, 1, 1, 2, "m", "z", 60, 0), + testRow(2, 1, 0, 2, "a", "m", 60, 0), + testRow(2, 1, 1, 2, "m", "z", 60, 0), + }, + } +} + +func hotMatrixColumn(at, start time.Time) keyviz.MatrixColumn { + column := hotColumn(at) + column.WindowStart = start + return column +} + +func dualRouteHotColumn(at time.Time) keyviz.MatrixColumn { + return keyviz.MatrixColumn{ + At: at, + Rows: []keyviz.MatrixRow{ + testRow(1, 1, 0, 2, "a", "g", 60, 0), + testRow(1, 1, 1, 2, "g", "m", 60, 0), + testRow(2, 1, 0, 2, "m", "t", 60, 0), + testRow(2, 1, 1, 2, "t", "z", 60, 0), + }, + } +} + +func packTestHLC(at time.Time) uint64 { + ms := at.UnixMilli() + if ms < 0 { + return 0 + } + return uint64(ms) << kv.HLCLogicalBits +} + +func requireFakeSampler(t *testing.T, scheduler *Scheduler) *fakeSampler { + t.Helper() + sampler, ok := scheduler.sampler.(*fakeSampler) + require.True(t, ok) + return sampler +} + +type fakeCatalogSnapshotSource struct { + snapshot distribution.CatalogSnapshot + err error + snapshotCalls int + failAt map[int]error +} + +func (f *fakeCatalogSnapshotSource) Snapshot(context.Context) (distribution.CatalogSnapshot, error) { + f.snapshotCalls++ + if err := f.failAt[f.snapshotCalls]; err != nil { + return distribution.CatalogSnapshot{}, err + } + if f.err != nil { + return distribution.CatalogSnapshot{}, f.err + } + return f.snapshot, nil +} + +type fakeSplitter struct { + requests []SplitRequest + err error + failAt map[int]error + normalizeSplitKey func([]byte) []byte + onSuccess func(SplitRequest, SplitResult) +} + +func (f *fakeSplitter) SplitRange(_ context.Context, req SplitRequest) (SplitResult, error) { + f.requests = append(f.requests, req) + if f.err != nil { + return SplitResult{}, f.err + } + if err := f.failAt[len(f.requests)]; err != nil { + return SplitResult{}, err + } + splitKey := req.SplitKey + if f.normalizeSplitKey != nil { + splitKey = f.normalizeSplitKey(splitKey) + } + baseID := req.RouteID*100 + uint64(len(f.requests))*2 + result := SplitResult{ + CatalogVersion: req.ExpectedCatalogVersion + 1, + Left: distribution.RouteDescriptor{ + RouteID: baseID, Start: distribution.CloneBytes(req.ParentStart), End: distribution.CloneBytes(splitKey), + GroupID: req.ParentGroupID, State: distribution.RouteStateActive, + }, + Right: distribution.RouteDescriptor{ + RouteID: baseID + 1, Start: distribution.CloneBytes(splitKey), End: distribution.CloneBytes(req.ParentEnd), + GroupID: req.ParentGroupID, State: distribution.RouteStateActive, + }, + } + if f.onSuccess != nil { + f.onSuccess(req, result) + } + return result, nil +} + +type fakeSampler struct { + step time.Duration + history int + cols []keyviz.MatrixColumn +} + +func (f *fakeSampler) Snapshot(from, to time.Time) []keyviz.MatrixColumn { + out := make([]keyviz.MatrixColumn, 0, len(f.cols)) + for _, col := range f.cols { + if col.At.Before(from) || !col.At.Before(to) { + continue + } + out = append(out, col) + } + return out +} + +func (f *fakeSampler) Step() time.Duration { + return f.step +} + +func (f *fakeSampler) HistoryColumns() int { + return f.history +} + +type fakeRegistrar struct { + registered []uint64 + registrations []fakeRegistration + removed []uint64 +} + +type fakeRegistration struct { + routeID uint64 + start []byte + end []byte + groupID uint64 +} + +func (f *fakeRegistrar) RegisterRoute(routeID uint64, start, end []byte, groupID uint64) bool { + f.registered = append(f.registered, routeID) + f.registrations = append(f.registrations, fakeRegistration{ + routeID: routeID, + start: distribution.CloneBytes(start), + end: distribution.CloneBytes(end), + groupID: groupID, + }) + return true +} + +func (f *fakeRegistrar) RemoveRoute(routeID uint64) { + f.removed = append(f.removed, routeID) +} diff --git a/distribution/watcher.go b/distribution/watcher.go index bb4ae095f..14a1730aa 100644 --- a/distribution/watcher.go +++ b/distribution/watcher.go @@ -19,6 +19,11 @@ var ( // CatalogWatcherOption customizes CatalogWatcher behavior. type CatalogWatcherOption func(*CatalogWatcher) +// CatalogSnapshotObserver receives each snapshot successfully applied by the +// watcher. Implementations run synchronously on the watcher goroutine and must +// not block. +type CatalogSnapshotObserver func(CatalogSnapshot) + // WithCatalogWatcherInterval sets the catalog polling interval. func WithCatalogWatcherInterval(interval time.Duration) CatalogWatcherOption { return func(w *CatalogWatcher) { @@ -47,13 +52,22 @@ func WithCatalogWatcherBatchSize(batchSize int) CatalogWatcherOption { } } +// WithCatalogWatcherSnapshotObserver installs a post-apply snapshot callback. +func WithCatalogWatcherSnapshotObserver(observer CatalogSnapshotObserver) CatalogWatcherOption { + return func(w *CatalogWatcher) { + w.snapshotObserver = observer + } +} + // CatalogWatcher periodically refreshes Engine from durable catalog snapshots. type CatalogWatcher struct { - catalog *CatalogStore - engine *Engine - interval time.Duration - batchSize int - logger *slog.Logger + catalog *CatalogStore + engine *Engine + interval time.Duration + batchSize int + logger *slog.Logger + snapshotObserver CatalogSnapshotObserver + observedVersion uint64 } // NewCatalogWatcher creates a watcher that polls the durable route catalog and @@ -75,11 +89,12 @@ func NewCatalogWatcher(catalog *CatalogStore, engine *Engine, opts ...CatalogWat } // RunCatalogWatcher runs CatalogWatcher with optional logger override. -func RunCatalogWatcher(ctx context.Context, catalog *CatalogStore, engine *Engine, logger *slog.Logger) error { +func RunCatalogWatcher(ctx context.Context, catalog *CatalogStore, engine *Engine, logger *slog.Logger, opts ...CatalogWatcherOption) error { if logger == nil { logger = slog.Default() } - catalogWatcher := NewCatalogWatcher(catalog, engine, WithCatalogWatcherLogger(logger)) + opts = append(opts, WithCatalogWatcherLogger(logger)) + catalogWatcher := NewCatalogWatcher(catalog, engine, opts...) return catalogWatcher.Run(ctx) } @@ -112,8 +127,8 @@ func (w *CatalogWatcher) Run(ctx context.Context) error { } } -// SyncOnce applies the latest durable snapshot when its version is newer than -// the current engine version. +// SyncOnce applies durable catalog deltas and notifies observers when the +// engine reaches a catalog snapshot that has not been observed yet. func (w *CatalogWatcher) SyncOnce(ctx context.Context) error { if err := w.validate(); err != nil { return err @@ -126,26 +141,82 @@ func (w *CatalogWatcher) SyncOnce(ctx context.Context) error { if err != nil { return err } + return w.applyCatalogChanges(ctx, changes) +} + +func (w *CatalogWatcher) applyCatalogChanges(ctx context.Context, changes CatalogChangeSet) error { if changes.Reset != nil { - if err := w.engine.ApplySnapshot(*changes.Reset); err != nil { - if errors.Is(err, ErrEngineSnapshotVersionStale) { - return nil - } - return err - } + return w.applyCatalogReset(*changes.Reset) + } + if len(changes.Deltas) == 0 { + return w.notifyLatestSnapshotObserver(ctx) + } + applied, err := w.applyCatalogDeltas(changes.Deltas) + if err != nil { + return err + } + if !applied { return nil } - for _, delta := range changes.Deltas { + return w.notifyLatestSnapshotObserver(ctx) +} + +func (w *CatalogWatcher) applyCatalogReset(snapshot CatalogSnapshot) error { + if err := w.engine.ApplySnapshot(snapshot); err != nil { + if errors.Is(err, ErrEngineSnapshotVersionStale) { + return nil + } + return err + } + w.notifySnapshotObserver(snapshot) + return nil +} + +func (w *CatalogWatcher) applyCatalogDeltas(deltas []CatalogDelta) (bool, error) { + applied := false + for _, delta := range deltas { if err := w.engine.ApplyDelta(delta); err != nil { if errors.Is(err, ErrEngineSnapshotVersionStale) { continue } - return err + return false, err } + applied = true + } + return applied, nil +} + +func (w *CatalogWatcher) notifySnapshotObserver(snapshot CatalogSnapshot) { + if w.snapshotObserver != nil { + w.snapshotObserver(snapshot) + } + w.observedVersion = snapshot.Version +} + +func (w *CatalogWatcher) notifyLatestSnapshotObserver(ctx context.Context) error { + if w.snapshotObserver == nil { + return nil + } + snapshot, err := w.catalog.Snapshot(ctx) + if err != nil { + return err + } + engineVersion := w.engine.Version() + if !shouldObserveCatalogSnapshot(snapshot.Version, engineVersion, w.observedVersion) { + return nil + } + if snapshot.Version != engineVersion { + return nil } + w.notifySnapshotObserver(snapshot) return nil } +func shouldObserveCatalogSnapshot(catalogVersion, engineVersion, observedVersion uint64) bool { + return catalogVersion >= engineVersion && + (catalogVersion > engineVersion || catalogVersion > observedVersion) +} + func (w *CatalogWatcher) validate() error { if err := ensureCatalogStore(w.catalog); err != nil { return err diff --git a/distribution/watcher_test.go b/distribution/watcher_test.go index 755f4c00e..2b66bed28 100644 --- a/distribution/watcher_test.go +++ b/distribution/watcher_test.go @@ -121,6 +121,66 @@ func TestCatalogWatcherNoOpWhenVersionUnchanged(t *testing.T) { require.True(t, bytes.Equal(before[1].Start, after[1].Start)) } +func TestCatalogWatcherNotifiesAfterApplyingSnapshot(t *testing.T) { + t.Parallel() + + ctx := context.Background() + catalog := NewCatalogStore(store.NewMVCCStore()) + _, err := catalog.Save(ctx, 0, []RouteDescriptor{{ + RouteID: 1, + Start: []byte(""), + End: nil, + GroupID: 1, + State: RouteStateActive, + }}) + require.NoError(t, err) + + var observed CatalogSnapshot + watcher := NewCatalogWatcher( + catalog, + NewEngine(), + WithCatalogWatcherSnapshotObserver(func(snapshot CatalogSnapshot) { + observed = snapshot + }), + ) + require.NoError(t, watcher.SyncOnce(ctx)) + require.Equal(t, uint64(1), observed.Version) + require.Len(t, observed.Routes, 1) + require.Equal(t, uint64(1), observed.Routes[0].RouteID) +} + +func TestCatalogWatcherNotifiesOnceWhenEngineAlreadyAppliedSnapshot(t *testing.T) { + t.Parallel() + + ctx := context.Background() + catalog := NewCatalogStore(store.NewMVCCStore()) + snapshot, err := catalog.Save(ctx, 0, []RouteDescriptor{{ + RouteID: 1, + Start: []byte(""), + End: nil, + GroupID: 1, + State: RouteStateActive, + }}) + require.NoError(t, err) + + engine := NewEngine() + require.NoError(t, engine.ApplySnapshot(snapshot)) + observed := 0 + watcher := NewCatalogWatcher( + catalog, + engine, + WithCatalogWatcherSnapshotObserver(func(got CatalogSnapshot) { + observed++ + require.Equal(t, snapshot.Version, got.Version) + require.Equal(t, snapshot.Routes, got.Routes) + }), + ) + + require.NoError(t, watcher.SyncOnce(ctx)) + require.NoError(t, watcher.SyncOnce(ctx)) + require.Equal(t, 1, observed) +} + func TestCatalogWatcherRetriesOnTransientReadError(t *testing.T) { t.Parallel() diff --git a/docs/design/2026_02_18_partial_hotspot_shard_split.md b/docs/design/2026_02_18_partial_hotspot_shard_split.md index e2c26ca01..42f2f213b 100644 --- a/docs/design/2026_02_18_partial_hotspot_shard_split.md +++ b/docs/design/2026_02_18_partial_hotspot_shard_split.md @@ -4,17 +4,18 @@ Elastickv already has shard boundaries, but it does not yet have the control-plane needed for safe automatic hotspot splitting. -Current implementation status (updated July 7, 2026): +Current implementation status (updated July 18, 2026): - M1 durable route catalog, watcher refresh, and same-group manual `SplitRange` are implemented. -- The old in-memory `RecordAccess` / midpoint `splitRange` path has been - removed by the M3-PR1a slice in - [`2026_06_11_partial_hotspot_split_milestone3_automation.md`](2026_06_11_partial_hotspot_split_milestone3_automation.md); - keyviz remains the only wired load-observation signal for future auto-split. +- Standalone M3 auto-split is implemented: keyviz supplies the sole load signal, + and the detector/scheduler issues guarded same-group `SplitRange` operations. + See + [`2026_06_11_implemented_hotspot_split_milestone3_automation.md`](2026_06_11_implemented_hotspot_split_milestone3_automation.md). - There is no data movement workflow to relocate part of a split range to another Raft group. -So practical “hotspot splitting” is currently not connected end-to-end. +Same-group hotspot splitting is connected end-to-end. Cross-group relocation +still depends on the M2 movement workflow and the M3-PR4 target policy. For a current runtime/component snapshot, see `docs/architecture_overview.md`. @@ -299,10 +300,11 @@ Add RPCs: 2. Hotspot detector 3. Auto-split scheduler (cooldown/hysteresis) -Status: partial. M3-PR1a removed the dead engine-local `RecordAccess` / -`splitRange` path and relaxed the route-catalog decoder for the later -`SplitAtHLC` schema tail. Detector and scheduler wiring remain open in -[`2026_06_11_partial_hotspot_split_milestone3_automation.md`](2026_06_11_partial_hotspot_split_milestone3_automation.md). +Status: implemented for standalone same-group automation. Detector, scheduler, +compound isolation, cooldown/hysteresis, leadership fencing, runtime control, +metrics, and three-node end-to-end coverage are complete in +[`2026_06_11_implemented_hotspot_split_milestone3_automation.md`](2026_06_11_implemented_hotspot_split_milestone3_automation.md). +Cross-group target selection remains the post-M2 M3-PR4 follow-on. ### Milestone 4: Hardening diff --git a/docs/design/2026_06_11_partial_hotspot_split_milestone3_automation.md b/docs/design/2026_06_11_implemented_hotspot_split_milestone3_automation.md similarity index 71% rename from docs/design/2026_06_11_partial_hotspot_split_milestone3_automation.md rename to docs/design/2026_06_11_implemented_hotspot_split_milestone3_automation.md index fae7d93e2..18491dd55 100644 --- a/docs/design/2026_06_11_partial_hotspot_split_milestone3_automation.md +++ b/docs/design/2026_06_11_implemented_hotspot_split_milestone3_automation.md @@ -1,12 +1,11 @@ # Hotspot Shard Split — Milestone 3: Automation -Status: Partial - M3-PR1a is implemented: the dead `RecordAccess` / -`splitRange` / threshold constructor path is removed, and the route-catalog -decoder now keeps v1 strict while accepting forward-version tails. M3-PR2a adds -the pure autosplit detector core. M3-PR2b-a adds the committed-window -`MemSampler.Snapshot` reader and observe-only detector bridge. M3-PR1b, the -remaining M3-PR2b Top-K / leadership-watermark work, and scheduler wiring remain -open. +Status: Implemented (standalone M3). The keyviz-backed detector, exact committed +window and Top-K alignment, same-group scheduler, compound single-key isolation, +catalog-key and shard-group leadership fences, durable cooldown reconstruction, +runtime control, bounded metrics, watcher/sampler reconciliation, and real +three-node leadership-transfer coverage are implemented. M3-PR4 cross-group +target selection remains a post-M2 follow-on and is not part of standalone M3. Author: bootjp Date: 2026-06-11 Updated: 2026-07-22 @@ -63,7 +62,7 @@ M3's job is to close the gap **without building a third counter pipeline**: driv 3. Detection state, scoring, hysteresis, and cooldown are **deterministic and unit-testable** in isolation from the data plane. 4. **Default OFF**, behind a flag (`--autoSplit`), with a runtime kill switch and bounded-cardinality metrics. 5. **Compose with M2**: when M2 lands, the scheduler picks a `target_group_id` (least-loaded group) and passes it to the same `SplitRange` RPC. M3 does not reimplement migration. -6. Guardrails: minimum route size, maximum route count, per-cycle split budget, and per-route cooldown to prevent split storms and detector flapping. +6. Guardrails: structural splittability, maximum route count, per-cycle split budget, and per-route cooldown to prevent split storms and detector flapping. ### 2.2 Non-Goals @@ -113,7 +112,7 @@ if *autoSplit && !keyBucketsExplicit && buckets <= 1 { } ``` -`flag.Visit` is the idiomatic Go answer here; no config-wrapper type is needed. **`flag.CommandLine.Lookup("keyvizKeyBucketsPerRoute")` is *not* an equivalent presence check and must not be substituted (codex P2):** Go's `flag` package defines `Visit` as iterating only flags that were *set*, whereas `Lookup` returns the named flag whenever it is *defined* (registered), regardless of whether the operator supplied it. Using `Lookup` for presence would make `keyBucketsExplicit` always `true`, so bare `--autoSplit` would keep the default bucket count of 1 and reintroduce the no-split-by-default trap this section exists to eliminate. The only correct presence checks are `flag.Visit` (above) or the equivalent `flag.CommandLine.Visit`; `Lookup` answers "is the flag defined," not "did the operator set it." The same `keyBucketsExplicit` computation is mirrored in `cmd/server/demo.go` (which has its own flag set, §7.1). With this, the implied-vs-explicit config is: +`flag.Visit` is the idiomatic Go answer here; no config-wrapper type is needed. **`flag.CommandLine.Lookup("keyvizKeyBucketsPerRoute")` is *not* an equivalent presence check and must not be substituted (review P2):** Go's `flag` package defines `Visit` as iterating only flags that were *set*, whereas `Lookup` returns the named flag whenever it is *defined* (registered), regardless of whether the operator supplied it. Using `Lookup` for presence would make `keyBucketsExplicit` always `true`, so bare `--autoSplit` would keep the default bucket count of 1 and reintroduce the no-split-by-default trap this section exists to eliminate. The only correct presence checks are `flag.Visit` (above) or the equivalent `flag.CommandLine.Visit`; `Lookup` answers "is the flag defined," not "did the operator set it." The same `keyBucketsExplicit` computation is mirrored in `cmd/server/demo.go` (which has its own flag set, §7.1). With this, the implied-vs-explicit config is: | Flag | Bare keyviz default | With `--autoSplit` (no other keyviz flags) | Effect on auto-split | |---|---|---|---| @@ -152,7 +151,7 @@ Open question OQ-1 (§9) asks reviewers whether to fully delete `Route.Load`/`St ### 4.1 Decision: leader-local consumption of the local sampler, no new collection RPC (M3); optional fan-out reuse later -The detector runs **only on the default-group leader** (where the scheduler must run anyway, since `SplitRange` mutates the catalog through the default group). The question is how the leader gets per-route load. +The detector runs **only on the catalog-key leader** (the Raft group that owns `distribution.CatalogVersionKey()`, where the scheduler must run because `SplitRange` mutates catalog metadata). The question is how the leader gets per-route load. Options considered: @@ -162,13 +161,13 @@ Options considered: **Recommendation: option 3 (leader-local).** It needs no new wire surface and composes with the existing `--autoSplit` leader-local design (§7.6). The detector aggregates the sampler's recent windows (last N flush columns, default N=3 to match the parent doc's "3 consecutive windows" candidate rule, §5.2) for routes with positive local evidence. -**Follower-forwarded write caveat (codex P2).** `ShardedCoordinator.observeMutation` runs on the node that received the client request before `ShardRouter.Commit` forwards (`kv/sharded_coordinator.go:1795-1844`), and the follower's `Internal.Forward` handler calls `transactionManager.Commit` directly (`adapter/internal.go:52`) rather than re-entering the leader's `ShardedCoordinator` observation hook. Therefore even a route whose shard group is led by the default-group leader can be **under-observed** if most clients connect through followers. M3 does not treat "missing local rows" as evidence of low load: local evidence can promote a split, but absence of local evidence only means "unknown" and is fail-closed for target selection (§6.2) and conservative for candidacy (fewer splits). Full coverage for follower-forwarded writes needs follower reports / fan-out into the same detector interface and is tracked under OQ-5. +**Follower-forwarded write caveat (review P2).** `ShardedCoordinator.observeMutation` runs on the node that received the client request before `ShardRouter.Commit` forwards (`kv/sharded_coordinator.go:1795-1844`), and the follower's `Internal.Forward` handler calls `transactionManager.Commit` directly (`adapter/internal.go:52`) rather than re-entering the leader's `ShardedCoordinator` observation hook. Therefore even a route whose shard group is led by the default-group leader can be **under-observed** if most clients connect through followers. M3 does not treat "missing local rows" as evidence of low load: local evidence can promote a split, but absence of local evidence only means "unknown" and is fail-closed for target selection (§6.2) and conservative for candidacy (fewer splits). Full coverage for follower-forwarded writes needs follower reports / fan-out into the same detector interface and is tracked under OQ-5. **Why not the keyviz cluster fan-out** (`docs/design/2026_04_27_implemented_keyviz_cluster_fanout.md`): that aggregates across nodes for the admin heatmap UI and dedupes write samples by `(groupID, leaderTerm)`. Pulling cross-node data in would add latency and a dependency on the fan-out being configured, so M3 stays leader-local and fail-closed on unknown load. If a future milestone wants complete cluster-wide scoring, including follower-forwarded writes, it can read the same fan-out aggregate behind the same detector interface (§5) — recorded as OQ-5. -**Known limitation — multi-leader (multi-group) deployments.** Leader-local consumption is only authoritative for routes whose shard group is led by the **same node** that runs the detector — i.e. the default-group leader (where the scheduler must run, §6.1). In a multi-group cluster (`--raftGroups` with G1, G2, …), the default-group leader is **not necessarily** the leader of G1/G2/…: each node runs a single `ShardedCoordinator` and `MemSampler` and only `Observe`s traffic it *receives* (`kv/sharded_coordinator.go:1795-1824`). Concretely, in a 3-node cluster where node A leads the default group and node B leads G1, **node A's sampler has no data for routes in G1 served through node B**, so the detector on A cannot score or split them. This is **broader than "heavy follower-forwarded reads"**: it is a structural gap for any route whose group leader differs from the default-group leader, not just an edge case of read forwarding. +**Known limitation — multi-leader (multi-group) deployments.** Leader-local consumption is only authoritative for routes whose shard group is led by the **same node** that runs the detector — i.e. the catalog-key leader (where the scheduler must run, §6.1). In a multi-group cluster (`--raftGroups` with G1, G2, …), the catalog-key leader is **not necessarily** the leader of G1/G2/…: each node runs a single `ShardedCoordinator` and `MemSampler` and only `Observe`s traffic it *receives* (`kv/sharded_coordinator.go:1795-1824`). Concretely, in a 3-node cluster where node A leads the catalog-key group and node B leads G1, **node A's sampler has no data for routes in G1 served through node B**, so the detector on A cannot score or split them. This is **broader than "heavy follower-forwarded reads"**: it is a structural gap for any route whose group leader differs from the catalog-key leader, not just an edge case of read forwarding. -M3 treats this as an explicit **non-goal / known limitation, not a defect**: in M3, routes whose shard group's leader is a different node than the default-group leader are **invisible to the detector** (no load data → no candidacy → never auto-split). Under-observation only biases the detector conservative (it splits fewer routes, never an unsafe one — §2.2.5), so the limitation is safe, just incomplete. Closing it requires cross-node scoring (reading per-route load from the group leaders, e.g. via the keyviz cluster fan-out aggregate behind the same detector interface, §5). That is deferred to a future milestone and tracked as the reframed OQ-5 below. M3-PR2/PR3 reviewers and operators must understand that a **single-default-group deployment is not automatically complete coverage**: M3 only covers traffic observed by the default-group leader itself. Requests that enter through followers can still be under-observed until follower reports / fan-out land, because the forwarded leader-side path bypasses the sampler hook (§4.1 caveat above). Multi-group deployments have the additional structural limit that only routes led by the default-group leader can produce local evidence at all. +M3 treats this as an explicit **non-goal / known limitation, not a defect**: in M3, routes whose shard group's leader is a different node than the catalog-key leader are **invisible to the detector** (no load data → no candidacy → never auto-split). Under-observation only biases the detector conservative (it splits fewer routes, never an unsafe one — §2.2.5), so the limitation is safe, just incomplete. Closing it requires cross-node scoring (reading per-route load from the group leaders, e.g. via the keyviz cluster fan-out aggregate behind the same detector interface, §5). That is deferred to a future milestone and tracked as the reframed OQ-5 below. M3-PR2/PR3 reviewers and operators must understand that a **single-catalog-key-leader deployment is not automatically complete coverage**: M3 only covers traffic observed by the catalog-key leader itself. Requests that enter through followers can still be under-observed until follower reports / fan-out land, because the forwarded leader-side path bypasses the sampler hook (§4.1 caveat above). Multi-group deployments have the additional structural limit that only routes led by the catalog-key leader can produce local evidence at all. ### 4.2 What the detector consumes per route @@ -180,7 +179,7 @@ For each route the local node leads, over the trailing window set: These come from `MemSampler.Snapshot(from, to)` `[]MatrixColumn` (`keyviz/sampler.go:1463`) filtered to routes whose `RaftGroupID` the local node currently leads (the `MatrixRow.RaftGroupID` field, `keyviz/sampler.go:449`, plus the engine's leader state per group). -**Per-column route aggregation before scoring (codex P2).** With `--keyvizKeyBucketsPerRoute > 1`, a single live route can emit multiple non-aggregate `MatrixRow`s in the same committed `MatrixColumn` — one per sub-bucket, all carrying the same `RouteID` / `RaftGroupID`. The detector must first normalize each committed column into route-level totals by grouping all **non-aggregate** rows by `(RouteID, RaftGroupID)` and summing `read_ops`, `write_ops`, `read_bytes`, and `write_bytes` before applying §5.1 scoring, §5.2 hysteresis, Top-K share gates, or target-group load aggregation. The original sub-bucket rows are retained separately only as split-key evidence for §5.4; they are not individually scored as route candidates. `MatrixRow.Aggregate` rows remain skipped per §5.0 and never contribute to a route total. This prevents a hot route whose writes are spread across buckets from being undercounted by a factor up to the bucket count. +**Per-column route aggregation before scoring (review P2).** With `--keyvizKeyBucketsPerRoute > 1`, a single live route can emit multiple non-aggregate `MatrixRow`s in the same committed `MatrixColumn` — one per sub-bucket, all carrying the same `RouteID` / `RaftGroupID`. The detector must first normalize each committed column into route-level totals by grouping all **non-aggregate** rows by `(RouteID, RaftGroupID)` and summing `read_ops`, `write_ops`, `read_bytes`, and `write_bytes` before applying §5.1 scoring, §5.2 hysteresis, Top-K share gates, or target-group load aggregation. The original sub-bucket rows are retained separately only as split-key evidence for §5.4; they are not individually scored as route candidates. `MatrixRow.Aggregate` rows remain skipped per §5.0 and never contribute to a route total. This prevents a hot route whose writes are spread across buckets from being undercounted by a factor up to the bucket count. **Deriving `from`/`to` (the time-range → committed-column recipe).** `Snapshot` takes a *time range*, not a column count, so the detector must convert "new committed columns since the last evaluation" and "trailing `N` columns for display" into `(from, to)` carefully, **excluding the current still-accumulating column**. Each emitted column carries its flush instant in `MatrixColumn.At` (`keyviz/sampler.go:424`); `Flush()` (`keyviz/sampler.go:1178`) commits a column at each `Step` boundary. The recipe: @@ -230,7 +229,7 @@ The §8.2 tests include the default-possible hot-hot-migrating-hot sequence wher ### 5.1 Scoring -There are **two** weighted ops/min scores, computed from the **same** committed flush columns but used for **different** purposes — keeping them separate is what makes the hysteresis counter mean exactly "N consecutive hot columns" (codex P2, review round 6). Both formulas are pinned (this is also what resolves OQ-8 — the normalization is no longer left to implementation): +There are **two** weighted ops/min scores, computed from the **same** committed flush columns but used for **different** purposes — keeping them separate is what makes the hysteresis counter mean exactly "N consecutive hot columns" (review P2, review round 6). Both formulas are pinned (this is also what resolves OQ-8 — the normalization is no longer left to implementation): ``` column_i.duration_s = (column_i.At - column_i.WindowStart).Seconds() @@ -253,7 +252,7 @@ smoothed_score = smoothed_write_rate × Ww + smoothed_read_rate × Wr Defaults from the parent doc §6.2: `Ww = 4`, `Wr = 1`, `threshold = 50_000 ops/min`. -**Why two scores, not one.** An earlier revision used the `N`-column average for *both* the display and the hysteresis test, which silently broke the parent doc §6.2 rule "Candidate after 3 consecutive windows over threshold" (codex P2): (a) a single **below-threshold** column could be hidden inside the moving average — e.g. with `N = 3` a hot/hot/below-threshold run still averages above threshold and wrongly advances the counter — and (b) cold start needed `2N−1 = 5` hot flushes before promotion (the average could not even be evaluated until `N` columns existed, then needed `N` more advances). Driving the counter off the **per-column** score (A) restores the literal "N consecutive hot columns" meaning: each individual flush window passes or fails the threshold on its own, a below-threshold column resets the counter immediately, and cold start promotes after exactly `N` hot columns. The averaged score (B) is retained **only** because operators want a smoothed rate in logs/diagnostics that does not jitter column-to-column; it has no role in any decision. +**Why two scores, not one.** An earlier revision used the `N`-column average for *both* the display and the hysteresis test, which silently broke the parent doc §6.2 rule "Candidate after 3 consecutive windows over threshold" (review P2): (a) a single **below-threshold** column could be hidden inside the moving average — e.g. with `N = 3` a hot/hot/below-threshold run still averages above threshold and wrongly advances the counter — and (b) cold start needed `2N−1 = 5` hot flushes before promotion (the average could not even be evaluated until `N` columns existed, then needed `N` more advances). Driving the counter off the **per-column** score (A) restores the literal "N consecutive hot columns" meaning: each individual flush window passes or fails the threshold on its own, a below-threshold column resets the counter immediately, and cold start promotes after exactly `N` hot columns. The averaged score (B) is retained **only** because operators want a smoothed rate in logs/diagnostics that does not jitter column-to-column; it has no role in any decision. **Why this normalization (both formulas).** Dividing by each column's actual committed duration (per-column) or by the sum of actual committed durations (smoothed) and rescaling by 60 makes each score an **ops-per-minute** rate, so the `50_000 ops/min` threshold means the same thing regardless of `--keyvizStep` and regardless of delayed/coalesced flusher commits — the `--keyvizStep` ↔ sensitivity coupling that OQ-8 worried about is removed by construction. `ops/min` is also the unit operators reason in when setting `--autoSplitThreshold`, so the flag and both formulas speak the same language. The smoothed score's `N` is for display smoothing only; the promotion decision is always a `window = 1` per-column test, so `candidateWindows` controls *how many consecutive hot columns* are required, never *what counts as one hot column*. (If a future need arises to widen the smoothing window independently of the hysteresis length, add an `--autoSplitSmoothColumns` flag defaulting to `candidateWindows` for the *display* score only — the per-column promotion test stays at `window = 1`. Not done in M3 to keep the surface small.) @@ -275,7 +274,7 @@ Down-side reset: the detector tracks a per-route `consecutiveOver` counter in it ### 5.3 Cooldown (per route) -After a split is **committed** for a route — meaning `SplitRange` returned success and the scheduler has either the response's child descriptors or a refreshed catalog snapshot proving the children exist — those child routes enter a **cooldown** of `splitCooldown` (parent doc §6.2 default 10 minutes). Cooldown is **not** started when the `SplitRange` RPC is merely issued. A `cas_conflict`, `rpc_error`, timeout, or `target_unavailable` failure leaves the parent route eligible to re-evaluate on the next committed column/cycle (subject only to normal candidate hysteresis and any optional short RPC backoff); it must not burn child cooldown for route IDs that were never created. During cooldown the detector will not promote the child to a candidate, regardless of score. Cooldown is keyed by route lineage (the child `RouteID`s produced by the split / observed via the next catalog snapshot), tracked in the leader-local state map with a wall-clock expiry that is **diagnostic-only** for logging but enforced via a monotonic deadline (`internal/monoclock`) so a wall-clock step cannot shorten or extend a cooldown (CLAUDE.md: no ordering-sensitive use of wall clock). +After a split is **committed** for a route — meaning `SplitRange` returned success and the scheduler has either the response's child descriptors or a refreshed catalog snapshot proving the children exist — those child routes enter a **cooldown** of `splitCooldown` (parent doc §6.2 default 10 minutes). Cooldown is **not** started when the `SplitRange` RPC is merely issued. A `cas_conflict`, `rpc_error`, timeout, or `target_unavailable` failure leaves the parent route eligible to re-evaluate on the next committed column/cycle (subject only to normal candidate hysteresis and any optional short RPC backoff); it must not burn child cooldown for route IDs that were never created. During cooldown the detector will not promote the child to a candidate, regardless of score. Cooldown is keyed by route lineage (the child `RouteID`s produced by the split / observed via the next catalog snapshot). After reconstruction from wall-clock `SplitAtHLC`, the remaining duration is represented by a Go `time.Time` deadline derived from `time.Now`; Go preserves its monotonic component for in-process comparisons, so later wall-clock steps do not shorten or extend enforcement. **Compound single-key isolation is cooldown-exempt for its own intermediate child on the same leader.** The §5.4.2 single-key-isolation flow first splits `[A,B) → [A,H) + [H,B)`, then immediately splits the *intermediate* child `[H,B)` again to isolate `{H}`. If the first split's cooldown applied to `[H,B)` while the same scheduler is still executing the compound, the second `SplitRange` could never fire and the hot key would never be isolated — which is the contradiction review flagged. The detector therefore treats the compound decision as a **single logical split** during that scheduler execution: cooldown is applied to the **final** children only — `[A,H)`, `{H}` = `[H, H·0x00)`, and `[H·0x00, B)` — and the *transient* intermediate child `[H,B)` is exempt because it exists only between the two calls of the same operation and is consumed by the second call. This exemption is scoped strictly to the compound operation's own intermediate child (matched by the leader-local compound execution record during the leader term that issued call 1); it does **not** weaken cooldown for any independently-scheduled split. @@ -287,7 +286,7 @@ If the second call does not run before leadership changes or the scheduler resta Per parent doc §3.1.3 and §6.3, the split key comes from the **observed key distribution**, not a byte midpoint. **Evaluation priority is Top-K single-key isolation first, then sub-range p50, then decline**: a route with both Top-K and sub-bucket evidence must first test whether one sampled key dominates enough to isolate; the detector uses p50 only when Top-K data is unavailable, disabled, misaligned for the scored column, or explicitly declined by the share/absolute-floor gates. This priority matters because bare `--autoSplit` enables sub-range buckets by default (§7.1), so a p50-first implementation would never isolate a true single-key hotspot. -1. **Top-K single-key skew — isolate the hot key (compound split).** When Top-K is enabled (`--keyvizHotKeysEnabled`) and one key's share of the route's writes is `>= topKeyShare` (parent doc §6.3 default 0.8) **AND** the hot key's absolute weighted contribution is at least `topKeyAbsoluteFloor` (default = `autoSplitThreshold / 2` = 25_000 weighted ops/min), split so the hot key `H` lands in its own single-key child. The absolute-weighted-contribution gate (codex round-7 P2) prevents a wasteful compound isolation on a read-promoted route where a single incidental write to `H` gives `H` 100% of route writes — share alone would pin a write key that did not cause the hotspot, burning the split budget and cooldown instead of using the sub-range p50 path below to relieve the read-heavy range. +1. **Top-K single-key skew — isolate the hot key (compound split).** When Top-K is enabled (`--keyvizHotKeysEnabled`) and one key's share of the route's writes is `>= topKeyShare` (parent doc §6.3 default 0.8) **AND** the hot key's absolute weighted contribution is at least `topKeyAbsoluteFloor` (default = `autoSplitThreshold / 2` = 25_000 weighted ops/min), split so the hot key `H` lands in its own single-key child. The absolute-weighted-contribution gate (review round-7 P2) prevents a wasteful compound isolation on a read-promoted route where a single incidental write to `H` gives `H` 100% of route writes — share alone would pin a write key that did not cause the hotspot, burning the split budget and cooldown instead of using the sub-range p50 path below to relieve the read-heavy range. The Top-K gate is evaluated against the **same committed flush column** that produced the per-column route score. `KeyvizHotKeyEntry.Count` is a raw sampled-stream count (`keyviz/hot_keys.go:46-49`), so the detector first scales it by the snapshot's `SampleRate`: `H_writes_est = entry.Count × snapshot.SampleRate`. The raw scaled share estimate uses the same route-level denominator as §4.2's aggregated per-column route total: `hot_key_share_est = min(H_writes_est, column_i.write_ops) / column_i.write_ops`, with `column_i.write_ops` taken from the summed non-aggregate rows for that route in the same committed column. If `column_i.write_ops == 0`, aligned Top-K data is missing, or the column's `(WindowStart, At]` boundary is not proven, the Top-K gate is unavailable and falls through to p50. The `min` clamp prevents sampling variance from producing a share above 1.0, but it does not remove the required sample-rate scaling; comparing raw `entry.Count / column_i.write_ops` would undercount by roughly `SampleRate` and make the default `HotKeysSampleRate = 16` fail the 0.8 share gate even for a key that caused every write. For diagnostics and for the uncertainty branch below, the detector also computes the upper-bound absolute contribution in the **same per-minute units as §5.1's score formula (A)** so it stays invariant under `--keyvizStep` and delayed/coalesced flushes (claude round-10 issue 1): `H_write_rate_est = H_writes_est / column_i.duration_s × 60` (ops/min, normalised consistently with §5.1) and `absolute_contribution_est = H_write_rate_est × Ww` (weighted ops/min). Writes-only because Top-K is currently write-only on the hot path; future read Top-K can extend with `+ H_read_rate × Wr`. Using the raw sampled count, or the raw unnormalised column count, would undercount and reintroduce the Step↔sensitivity coupling OQ-8 was resolved to eliminate. @@ -297,21 +296,21 @@ Per parent doc §3.1.3 and §6.3, the split key comes from the **observed key di When the share-only gate fires but the absolute floor does not, the detector logs `autosplit_isolation_declined_total{reason="absolute_floor"}` and falls through to the sub-range p50 path below instead. Isolating `H` from a route `[A, B)` normally takes **two boundaries** — one at `H` and one at `H`'s successor `H·0x00` — so the middle child is exactly `[H, H·0x00)` = `{H}`. The detector emits this as **one atomic compound decision** carrying both split keys, not two independent decisions across cycles. The scheduler executes it as a back-to-back pair of `SplitRange` calls (first `[A,B) → [A,H) + [H,B)`, then `[H,B) → [H, H·0x00) + [H·0x00, B)`), counted and budgeted as described in §6.4 (compound = 1 budget unit, cooldown exempt for the intermediate child). This is the single-hot-key hotspot case the whole feature targets. - - **Edge cases — `H` touches a route boundary (review, codex P2).** `validateSplitKey` rejects `splitKey == parent.Start` or `splitKey == parent.End` (`adapter/distribution_server.go:358-371`), so the two-boundary form must collapse to a one-boundary form whenever one of the two isolation boundaries is already at, or beyond, the live route edge. Lower edge: if `H == route.Start`, `{H}` is already pinned to the left edge and needs only **one** boundary at `H·0x00`: a single `SplitRange([A,B), split_key=H·0x00)` yields `[A, H·0x00) = [H, H·0x00) = {H}` (since `A == H`) plus `[H·0x00, B)`. The "already isolated" decline is limited to finite ends: only when `route.End != nil && H·0x00 >= route.End` in that lower-edge case is the route already exactly `{H}` (or smaller/invalid) and declined. A rightmost route with `route.End == nil` treats the end as `+inf`, so `H·0x00` is a valid finite interior boundary and must not be rejected by comparing it to nil/empty bytes. The lower-edge one-boundary decision emits `split_origin = isolation_single_lower_edge` and is **same-group-only** under §6.2, because M2 can only move the right child and the hot singleton is the left child. Upper edge: if `route.Start < H` and `route.End != nil && H·0x00 >= route.End` (for example the finite route `[A, H·0x00)`), the second boundary would be invalid at the upper edge, but a single `SplitRange([A,B), split_key=H)` is valid and yields `[A,H)` plus `[H,B) = {H}` as the right child. The upper-edge one-boundary decision emits `split_origin = isolation_single_upper_edge`; unlike the lower-edge case, the hot singleton is the right child, so §6.2 may apply cross-group target selection if every target-admission gate passes. Both one-boundary cases have `routeDelta = +1`; the two-boundary compound is emitted only when `route.Start < H` and (`route.End == nil || H·0x00 < route.End`). The §8.2 split-key tests add the lower-edge single-split case, the rightmost lower-edge case where `route.End == nil` keeps `H·0x00` valid, the finite upper-edge single-split case, the lower-edge-cross-group-forced-same-group case, and the already-isolated finite `[H, H·0x00)` decline case. + - **Edge cases — `H` touches a route boundary (review, review P2).** `validateSplitKey` rejects `splitKey == parent.Start` or `splitKey == parent.End` (`adapter/distribution_server.go:358-371`), so the two-boundary form must collapse to a one-boundary form whenever one of the two isolation boundaries is already at, or beyond, the live route edge. Lower edge: if `H == route.Start`, `{H}` is already pinned to the left edge and needs only **one** boundary at `H·0x00`: a single `SplitRange([A,B), split_key=H·0x00)` yields `[A, H·0x00) = [H, H·0x00) = {H}` (since `A == H`) plus `[H·0x00, B)`. The "already isolated" decline is limited to finite ends: only when `route.End != nil && H·0x00 >= route.End` in that lower-edge case is the route already exactly `{H}` (or smaller/invalid) and declined. A rightmost route with `route.End == nil` treats the end as `+inf`, so `H·0x00` is a valid finite interior boundary and must not be rejected by comparing it to nil/empty bytes. The lower-edge one-boundary decision emits `split_origin = isolation_single_lower_edge` and is **same-group-only** under §6.2, because M2 can only move the right child and the hot singleton is the left child. Upper edge: if `route.Start < H` and `route.End != nil && H·0x00 >= route.End` (for example the finite route `[A, H·0x00)`), the second boundary would be invalid at the upper edge, but a single `SplitRange([A,B), split_key=H)` is valid and yields `[A,H)` plus `[H,B) = {H}` as the right child. The upper-edge one-boundary decision emits `split_origin = isolation_single_upper_edge`; unlike the lower-edge case, the hot singleton is the right child, so §6.2 may apply cross-group target selection if every target-admission gate passes. Both one-boundary cases have `routeDelta = +1`; the two-boundary compound is emitted only when `route.Start < H` and (`route.End == nil || H·0x00 < route.End`). The §8.2 split-key tests add the lower-edge single-split case, the rightmost lower-edge case where `route.End == nil` keeps `H·0x00` valid, the finite upper-edge single-split case, the lower-edge-cross-group-forced-same-group case, and the already-isolated finite `[H, H·0x00)` decline case. If Top-K is **not** enabled, single-key isolation is unavailable; the detector falls through to the §5.4 sub-range-p50 path (which still relieves a *range* skew but cannot pin a single key), so the operator loses only this path, not all splits. 2. **Sub-range p50 fallback after Top-K.** When Top-K isolation is unavailable/declined and `--keyvizKeyBucketsPerRoute > 1` **and the row's effective `SubBucketCount > 1`**, use the route's sub-range bucket rows to find the bucket whose **cumulative load first crosses 50%**, and use the **`hi` (upper boundary)** of that bucket — `SubBucketBoundsFor(routeID, bucket)` returns `(lo, hi []byte, ok bool)` (`keyviz/sampler.go:559`); the split key is the `hi` of the median bucket, i.e. the boundary *between* the p50 bucket and the next one, so all of the p50 bucket's load stays in the left child. This places the boundary where load is actually halved. If the sampler collapses the route to an **effective single sub-bucket** (`SubBucketCount == 1`) despite the configured bucket count — for example a long common-prefix range or an unbounded tail whose first bucket spans the whole live interval — the sub-range row carries no interior boundary evidence. The detector treats that as **no sub-range evidence** and declines via §5.4.3 rather than trying to split at the route boundary. - - **Boundary-bucket fallback (review, codex P2).** `SubBucketBoundsFor` returns the **route's own `End`** as `hi` for the last sub-bucket, and `nil` only when the last sub-bucket is the unbounded tail (`keyviz/sampler.go:570-575` — the single-bucket case at `:571-573` likewise returns `start, end`). When the median (p50) bucket is the **last** bucket — an upper-tail hotspot where the final bucket alone carries `> 50%` of the load — its `hi` is either `route.End` or `nil`, and §5.4's `split_key < route.End` rule plus `validateSplitKey`'s boundary rejection (`adapter/distribution_server.go:359-360,370-371`: `splitKey == parent.End` → `errDistributionSplitKeyAtBoundary`) would make that key invalid, so the detector would drop the decision and never split a genuinely hot tail. The detector therefore **falls back to the median bucket's `lo` (its lower boundary) only when `hi == nil` or (`route.End != nil && hi >= route.End`)**: an interior boundary that still places the dominant last bucket in the *right* child while satisfying `route.Start < split_key < route.End`. For a rightmost route with `route.End == nil`, any non-last median bucket whose `hi` is finite remains a valid interior `hi` split; the detector must not take the `lo` fallback merely because the parent route is unbounded. Symmetrically, if the p50 bucket is the **first** bucket and its `lo` equals `route.Start`, the detector uses that bucket's `hi` instead (it is interior because this path is gated on effective `SubBucketCount > 1`). + - **Boundary-bucket fallback (review, review P2).** `SubBucketBoundsFor` returns the **route's own `End`** as `hi` for the last sub-bucket, and `nil` only when the last sub-bucket is the unbounded tail (`keyviz/sampler.go:570-575` — the single-bucket case at `:571-573` likewise returns `start, end`). When the median (p50) bucket is the **last** bucket — an upper-tail hotspot where the final bucket alone carries `> 50%` of the load — its `hi` is either `route.End` or `nil`, and §5.4's `split_key < route.End` rule plus `validateSplitKey`'s boundary rejection (`adapter/distribution_server.go:359-360,370-371`: `splitKey == parent.End` → `errDistributionSplitKeyAtBoundary`) would make that key invalid, so the detector would drop the decision and never split a genuinely hot tail. The detector therefore **falls back to the median bucket's `lo` (its lower boundary) only when `hi == nil` or (`route.End != nil && hi >= route.End`)**: an interior boundary that still places the dominant last bucket in the *right* child while satisfying `route.Start < split_key < route.End`. For a rightmost route with `route.End == nil`, any non-last median bucket whose `hi` is finite remains a valid interior `hi` split; the detector must not take the `lo` fallback merely because the parent route is unbounded. Symmetrically, if the p50 bucket is the **first** bucket and its `lo` equals `route.Start`, the detector uses that bucket's `hi` instead (it is interior because this path is gated on effective `SubBucketCount > 1`). - **P50 cross-group requires the right child to carry the hotter side.** The detector marks each emitted decision with a `split_origin ∈ {p50_mid, p50_last_bucket_lo, p50_first_bucket_hi, isolation_compound, isolation_single_lower_edge, isolation_single_upper_edge, decline}` enum and p50 child-load estimates derived from the same sub-bucket histogram. For `hi` splits, the median bucket's load stays in the left child; for `lo` last-bucket fallback, the dominant last bucket moves to the right child. Because M2's `SplitRange` semantics make the **right** child the migration target, §6.2 may apply cross-group target selection to a p50 decision only when the estimated right-child weighted load is at least the left-child load (or the detector otherwise proves the right child carries the observed hot side). If the left child is hotter or the estimate is unknown, the scheduler issues a same-group split (`target_group_id = 0`) even under `--autoSplitCrossGroup`. This generalizes the earlier first-bucket rule: `p50_first_bucket_hi` is always same-group because the first bucket remains left, while `p50_mid` is cross-group only when the right child is the hot child. The same-group split still relieves range pressure, but M3 **does not promise** that repeated lower-tail or left-heavy p50 decisions will later become cross-group migratable without a left-child relocation / `MoveRange`-style primitive. If both candidate boundaries coincide with a route edge, the detector declines per §5.4.3. The §8.2 split-key/target tests add the "effective `SubBucketCount == 1` → no sub-range evidence", "rightmost route with interior p50 bucket → finite `hi`, not `lo` fallback", "p50 in the last bucket → interior `lo` fallback", "p50 in the first bucket → interior `hi` fallback and same-group", and "p50_mid with left-estimated load > right-estimated load under `--autoSplitCrossGroup` → same-group" cases. -3. **Fallback — coarse boundary.** When neither usable Top-K nor sub-range buckets are enabled (route-granular sampling only), the detector has no intra-route distribution. Rather than fall back to a blind midpoint (which the parent doc forbids and which can produce an empty child), the detector **declines to split** and emits a metric/log (`autosplit_skipped_total{reason="no_split_key"}`) telling the operator to raise `--keyvizKeyBucketsPerRoute`. OQ-3 asks whether a midpoint-of-observed-min/max-key fallback is acceptable here instead of declining. +3. **Fallback — decline without evidence.** When neither usable Top-K nor sub-range buckets are enabled (route-granular sampling only), the detector has no intra-route distribution. It **declines to split** and emits `autosplit_skipped_total{reason="no_split_key"}` rather than manufacturing a midpoint that can create an empty child. OQ-3 is resolved in favor of this fail-closed behavior. 4. **Unbounded-tail routes (`End == nil`).** Derive the split key from observed keys (sub-range layout over `[start, MaxUint64]` already supports this, `keyviz/sampler.go:833-848`), never from a synthetic byte. The selected split key must satisfy `route.Start < split_key < route.End` (or `< +inf`); the detector validates this against the live engine route before emitting a decision and drops the decision (with a metric) if the key no longer falls inside the route (the route may have shifted under a concurrent split). ### 5.5 Guardrails -- **Minimum route size** (`minRouteSpan`, OQ-4): a route whose observed key span (distance between min and max observed key, or sub-range extent) is below a floor is not split — splitting a tiny range yields children too small to relieve load and churns the catalog. Because the engine has no per-route key-count, this is approximated from the sub-range/Top-K evidence; when that evidence is absent the guard is conservatively "do not split" (matches §5.4.3). -- **Max routes** (`maxAutoRoutes`, default e.g. 1024): the cap bounds the **live** route count after a decision lands, so the detector must check it against the decision's actual **route delta** AND against any deltas it has already committed to within the same cycle (codex P2 round-7). A normal split turns one parent into two children — net **+1** route; a §5.4.2 compound single-key isolation turns one parent into **three** children (`[A,H)` + `{H}` + `[H·0x00,B)`) — net **+2** routes (the `H == route.Start` single-split special case in §5.4.2 is two children, net **+1**). With the default `maxSplitsPerCycle = 1` only one decision is ever emitted per cycle, but an operator who raises `maxSplitsPerCycle > 1` can have multiple decisions selected against the *same* starting `liveRoutes`. The detector therefore maintains a **cycle-local `reservedDelta` accumulator** initialised to 0 at the start of each evaluation cycle and incremented by each emitted decision's `routeDelta`; the admission check is **`liveRoutes + reservedDelta + decision.routeDelta <= maxAutoRoutes`** at every decision, not the bare per-decision check. Example: with `liveRoutes == maxAutoRoutes - 1` and `maxSplitsPerCycle == 2`, two normal splits each satisfy the per-decision check (`+1` ≤ remaining 1), but together would leave the catalog at `maxAutoRoutes + 1`; the accumulator catches the second decision at `liveRoutes + 1 + 1 > maxAutoRoutes` and skips it with `autosplit_skipped_total{reason="route_cap"}`. Otherwise it skips and emits the same metric. Checking the bare `liveRoutes >= maxAutoRoutes` would wrongly admit a compound isolation at `liveRoutes == maxAutoRoutes - 1`, leaving the catalog at `maxAutoRoutes + 1` and breaching the cap. This bounds catalog growth (and the watcher fan-out and the Composed-1 history-ring pressure noted at `distribution/engine.go:53-58`). The §8.2 guardrail tests add: a compound isolation scheduled at `liveRoutes == maxAutoRoutes - 1` is **skipped** with `route_cap` (delta `+2` would breach), a normal split at the same live count **proceeds** (delta `+1` is exactly at the cap), AND a per-cycle "two normal splits at `liveRoutes == maxAutoRoutes - 1` with `maxSplitsPerCycle == 2`" admits the first and skips the second via the cycle-local reservation accumulator. +- **Structural splittability** (OQ-4): M3 does not define a numeric `minRouteSpan`. Variable-length lexicographic keys have no generic byte-distance that corresponds to key count or useful child size, and the engine has no per-route key-count estimate. Instead, every decision must carry observed evidence for a strict interior boundary: an effective multi-bucket sub-range boundary or a Top-K isolation boundary. Missing evidence declines with `no_split_key`; an already-isolated singleton declines with `unsplittable_hot_key`. This is the implementable, fail-closed form of the minimum-size guard. +- **Max routes** (`maxAutoRoutes`, default e.g. 1024): the cap bounds the **live** route count after a decision lands, so the detector must check it against the decision's actual **route delta** AND against any deltas it has already committed to within the same cycle (review P2 round-7). A normal split turns one parent into two children — net **+1** route; a §5.4.2 compound single-key isolation turns one parent into **three** children (`[A,H)` + `{H}` + `[H·0x00,B)`) — net **+2** routes (the `H == route.Start` single-split special case in §5.4.2 is two children, net **+1**). With the default `maxSplitsPerCycle = 1` only one decision is ever emitted per cycle, but an operator who raises `maxSplitsPerCycle > 1` can have multiple decisions selected against the *same* starting `liveRoutes`. The detector therefore maintains a **cycle-local `reservedDelta` accumulator** initialised to 0 at the start of each evaluation cycle and incremented by each emitted decision's `routeDelta`; the admission check is **`liveRoutes + reservedDelta + decision.routeDelta <= maxAutoRoutes`** at every decision, not the bare per-decision check. Example: with `liveRoutes == maxAutoRoutes - 1` and `maxSplitsPerCycle == 2`, two normal splits each satisfy the per-decision check (`+1` ≤ remaining 1), but together would leave the catalog at `maxAutoRoutes + 1`; the accumulator catches the second decision at `liveRoutes + 1 + 1 > maxAutoRoutes` and skips it with `autosplit_skipped_total{reason="route_cap"}`. Otherwise it skips and emits the same metric. Checking the bare `liveRoutes >= maxAutoRoutes` would wrongly admit a compound isolation at `liveRoutes == maxAutoRoutes - 1`, leaving the catalog at `maxAutoRoutes + 1` and breaching the cap. This bounds catalog growth (and the watcher fan-out and the Composed-1 history-ring pressure noted at `distribution/engine.go:53-58`). The §8.2 guardrail tests add: a compound isolation scheduled at `liveRoutes == maxAutoRoutes - 1` is **skipped** with `route_cap` (delta `+2` would breach), a normal split at the same live count **proceeds** (delta `+1` is exactly at the cap), AND a per-cycle "two normal splits at `liveRoutes == maxAutoRoutes - 1` with `maxSplitsPerCycle == 2`" admits the first and skips the second via the cycle-local reservation accumulator. - **Per-cycle split budget** (`maxSplitsPerCycle`, default 1): at most this many **logical split decisions** (distinct hot routes relieved) are scheduled per evaluation cycle, so a cluster-wide load spike across many routes cannot trigger a split storm in one tick. This budgets *logical decisions*, not raw `SplitRange` RPCs (§6.3): a §5.4.2 compound single-key isolation is one decision that issues two RPCs but consumes one budget unit, so the default of 1 still completes a single-key isolation. Remaining candidates carry their `consecutiveOver` state to the next cycle. ## 6. Scheduler @@ -324,23 +323,23 @@ The same catalog watcher propagation must update the keyviz sampler membership. A `SplitRangeRequest` carries `expected_catalog_version` (CAS) so a scheduler issuing a split against a catalog that shifted under it fails closed; the scheduler re-reads the engine version and re-evaluates next cycle rather than retrying blindly. -### 6.2 Same-group today; least-loaded target once M2 lands +### 6.2 Same-group standalone; M3-PR4 target policy - **M3 standalone (no M2):** `SplitRange` does a same-group split — both children stay in the source group. This already relieves a hot **range** by halving the key span each child serves and is the first-value deliverable. (It does not relieve a single-group CPU/IO hotspot, since both children stay on the same Raft group — that needs M2's cross-group move.) -- **Once M2 lands:** the scheduler selects `target_group_id` for the right child via a **least-loaded-group policy** and passes it on the same `SplitRange` RPC (M2's new field). "Least-loaded" is computed from the same keyviz per-group aggregate (sum of route scores per group) the detector already has, choosing the group with the lowest aggregate score among groups in `--raftGroups`, excluding the source group and any group already targeted by an in-flight `SplitJob`. - - **Unknown-load exclusion (codex round-7 P2).** In multi-group deployments, §4.1 documents that the default-group leader's local sampler **only sees the routes it serves directly** — routes served through *another* group's leader are not in its sampler at all, so a per-group aggregate computed from the local sampler reports `0` for every "unknown" group (any group the default-group leader does not co-host as a route owner). A least-loaded policy on that aggregate would therefore migrate a hot range into an already-overloaded group whose load is simply invisible to the default leader, defeating the entire balancing intent. Target-group eligibility therefore requires **positive evidence of low load**, not absence of evidence: a group is admitted as a candidate target only when its load is known. M3 ships **two** ways to obtain that evidence, with the second being the future-work hook: - 1. **Local-view fast path (M3 default):** a non-empty group is admitted only if the evaluated window has authoritative load knowledge for **every live active route in that group** from the default-group leader's local view (after §4.2 per-column route aggregation). A committed `MatrixRow` contributes observed load only when the current catalog plus local serving state prove the default-group leader was the authoritative serving node for that route for the whole committed column and the sampler membership was registered before the column opened. A row that exists merely because this node was an ingress/follower and sampled before forwarding is useful for local diagnostics and candidacy, but it is **not** target-admission evidence for another group's full load: the actual group leader may have served unobserved traffic. A missing row is **not** synthesized as zero for a non-empty group in M3, even when the route appears locally authoritative: follower-received writes can be sampled on the ingress follower and forwarded through a leader path that bypasses this node's sampler (§4.1), so absence of a local row is not proof of idle load. Missing rows or non-authoritative rows therefore remain unknown and exclude the group with `autosplit_target_skipped_total{reason="load_unknown"}` unless a future explicit idle certificate or quorum aggregate covers that route for the same committed window. Evidence for only one route in a group with several live routes is insufficient; treating non-authoritative rows or missing rows as low load would pick an overloaded but partially observed group as least-loaded. The only no-row exception in M3 is when the current catalog snapshot proves the group owns **zero live routes**. The zero-route case is known idle, not unknown: no user route can currently be serving traffic there, so it is a valid bootstrap target for cross-group auto-split. This keeps M3 standalone without a new fan-out RPC while distinguishing "authoritatively observed low" from cross-node partial observation. +- **M3-PR4 after M2 lands:** the scheduler may select `target_group_id` for the right child via a **least-loaded-group policy** and pass it on the same `SplitRange` RPC. The remainder of this subsection is normative for that follow-on, not implemented standalone behavior. + - **Unknown-load exclusion (review round-7 P2).** In multi-group deployments, §4.1 documents that the default-group leader's local sampler **only sees the routes it serves directly** — routes served through *another* group's leader are not in its sampler at all, so a per-group aggregate computed from the local sampler reports `0` for every "unknown" group (any group the default-group leader does not co-host as a route owner). A least-loaded policy on that aggregate would therefore migrate a hot range into an already-overloaded group whose load is simply invisible to the default leader, defeating the entire balancing intent. Target-group eligibility therefore requires **positive evidence of low load**, not absence of evidence: a group is admitted as a candidate target only when its load is known. M3-PR4 must obtain that evidence through one of these mechanisms: + 1. **Complete local view:** a non-empty group is admitted only if the evaluated window has authoritative load knowledge for **every live active route in that group** from the default-group leader's local view (after §4.2 per-column route aggregation). A committed `MatrixRow` contributes observed load only when the current catalog plus local serving state prove the default-group leader was the authoritative serving node for that route for the whole committed column and the sampler membership was registered before the column opened. Missing rows or non-authoritative ingress/follower rows remain unknown and exclude the group with `autosplit_target_skipped_total{reason="load_unknown"}`. The only no-row exception is a group with zero live routes in the current catalog, which is known idle. 2. **Cluster-wide fan-out (M3.x / OQ-12):** the M2 cap_migration_v2 heartbeat already carries per-node capability bits; an extension can carry each node's per-group aggregate so the default-group leader has a cluster-wide load view. The scheduler then admits any group whose load is reported by quorum, or whose group-level quorum aggregate explicitly reports zero/low load for the evaluated committed window (defending against single-node staleness and distinguishing "reported zero" from "not reported"). This removes the local-view limitation and is recorded under OQ-12 as the eventual default once the fan-out lands. - The §8.2 target-selection tests add the local-view rule: "node A (default-group leader) locally owns and had registered an idle route in group B for the committed column, and the evaluated window has no row for it → skip B as `load_unknown`", "group B has authoritative fresh rows covering all live routes → admit B as observed", "group B has a row for one of two live routes and the other route has no authoritative row → skip B as `load_unknown`", "group B has a fresh row for a route that node A only observed as ingress/follower before forwarding → skip B as `load_unknown`", and "group C owns zero live routes in the catalog → admit C as known idle." The migration mechanics (the resumable job) are entirely M2's; the scheduler just chooses the target and fires the RPC, then tracks the returned `job_id` for observability. - - **Compound single-key isolation is same-group-only, even under `--autoSplitCrossGroup` (codex P2).** Cross-group target selection is **never** applied to a §5.4.2 compound isolation in progress. The two-`SplitRange`-call chain (§6.4) must both fire with `target_group_id = 0` (same-group). If the *first* call carried a `target_group_id`, its intermediate right child `[H,B)` would become a cross-group migration target — an in-flight `SplitJob` whose route §6.3 excludes from candidacy and on which a second `SplitRange` is rejected — so the second call (§6.4 step 2) could not proceed and the isolation would strand at `compound_partial` on its very first execution. Cross-group relocation of an isolated child is therefore a **separate, later, same-group-complete-first step**: the **two non-singleton final children (`[A,H)` and `[H·0x00,B)`) are eligible** for cross-group migration via a subsequent `SplitRange` call, as **independent decisions in subsequent cycles** once the compound operation has fully resolved same-group (and each such child is `RouteStateActive` with no in-flight `SplitJob`, §5.0/§6.3). **The isolated singleton child `{H} = [H, H·0x00)` is NOT migratable via `SplitRange` — a known limitation (closes claude round-13 Issue 2).** M2's `SplitRange` requires a split key strictly between the route's `Start` and `End` (`validateSplitKey` at `adapter/distribution_server.go:358-371`), and there is no byte strictly between `H` and `H·0x00` in the key space — `H·0x00` is the immediate lexicographic successor of `H`, so the singleton route has no interior split key, so `SplitRange` against `{H}` always returns `errDistributionSplitKeyAtBoundary`. The M2 API cannot relocate `{H}` to another group; the hot key's group ID is fixed at isolation time. The isolation still reduces per-range CPU/IO contention on the source group (the hot key is in a dedicated single-key route, no longer sharing a range with its neighbors and not stealing budget from their reads/writes), but the key's *group* cannot change without a new control-plane primitive. **A future `MoveRange(routeID, target_group_id)` RPC** (distinct from `SplitRange` — moves an existing route to another group without requiring a split) would close this gap; it is recorded as **OQ-16** ("MoveRange primitive for singleton routes") and is **out of scope for M3**. Operators with a truly single-key hotspot whose group itself is overloaded should use application-level key sharding until OQ-16 lands. This restriction is the M3-PR4 hook constraint. - - **Right-child-only migration gates p50 and lower-edge single-key decisions (codex P2).** Per §5.4's `split_origin` enum and p50 child-load estimates on `SplitDecision`, the scheduler may pass `target_group_id != 0` only when the child M2 will move — the right child — is the child that carries the observed hot load. Therefore `decision.split_origin == p50_first_bucket_hi` is always same-group, and any `p50_mid` whose estimated left-child weighted load is greater than the right-child load is also same-group. A `p50_last_bucket_lo` decision can be cross-group only after the target-admission gates above pass, because the last bucket is placed in the right child by the `lo` fallback. Likewise, `isolation_single_lower_edge` is same-group-only: when `H == route.Start`, the split at `H·0x00` leaves the hot singleton in the **left** child and M2 can only move the cooler right child. `isolation_single_upper_edge` may use cross-group target selection because the singleton is the right child. The same-group split still relieves range pressure, but M3 treats persistent lower-tail/left-heavy ranges as **not cross-group migratable through the right-child-only `SplitRange` hook**. Guaranteed cross-group relief for those cases needs a left-child move / `MoveRange(routeID, target_group_id)` primitive (OQ-16/future work). The §8.2 target-selection tests add "p50_first_bucket_hi under --autoSplitCrossGroup → target_group_id=0", "p50_mid with left-estimated load > right-estimated load → target_group_id=0", "isolation_single_lower_edge → target_group_id=0", and "isolation_single_upper_edge may target a group only when target evidence is complete" alongside the existing compound-isolation-stays-same-group case. + - **Compound single-key isolation is same-group-only, even under `--autoSplitCrossGroup` (review P2).** Cross-group target selection is **never** applied to a §5.4.2 compound isolation in progress. The two-`SplitRange`-call chain (§6.4) must both fire with `target_group_id = 0` (same-group). If the *first* call carried a `target_group_id`, its intermediate right child `[H,B)` would become a cross-group migration target — an in-flight `SplitJob` whose route §6.3 excludes from candidacy and on which a second `SplitRange` is rejected — so the second call (§6.4 step 2) could not proceed and the isolation would strand at `compound_partial` on its very first execution. Cross-group relocation of an isolated child is therefore a **separate, later, same-group-complete-first step**: the **two non-singleton final children (`[A,H)` and `[H·0x00,B)`) are eligible** for cross-group migration via a subsequent `SplitRange` call, as **independent decisions in subsequent cycles** once the compound operation has fully resolved same-group (and each such child is `RouteStateActive` with no in-flight `SplitJob`, §5.0/§6.3). **The isolated singleton child `{H} = [H, H·0x00)` is NOT migratable via `SplitRange` — a known limitation (closes claude round-13 Issue 2).** M2's `SplitRange` requires a split key strictly between the route's `Start` and `End` (`validateSplitKey` at `adapter/distribution_server.go:358-371`), and there is no byte strictly between `H` and `H·0x00` in the key space — `H·0x00` is the immediate lexicographic successor of `H`, so the singleton route has no interior split key, so `SplitRange` against `{H}` always returns `errDistributionSplitKeyAtBoundary`. The M2 API cannot relocate `{H}` to another group; the hot key's group ID is fixed at isolation time. The isolation still reduces per-range CPU/IO contention on the source group (the hot key is in a dedicated single-key route, no longer sharing a range with its neighbors and not stealing budget from their reads/writes), but the key's *group* cannot change without a new control-plane primitive. **A future `MoveRange(routeID, target_group_id)` RPC** (distinct from `SplitRange` — moves an existing route to another group without requiring a split) would close this gap; it is recorded as **OQ-16** ("MoveRange primitive for singleton routes") and is **out of scope for M3**. Operators with a truly single-key hotspot whose group itself is overloaded should use application-level key sharding until OQ-16 lands. This restriction is the M3-PR4 hook constraint. + - **Right-child-only migration gates p50 and lower-edge single-key decisions (review P2).** Per §5.4's `split_origin` enum and p50 child-load estimates on `SplitDecision`, the scheduler may pass `target_group_id != 0` only when the child M2 will move — the right child — is the child that carries the observed hot load. Therefore `decision.split_origin == p50_first_bucket_hi` is always same-group, and any `p50_mid` whose estimated left-child weighted load is greater than the right-child load is also same-group. A `p50_last_bucket_lo` decision can be cross-group only after the target-admission gates above pass, because the last bucket is placed in the right child by the `lo` fallback. Likewise, `isolation_single_lower_edge` is same-group-only: when `H == route.Start`, the split at `H·0x00` leaves the hot singleton in the **left** child and M2 can only move the cooler right child. `isolation_single_upper_edge` may use cross-group target selection because the singleton is the right child. The same-group split still relieves range pressure, but M3 treats persistent lower-tail/left-heavy ranges as **not cross-group migratable through the right-child-only `SplitRange` hook**. Guaranteed cross-group relief for those cases needs a left-child move / `MoveRange(routeID, target_group_id)` primitive (OQ-16/future work). The §8.2 target-selection tests add "p50_first_bucket_hi under --autoSplitCrossGroup → target_group_id=0", "p50_mid with left-estimated load > right-estimated load → target_group_id=0", "isolation_single_lower_edge → target_group_id=0", and "isolation_single_upper_edge may target a group only when target evidence is complete" alongside the existing compound-isolation-stays-same-group case. -The target-group selection slots **behind the same `SplitRange` interface** — the scheduler code path is identical; only the `target_group_id` field flips from zero (same-group) to a chosen group once M2 is present. M3 ships the policy hook disabled (always same-group) and a follow-on M3.x / M2-completion PR enables target selection. OQ-6 asks whether to gate target selection on an explicit `--autoSplitCrossGroup` sub-flag. +The target-group selection slots **behind the same `SplitRange` interface** — the scheduler code path is identical; only the `target_group_id` field flips from zero (same-group) to a chosen group once M2 is present. Standalone M3 ships the policy hook disabled (always same-group); M3-PR4 must gate it behind the resolved explicit `--autoSplitCrossGroup` flag. ### 6.3 Concurrency with in-flight splits -The scheduler executes at most `maxSplitsPerCycle` (§5.5) **logical split decisions** per cycle and waits for each to return (same-group splits are synchronous; cross-group returns a `job_id` immediately per M2 §5.1). **`maxSplitsPerCycle` budgets *logical split decisions* (one hot route relieved), not raw `SplitRange` RPCs (codex P2):** most decisions are one RPC, but a §5.4.2 compound single-key isolation is one logical decision that issues *two* back-to-back `SplitRange` calls (§6.4) — it still consumes exactly **one** budget unit, so the default `maxSplitsPerCycle = 1` does not block the second RPC the isolation needs. The anti-storm guarantee is "at most `maxSplitsPerCycle` distinct hot routes are acted on per tick"; the bounded RPC count per cycle is then `maxSplitsPerCycle` in the common case and at most `2 × maxSplitsPerCycle` when every decision in the cycle is a compound isolation. A route with an in-flight cross-group `SplitJob` (not yet `DONE`) is excluded from candidacy until the job completes, so the detector cannot stack a second split on a route mid-migration. +The scheduler executes at most `maxSplitsPerCycle` (§5.5) **logical split decisions** per cycle and waits for each to return (same-group splits are synchronous; cross-group returns a `job_id` immediately per M2 §5.1). **`maxSplitsPerCycle` budgets *logical split decisions* (one hot route relieved), not raw `SplitRange` RPCs (review P2):** most decisions are one RPC, but a §5.4.2 compound single-key isolation is one logical decision that issues *two* back-to-back `SplitRange` calls (§6.4) — it still consumes exactly **one** budget unit, so the default `maxSplitsPerCycle = 1` does not block the second RPC the isolation needs. The anti-storm guarantee is "at most `maxSplitsPerCycle` distinct hot routes are acted on per tick"; the bounded RPC count per cycle is then `maxSplitsPerCycle` in the common case and at most `2 × maxSplitsPerCycle` when every decision in the cycle is a compound isolation. A route with an in-flight cross-group `SplitJob` (not yet `DONE`) is excluded from candidacy until the job completes, so the detector cannot stack a second split on a route mid-migration. When `maxSplitsPerCycle > 1`, decisions selected from the same starting catalog snapshot must be **serialized through fresh catalog versions**, not all issued with the same `expected_catalog_version`. After each successful `SplitRange`, the scheduler updates its working catalog version from the response (or re-reads the catalog if the response cannot carry the new version), revalidates remaining decisions against the new snapshot, and only then issues the next split. If a decision's parent route disappeared, changed bounds/state, or no longer contains the selected split key, it is dropped and the scheduler continues with the next still-valid decision or ends the cycle. This chaining is the general form of §6.4's compound CAS chaining and prevents a multi-split budget from degenerating into predictable CAS failures after the first successful split. @@ -357,19 +356,19 @@ Budget and cooldown treatment: - **CAS chaining.** The second call uses the `v1` returned by the first (not the cycle's starting `v0`), so the pair is serialized against the live catalog version. If the first call's CAS fails (a concurrent manual split landed), the scheduler abandons the compound, emits `autosplit_splits_failed_total{reason="cas_conflict"}`, and re-evaluates next cycle — it does **not** issue the second call against a stale version. - **Partial-completion safety.** If the first call commits but the second fails (CAS conflict or RPC error), the catalog is left in a valid two-child state `[A,H) + [H,B)`. On the same leader, the leader-local pending-finalization record from §5.3 keeps the exact `(intermediate_route_id, [H,B), split_key=H·0x00, target_group_id=0)` and is retried before normal detector candidates on later cycles, after revalidating the current catalog shape and route cap. The retry does not depend on `[H,B)` accumulating new `candidateWindows` evidence; a cycle with no new sampler flush can still finish the isolation. If leadership changes or the scheduler restarts before that retry, there is no durable compound-intermediate marker in M3; the new leader reconstructs normal cooldown from `[H,B)`'s `SplitAtHLC` and the retry waits until cooldown expires. No invariant is broken by stopping after the first call; the worst case is delayed relief by up to `splitCooldown`, which is documented and tested. This is logged `autosplit_compound_partial` while the pending record remains live. - Cooldown applies to the three **final** children only, per §5.3. -- **Same-group-only, even under `--autoSplitCrossGroup` (codex P2).** Both `SplitRange` calls always pass `target_group_id = 0`; cross-group target selection (§6.2) is **incompatible with a compound operation in progress** and is never applied to it. If call 1 carried a `target_group_id`, the intermediate child `[H,B)` would become a cross-group `SplitJob` target — a route §6.3 excludes from candidacy and on which `SplitRange` is rejected — so call 2 could not run and the isolation would strand at `compound_partial` immediately. Cross-group relocation after isolation is a separate, later decision (§6.2): only the **two non-singleton final children** (`[A,H)` and `[H·0x00,B)`), once resolved same-group and `RouteStateActive` with no in-flight `SplitJob`, may be independently selected for cross-group migration in a subsequent cycle. Under the M2 hook that later relocation is still another `SplitRange` decision on the existing child, so it creates an additional child and has `routeDelta = +1`; it must pass the same `maxAutoRoutes` / `reservedDelta` accounting as any other normal split. It becomes a zero-route-count move only if the future `MoveRange(routeID, target_group_id)` primitive exists. The singleton `{H} = [H, H·0x00)` is explicitly non-targetable through `SplitRange` because it has no valid interior split key (§6.2 / OQ-16). The route-cap budget for the original compound isolation remains `+2` routes (§5.5). +- **Same-group-only, even under `--autoSplitCrossGroup` (review P2).** Both `SplitRange` calls always pass `target_group_id = 0`; cross-group target selection (§6.2) is **incompatible with a compound operation in progress** and is never applied to it. If call 1 carried a `target_group_id`, the intermediate child `[H,B)` would become a cross-group `SplitJob` target — a route §6.3 excludes from candidacy and on which `SplitRange` is rejected — so call 2 could not run and the isolation would strand at `compound_partial` immediately. Cross-group relocation after isolation is a separate, later decision (§6.2): only the **two non-singleton final children** (`[A,H)` and `[H·0x00,B)`), once resolved same-group and `RouteStateActive` with no in-flight `SplitJob`, may be independently selected for cross-group migration in a subsequent cycle. Under the M2 hook that later relocation is still another `SplitRange` decision on the existing child, so it creates an additional child and has `routeDelta = +1`; it must pass the same `maxAutoRoutes` / `reservedDelta` accounting as any other normal split. It becomes a zero-route-count move only if the future `MoveRange(routeID, target_group_id)` primitive exists. The singleton `{H} = [H, H·0x00)` is explicitly non-targetable through `SplitRange` because it has no valid interior split key (§6.2 / OQ-16). The route-cap budget for the original compound isolation remains `+2` routes (§5.5). ## 7. Safety and Operations ### 7.1 Default OFF behind a flag - `--autoSplit` (bool, default `false`). When false, none of the detector/scheduler goroutines start; behavior is identical to today. When true, the default-group leader runs the detector + scheduler loop, and the keyviz sampler is constructed and flushed if it is not already. -- **Sampler-wiring condition.** The sampler is built when `keyvizEnabled || autoSplit` (today it is built only on `keyvizEnabled`; see `main.go:2005` `newKeyvizSampler` and the analogous wiring in `cmd/server/demo.go`). M3-PR3's flag-wiring change touches both `main.go` and `cmd/server/demo.go`. When auto-split forces the sampler on and the operator **did not explicitly set** `--keyvizKeyBucketsPerRoute` (detected via `flag.Visit` after `flag.Parse()`, see §3.1 — plain `flag.Int` cannot distinguish an omitted flag from an explicit `1`), auto-split raises it to `autoSplitDefaultBuckets` (default 16) so split-key evidence exists by default (§3.1 table). An explicit `--keyvizKeyBucketsPerRoute 1` is honored as-is (operator opts into route-granular-only → §5.4.3 decline). `--autoSplitDefaultBuckets` is itself clamped to `[2, MaxKeyBucketsPerRoute]` (256, `keyviz/sampler.go:115`) and validated at startup — a value below 2 would defeat the purpose, and above the cap is rejected like the other keyviz bucket flags. +- **Sampler-wiring condition.** The sampler is built when `keyvizEnabled || autoSplit` in both `main.go` and `cmd/server/demo.go`. When auto-split forces the sampler on and the operator **did not explicitly set** `--keyvizKeyBucketsPerRoute` (detected via `flag.Visit` after `flag.Parse()`, see §3.1), auto-split raises it to `autoSplitDefaultBuckets` (default 16) so split-key evidence exists by default. An explicit `--keyvizKeyBucketsPerRoute 1` is honored as-is (operator opts into route-granular-only → §5.4.3 decline). `--autoSplitDefaultBuckets` is validated in `[2, MaxKeyBucketsPerRoute]` (256) only when that implied default bucket count will be used; values outside the range fail startup in that mode. - Tuning flags (all with parent-doc-derived defaults): `--autoSplitWriteWeight` (4), `--autoSplitReadWeight` (1), `--autoSplitThreshold` (50000 ops/min), `--autoSplitCandidateWindows` (3 — number of **consecutive committed flush columns** whose **per-column** score must clear the threshold to promote, §5.2; it sets the hysteresis length, not the per-column rate), `--autoSplitCooldown` (10m), `--autoSplitMaxRoutes` (1024), `--autoSplitMaxPerCycle` (1 — bounds **logical split decisions** per cycle, not raw `SplitRange` RPCs; a compound single-key isolation is one decision / two RPCs, §6.3), `--autoSplitEvalInterval` (evaluation cycle period, default = `--keyvizStep`), `--autoSplitDefaultBuckets` (sub-range buckets auto-split implies when keyviz bucketing is left at the default, default 16). ### 7.2 Runtime kill switch -The detector/scheduler loop checks an `atomic.Bool` enable flag each cycle. An admin RPC / endpoint (reuse the existing admin surface that hosts keyviz/distribution operator calls) toggles it without a restart. Flipping it off cancels in-progress scheduling at the next cycle boundary; it does **not** abort an already-issued cross-group `SplitJob` (that is M2's `AbandonSplitJob`). The kill switch is the operator's "stop making new splits NOW" lever. +The detector/scheduler loop checks an `atomic.Bool` enable flag each cycle. The authenticated admin RPC `Admin.SetAutoSplitEnabled` toggles it without a restart; the RPC is registered only when auto-split was enabled at startup. Flipping it off cancels in-progress scheduling at the next cycle boundary; it does **not** abort an already-issued cross-group `SplitJob` (that is M2's `AbandonSplitJob`). The kill switch is the operator's "stop making new splits NOW" lever. ### 7.3 Metrics (bounded cardinality) @@ -380,10 +379,11 @@ Counters (the two `{reason}` counters partition cleanly by **whether a `SplitRan - `autosplit_splits_scheduled_total` - `autosplit_splits_failed_total{reason}` — a `SplitRange` RPC **was issued** and failed; `reason` ∈ {`cas_conflict`, `rpc_error`, `target_unavailable`}. - `autosplit_skipped_total{reason}` — the detector **did not issue** a `SplitRange` RPC; `reason` ∈ {`no_split_key`, `route_cap`, `budget_exhausted`, `non_active_state`, `aggregate_row`, `unsplittable_hot_key`} (fixed enum). -- `autosplit_isolation_declined_total{reason}` — the §5.4 compound single-key isolation gate was reached but its preconditions failed; `reason` ∈ {`absolute_floor`} (the share gate fired but the absolute-weighted-contribution floor did not — the detector falls through to the §5.4 sub-range p50 path, so a `SplitRange` RPC may still be issued for the *same* route in the same cycle; this is **not** the same outcome as `autosplit_skipped_total`). -- `autosplit_target_skipped_total{reason}` — a cross-group target candidate was excluded from §6.2's least-loaded selection without affecting whether a split is scheduled; `reason` ∈ {`load_unknown`} (the group owns existing routes but at least one route lacks an authoritative fresh row for the committed column and no future explicit idle certificate or quorum aggregate is available; a zero-route group is admitted as known idle, not skipped). +- `autosplit_isolation_declined_total{reason}` — the §5.4 compound single-key isolation gate was reached but its preconditions failed; `reason` ∈ {`absolute_floor`, `topk_degraded`, `topk_insufficient`, `topk_error_bound`}. The detector falls through to sub-range p50 when possible, so a `SplitRange` RPC may still be issued for the same route in the same cycle. - `autosplit_compound_partial_total` — a compound single-key isolation (§6.4) committed its first split but not the second; the hot key will be isolated on a later cycle. +`autosplit_target_skipped_total{reason="load_unknown"}` is reserved for M3-PR4 and is deliberately not registered by standalone M3 because cross-group target selection is disabled. + Gauges: - `autosplit_enabled` (0/1) - `autosplit_tracked_routes` (size of the leader-local state map) @@ -402,7 +402,7 @@ Structured `slog` with stable keys per CLAUDE.md conventions: `route_id`, `group - **Split storms** (many routes hot at once): mitigated by `maxSplitsPerCycle` per-cycle budget (§5.5) and `maxAutoRoutes` cap. - **Empty/degenerate child** (split key produces a child with no keys): prevented by distribution-based split-key selection (§5.4) — the key sits at the observed load median, never a synthetic byte; plus the `route.Start < split_key < route.End` validation. When no distribution evidence exists, the detector declines (§5.4.3) rather than risk an empty child. - **Catalog churn under the scheduler** (route shifted between detection and `SplitRange`): the `expected_catalog_version` CAS fails closed; the scheduler re-evaluates next cycle (§6.1). -- **Repeated re-split of the same hotspot** that a single split cannot relieve (genuinely un-splittable single-key hotspot, e.g. one key taking all writes): single-key isolation (§5.4.2) splits the key into its own child once; cooldown + the `minRouteSpan` guard then prevent infinitely re-splitting a child that is already a single key (a one-key range cannot be split further — `split_key` validation has no valid interior key, the detector declines and counts `autosplit_skipped_total{reason="unsplittable_hot_key"}`). This is the limit of split-based mitigation; truly relieving a single-key hotspot is an application/data-model problem, not a sharding one. +- **Repeated re-split of the same hotspot** that a single split cannot relieve (genuinely un-splittable single-key hotspot, e.g. one key taking all writes): single-key isolation (§5.4.2) splits the key into its own child once; cooldown plus the structural-splittability guard then prevent infinitely re-splitting a child that has no strict interior key. The detector declines and counts `autosplit_skipped_total{reason="unsplittable_hot_key"}`. This is the limit of split-based mitigation; truly relieving a single-key hotspot is an application/data-model problem, not a sharding one. ### 7.6 Interaction with leader changes @@ -433,7 +433,7 @@ M3 adds one durable field to `RouteDescriptor`: - **`SplitAtHLC uint64`** — the **committed HLC timestamp of the `SplitRange` transaction that produced this route**. This must be a leader-fenced, Raft-committed timestamp, **not** an arbitrary `HLC.Next()` allocation (review correctly flagged this: `HLC.Next()` bypasses the physical-ceiling fence — `kv/hlc.go:110-128` — so a stale leader could mint a `SplitAtHLC` inside a window the ceiling fence is meant to forbid, the exact case the fence exists to prevent). The safe source is already at hand: the catalog mutation commits through `coordinator.Dispatch` (`adapter/distribution_server.go:229-235`), and the coordinator allocates that transaction's commit ts via the **fenced** path (`resolveTxnCommitTS`, `kv/sharded_coordinator.go:1091-1107` → `nextTxnTSAfter`, `:1365-1379` → `NextFenced()`). So `SplitAtHLC` is **defined to be that transaction's commit ts** — the same value the split mutation is durably applied at. - **Mechanism (so the stored value is allocated by the committing leader).** The `SplitRange` handler must not pre-allocate `SplitAtHLC` before `Dispatch`. The current `LeaderProxy.Commit` path may forward a request after `verifyCatalogLeader` succeeds but before the write commits; a timestamp minted by the deposed handler would then become a caller-supplied `CommitTS` on the new leader, violating the repo invariant that persistence timestamps originate from the Raft leader that commits them. M3-PR1b therefore adds a coordinator-side catalog transaction entrypoint for `SplitRange`: the request is forwarded as a whole to the default-group leader; the committing coordinator reads the catalog snapshot, calls the same `nextTxnTSAfter(snapshot.ReadTS)` fenced allocator it uses for ordinary transactions, builds the two child descriptors with `SplitAtHLC = commit_ts`, commits them at that exact `commit_ts`, and returns that `commit_ts` in `CoordinateResponse` / `SplitRangeResponse` for tests and cooldown diagnostics. The adapter supplies no `OperationGroup.CommitTS` for this path. If leadership changes before allocation, the forwarded leader performs the allocation; if leadership is lost after allocation but before commit, the operation fails/retries without publishing descriptors stamped by the old process. - - **Commit ts must be `> StartTS` (review, codex P2).** The committing coordinator reuses `nextTxnTSAfter(startTS)` (`kv/sharded_coordinator.go:1365-1388`): allocate `ts = HLC.NextFenced()`; if `ts <= snapshot.ReadTS`, `HLC.Observe(snapshot.ReadTS)` and re-allocate `ts = HLC.NextFenced()`; only then build the descriptors and commit. After the Observe-and-retry the clock has been bumped strictly past `ReadTS`, so the stored `SplitAtHLC` is both leader-fenced and `> StartTS`. This logic stays in the coordinator rather than being reimplemented in the adapter. + - **Commit ts must be `> StartTS` (review, review P2).** The committing coordinator reuses `nextTxnTSAfter(startTS)` (`kv/sharded_coordinator.go:1365-1388`): allocate `ts = HLC.NextFenced()`; if `ts <= snapshot.ReadTS`, `HLC.Observe(snapshot.ReadTS)` and re-allocate `ts = HLC.NextFenced()`; only then build the descriptors and commit. After the Observe-and-retry the clock has been bumped strictly past `ReadTS`, so the stored `SplitAtHLC` is both leader-fenced and `> StartTS`. This logic stays in the coordinator rather than being reimplemented in the adapter. - **Ceiling-expired path.** If `NextFenced()` returns `ErrCeilingExpired` (the committing leader's physical ceiling has lapsed), the coordinator fails the `SplitRange` closed (`FailedPrecondition`) instead of writing an unfenced timestamp — a brief, safe rejection, not a corrupt record. - **Safety across leader changes.** Because the ts is minted inside the committing coordinator, a newly elected leader uses its own fenced allocator, and a stale/deposed process can never smuggle a persistence timestamp through a caller-supplied field. Combined with the physical-ceiling clamp on `Next()`, `SplitAtHLC` is monotonic across elections and never under-counts a cooldown. This is what makes the §7.6 cooldown reconstruction trustworthy after an election. - **Encoding / back-compat.** `SplitAtHLC` is encoded/decoded alongside `ParentRouteID` in `EncodeRouteDescriptor`/`DecodeRouteDescriptor` (`distribution/catalog.go:168-217`). See §7.7c for the codec-version handling and the rolling-upgrade fence — the decoder treats a record without the field as `SplitAtHLC = 0` (= "unknown / pre-upgrade", never in cooldown), so new code reading old records is safe by construction. @@ -463,7 +463,7 @@ We resolve this by **relaxing the decoder to accept-and-ignore unknown trailing - **M3-PR1a (decoder relaxation only, `catalogRouteCodecVersion = 1`).** A `version = 2` record is `> catalogRouteCodecVersion` from PR1a's perspective and is drained — the relaxed v1-era decoder reads through `End`, then drains the 8-byte `SplitAtHLC` tail. PR1a has no `SplitAtHLC` struct field, so tests assert decode success and equality of the preexisting fields only; semantically, the lost cooldown hint is equivalent to "unknown / never in cooldown" and is safe per (b). - **M3-PR1b (v2 encoder, `catalogRouteCodecVersion = 2`).** A `version = 2` record falls into the "known version" branch and gets the **strict** `r.Len() != 0` check — any tail beyond the `SplitAtHLC` 8 bytes is rejected as corruption. A `version = 3+` record is drained for forward compatibility. - This catches the codex-flagged case: a v2 record with an extra byte after `SplitAtHLC` (or a malformed encoder that emitted bytes after the documented layout) is rejected by the PR1b decoder, not silently normalised. **Codec matrix Case 1b (claude round-12 Issue 1, added to the numbered table in §8.1 / §8.2):** + This catches the review-flagged case: a v2 record with an extra byte after `SplitAtHLC` (or a malformed encoder that emitted bytes after the documented layout) is rejected by the PR1b decoder, not silently normalised. **Codec matrix Case 1b (claude round-12 Issue 1, added to the numbered table in §8.1 / §8.2):** | # | Encoder | Decoder | Expectation | |---|---|---|---| @@ -486,7 +486,7 @@ We resolve this by **relaxing the decoder to accept-and-ignore unknown trailing ``` The v2 decoder needs no special-casing here: it reads the 8-byte tail *after* `decodeRouteDescriptorEnd` (`distribution/catalog.go:209`), which already consumes the end-flag and the optional `End` block for both `endFlag == 0` and `endFlag == 1`, so the tail lands at the same offset regardless of the `End == nil` / non-nil case. -- **Upgrade ordering fence — technically enforced by splitting the release, not a documented ops constraint (codex `afc95e3b`, review round 5 Option A).** A documented "complete the rollout before issuing a split" instruction is **not enforcement**: the encoder bump and the relaxed decoder ship in the same binary, so as soon as one M3 node becomes the default-group leader, an operator's manual `SplitRange` — which `buildCatalogSplitOps` always encodes through `distribution.EncodeRouteDescriptor` (`adapter/distribution_server.go:254`) — can write a `version = 2` record while not-yet-upgraded nodes still run the strict decoder (`raw[0] != catalogRouteCodecVersion` *and* `r.Len() != 0`, `distribution/catalog.go:200,213`). Those nodes' `Snapshot`/watcher then fail with `ErrCatalogInvalidRouteRecord`, surfacing as route-not-found on every pre-M3 node until it is upgraded. `SplitRange` is operator-initiated and exists today, so the window is real even before M3-PR3 wires the auto-scheduler. **We make the ordering safe by construction by splitting the codec work across two separately-deployed PRs** (codex: "split decoder relaxation and v2 emission into separate releases"): +- **Upgrade ordering fence — technically enforced by splitting the release, not a documented ops constraint (review `afc95e3b`, review round 5 Option A).** A documented "complete the rollout before issuing a split" instruction is **not enforcement**: the encoder bump and the relaxed decoder ship in the same binary, so as soon as one M3 node becomes the default-group leader, an operator's manual `SplitRange` — which `buildCatalogSplitOps` always encodes through `distribution.EncodeRouteDescriptor` (`adapter/distribution_server.go:254`) — can write a `version = 2` record while not-yet-upgraded nodes still run the strict decoder (`raw[0] != catalogRouteCodecVersion` *and* `r.Len() != 0`, `distribution/catalog.go:200,213`). Those nodes' `Snapshot`/watcher then fail with `ErrCatalogInvalidRouteRecord`, surfacing as route-not-found on every pre-M3 node until it is upgraded. `SplitRange` is operator-initiated and exists today, so the window is real even before M3-PR3 wires the auto-scheduler. **We make the ordering safe by construction by splitting the codec work across two separately-deployed PRs** (review: "split decoder relaxation and v2 emission into separate releases"): | PR | Codec scope | `catalogRouteCodecVersion` | Encoder writes | |---|---|---|---| @@ -499,7 +499,7 @@ We resolve this by **relaxing the decoder to accept-and-ignore unknown trailing - The deployment ordering is restated in the M3-PR1a/M3-PR1b PR descriptions and pinned by §8.4 acceptance criterion 10. - **Rejected alternative — separate catalog key.** Storing `SplitAtHLC` under a sibling key (`{routePrefix}{routeID}/split_at_hlc`) avoids any codec change, but it splits one logical record across two keys (extra read, non-atomic update relative to the descriptor, and a second key to GC on route retirement). The single-record append with a relaxed decoder is simpler and keeps the descriptor atomic, so we take it. -**Codec round-trip test matrix (§8.1) — both directions explicit.** The relaxation must be locked down by tests written *before* the encoder bump (so the relaxed decoder is proven against a v2 record built by hand, not only against its own encoder). The matrix is split across the two PRs: the decoder-direction cases that prove the relaxation and preserve current-version strictness (3, 3a, 4, 4a, 5, 6) are authored in **M3-PR1a** against hand-built records; the v2-encoder round-trip cases (1, 1a, 2), the re-runs of 4/4a against the real encoder, **and case 7 (truncated-tail reject)** are authored in **M3-PR1b**. **Case 7 is PR1b-scope** because the PR1a forward-version drain has no `version >= 2` branch and so accepts a `< 8`-byte v2 tail by design (no field to populate, no error); only PR1b's exact-8-byte read can surface an `io.ErrUnexpectedEOF` and reject the truncation (codex P2): +**Codec round-trip test matrix (§8.1) — both directions explicit.** The relaxation must be locked down by tests written *before* the encoder bump (so the relaxed decoder is proven against a v2 record built by hand, not only against its own encoder). The matrix is split across the two PRs: the decoder-direction cases that prove the relaxation and preserve current-version strictness (3, 3a, 4, 4a, 5, 6) are authored in **M3-PR1a** against hand-built records; the v2-encoder round-trip cases (1, 1a, 2), the re-runs of 4/4a against the real encoder, **and case 7 (truncated-tail reject)** are authored in **M3-PR1b**. **Case 7 is PR1b-scope** because the PR1a forward-version drain has no `version >= 2` branch and so accepts a `< 8`-byte v2 tail by design (no field to populate, no error); only PR1b's exact-8-byte read can surface an `io.ErrUnexpectedEOF` and reject the truncation (review P2): | # | Encoder | Decoder | Expectation | |---|---|---|---| @@ -512,7 +512,7 @@ We resolve this by **relaxing the decoder to accept-and-ignore unknown trailing | 4a | v2, **`End == nil`**, `SplitAtHLC = T ≠ 0` (hand-built with **version byte `2`**) | **M3-PR1a relaxed decoder** (`catalogRouteCodecVersion` still `1`, no `version >= 2` branch, drains tail) | decodes without error; `End == nil` and all preexisting fields equal (PR1a has no `SplitAtHLC` field to assert); covers the **rightmost-route** mid-upgrade direction — the combination (PR1a decoder × `End == nil` v2 record) that fires on the live write path when the rightmost route or its split right child is encoded by a PR1b leader | | 5 | future v3 (hand-built: v2 layout + extra appended bytes, version byte `3`) | v2 (relaxed) | decodes, `SplitAtHLC == T`, extra trailing bytes drained without error — forward-version tolerance | | 6 | corrupt (version byte `0`, i.e. `< catalogRouteCodecVersionMin`) | v2 (relaxed) | rejected with `ErrCatalogInvalidRouteRecord` — relaxation does **not** weaken the floor | -| 7 | v2 with a **truncated** tail (`< 8` bytes after `End`) | **v2 (PR1b `version >= 2` exact-8-byte read)** | rejected (`io.ErrUnexpectedEOF → ErrCatalogInvalidRouteRecord`) — a present-but-short `SplitAtHLC` is corruption, not a forward-version tail. **PR1b-scope:** the PR1a drain-unknown-tail decoder has no `version >= 2` branch and accepts a short tail with no error, so it cannot detect truncation; only PR1b's exact read can (codex P2) | +| 7 | v2 with a **truncated** tail (`< 8` bytes after `End`) | **v2 (PR1b `version >= 2` exact-8-byte read)** | rejected (`io.ErrUnexpectedEOF → ErrCatalogInvalidRouteRecord`) — a present-but-short `SplitAtHLC` is corruption, not a forward-version tail. **PR1b-scope:** the PR1a drain-unknown-tail decoder has no `version >= 2` branch and accepts a short tail with no error, so it cannot detect truncation; only PR1b's exact read can (review P2) | Cases 4, 4a, and 5 are the load-bearing ones for the rolling-upgrade fence; case 3a is the load-bearing guard that PR1a did not accidentally make the current v1 layout lax. Case 7 (authored in **M3-PR1b**, not PR1a — see the per-PR split above and §8.1) ensures the PR1b `version >= 2` exact-read treats a truncated tail as corruption rather than letting "skip unknown tail" be mistaken for "ignore truncation of a field this version requires." The PR1a relaxed decoder has no such read and intentionally drains forward-version tail bytes, so it is *not* the decoder under test for case 7. Because the two-PR split (above) guarantees PR1a is cluster-wide before PR1b emits any v2 record, cases 4 and 4a model the *only* old-reads-new combination that can occur in a correctly-ordered rollout (PR1a decoder × PR1b v2 record); a *pre-PR1a* strict decoder reading a v2 record is the violated-gate failure mode (transient, non-corrupting) described above, not a supported path. **Cases 3a, 4, and 4a must be authored in M3-PR1a** (the relaxed decoder is proven against a hand-built current-version corruption case and hand-built v2 records before the v2 encoder exists in M3-PR1b), so the decoder relaxation is locked down by test before any code can emit the tail it must tolerate. @@ -538,7 +538,7 @@ inCooldown := elapsedMs < splitCooldownMs The fence that makes this safe lives on `SplitAtHLC` (a leader-fenced, Raft-committed ts, §7.7b), not on `now`: even with a modestly skewed wall clock the elapsed estimate stays within clock-skew of the true value, which CLAUDE.md already requires ("keep wall clocks reasonably synchronized"). If the HLC physical component is **ahead** of `time.Now().UnixMilli()` — possible because `NextFenced` can encode the Raft physical ceiling when it is ahead of the local wall clock — the detector clamps negative elapsed to zero and treats the route as still cooling down. That is the conservative direction: an early post-election re-split is prevented, while the route becomes eligible after at most the configured cooldown plus bounded clock skew. The raw HLC is retained only for the durable `SplitAtHLC` field and for diagnostics; **all** conversions of `SplitAtHLC` go through `hlcPhysicalMs(...)`. (The actual enforcement deadline is then pinned on the leader's monotonic clock per §5.3 — the elapsed-ms delta only decides *whether* a route is still in cooldown at reconstruction time and *how much* monotonic time remains.) -**Scoping.** This is a catalog schema change, and to make the decoder-before-encoder ordering safe by construction (not advisory) it splits across **two separately-deployed PRs** (§7.7c "Upgrade ordering fence", codex `afc95e3b`): **M3-PR1a** ships the decoder relaxation alone (`catalogRouteCodecVersion` stays `1`, no v2 emission, current-version v1 remains strict, forward versions drain unknown tails, with the engine/catalog cleanup) and rolls out cluster-wide first; **M3-PR1b** then bumps the encoder to v2, adds `SplitAtHLC` + the committing-coordinator `SplitRange` timestamp builder + the HLC→ms conversion helper, and re-runs the codec matrix against the real encoder. Both land ahead of the detector that consumes them (M3-PR2/PR3). The §8.1 table and §8.4 acceptance criteria are updated accordingly. +**Scoping.** This is a catalog schema change, and to make the decoder-before-encoder ordering safe by construction (not advisory) it splits across **two separately-deployed PRs** (§7.7c "Upgrade ordering fence", review `afc95e3b`): **M3-PR1a** ships the decoder relaxation alone (`catalogRouteCodecVersion` stays `1`, no v2 emission, current-version v1 remains strict, forward versions drain unknown tails, with the engine/catalog cleanup) and rolls out cluster-wide first; **M3-PR1b** then bumps the encoder to v2, adds `SplitAtHLC` + the committing-coordinator `SplitRange` timestamp builder + the HLC→ms conversion helper, and re-runs the codec matrix against the real encoder. Both land ahead of the detector that consumes them (M3-PR2/PR3). The §8.1 table and §8.4 acceptance criteria are updated accordingly. ## 8. Milestone / PR Breakdown, Test Strategy, Open Questions @@ -547,19 +547,18 @@ The fence that makes this safe lives on `SplitAtHLC` (a leader-fenced, Raft-comm | PR | Scope | Tests | Independently shippable? | |---|---|---|---| | **M3-PR1a** (decoder relaxation, ship **first**) | **Implemented.** Retired engine `RecordAccess`/`splitRange` threshold path (§3.4); confirmed keyviz is the sole signal; introduced `catalogRouteCodecVersionMin` (`= 1`), accepted version byte `>= catalogRouteCodecVersionMin`, and made trailing bytes version-conditional: **current `version == 1` remains strict**, while `version > catalogRouteCodecVersion` drains unknown forward-version tail bytes (§7.7c). **No `SplitAtHLC` struct field, no encoder change, no helper change; `catalogRouteCodecVersion` stays `1` and the encoder still writes v1.** | `distribution/engine_test.go` updated (dead-code removal); codec matrix cases authored before the encoder exists — v1 still round-trips, v1 plus extra trailing byte is rejected, hand-built v2 records with finite and nil `End` drain the 8-byte tail, and version `< min` is rejected. | Shipped. | -| **M3-PR1b** (v2 emission, deploy **after** PR1a is cluster-wide) | **Add the durable `RouteDescriptor.SplitAtHLC` field**; bump the encoder to `catalogRouteCodecVersion = 2` and append the 8-byte big-endian tail on **both** End paths (§7.7c if/else); add the version-keyed `version >= 2` tail read in the decoder; update `CloneRouteDescriptor` and `routeDescriptorEqual` to include `SplitAtHLC` (§7.7b, Issue 2); add a coordinator-side `SplitRange` catalog transaction builder that allocates the fenced commit ts on the committing default-group leader with `nextTxnTSAfter(snapshot.ReadTS)`, stamps child descriptors with that exact ts, commits at that ts, and returns it; add the `hlcPhysicalMs` conversion helper (§7.7d); add `autosplit_*` metric scaffolding (no detector yet). | **codec round-trip matrix cases 1, 1a, 2 (§7.7c)** — v2 round-trip preserves `SplitAtHLC` (incl. `=0` and incl. at least one `End == nil` route, case 1a, guarding the encoder early-return-on-`End == nil` path); re-run cases 4/4a now against the actual v2 encoder output; **case 7 (truncated tail rejected) — only the PR1b `version >= 2` exact-8-byte read can detect a `< 8`-byte tail (`io.ErrUnexpectedEOF → ErrCatalogInvalidRouteRecord`), so it is authored here, not in PR1a (§7.7c, codex P2)**; helper tests — clone preserves `SplitAtHLC` and `routeDescriptorEqual` distinguishes routes differing only in `SplitAtHLC`; fenced-ts tests — the committing leader's allocator retries after `Observe(ReadTS)` when a first `NextFenced()` would be `<= ReadTS`, forwarded SplitRange requests allocate on the new leader rather than accepting an adapter-supplied `CommitTS`, and `ErrCeilingExpired` fails closed; `hlcPhysicalMs` unit test incl. the `1<<16`-per-ms regression case; metric registration test. | Yes — additive schema field with forward/back-compat decode + metric names; no behavior change. **Deployment-ordered after M3-PR1a** (the only operational gate; see §7.7c failure mode if violated). Carries the deploy-after-PR1a rollout note in its PR description. | -| **M3-PR2a** | **Pure autosplit detector core.** Add `distribution/autosplit` with route-row aggregation by `(RouteID, RaftGroupID)`, `RouteStateActive`-only candidacy, per-window weighted ops/min scoring, consecutive-window hysteresis, cooldown state, p50 sub-bucket split-key selection, aggregate-row skips, route cap accounting, per-cycle split budget, and state-map GC reconciliation against the live catalog each evaluation (§7.7a). No sampler reader, no scheduler wiring, no Top-K isolation, and no runtime side effects. | Table-driven unit tests for row aggregation, active vs non-active candidacy reset, consecutive-window promotion/reset, cooldown, state-map shrink after parent retirement, no-evidence decline, route cap reservation, per-cycle budget, and p50 boundary selection including finite and unbounded right edges. | Yes — library-only detector core; no runtime behavior change. | -| **M3-PR2b-a** | **Committed-window reader and observe-only detector bridge.** Add `MatrixColumn.WindowStart` at `MemSampler.Flush`, preserve it through `Snapshot` deep copies, and add a `distribution/autosplit` reader that consumes `MemSampler.Snapshot`, fetches from `lastProcessedAt - Step`, excludes already-processed columns, accepts only proven committed window boundaries, falls back only to the immediately previous column for legacy in-memory rows, and runs the pure detector without calling `SplitRange`. | `keyviz` tests for persisted/deep-copied `WindowStart`; autosplit tests for committed-window derivation, legacy previous-boundary fallback, `lastProcessedAt - Step` fetches, and missing sampler rows resetting confidence in observe-only evaluation. | Yes — no runtime scheduler, no catalog mutation, no split RPC. | -| **M3-PR2b** | Remaining observe-only detector integration: compound single-key isolation (§5.4.2), Top-K alignment/error gates, and the default-group plus shard-group leadership-start processed-column watermark/straddling-column skip (§4.2/§7.6). No scheduler wiring — detector emits decisions to a no-op/log sink in this PR. | Unit tests for leadership watermarks, Top-K lower-bound/share/absolute-floor gates, compound split cooldown exemption, unknown-load target exclusion, and observe-only sink output. | Yes — detector runs and logs would-be decisions; nothing splits. Safe to ship "observe-only." | -| **M3-PR3** | Scheduler wiring to `SplitRange` (§6, incl. compound single-key isolation §6.4) + `--autoSplit` flag + sampler-wiring condition `keyvizEnabled \|\| autoSplit` and `autoSplitDefaultBuckets` implied bucketing in `main.go` and `cmd/server/demo.go` (§7.1) + runtime kill switch + slog + `SplitAtHLC`-seeded cooldown reconstruction on election (§7.7b). Same-group target only (cross-group target hook present but disabled). | Integration: detector→`SplitRange` end-to-end in the 3-node demo (`cmd/server/demo.go`); kill-switch + leader-change reset + seeded-cooldown-from-`SplitAtHLC` tests; failed `SplitRange` does **not** seed cooldown; a same-leader compound partial stores the pending second-call details and retries the final split before normal candidates even with no new sampler column, while a restart/leader change reconstructs normal cooldown for that intermediate child. | Yes — completes standalone auto-split. | -| **M3-PR4** (post-M2) | Enable least-loaded `target_group_id` selection on the existing scheduler hook once M2's cross-group `SplitRange` lands (§6.2); exclude in-flight `SplitJob` routes. **Compound single-key isolation stays same-group-only** — both its `SplitRange` calls always pass `target_group_id = 0`; cross-group selection applies only when the right child carries the hot load and target knowledge covers every live route in the target group (§6.2/§6.4, codex P2). The singleton `{H}` is non-targetable after isolation until a future `MoveRange`-style primitive exists. | Integration: cross-group auto-split against M2 migrator; target-selection unit tests; **a test that a compound isolation under `--autoSplitCrossGroup` still issues both calls same-group and only a non-singleton final child is migrated in a later independent decision (never call 1's intermediate child, never the `{H}` singleton)**; p50 left-hot and `isolation_single_lower_edge` decisions force same-group; target admission with authoritative fresh rows for every live route admits the group, while a non-authoritative fresh row or any missing live-route row skips as `load_unknown`; zero-live-route groups remain admissible as known idle. | Depends on M2; the M3 scheduler interface is unchanged. | -| **M3-PR5** | Lifecycle: this doc is renamed `*_partial_*` after M3-PR1a; → `*_implemented_*` after M3-PR3 (standalone) with M3-PR4 tracked as the cross-group follow-on. | — | Doc-only. | +| **M3-PR1b** (v2 emission, deploy **after** PR1a is cluster-wide) | **Implemented.** Added durable `RouteDescriptor.SplitAtHLC`, v2 encoding/decoding, clone/equality propagation, committing-leader fenced timestamp allocation, and HLC-to-physical-ms conversion. | Codec matrix, helper, fenced timestamp, forwarding, ceiling-expiry, and conversion tests. | Shipped after M3-PR1a. | +| **M3-PR2a** | **Implemented in PR #1097.** Pure `distribution/autosplit` detector: per-label route aggregation, per-column and smoothed rates, strict consecutive-window hysteresis, cooldown, p50, compound isolation decision forms, Top-K uncertainty gates, route-cap reservation, per-cycle budget, and state GC. | Table-driven and rapid invariant tests in `distribution/autosplit/detector_test.go` and `detector_rapid_test.go`. | Yes — library-only detector core. | +| **M3-PR2b** | **Implemented in PR #1104.** Exact committed `(WindowStart, At]` evidence is stored with each matrix column; aligned Top-K snapshots are attached to that column. Missing-row resets, catalog/default-group/shard-group leadership fences, history-gap confidence reset, and route-only locally-led scoring are wired into the scheduler input. | `keyviz/flusher_test.go`, detector alignment/gap tests, and scheduler leadership/watermark tests. | Integrated with M3-PR3 so the proven evidence path is the path that schedules splits. | +| **M3-PR3** | **Implemented in PR #1104.** Same-group `SplitRange` scheduling, two-call compound finalization/retry, CAS/version chaining, catalog-key leadership gate, sampler reconciliation on every node, startup/runtime controls, fixed-cardinality metrics, demo wiring, and stale snapshot/catalog-gap fencing. | Scheduler, watcher, admin RPC, metrics, demo, rapid, and real three-node etcd-Raft end-to-end tests, including runtime disable/re-enable and actual leadership transfer. | Yes — standalone M3 is complete; cross-group targeting stays disabled. | +| **M3-PR4** (post-M2) | Enable least-loaded `target_group_id` selection on the existing scheduler hook once M2's cross-group `SplitRange` lands (§6.2); exclude in-flight `SplitJob` routes. **Compound single-key isolation stays same-group-only** — both its `SplitRange` calls always pass `target_group_id = 0`; cross-group selection applies only when the right child carries the hot load and target knowledge covers every live route in the target group (§6.2/§6.4, review P2). The singleton `{H}` is non-targetable after isolation until a future `MoveRange`-style primitive exists. | Integration: cross-group auto-split against M2 migrator; target-selection unit tests; **a test that a compound isolation under `--autoSplitCrossGroup` still issues both calls same-group and only a non-singleton final child is migrated in a later independent decision (never call 1's intermediate child, never the `{H}` singleton)**; p50 left-hot and `isolation_single_lower_edge` decisions force same-group; target admission with authoritative fresh rows for every live route admits the group, while a non-authoritative fresh row or any missing live-route row skips as `load_unknown`; zero-live-route groups remain admissible as known idle. | Depends on M2; the M3 scheduler interface is unchanged. | +| **M3-PR5** | **Implemented.** This document is renamed `*_implemented_*`; M3-PR4 remains explicitly tracked as the post-M2 cross-group follow-on. | Requirement audit and implementation evidence in §8.3. | Doc-only. | Each PR carries the five-lens self-review and is gated by its tests + `make lint`. ### 8.2 Test strategy -- **Unit (table-driven, co-located `*_test.go`):** scoring math against the **pinned §5.1 formulas** (given per-column `(WindowStart, At, read_ops, write_ops)` vectors, both the **per-column** score sequence and the **smoothed** score are fully determined, including the configured-step invariance check across two different `--keyvizStep` values, a delayed/coalesced flusher regression where a column duration exceeds one configured step but still normalizes by actual duration, and a case asserting the per-column score differs from the smoothed average for a hot/hot/below-threshold column run); consecutive-column promotion and reset driven by the **per-column** score (§5.1 (A), §5.2), **including (codex P2 / round 6): (i) cold start promotes after exactly `candidateWindows` = 3 hot columns — not `2N−1` = 5 — proving the cold-start guard is gone; (ii) `N−1` hot columns then 1 below-threshold column then `N` hot columns: the below-threshold column resets the counter, total hot columns to promotion is `N`, not `2N−1`; (iii) a hot/hot/below-threshold run whose 3-column smoothed average stays above threshold must NOT promote (the below-threshold column's per-column score resets the counter — the regression that catches an averaged-score hysteresis); (iv) hot, hold-band, hot, hot with `candidateWindows = 3` must NOT promote because the hold-band/near-miss column resets confidence**; the `evalInterval < Step` case where extra cycles with no new flush column must not advance `consecutiveOver` (§5.2), while a pending compound finalization record still retries its stored second call on such a cycle; the **`evalInterval > Step` multi-column accumulation** case (the forward direction referenced in §5.2): with `evalInterval = 2 × Step`, `candidateWindows = 3`, after one evaluation cycle where 2 new flush columns have committed since `lastProcessedAt`, the detector evaluates each of the 2 new columns in chronological order and advances `consecutiveOver` twice (both per-column scores above threshold); a further single hot column on a subsequent cycle reaches `consecutiveOver = 3` and promotes — confirming the counter counts **columns**, not **cycles**, in the forward direction; strict below-threshold reset applied per column; cooldown enforcement and expiry (monotonic enforcement deadline) **including the `SplitAtHLC` → physical-ms conversion (§7.7d): a raw-HLC subtraction must not pass where the `hlcPhysicalMs(SplitAtHLC)` form fails (the `1<<16`-per-ms regression guard), with `now` taken from the wall clock per the §7.7d `now`-source decision**; split-key selection across the §5.4 cases — sub-range p50 using the bucket `hi` boundary, **plus the boundary-bucket fallbacks (codex P2): p50 in the last bucket → interior `lo`, p50 in the first bucket → interior `hi`**; single-key skew isolation (two-boundary compound), **plus the `H == route.Start` lower-edge single-split case, the rightmost lower-edge case where `route.End == nil` keeps `H·0x00` valid, the finite `route.End != nil && H·0x00 >= route.End && route.Start < H` upper-edge single-split case, and the already-isolated finite `[H, H·0x00)` decline (codex P2)**; no-evidence decline; unbounded tail; guardrails (min span, **max routes — including the route-delta accounting: a compound isolation at `liveRoutes == maxAutoRoutes - 1` is skipped with `route_cap` because its delta is `+2`, while a normal split at the same count proceeds since its delta is `+1`; AND the cycle-local `reservedDelta` accumulator (codex round-7 P2 / claude round-10 issue 2): with `liveRoutes == maxAutoRoutes - 1` and `maxSplitsPerCycle == 2`, two normal splits — admit the first, skip the second under `autosplit_skipped_total{reason="route_cap"}`**, per-cycle budget); **`topKeyAbsoluteFloor` duration-invariance (claude round-10 issue 1): the same hot key at the same per-minute rate fires/doesn't fire the gate identically across `Step=30s`/`60s`/`120s` and across delayed/coalesced committed windows because the floor is computed in normalised ops/min from actual duration, not raw column counts**; **share-fires-but-floor-doesn't fallthrough (claude round-10 issue 3): a route whose hot key has share >= `topKeyShare` but `absolute_contribution < topKeyAbsoluteFloor` emits a sub-range p50 decision (not a compound isolation) and increments `autosplit_isolation_declined_total{reason="absolute_floor"}`**; **unknown-load target exclusion (codex round-7 P2): cross-group target selection skips a candidate group when any live route lacks an authoritative fresh row or a future explicit idle certificate/quorum aggregate, and increments `autosplit_target_skipped_total{reason="load_unknown"}`**). The detector being a pure function (§5) makes these deterministic with no data plane. +- **Unit (table-driven, co-located `*_test.go`):** scoring math against the pinned §5.1 formulas; strict consecutive-column promotion/reset; multi-column chronological consumption; missing-row and history-gap resets; `SplitAtHLC` cooldown reconstruction; p50 and Top-K boundary forms; Top-K sample scaling, lower-bound, degradation, and absolute-floor gates; structural splittability; route-delta reservations; per-cycle budget; compound partial retry and revalidation; and final catalog-version chaining. `distribution/autosplit/detector_rapid_test.go` adds randomized invariants over promotion, cooldown, boundary validity, and cycle budget. - **Missing-row / skipped-column regressions:** with `evalInterval == Step`, `N-1` hot columns followed by an absent sampler row for the live route must reset `consecutiveOver` exactly like an explicit zero/below-threshold row. With `evalInterval > Step`, a single evaluation that receives hot column 1, absent column 2, and hot column 3 must process all three in chronological order and end with `consecutiveOver = 1`, not 2; looking only at the freshest committed column is a red control and must fail. - **Long-gap / state-transition regressions:** with `evalInterval` or detector stalls longer than `candidateWindows × Step`, the detector fetches and processes every committed column since `lastProcessedAt`; a below-threshold/absent column outside the trailing display window still resets confidence. A route that goes active → migrating → active entirely between evaluations clears `consecutiveOver` through watcher transition events or the catalog-version-gap fallback (§5.0). In the snapshot-only fallback, the ambiguous buffered columns up to the current freshest committed boundary are skipped for that route after the reset; a hot-hot-migrating-hot sequence that fits entirely between evaluations must not promote by scoring pre-gap hot columns after clearing. On default-group leadership start and on shard-group leadership start for a route's group, sampler columns with `At <= leaderStartProcessedAt` / `groupLeaderStartProcessedAt` and the first later column whose proven `(WindowStart, At]` window opened before the relevant leadership-start time are ignored for hysteresis; a node that had `candidateWindows` hot columns in local history before election/transfer, or a hot straddling column with no post-start traffic, must end with `consecutiveOver = 0` until fully post-start columns arrive. - **Route-row aggregation regressions:** a committed column with four sub-bucket rows for the same `(RouteID, RaftGroupID)` sums all non-aggregate `read_ops`/`write_ops` before scoring; no single bucket row can be treated as the route total. A hot route whose bucket rows are each below threshold but whose summed route total is above threshold must promote after `candidateWindows` qualifying columns. Synthetic `Aggregate=true` rows are still skipped and do not participate in the sum. @@ -569,12 +568,22 @@ Each PR carries the five-lens self-review and is gated by its tests + `make lint - **P50 boundary / hot-child regressions:** rightmost routes with `End == nil` and an interior median bucket split at the finite `hi`, not the bucket `lo`; only an actual last-bucket `hi == nil` or `hi >= route.End` uses the `lo` fallback. Cross-group p50 is allowed only when the estimated right-child load is at least the left-child load; first-bucket fallback and left-hot `p50_mid` decisions force same-group. - **Multi-split version chaining:** with `maxSplitsPerCycle = 2`, two disjoint valid decisions selected from one snapshot are issued serially: the second uses the first split's returned/re-read catalog version and is revalidated against the updated snapshot, not the starting `expected_catalog_version`. - **Cooldown scheduling regressions:** failed `SplitRange` responses do not seed cooldown for predicted children and leave the parent eligible to re-evaluate; cooldown starts only after committed children are observed. A same-leader compound partial keeps a pending-finalization record so the second isolation call retries from stored `(route_id, bounds, split_key)` before normal candidates even when no new flush column arrives, while a leader restart with only `[H,B)` committed reconstructs normal cooldown from that route's `SplitAtHLC`. -- **Cross-group target evidence regressions:** a target group with zero live routes is admissible as known idle; a non-empty group is admissible only when every live active route has an authoritative fresh row for the committed column or a future explicit idle certificate/quorum aggregate. A group with two live routes and an authoritative row for only one is skipped as `load_unknown`; the same skip applies if any row is non-authoritative ingress/follower evidence. `isolation_single_lower_edge` always uses `target_group_id = 0`; `isolation_single_upper_edge` may target only after the complete-evidence gate passes. +- **Cross-group target evidence regressions (M3-PR4):** these remain follow-on acceptance tests and are not claimed by standalone M3. Standalone scheduling always uses `target_group_id = 0`. - **Property tests (`pgregory.net/rapid`):** feed randomized per-column sequences and assert invariants — never schedule during cooldown; never exceed `maxSplitsPerCycle` logical splits per cycle (§6.3); never emit a `split_key` outside `(route.Start, route.End)`; a route is promoted iff its **per-column** score cleared threshold for `candidateWindows` consecutive committed columns (so any single column whose per-column score is below threshold within the would-be run prevents promotion, regardless of the smoothed average or former hold band — the per-column-vs-average invariant); promotion never requires more than `candidateWindows` hot columns from a cold start. -- **Integration:** detector → `SplitRange` end-to-end in the 3-node demo (`cmd/server/demo.go`) — drive a hot key, assert a same-group split appears in `ListRoutes` and the catalog version bumped; kill-switch flips off mid-load and no further splits occur; default-group leader change and shard-group leadership transfer reset detector confidence for affected routes, ignore pre-start/straddling sampler history, and re-earn a candidate before re-splitting; seeded cooldown from catalog lineage prevents immediate re-split after election (§7.6). +- **Integration:** `main_autosplit_e2e_test.go` boots three real etcd-Raft nodes, drives a hot route through the production coordinator/sampler path, observes a committed same-group split and final catalog version, disables and re-enables scheduling through the runtime switch, transfers actual leadership, rejects pre-transfer history, and requires fully post-transfer evidence before another split (§7.6). - **Jepsen:** the hotspot workload (hot-key load + partition/kill nemeses, linearizability during split) is **deferred to M4** per the parent doc §12 (M4: "Jepsen hotspot workloads") and §13.3. M3 does not add a Jepsen suite; M3-PR3's integration test is the end-to-end gate. -### 8.3 Five-lens self-review (to be filled per PR) +### 8.3 Implementation evidence and completed self-review + +| Normative area | Implementation evidence | Proof | +|---|---|---| +| Signal and exact windows (§3-§4) | `keyviz/flusher.go`, `keyviz/sampler.go`, `main.go`, `cmd/server/demo.go` | Exact `(WindowStart, At]` columns carry aligned Top-K snapshots; bare `--autoSplit` enables sampling and implied buckets. | +| Detection and guardrails (§5) | `distribution/autosplit/detector.go` | Per-label aggregation, normalized per-column/smoothed scores, strict hysteresis, p50/Top-K/compound forms, structural splittability, route cap, cooldown, state GC. | +| Scheduling and catalog safety (§6) | `distribution/autosplit/scheduler.go`, `adapter/distribution_server.go` | Catalog-key leadership gate, same-group `SplitRange` only, fresh-version chaining, compound retry/revalidation, committed final catalog version. | +| Leadership and missed state (§4.2, §7.6) | `kv/coordinator.go`, `kv/sharded_coordinator.go`, scheduler leadership fences | Default-group and shard-group terms reset confidence; stale/straddling/history-gap evidence is fenced. | +| Watcher and sampler membership (§6.1) | `distribution/watcher.go`, `main_autosplit.go` | Thread-safe reconciliation registers changed children and removes retired routes on every node. | +| Operations (§7) | `proto/admin.proto`, `adapter/admin_grpc.go`, `distribution/autosplit/metrics.go` | Authenticated `Admin.SetAutoSplitEnabled`, validated startup flags, runtime-enabled gauge, fixed metric label enums. | +| End-to-end behavior (§8.2) | `main_autosplit_e2e_test.go` | Real three-node Raft split, runtime disable/re-enable, leadership transfer, stale-history rejection, confidence re-earn. | | Lens | M3-specific risk to check | |---|---| @@ -596,9 +605,9 @@ Each PR carries the five-lens self-review and is gated by its tests + `make lint 8. Routes in `RouteStateWriteFenced`/`RouteStateMigratingSource`/`RouteStateMigratingTarget` are never auto-split (§5.0); only `RouteStateActive` routes are candidates, and any observed or unprovable-between-evaluations non-active interval clears stale confidence before a route can promote. 9. The leader-local state map is bounded: after a split retires a parent `RouteID`, its entry is evicted within one cycle and `autosplit_tracked_routes` never exceeds the live route count (§7.7a). 9a. Synthetic aggregate/coarsened sampler rows are never split: auto-split either validates enough tracked-route capacity up front or skips `MatrixRow.Aggregate` with `autosplit_skipped_total{reason="aggregate_row"}` (§5.0). -9b. Missing or non-authoritative local load is never treated as low load: follower-forwarded-write and multi-group under-observation are documented fail-closed limitations, and cross-group target selection requires complete knowledge for every live route in a non-empty target group: authoritative fresh rows for the committed column, or a future explicit idle certificate/quorum group aggregate. Zero-live-route proof remains a known-idle no-row case (§4.1, §6.2). -10. **Rolling-upgrade safe by construction (technically enforced, codex `afc95e3b`):** the codec work ships as two separately-deployed PRs — **M3-PR1a (decoder relaxation, `catalogRouteCodecVersion` stays `1`, no v2 emission, current v1 remains strict) rolls out cluster-wide before M3-PR1b (v2 encoder + `SplitAtHLC`) deploys** (§7.7c "Upgrade ordering fence"). After PR1a is everywhere, no strict pre-relaxation decoder remains for forward versions, so the first v2 record PR1b emits is decodable on every node — a node with the PR1a relaxed decoder reads a v2 `RouteDescriptor` (with `SplitAtHLC`) without error, draining the 8-byte tail and treating the route as never-in-cooldown (codec matrix **cases 3a, 4, 4a, 1b, and 7** load-bearing — PR1a covers current-version strictness in 3a and forward-version drain in 4/4a; PR1b covers 1b/7). PR1a relaxes the version-byte check but keeps trailing bytes strict for `version == 1`; PR1b keeps the same known-version strictness for `version == 2` (Case 1b) and adds the truncated-tail rejection (Case 7), §7.7c Issue 1. If the deploy gate is violated (PR1b before PR1a is cluster-wide) and a `SplitRange` runs, the only consequence is transient route-not-found on the lagging node until it receives PR1a — no data loss, the v2 record is well-formed (§7.7c failure mode). Codec matrix case 5 (forward version) and the §7.7b helper updates (`CloneRouteDescriptor` / `routeDescriptorEqual` preserve `SplitAtHLC`) pass. -11. **Split-key boundary cases (codex P2):** an effective `SubBucketCount == 1` row is treated as no sub-range evidence; a rightmost `End == nil` route whose p50 bucket has a finite `hi` uses that finite `hi`, not the `lo` fallback; an upper-tail hotspot whose p50 falls in the actual last sub-bucket splits at the interior `lo` fallback (never drops the decision on a `hi == route.End`/`nil` boundary, §5.4); a lower-tail first-bucket fallback and any p50 decision whose left child is estimated hotter are same-group-only and do not promise later cross-group relief without a move primitive; a single-key hotspot whose hot key is already the route start (`H == route.Start`) isolates `{H}` with the lower-edge single-split form and forces same-group, including on a rightmost `End == nil` route where `H·0x00` is finite and valid; only a finite route whose upper edge is at/before `H·0x00` isolates `{H}` with an upper-edge single split that may target cross-group if target evidence is complete; neither case attempts the invalid two-boundary form (§5.4.2). +9b. Missing local evidence is never fabricated as hotspot evidence: follower-forwarded-write and multi-group under-observation remain documented conservative limitations. Standalone M3 does not perform cross-group target selection; its complete-load admission rules remain M3-PR4 requirements (§4.1, §6.2). +10. **Rolling-upgrade safe by construction (technically enforced, review `afc95e3b`):** the codec work ships as two separately-deployed PRs — **M3-PR1a (decoder relaxation, `catalogRouteCodecVersion` stays `1`, no v2 emission, current v1 remains strict) rolls out cluster-wide before M3-PR1b (v2 encoder + `SplitAtHLC`) deploys** (§7.7c "Upgrade ordering fence"). After PR1a is everywhere, no strict pre-relaxation decoder remains for forward versions, so the first v2 record PR1b emits is decodable on every node — a node with the PR1a relaxed decoder reads a v2 `RouteDescriptor` (with `SplitAtHLC`) without error, draining the 8-byte tail and treating the route as never-in-cooldown (codec matrix **cases 3a, 4, 4a, 1b, and 7** load-bearing — PR1a covers current-version strictness in 3a and forward-version drain in 4/4a; PR1b covers 1b/7). PR1a relaxes the version-byte check but keeps trailing bytes strict for `version == 1`; PR1b keeps the same known-version strictness for `version == 2` (Case 1b) and adds the truncated-tail rejection (Case 7), §7.7c Issue 1. If the deploy gate is violated (PR1b before PR1a is cluster-wide) and a `SplitRange` runs, the only consequence is transient route-not-found on the lagging node until it receives PR1a — no data loss, the v2 record is well-formed (§7.7c failure mode). Codec matrix case 5 (forward version) and the §7.7b helper updates (`CloneRouteDescriptor` / `routeDescriptorEqual` preserve `SplitAtHLC`) pass. +11. **Split-key boundary cases (review P2):** an effective `SubBucketCount == 1` row is treated as no sub-range evidence; a rightmost `End == nil` route whose p50 bucket has a finite `hi` uses that finite `hi`, not the `lo` fallback; an upper-tail hotspot whose p50 falls in the actual last sub-bucket splits at the interior `lo` fallback (never drops the decision on a `hi == route.End`/`nil` boundary, §5.4); a single-key hotspot whose hot key is already the route start (`H == route.Start`) isolates `{H}` with the lower-edge single-split form, including on a rightmost `End == nil` route where `H·0x00` is finite and valid; a finite route whose upper edge is at/before `H·0x00` isolates `{H}` with the upper-edge single-split form; neither case attempts the invalid two-boundary form. Standalone M3 sends every form same-group; right-child target eligibility remains M3-PR4 scope (§5.4.2, §6.2). 12. **Top-K isolation priority, alignment, sampling scale, and error gating:** when Top-K and sub-range buckets are both available, the detector evaluates Top-K single-key isolation first only when the Top-K sketch is attached to the scored column or tagged with the same explicit window boundaries, the snapshot is not degraded, and the Space-Saving lower-bound estimate passes both gates. Timestamp-only `SnapshotAt` tolerance is rejected. The share gate uses `max(0, entry.Count - ceil(SampledN/Capacity)) × SampleRate / column_i.write_ops` over the same committed column and route-level aggregated denominator, so changing `HotKeysSampleRate` does not silently disable isolation, while sketch error or degraded windows fall through to p50 instead of burning a compound split. ## 9. Open Questions @@ -607,20 +616,15 @@ Per the design-doc-first workflow this section **converges**: where review (or t 1. **OQ-1 — `Route.Load` / `Stats` fate. (RESOLVED — keep `Stats`, drop `RecordAccess`/`splitRange`/threshold ctor.)** `Engine.Stats()` is a useful read-only debugging surface and costs nothing when unused, so M3-PR1a keeps `Route.Load`/`Stats` but deletes the dead `RecordAccess`/`splitRange`/`NewEngineWithThreshold` write path (§3.4). Full deletion of `Load`/`Stats` is not worth losing the diagnostic. (§3.4) 2. **OQ-2 — Down-side hysteresis shape. (RESOLVED — reset every below-threshold column.)** Promotion uses the literal parent-doc rule: `candidateWindows` consecutive committed columns must each have `per_col_score >= threshold`. Scores in the former `[hysteresisDownFactor×threshold, threshold)` hold band reset `consecutiveOver` just like any other below-threshold score. Holding would allow hot/hold/hot/hot to promote with `candidateWindows = 3`, while decay would make promotion depend on the prior counter depth; both weaken the invariant. The factor may remain only as diagnostic classification, not as promotion state. (§5.2) -3. **OQ-3 — No-evidence fallback. (OPEN — implementer call in M3-PR2.)** When neither usable sub-range buckets nor Top-K are available, decline to split (current proposal) or allow a midpoint-of-observed-min/max-key split? Declining is safer and is the proposal. This is most commonly reached when the operator explicitly sets `--keyvizKeyBucketsPerRoute 1` alongside `--autoSplit` (§3.1, §7.1 — `--autoSplit` otherwise implies `autoSplitDefaultBuckets` sub-range buckets), but it can also occur when the sampler's effective `SubBucketCount` collapses to 1 for a long-common-prefix / unbounded-tail route (§5.4). The detector treats both as no split-key evidence rather than manufacturing a boundary at the route edge. Left open only because a future operator might want an opt-in `--autoSplitMidpointFallback` escape hatch; not needed for M3. (§5.4) -4. **OQ-4 — Minimum route size without per-route key counts. (OPEN — implementer call in M3-PR2.)** The engine has no per-route key count. Approximate `minRouteSpan` from sub-range/Top-K evidence (current proposal, conservative "do not split" when evidence is absent), or add a cheap per-route key-count estimate to the sampler? The proposal (approximate, fail-conservative) is sufficient for M3; the sampler-side estimate is a keyviz follow-up, not an M3 blocker. (§5.5) +3. **OQ-3 — No-evidence fallback. (RESOLVED — decline.)** When neither usable sub-range buckets nor Top-K evidence supplies a strict interior boundary, the detector emits `no_split_key`. It never invents a midpoint. (§5.4) +4. **OQ-4 — Minimum route size without per-route key counts. (RESOLVED — structural splittability.)** M3 does not expose `minRouteSpan`: lexicographic byte distance is not a meaningful generic size measure. A route is eligible only when observed evidence supplies a strict interior boundary; a singleton is terminally unsplittable. (§5.5) 5. **OQ-5 — Cross-node / cross-ingress scoring. (OPEN — deferred to a future milestone; reframed.)** *Reframed from "heavy follower-forwarded reads" — the real gap is broader (§4.1).* In a multi-group cluster, routes whose shard group leader is a different node than the default-group leader are invisible to the leader-local detector. Even in a single-default-group deployment, coverage is complete only for traffic observed by the default-group leader itself; writes received by followers can be observed only on the ingress follower because `Internal.Forward` does not re-enter the leader's sampler hook. M3 ships leader-local and fail-closed on unknown load: positive local evidence can promote a split, but missing local evidence is not treated as "low load." Closing full coverage needs cross-node scoring or follower reports (e.g. reading the keyviz cluster fan-out aggregate behind the same detector interface, §5). Deferred; not an M3 blocker because under-observation is conservative (fewer splits, never an unsafe one). (§4.1) -6. **OQ-6 — Cross-group activation gate. (RESOLVED — gate behind `--autoSplitCrossGroup`; only move the right child when it is hot.)** Once M2 lands, do **not** auto-enable least-loaded `target_group_id`; gate it behind an explicit `--autoSplitCrossGroup` sub-flag. Cross-group splits trigger M2's `SplitJob` data movement, whose blast radius warrants a separate, explicit opt-in distinct from same-group range splitting. M3 ships the hook disabled; M3-PR4 adds the flag. **Even with the flag on, a §5.4.2 compound single-key isolation is always executed same-group (both `SplitRange` calls with `target_group_id = 0`)** — applying a target on call 1 would make the intermediate child a migrating `SplitJob` route that §6.3 excludes, blocking call 2 (codex P2). P50 and one-boundary single-key decisions may target cross-group only when the M2-moved right child carries the hot load: first-bucket/left-hot p50 and `isolation_single_lower_edge` are same-group-only, while `isolation_single_upper_edge` may target after the complete target-evidence gate passes. Cross-group relocation after compound applies only to the two non-singleton final children as independent later decisions; the `{H}` singleton is non-targetable through `SplitRange` until a future `MoveRange(routeID, target_group_id)` primitive exists (§6.2/§6.4/OQ-16). (§6.2) +6. **OQ-6 — Cross-group activation gate. (RESOLVED — gate behind `--autoSplitCrossGroup`; only move the right child when it is hot.)** Once M2 lands, do **not** auto-enable least-loaded `target_group_id`; gate it behind an explicit `--autoSplitCrossGroup` sub-flag. Cross-group splits trigger M2's `SplitJob` data movement, whose blast radius warrants a separate, explicit opt-in distinct from same-group range splitting. M3 ships the hook disabled; M3-PR4 adds the flag. **Even with the flag on, a §5.4.2 compound single-key isolation is always executed same-group (both `SplitRange` calls with `target_group_id = 0`)** — applying a target on call 1 would make the intermediate child a migrating `SplitJob` route that §6.3 excludes, blocking call 2 (review P2). P50 and one-boundary single-key decisions may target cross-group only when the M2-moved right child carries the hot load: first-bucket/left-hot p50 and `isolation_single_lower_edge` are same-group-only, while `isolation_single_upper_edge` may target after the complete target-evidence gate passes. Cross-group relocation after compound applies only to the two non-singleton final children as independent later decisions; the `{H}` singleton is non-targetable through `SplitRange` until a future `MoveRange(routeID, target_group_id)` primitive exists (§6.2/§6.4/OQ-16). (§6.2) 7. **OQ-7 — Durable cooldown across elections. (RESOLVED.)** Resolved by adding `RouteDescriptor.SplitAtHLC` — a leader-fenced commit ts (§7.7b) used by the new leader to reconstruct cooldown from a durable timestamp (the elapsed estimate compares `time.Now().UnixMilli()` against `hlcPhysicalMs(SplitAtHLC)`, §7.7d; `now` is the wall clock because reconstruction is a non-ordering-sensitive throttle, robust to bounded clock skew). The remaining sub-question — is a single `SplitAtHLC` (the split that *created* the route) sufficient, or should it refresh on later same-route events? — is **answered: single is enough**, because cooldown is only ever started by a split, and a split always creates fresh child routes each with their own `SplitAtHLC`. (§7.7b) -8. **OQ-8 — Threshold units across `--keyvizStep`. (RESOLVED — pin the ops/min formulas; per-column score drives hysteresis.)** Resolved by pinning two normalization formulas in §5.1: the **per-column** score `per_col_score = (write_ops/column_i.duration_s×60)×Ww + (read_ops/column_i.duration_s×60)×Wr` (a single column's weighted ops/min, the input to the §5.2 hysteresis counter) and the **smoothed** score over the trailing `N = candidateWindows` columns divided by the sum of those columns' actual durations (display/slog only). Both are weighted ops/min, so the `50_000 ops/min` threshold is invariant to `--keyvizStep` and delayed/coalesced flusher commits (the coupling OQ-8 worried about) and keeps the unit the one operators reason in. Driving hysteresis off the per-column score — not the smoothed average — is the reconciliation with the parent doc §6.2 "3 consecutive windows over threshold" rule (codex P2, review round 6); the averaged score is kept only because operators want a non-jittering rate in logs. Both formulas are the reference for the §8.2 scoring tests (incl. a configured-step invariance case, a delayed/coalesced duration red control, and the per-column vs smoothed distinction). Expressing the threshold in ops-per-`Step` instead is rejected — it would re-introduce the `Step`↔sensitivity coupling. (§5.1, §5.2) -12. **OQ-12 — Cluster-wide target-load fan-out. (OPEN — future M3.x/M2 hook.)** M3's local-view target selection admits a non-empty group only when the default-group leader has complete committed-window knowledge for every live route in that group: authoritative fresh rows for all live routes in the evaluated window. Any non-authoritative row or missing route fails closed as `load_unknown` (§6.2), even if the local node otherwise appears to serve the route, because follower-forwarded writes can bypass the sampler hook. A future fan-out should publish per-group aggregates from every serving node with a committed-window timestamp and enough freshness/quorum metadata for the default-group leader to distinguish "reported zero" from "not reported." Once that exists, target eligibility can accept cluster-wide zero/low reports for groups with live routes instead of relying on the default leader's local serving set. Until then, cross-node missing or ingress-only rows are unknown, not low. +8. **OQ-8 — Threshold units across `--keyvizStep`. (RESOLVED — pin the ops/min formulas; per-column score drives hysteresis.)** Resolved by pinning two normalization formulas in §5.1: the **per-column** score `per_col_score = (write_ops/column_i.duration_s×60)×Ww + (read_ops/column_i.duration_s×60)×Wr` (a single column's weighted ops/min, the input to the §5.2 hysteresis counter) and the **smoothed** score over the trailing `N = candidateWindows` columns divided by the sum of those columns' actual durations (display/slog only). Both are weighted ops/min, so the `50_000 ops/min` threshold is invariant to `--keyvizStep` and delayed/coalesced flusher commits (the coupling OQ-8 worried about) and keeps the unit the one operators reason in. Driving hysteresis off the per-column score — not the smoothed average — is the reconciliation with the parent doc §6.2 "3 consecutive windows over threshold" rule (review P2, review round 6); the averaged score is kept only because operators want a non-jittering rate in logs. Both formulas are the reference for the §8.2 scoring tests (incl. a configured-step invariance case, a delayed/coalesced duration red control, and the per-column vs smoothed distinction). Expressing the threshold in ops-per-`Step` instead is rejected — it would re-introduce the `Step`↔sensitivity coupling. (§5.1, §5.2) +12. **OQ-12 — Cluster-wide target-load fan-out. (OPEN — M3-PR4/M2 hook.)** Standalone M3 has no cross-group target selection. Before M3-PR4 enables it, a fan-out must publish per-group aggregates with committed-window freshness and enough quorum metadata to distinguish "reported zero" from "not reported"; missing or ingress-only rows remain unknown, not low. 16. **OQ-16 — MoveRange primitive for singleton routes. (OPEN — future control-plane primitive.)** M2's cross-group path is attached to `SplitRange`, so it can only move the right child produced by a split with an interior `split_key`. Once a singleton `{H} = [H, H·0x00)` already exists, it has no interior key and is therefore non-targetable through another `SplitRange`; M3 must not repeatedly try to migrate it after isolation. A future `MoveRange(routeID, target_group_id)` RPC could relocate an existing route without splitting it first, but that primitive is out of scope for M3 (§6.2/§6.4). ## 10. Lifecycle -This document begins as `*_proposed_*`. Per CLAUDE.md / `docs/design/README.md`: - -- Renamed to `*_partial_*` after M3-PR1a; track per-PR landing under §8.1 and update the parent partial doc's M3 row. -- Rename to `*_implemented_*` after M3-PR3 ships (standalone auto-split complete), with M3-PR4 (cross-group target selection, depends on M2) tracked as the cross-group follow-on. - -Use `git mv` so history follows the rename. The propose date (2026-06-11) and slug stay fixed. +This document progressed from `*_proposed_*` to `*_partial_*` after M3-PR1a and is now `*_implemented_*` after standalone M3-PR2b/M3-PR3 completion. M3-PR4 remains tracked here as the post-M2 cross-group follow-on and does not change standalone M3's implemented lifecycle state. The proposal date (2026-06-11) and slug remain fixed. diff --git a/docs/design/2026_06_11_implemented_leader_balance_scheduler.md b/docs/design/2026_06_11_implemented_leader_balance_scheduler.md index 36305731a..bb38a8632 100644 --- a/docs/design/2026_06_11_implemented_leader_balance_scheduler.md +++ b/docs/design/2026_06_11_implemented_leader_balance_scheduler.md @@ -4,7 +4,7 @@ Status: Implemented Author: bootjp Date: 2026-06-11 -Sibling: [2026_06_11_partial_hotspot_split_milestone3_automation.md](2026_06_11_partial_hotspot_split_milestone3_automation.md) — shares the "scheduler lives on the default-group leader, default OFF behind a flag, runtime kill switch, bounded-cardinality metrics, leader-local state reset on election" conventions. This doc reuses those conventions but does **not** depend on M3 landing. +Sibling: [2026_06_11_implemented_hotspot_split_milestone3_automation.md](2026_06_11_implemented_hotspot_split_milestone3_automation.md) — shares the "scheduler lives on the default-group leader, default OFF behind a flag, runtime kill switch, bounded-cardinality metrics, leader-local state reset on election" conventions. This doc reuses those conventions but does **not** depend on M3 landing. ## Implementation status diff --git a/docs/design/2026_06_12_proposed_scaling_roadmap.md b/docs/design/2026_06_12_proposed_scaling_roadmap.md index 76a179915..844a3d3cc 100644 --- a/docs/design/2026_06_12_proposed_scaling_roadmap.md +++ b/docs/design/2026_06_12_proposed_scaling_roadmap.md @@ -30,7 +30,7 @@ production envelope within one growth cycle: | Dimension | Today's comfortable ceiling | Where the next operator wants to go | Triggering signal | | --- | --- | --- | --- | -| Routes (shards) per cluster | ~10 k (binary search + 100 ms watcher + history ring of 32 versions) | 100 k–1 M routes | hotspot-split-M3 automation (`docs/design/2026_06_11_partial_hotspot_split_milestone3_automation.md`) generates routes faster than the catalog watcher can fan out at scale | +| Routes (shards) per cluster | ~10 k (binary search + 100 ms watcher + history ring of 32 versions) | 100 k–1 M routes | hotspot-split-M3 automation (`docs/design/2026_06_11_implemented_hotspot_split_milestone3_automation.md`) generates routes faster than the catalog watcher can fan out at scale | | Region/DC fan-out | Single DC, LAN-tuned etcd/raft (10 ms tick, 100 ms heartbeat, 1 s election), single default-group HLC ceiling | 2–3 regions with active-active per-shard | Disaster-recovery SLO ("survive a region outage with < 30 s data-plane reconvergence") | | Bytes per shard | ~1 TB before snapshot transfer / WAL replay becomes operational pain | 5–10 TB per shard, with shard counts in the 100 k range | DynamoDB-compatibility customers ingesting at multi-GiB/s per table | | Per-node QPS | Leader-bound write, lease-read scales across shards but only on each shard's leader | 5–10x current per-node throughput; sustained reads from non-leader replicas | Redis-compatibility migration projects ingesting from upstream Redis where the upstream serves reads from replicas | diff --git a/docs/design/2026_06_23_proposed_scaling_roadmap.md b/docs/design/2026_06_23_proposed_scaling_roadmap.md index 4b6634c3a..7d5b5a10e 100644 --- a/docs/design/2026_06_23_proposed_scaling_roadmap.md +++ b/docs/design/2026_06_23_proposed_scaling_roadmap.md @@ -110,7 +110,7 @@ memory each group's private cache/memtable pins. over-full node. Actual per-node volume relief therefore also depends on Gap 1 plus the replica-placement / region-balance work in Gap 4. Auto-detection / scheduling is **PR #951** - (`docs/design/2026_06_11_partial_hotspot_split_milestone3_automation.md`, + (`docs/design/2026_06_11_implemented_hotspot_split_milestone3_automation.md`, branch `design/hotspot-split-m3-automation`), which drives detection off the already-wired keyviz sampler rather than the dead `RecordAccess` counters. - **Range MERGE — missing, no design exists.** The inverse of split. Both the diff --git a/keyviz/flusher.go b/keyviz/flusher.go index 2dfc460f7..74dbc5c4a 100644 --- a/keyviz/flusher.go +++ b/keyviz/flusher.go @@ -30,18 +30,16 @@ func RunFlusher(ctx context.Context, s *MemSampler, step time.Duration) { select { case <-ctx.Done(): return - case <-t.C: - s.Flush() + case at := <-t.C: + if !s.flushWindow(ctx, at) { + return + } } } } -// RunHotKeysAggregator drives the per-route Top-K aggregator goroutine -// until ctx is cancelled, returning afterwards. Mirrors RunFlusher; the -// caller in main.go is expected to launch one of each in the same -// errgroup. No-op (blocks on ctx) when the sampler is nil or -// HotKeysEnabled is false — keeping the call site uniform so a future -// flip of the flag does not require restructuring startup wiring. +// RunHotKeysAggregator drains sampled events and services exact-boundary +// snapshot requests from RunFlusher until ctx is cancelled. func RunHotKeysAggregator(ctx context.Context, s *MemSampler) { if s == nil || s.hotKeys == nil { <-ctx.Done() diff --git a/keyviz/flusher_test.go b/keyviz/flusher_test.go index 8ce3ba69c..df82a1de2 100644 --- a/keyviz/flusher_test.go +++ b/keyviz/flusher_test.go @@ -4,6 +4,8 @@ import ( "context" "testing" "time" + + "github.com/stretchr/testify/require" ) func TestRunFlusherTicksUntilCancel(t *testing.T) { @@ -37,6 +39,103 @@ func TestRunFlusherTicksUntilCancel(t *testing.T) { } } +func TestRunFlusherAttachesAlignedHotKeysWindow(t *testing.T) { + t.Parallel() + s := NewMemSampler(MemSamplerOptions{ + Step: 5 * time.Millisecond, + HistoryColumns: 16, + HotKeysEnabled: true, + HotKeysPerRoute: 8, + HotKeysSampleRate: 1, + HotKeysQueueSize: 32, + HotKeysMaxKeyLen: 64, + }) + require.True(t, s.RegisterRoute(1, []byte("a"), []byte("z"), 1)) + + ctx, cancel := context.WithCancel(context.Background()) + var done = make(chan struct{}, 2) + go func() { + RunHotKeysAggregator(ctx, s) + done <- struct{}{} + }() + go func() { + RunFlusher(ctx, s, s.Step()) + done <- struct{}{} + }() + for i := 0; i < 8; i++ { + s.Observe(1, []byte("hot"), OpWrite, 0, LabelLegacy) + } + + require.Eventually(t, func() bool { + for _, col := range s.Snapshot(time.Time{}, time.Time{}) { + if len(col.HotKeys) == 0 { + continue + } + snapshot := col.HotKeys[0] + return snapshot.WindowStart.Equal(col.WindowStart) && + snapshot.WindowEnd.Equal(col.At) && + len(snapshot.Entries) > 0 + } + return false + }, time.Second, 5*time.Millisecond) + cancel() + <-done + <-done +} + +func TestFlushWindowKeepsHotKeyAndCounterInSameColumn(t *testing.T) { + t.Parallel() + now := time.Unix(1_700_000_000, 0) + s := NewMemSampler(MemSamplerOptions{ + Step: time.Minute, + HistoryColumns: 4, + HotKeysEnabled: true, + HotKeysPerRoute: 8, + HotKeysSampleRate: 1, + HotKeysQueueSize: 32, + HotKeysMaxKeyLen: 64, + Now: func() time.Time { return now }, + }) + require.True(t, s.RegisterRoute(1, []byte("a"), []byte("z"), 1)) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + defer close(done) + RunHotKeysAggregator(ctx, s) + }() + + s.flushMu.Lock() + observed := make(chan struct{}) + go func() { + s.Observe(1, []byte("hot"), OpWrite, 0, LabelLegacy) + close(observed) + }() + select { + case <-observed: + t.Fatal("Observe crossed a hot-key flush boundary") + case <-time.After(20 * time.Millisecond): + } + s.flushMu.Unlock() + select { + case <-observed: + case <-time.After(time.Second): + t.Fatal("Observe remained blocked after the flush boundary opened") + } + require.True(t, s.flushWindow(ctx, now.Add(time.Minute))) + + columns := s.Snapshot(time.Time{}, time.Time{}) + require.Len(t, columns, 1) + require.Len(t, columns[0].Rows, 1) + require.Equal(t, uint64(1), columns[0].Rows[0].Writes) + require.Len(t, columns[0].HotKeys, 1) + require.Equal(t, uint64(1), columns[0].HotKeys[0].SampledN) + require.Len(t, columns[0].HotKeys[0].Entries, 1) + require.Equal(t, []byte("hot"), columns[0].HotKeys[0].Entries[0].Key) + + cancel() + <-done +} + // TestRunFlusherNilSamplerWaitsCtx asserts the nil-sampler contract // (RunFlusher just blocks on ctx.Done so callers can hard-wire it // regardless of whether keyviz is enabled). diff --git a/keyviz/hot_keys.go b/keyviz/hot_keys.go index 8de2ed27e..33f6acc10 100644 --- a/keyviz/hot_keys.go +++ b/keyviz/hot_keys.go @@ -39,6 +39,8 @@ type KeyvizHotKeysSnapshot struct { DroppedSamples uint64 // node-global: bounded-queue back-pressure drops SkippedLongKeys uint64 // node-global: pre-sample length-cap rejects SnapshotAt time.Time // when the aggregator deep-copied this snapshot + WindowStart time.Time // committed matrix-window lower boundary + WindowEnd time.Time // committed matrix-window upper boundary SampleRate int // R at the time of snapshot Capacity int // m at the time of snapshot Entries []KeyvizHotKeyEntry @@ -60,6 +62,7 @@ type KeyvizHotKeyEntry struct { type hotKeysAggregator struct { s *MemSampler // back-reference, for table.Load() at publish time ch chan hotKeyEvent + flush chan hotKeyFlushRequest keyPool *sync.Pool // []byte buffers for hot-path key clones capacity int // m sampleRate int // R (used by Observe) @@ -88,6 +91,12 @@ type hotKeysAggregator struct { rngState atomic.Uint64 } +type hotKeyFlushRequest struct { + windowStart time.Time + windowEnd time.Time + reply chan []KeyvizHotKeysSnapshot +} + // splitmix64 standard parameters — public-domain mixer used unchanged // across implementations. The golden-ratio increment is coprime with // every power of two so the atomic counter walks the whole uint64 space. @@ -152,6 +161,7 @@ func newHotKeysAggregator(s *MemSampler, opts MemSamplerOptions) *hotKeysAggrega a := &hotKeysAggregator{ s: s, ch: make(chan hotKeyEvent, opts.HotKeysQueueSize), + flush: make(chan hotKeyFlushRequest), capacity: opts.HotKeysPerRoute, sampleRate: opts.HotKeysSampleRate, maxKeyLen: maxKeyLen, @@ -239,24 +249,10 @@ func (a *hotKeysAggregator) observe(routeID uint64, label Label, key []byte) boo } } -// run is the aggregator goroutine's main loop. Single select with two -// arms (drain events / tick), matching the design §4 pseudocode: -// -// for { -// select { -// case e := <-ch: updateSketch(e) -// case <-tick: -// for len(ch) > 0 { updateSketch(<-ch) } // best-effort pre-drain -// publishAndReset() -// } -// } -// -// Single-writer to every per-route/label SS sketch and to perSlotN, so no -// lock is needed on the update path. ctx cancellation drains a final -// publish so the last window's data isn't lost on shutdown. +// run serializes sampled events and matrix-flush snapshot requests. The matrix +// flusher supplies the exact committed boundaries so Top-K evidence can be +// attached to the matching MatrixColumn without timestamp heuristics. func (a *hotKeysAggregator) run(ctx context.Context) { - tick := time.NewTicker(a.step) - defer tick.Stop() for { select { case <-ctx.Done(): @@ -270,21 +266,41 @@ func (a *hotKeysAggregator) run(ctx context.Context) { return case e := <-a.ch: a.consume(e) - case <-tick.C: - // Pre-drain: process every event currently queued before we - // publish + reset, so the just-finished window's tail-end - // observations land in this snapshot rather than the next - // (Codex P2 round-7 L193 — boundary smear is now bounded by - // arrivals during publishAndReset, not by the channel depth - // at the tick). + case req := <-a.flush: for len(a.ch) > 0 { a.consume(<-a.ch) } - a.publishAndReset() + req.reply <- a.publishAndResetAt(req.windowStart, req.windowEnd) } } } +func (a *hotKeysAggregator) snapshotWindow( + ctx context.Context, + windowStart time.Time, + windowEnd time.Time, +) ([]KeyvizHotKeysSnapshot, bool) { + if a == nil { + return nil, true + } + req := hotKeyFlushRequest{ + windowStart: windowStart, + windowEnd: windowEnd, + reply: make(chan []KeyvizHotKeysSnapshot, 1), + } + select { + case <-ctx.Done(): + return nil, false + case a.flush <- req: + } + select { + case <-ctx.Done(): + return nil, false + case snapshots := <-req.reply: + return snapshots, true + } +} + // consume handles one event: drains the sample-N counter for the route, // updates the Space-Saving sketch, and releases the pool buffer. func (a *hotKeysAggregator) consume(e hotKeyEvent) { @@ -314,6 +330,11 @@ func (a *hotKeysAggregator) consume(e hotKeyEvent) { // per-route / node-global counters so the next keyvizStep window // starts empty (design §4 sketch reset; Codex P2 round-6 L186). func (a *hotKeysAggregator) publishAndReset() { + now := a.clock() + a.publishAndResetAt(now.Add(-a.step), now) +} + +func (a *hotKeysAggregator) publishAndResetAt(windowStart, windowEnd time.Time) []KeyvizHotKeysSnapshot { // When no per-route sketch exists this window (e.g. every observe // was filtered by the length cap, or no traffic at all on any // route), we have nowhere to attach the node-global drop / skip @@ -321,9 +342,8 @@ func (a *hotKeysAggregator) publishAndReset() { // the signal. Carry them forward into the next window instead; // whichever publish tick first has a sketch will surface them. if len(a.sketches) == 0 { - return + return nil } - now := a.clock() // Swap-and-capture (not Load + later Store(0)) — the hot path can // still call a.dropped.Add(1) / a.skipped.Add(1) between a Load and // a Store, and those increments would belong to neither the snapshot @@ -333,6 +353,7 @@ func (a *hotKeysAggregator) publishAndReset() { dropped := a.dropped.Swap(0) skipped := a.skipped.Swap(0) tbl := a.s.table.Load() + out := make([]KeyvizHotKeysSnapshot, 0, len(a.sketches)) for skey, sk := range a.sketches { slot := lookupSlotForRoute(tbl, skey.RouteID, skey.Label) if slot == nil { @@ -355,12 +376,15 @@ func (a *hotKeysAggregator) publishAndReset() { SampledN: n, DroppedSamples: dropped, SkippedLongKeys: skipped, - SnapshotAt: now, + SnapshotAt: windowEnd, + WindowStart: windowStart, + WindowEnd: windowEnd, SampleRate: a.sampleRate, Capacity: a.capacity, Entries: toSnapshotEntries(sk.snapshot()), } slot.hotKeysSnap.Store(snap) + out = append(out, cloneHotKeysSnapshot(*snap)) sk.reset() } // Per-route counters reset; the node-global ones were already @@ -370,6 +394,17 @@ func (a *hotKeysAggregator) publishAndReset() { for k := range a.perSlotN { *a.perSlotN[k] = 0 } + return out +} + +func cloneHotKeysSnapshot(snapshot KeyvizHotKeysSnapshot) KeyvizHotKeysSnapshot { + out := snapshot + out.Entries = make([]KeyvizHotKeyEntry, len(snapshot.Entries)) + for i, entry := range snapshot.Entries { + out.Entries[i] = entry + out.Entries[i].Key = cloneBytes(entry.Key) + } + return out } func toSnapshotEntries(es []ssEntrySnap) []KeyvizHotKeyEntry { diff --git a/keyviz/hot_keys_test.go b/keyviz/hot_keys_test.go index 9a2b3d002..2f7f3d1e1 100644 --- a/keyviz/hot_keys_test.go +++ b/keyviz/hot_keys_test.go @@ -355,11 +355,15 @@ func TestHotKeysAggregatorRaceFree(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) var wgRun sync.WaitGroup - wgRun.Add(1) + wgRun.Add(2) go func() { defer wgRun.Done() RunHotKeysAggregator(ctx, s) }() + go func() { + defer wgRun.Done() + RunFlusher(ctx, s, s.Step()) + }() var wg sync.WaitGroup for w := 0; w < 8; w++ { @@ -377,8 +381,8 @@ func TestHotKeysAggregatorRaceFree(t *testing.T) { }(w) } wg.Wait() - // Wait for the aggregator to drain the queue and publish at least - // one tick-driven snapshot for each route. The previous + // Wait for the matrix flusher to request an aligned Top-K snapshot for each + // route. The previous // time.Sleep(50ms) raced the aggregator's tick (Step=5ms) on slow // -race CI runners — the Run goroutine could be slow to schedule // at all, leaving hotKeysSnap as its initial nil atomic.Pointer diff --git a/keyviz/ring_buffer.go b/keyviz/ring_buffer.go index 8e81b3c23..4d77e7adf 100644 --- a/keyviz/ring_buffer.go +++ b/keyviz/ring_buffer.go @@ -83,7 +83,16 @@ func cloneColumn(col MatrixColumn) MatrixColumn { rows[i].MemberRoutes = append([]uint64(nil), row.MemberRoutes...) } } - return MatrixColumn{WindowStart: col.WindowStart, At: col.At, Rows: rows} + hotKeys := make([]KeyvizHotKeysSnapshot, len(col.HotKeys)) + for i, snapshot := range col.HotKeys { + hotKeys[i] = cloneHotKeysSnapshot(snapshot) + } + return MatrixColumn{ + WindowStart: col.WindowStart, + At: col.At, + Rows: rows, + HotKeys: hotKeys, + } } // snapshotOrdered returns a chronologically ordered (oldest first) diff --git a/keyviz/sampler.go b/keyviz/sampler.go index 20aa01a93..0e4523f39 100644 --- a/keyviz/sampler.go +++ b/keyviz/sampler.go @@ -14,11 +14,11 @@ // // Hot-path properties (see design §5.1, §10): // -// - Observe is a single atomic.Pointer[routeTable].Load, a plain map -// lookup against an immutable snapshot, and at most two -// atomic.AddUint64 calls (one for the count, one for bytes — -// skipped when both keyLen and valueLen are zero). No allocation, -// no mutex. +// - With hot-key tracking disabled, Observe is a single +// atomic.Pointer[routeTable].Load, a plain map lookup against an immutable +// snapshot, and at most two atomic.AddUint64 calls. Hot-key tracking adds a +// read lock to writes so sampled events stay aligned with their counter +// window. // - Flush drains the per-route counters with atomic.SwapUint64; no // pointer retirement, so a late writer cannot race past the snapshot // and lose counts. @@ -31,6 +31,7 @@ package keyviz import ( "bytes" + "context" "encoding/binary" "log/slog" "math" @@ -201,6 +202,11 @@ type MemSamplerOptions struct { type MemSampler struct { opts MemSamplerOptions now func() time.Time + // flushMu serializes explicit and background flushes, protects the + // committed window lower boundary, and aligns enabled hot-key events with + // the counter window drained by that flush. + flushMu sync.RWMutex + lastFlushAt time.Time // table holds the immutable map of currently-tracked routes. The // hot path Observe() does a single Load + map lookup; mutations @@ -212,19 +218,10 @@ type MemSampler struct { // Held only by non-hot-path callers. routesMu sync.Mutex - // flushMu serialises the full Flush flow from timestamp capture - // through history append. That keeps column order and window - // boundaries aligned when callers accidentally run concurrent - // flushers. - flushMu sync.Mutex - // historyMu guards the ring buffer. Reads (Snapshot) and writes // (Flush) acquire it; Observe never touches it. historyMu sync.Mutex history *ringBuffer - // lastFlushAt is the lower boundary for the next committed matrix - // column. Guarded by historyMu with history. - lastFlushAt time.Time // retiredSlots holds slots that RemoveRoute removed from the live // table. Each entry is drained for `remaining` Flushes — a grace @@ -437,10 +434,12 @@ func (s *routeSlot) snapshotMeta() (start, end []byte, aggregate bool, members [ // MatrixColumn is one committed heatmap window. type MatrixColumn struct { // WindowStart and At delimit the committed half-open sampler - // interval. Rows contain counters drained for (WindowStart, At]. + // interval. Rows and HotKeys contain evidence drained for + // (WindowStart, At]. WindowStart time.Time At time.Time Rows []MatrixRow + HotKeys []KeyvizHotKeysSnapshot } // MatrixRow is a single route or virtual bucket's counter snapshot @@ -524,8 +523,8 @@ func NewMemSampler(opts MemSamplerOptions) *MemSampler { opts: opts, now: now, history: newRingBuffer(opts.HistoryColumns), - lastFlushAt: now(), groupTerms: map[uint64]uint64{}, + lastFlushAt: now(), } s.table.Store(newEmptyRouteTable()) if opts.HotKeysEnabled { @@ -664,6 +663,14 @@ func (s *MemSampler) Observe(routeID uint64, key []byte, op Op, valueLen int, la if s == nil { return } + if s.hotKeys != nil && op == OpWrite { + s.flushMu.RLock() + defer s.flushMu.RUnlock() + } + s.observe(routeID, key, op, valueLen, label) +} + +func (s *MemSampler) observe(routeID uint64, key []byte, op Op, valueLen int, label Label) { if !s.opts.KeyVizLabelsEnabled { label = LabelLegacy } @@ -1220,8 +1227,35 @@ func (s *MemSampler) Flush() { } s.flushMu.Lock() defer s.flushMu.Unlock() + s.flushAtLocked(s.now(), nil) +} + +func (s *MemSampler) flushWindow(ctx context.Context, at time.Time) bool { + if s == nil { + return true + } + s.flushMu.Lock() + defer s.flushMu.Unlock() - col := MatrixColumn{At: s.now()} + var hotKeys []KeyvizHotKeysSnapshot + if s.hotKeys != nil { + var ok bool + hotKeys, ok = s.hotKeys.snapshotWindow(ctx, s.lastFlushAt, at) + if !ok { + return false + } + } + s.flushAtLocked(at, hotKeys) + return true +} + +func (s *MemSampler) flushAtLocked(at time.Time, hotKeys []KeyvizHotKeysSnapshot) { + col := MatrixColumn{ + WindowStart: s.lastFlushAt, + At: at, + HotKeys: hotKeys, + } + s.lastFlushAt = at // Snapshot the per-group leader-term map once at the top of // Flush so every row in this column sees a consistent view, even // if SetLeaderTerm fires concurrently. Later rows can never @@ -1245,11 +1279,9 @@ func (s *MemSampler) Flush() { }) s.historyMu.Lock() - col.WindowStart = s.lastFlushAt if !col.WindowStart.Before(col.At) { col.WindowStart = col.At.Add(-s.opts.Step) } - s.lastFlushAt = col.At s.history.Push(col) s.historyMu.Unlock() } diff --git a/kv/coordinator.go b/kv/coordinator.go index 093d77686..a06932158 100644 --- a/kv/coordinator.go +++ b/kv/coordinator.go @@ -8,10 +8,12 @@ import ( "reflect" "strings" "sync" + "sync/atomic" "time" "github.com/bootjp/elastickv/internal/monoclock" "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/keyviz" pb "github.com/bootjp/elastickv/proto" "github.com/cockroachdb/errors" ) @@ -112,6 +114,36 @@ func (c *Coordinate) SetHLCLeaseRenewalBlocker(blocked func() bool) { c.hlcRenewalBlocked = blocked } +// WithSampler wires a keyviz sampler onto the single-group coordinator. +// routeID must be the catalog route ID covering the single group; a zero routeID +// disables sampling so callers cannot accidentally emit unroutable rows. +func (c *Coordinate) WithSampler(s keyviz.Sampler, routeID uint64) *Coordinate { + if c == nil { + return c + } + if routeID == 0 { + c.samplerConfig.Store(nil) + return c + } + c.samplerConfig.Store(&samplerConfig{sampler: s, routeID: routeID}) + return c +} + +// WithSamplerRouteResolver wires keyviz sampling with a route lookup evaluated +// for every observed key. Single-group deployments need this after a range +// split, when one fixed startup RouteID no longer covers the keyspace. +func (c *Coordinate) WithSamplerRouteResolver(s keyviz.Sampler, resolve func(key []byte) (uint64, bool)) *Coordinate { + if c == nil { + return c + } + if s == nil || resolve == nil { + c.samplerConfig.Store(nil) + return c + } + c.samplerConfig.Store(&samplerConfig{sampler: s, resolve: resolve}) + return c +} + // normalizeLeaseObserver flattens a typed-nil LeaseReadObserver to an // untyped nil interface so downstream `observer != nil` checks behave // as expected. @@ -229,10 +261,19 @@ type Coordinate struct { // nil check so production does not pay an interface call when // monitoring is disabled). leaseObserver LeaseReadObserver + samplerConfig atomic.Pointer[samplerConfig] } var _ Coordinator = (*Coordinate)(nil) +type samplerRouteResolveFunc func(key []byte) (uint64, bool) + +type samplerConfig struct { + sampler keyviz.Sampler + routeID uint64 + resolve samplerRouteResolveFunc +} + type Coordinator interface { Dispatch(ctx context.Context, reqs *OperationGroup[OP]) (*CoordinateResponse, error) IsLeader() bool @@ -571,6 +612,7 @@ func prepareDispatchRetry(ctx context.Context, reqs *OperationGroup[OP], leaderA // monotonicity across the leader transition. func (c *Coordinate) dispatchOnce(ctx context.Context, reqs *OperationGroup[OP]) (*CoordinateResponse, error) { if !c.IsLeader() { + c.observeElems(reqs.Elems) return c.redirect(ctx, reqs) } @@ -895,6 +937,17 @@ func (c *Coordinate) IsLeaderForKey(_ []byte) bool { return c.IsLeader() } +// LeadershipForKey reports local leadership and the current Raft term for the +// single group that owns every key handled by Coordinate. +func (c *Coordinate) LeadershipForKey(_ []byte) (bool, uint64) { + return leadershipForEngine(c.engine) +} + +// GroupLeadership reports leadership for Coordinate's single Raft group. +func (c *Coordinate) GroupLeadership(_ uint64) (bool, uint64) { + return leadershipForEngine(c.engine) +} + func (c *Coordinate) VerifyLeaderForKey(ctx context.Context, _ []byte) error { return c.VerifyLeader(ctx) } @@ -907,7 +960,8 @@ func (c *Coordinate) LinearizableRead(ctx context.Context) (uint64, error) { return linearizableReadEngineCtx(ctx, c.engine) } -func (c *Coordinate) LinearizableReadForKey(ctx context.Context, _ []byte) (uint64, error) { +func (c *Coordinate) LinearizableReadForKey(ctx context.Context, key []byte) (uint64, error) { + c.observeRead(key) return c.LinearizableRead(ctx) } @@ -1012,7 +1066,8 @@ func engineLeaseAckValid(state raftengine.State, ack, now monoclock.Instant, lea return now.Sub(ack) < leaseDur } -func (c *Coordinate) LeaseReadForKey(ctx context.Context, _ []byte) (uint64, error) { +func (c *Coordinate) LeaseReadForKey(ctx context.Context, key []byte) (uint64, error) { + c.observeRead(key) return c.LeaseRead(ctx) } @@ -1119,6 +1174,7 @@ func (c *Coordinate) dispatchTxn(ctx context.Context, reqs []*Elem[OP], readKeys if err := ValidateElemCommitTSPatches(reqs, commitTS); err != nil { return nil, err } + c.observeElems(reqs) // ReadKeys are included in the Raft log entry so the FSM validates // read-write conflicts atomically under applyMu, eliminating the TOCTOU @@ -1144,7 +1200,9 @@ func (c *Coordinate) dispatchTxn(ctx context.Context, reqs []*Elem[OP], readKeys func (c *Coordinate) dispatchRaw(ctx context.Context, req []*Elem[OP]) (*CoordinateResponse, error) { muts := make([]*pb.Mutation, 0, len(req)) for _, elem := range req { - muts = append(muts, elemToMutation(elem)) + mut := elemToMutation(elem) + c.observeMutation(mut) + muts = append(muts, mut) } ts, err := c.allocateTimestamp(ctx, "allocate raw dispatch ts") @@ -1168,6 +1226,78 @@ func (c *Coordinate) dispatchRaw(ctx context.Context, req []*Elem[OP]) (*Coordin }, nil } +func (c *Coordinate) observeElems(elems []*Elem[OP]) { + if c == nil { + return + } + for _, elem := range elems { + c.observeMutation(elemToMutation(elem)) + } +} + +// ObserveForwardedRequests records leader-side sampling evidence for writes +// that entered through a follower and were committed via Internal.Forward. +func (c *Coordinate) ObserveForwardedRequests(reqs []*pb.Request) { + if c == nil { + return + } + for _, req := range reqs { + if req == nil { + continue + } + for _, mut := range req.Mutations { + if mut == nil || isTxnMetaKey(mut.Key) { + continue + } + c.observeMutation(mut) + } + } +} + +func (c *Coordinate) observeMutation(mut *pb.Mutation) { + cfg := c.samplerSnapshot() + if cfg == nil || mut == nil { + return + } + routeID, sampleKey, ok := cfg.routeForKey(mut.Key) + if !ok { + return + } + cfg.sampler.Observe(routeID, sampleKey, keyviz.OpWrite, len(mut.Value), keyviz.LabelLegacy) +} + +func (c *Coordinate) observeRead(key []byte) { + cfg := c.samplerSnapshot() + if cfg == nil { + return + } + routeID, sampleKey, ok := cfg.routeForKey(key) + if !ok { + return + } + cfg.sampler.Observe(routeID, sampleKey, keyviz.OpRead, 0, keyviz.LabelLegacy) +} + +func (c *Coordinate) samplerSnapshot() *samplerConfig { + if c == nil { + return nil + } + cfg := c.samplerConfig.Load() + if cfg == nil || cfg.sampler == nil { + return nil + } + return cfg +} + +func (cfg *samplerConfig) routeForKey(key []byte) (uint64, []byte, bool) { + sampleKey := RouteKey(key) + if cfg.resolve != nil { + routeID, ok := cfg.resolve(sampleKey) + return routeID, sampleKey, ok && routeID != 0 + } + return cfg.routeID, sampleKey, cfg.routeID != 0 +} + // toRawRequest builds a forwarded raw Request for the redirect path // (follower → leader Internal.Forward). The leader stamps Ts on // arrival via adapter.Internal.stampRawTimestamps, so the follower diff --git a/kv/coordinator_dispatch_test.go b/kv/coordinator_dispatch_test.go index a559c6405..ebbea3979 100644 --- a/kv/coordinator_dispatch_test.go +++ b/kv/coordinator_dispatch_test.go @@ -1,9 +1,15 @@ package kv import ( + "bytes" "context" "testing" + "time" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/internal/s3keys" + "github.com/bootjp/elastickv/keyviz" + pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" ) @@ -180,6 +186,145 @@ func TestCoordinateDispatchRaw_CallsTransactionManager(t *testing.T) { require.Equal(t, 1, tx.commits) } +func TestCoordinateSamplerObservesRawTxnAndRead(t *testing.T) { + t.Parallel() + + const routeID = uint64(7) + sampler := keyviz.NewMemSampler(keyviz.MemSamplerOptions{ + Step: time.Second, + HistoryColumns: 4, + MaxTrackedRoutes: 8, + }) + sampler.RegisterRoute(routeID, []byte(""), nil, 1) + clock := NewHLC() + clock.SetPhysicalCeiling(time.Now().Add(time.Minute).UnixMilli()) + c := (&Coordinate{ + transactionManager: &stubTransactional{}, + engine: stubLeaderEngine{}, + clock: clock, + }).WithSampler(sampler, routeID) + + _, err := c.Dispatch(context.Background(), &OperationGroup[OP]{ + Elems: []*Elem[OP]{ + {Op: Put, Key: []byte("raw"), Value: []byte("value")}, + }, + }) + require.NoError(t, err) + startTS, err := c.nextStartTS(context.Background()) + require.NoError(t, err) + _, err = c.Dispatch(context.Background(), &OperationGroup[OP]{ + IsTxn: true, + StartTS: startTS, + Elems: []*Elem[OP]{ + {Op: Put, Key: []byte("txn"), Value: []byte("value")}, + }, + }) + require.NoError(t, err) + _, err = c.LeaseReadForKey(context.Background(), []byte("read")) + require.NoError(t, err) + + sampler.Flush() + row := requireKeyVizRouteRow(t, sampler, routeID) + const expectedWriteBytes uint64 = 16 // raw+value and txn+value. + require.Equal(t, uint64(2), row.Writes) + require.Equal(t, uint64(1), row.Reads) + require.Equal(t, expectedWriteBytes, row.WriteBytes) +} + +func TestCoordinateSamplerResolvesRouteForEveryObservedKey(t *testing.T) { + t.Parallel() + + sampler := &recordingSampler{} + rawS3Key := s3keys.ObjectManifestKey("bucket", 1, "z-object") + normalizedS3Key := RouteKey(rawS3Key) + var resolverKeys [][]byte + clock := NewHLC() + clock.SetPhysicalCeiling(time.Now().Add(time.Minute).UnixMilli()) + c := (&Coordinate{ + transactionManager: &stubTransactional{}, + engine: stubLeaderEngine{}, + clock: clock, + }).WithSamplerRouteResolver(sampler, func(key []byte) (uint64, bool) { + resolverKeys = append(resolverKeys, append([]byte(nil), key...)) + if bytes.Equal(key, normalizedS3Key) { + return 8, true + } + return 7, true + }) + + _, err := c.Dispatch(context.Background(), &OperationGroup[OP]{Elems: []*Elem[OP]{ + {Op: Put, Key: []byte("a"), Value: []byte("left")}, + {Op: Put, Key: rawS3Key, Value: []byte("right")}, + }}) + require.NoError(t, err) + + require.Equal(t, [][]byte{[]byte("a"), normalizedS3Key}, resolverKeys) + calls := sampler.snapshot() + require.Len(t, calls, 2) + require.Equal(t, uint64(7), calls[0].routeID) + require.Equal(t, []byte("a"), calls[0].key) + require.Equal(t, uint64(8), calls[1].routeID) + require.Equal(t, normalizedS3Key, calls[1].key) + require.NotEqual(t, rawS3Key, calls[1].key) +} + +func TestCoordinateObserveForwardedRequestsSkipsTxnMetadata(t *testing.T) { + t.Parallel() + + sampler := &recordingSampler{} + var resolverKeys [][]byte + c := (&Coordinate{}).WithSamplerRouteResolver(sampler, func(key []byte) (uint64, bool) { + resolverKeys = append(resolverKeys, append([]byte(nil), key...)) + return 7, true + }) + + c.ObserveForwardedRequests([]*pb.Request{{ + IsTxn: true, + Phase: pb.Phase_NONE, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(TxnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("hot")})}, + {Op: pb.Op_PUT, Key: []byte("hot"), Value: []byte("value")}, + }, + }}) + + require.Equal(t, [][]byte{[]byte("hot")}, resolverKeys) + calls := sampler.snapshot() + require.Len(t, calls, 1) + require.Equal(t, uint64(7), calls[0].routeID) + require.Equal(t, []byte("hot"), calls[0].key) +} + +func TestCoordinateSamplerObservesFollowerIngressBeforeRedirect(t *testing.T) { + t.Parallel() + + const routeID = uint64(7) + sampler := keyviz.NewMemSampler(keyviz.MemSamplerOptions{ + Step: time.Second, + HistoryColumns: 4, + MaxTrackedRoutes: 8, + }) + require.True(t, sampler.RegisterRoute(routeID, []byte(""), nil, 1)) + c := (&Coordinate{ + engine: redirectSamplerFollowerEngine{}, + }).WithSampler(sampler, routeID) + + _, err := c.dispatchOnce(context.Background(), &OperationGroup[OP]{Elems: []*Elem[OP]{ + {Op: Put, Key: []byte("forwarded"), Value: []byte("value")}, + }}) + require.ErrorIs(t, err, ErrLeaderNotFound) + sampler.Flush() + require.Equal(t, uint64(1), requireKeyVizRouteRow(t, sampler, routeID).Writes) +} + +type redirectSamplerFollowerEngine struct { + stubLeaderEngine +} + +func (redirectSamplerFollowerEngine) State() raftengine.State { return raftengine.StateFollower } +func (redirectSamplerFollowerEngine) Leader() raftengine.LeaderInfo { + return raftengine.LeaderInfo{} +} + // TestToRawRequestLeavesTsForLeaderStamping is the regression for the // codex P2 review on PR #867. // @@ -221,6 +366,20 @@ func TestToRawRequestLeavesTsForLeaderStamping(t *testing.T) { } } +func requireKeyVizRouteRow(t *testing.T, sampler *keyviz.MemSampler, routeID uint64) keyviz.MatrixRow { + t.Helper() + cols := sampler.Snapshot(time.Now().Add(-time.Hour), time.Now().Add(time.Hour)) + for _, col := range cols { + for _, row := range col.Rows { + if row.RouteID == routeID { + return row + } + } + } + require.Failf(t, "missing keyviz row", "route_id=%d", routeID) + return keyviz.MatrixRow{} +} + // TestBuildRedirectRequestsSurvivesStaleFollowerCeiling exercises the // follower's redirect path end-to-end with an expired ceiling: it // confirms the follower hands off a Ts==0 request to the leader diff --git a/kv/raft_engine.go b/kv/raft_engine.go index 4eedebe81..5268cccd1 100644 --- a/kv/raft_engine.go +++ b/kv/raft_engine.go @@ -37,6 +37,17 @@ func isLeaderEngine(engine raftengine.LeaderView) bool { return engine != nil && engine.State() == raftengine.StateLeader } +func leadershipForEngine(engine interface { + raftengine.LeaderView + raftengine.StatusReader +}) (bool, uint64) { + if engine == nil { + return false, 0 + } + status := engine.Status() + return status.State == raftengine.StateLeader, status.Term +} + // isLeaderAcceptingWrites reports whether the engine is currently the leader // AND not mid leadership-transfer. During transfer, etcd/raft silently drops // proposals; callers that poll on a timer (HLC lease, lock resolver) should diff --git a/kv/sharded_coordinator.go b/kv/sharded_coordinator.go index 8c6a81ad4..335fa13ec 100644 --- a/kv/sharded_coordinator.go +++ b/kv/sharded_coordinator.go @@ -655,6 +655,13 @@ func (c *ShardedCoordinator) WithPartitionResolver(r PartitionResolver) *Sharded // The defaultGroup is used for non-keyed leader checks. func NewShardedCoordinator(engine *distribution.Engine, groups map[uint64]*ShardGroup, defaultGroup uint64, clock *HLC, st store.MVCCStore) *ShardedCoordinator { router := NewShardRouter(engine) + ownedGroups := map[uint64]*ShardGroup(nil) + if groups != nil { + ownedGroups = make(map[uint64]*ShardGroup, len(groups)) + for gid, group := range groups { + ownedGroups[gid] = group + } + } // Construct the coordinator before wrapping the per-shard // Transactionals so each leaseRefreshingTxn can back-reference it // for the §4.1 registration barrier (the gate is installed later @@ -662,14 +669,14 @@ func NewShardedCoordinator(engine *distribution.Engine, groups map[uint64]*Shard c := &ShardedCoordinator{ engine: engine, router: router, - groups: groups, + groups: ownedGroups, defaultGroup: defaultGroup, clock: clock, store: st, log: slog.Default(), } var deregisters []func() - for gid, g := range groups { + for gid, g := range ownedGroups { // Wrap Txn so every successful Commit/Abort refreshes the // per-shard lease. Leave nil transactions unchanged, and skip // if already wrapped so repeat calls don't stack wrappers. @@ -679,7 +686,7 @@ func NewShardedCoordinator(engine *distribution.Engine, groups map[uint64]*Shard // supports repeat construction by not stacking // wrappers): point the existing wrapper at the NEW // coordinator so Commit consults the freshly-installed - // registration gate, not a stale one (codex P2). + // registration gate, not a stale one. existing.coord = c } else { g.Txn = &leaseRefreshingTxn{inner: g.Txn, g: g, coord: c} @@ -1717,6 +1724,24 @@ func (c *ShardedCoordinator) IsLeaderForKey(key []byte) bool { return isLeaderEngine(engineForGroup(g)) } +// LeadershipForKey reports local leadership and Raft term for key's group. +func (c *ShardedCoordinator) LeadershipForKey(key []byte) (bool, uint64) { + g, ok := c.groupForKey(key) + if !ok { + return false, 0 + } + return leadershipForEngine(engineForGroup(g)) +} + +// GroupLeadership reports local leadership and Raft term for groupID. +func (c *ShardedCoordinator) GroupLeadership(groupID uint64) (bool, uint64) { + g, ok := c.groups[groupID] + if !ok { + return false, 0 + } + return leadershipForEngine(engineForGroup(g)) +} + func (c *ShardedCoordinator) VerifyLeaderForKey(ctx context.Context, key []byte) error { g, ok := c.groupForKey(key) if !ok { @@ -2180,12 +2205,36 @@ func (c *ShardedCoordinator) txnLogs(ctx context.Context, reqs *OperationGroup[O // path allocation-free. Reads have their own observeReadKey helper // (LinearizableReadForKey / LeaseReadForKey). func (c *ShardedCoordinator) observeMutation(routeID uint64, mut *pb.Mutation, label keyviz.Label) { - if c.sampler == nil { + if c == nil || c.sampler == nil || mut == nil { return } c.sampler.Observe(routeID, mut.Key, keyviz.OpWrite, len(mut.Value), c.keyVizObserveLabel(label)) } +// ObserveForwardedRequests records committed leader-side sampling evidence for +// writes that entered through a shard follower and were committed via +// Internal.Forward on this shard leader. +func (c *ShardedCoordinator) ObserveForwardedRequests(reqs []*pb.Request) { + if c == nil || c.sampler == nil { + return + } + for _, req := range reqs { + if req == nil { + continue + } + for _, mut := range req.Mutations { + if mut == nil || isTxnMetaKey(mut.Key) { + continue + } + routeID, _, ok := c.routeAndGroupForKey(mut.Key) + if !ok { + continue + } + c.observeMutation(routeID, mut, keyviz.LabelLegacy) + } + } +} + // observeRead records a single linearizable / lease read against the // route. The full key is passed (not just its length) so the sampler // can bucket the read into the key's sub-range for the hot-key heatmap; diff --git a/kv/sharded_coordinator_sampler_test.go b/kv/sharded_coordinator_sampler_test.go index 080a29027..5c95c461f 100644 --- a/kv/sharded_coordinator_sampler_test.go +++ b/kv/sharded_coordinator_sampler_test.go @@ -7,6 +7,7 @@ import ( "github.com/bootjp/elastickv/distribution" "github.com/bootjp/elastickv/keyviz" + pb "github.com/bootjp/elastickv/proto" "github.com/bootjp/elastickv/store" "github.com/stretchr/testify/require" ) @@ -62,8 +63,13 @@ func TestShardedCoordinatorObservesEveryDispatchedMutation(t *testing.T) { ctx := context.Background() engine := distribution.NewEngine() - engine.UpdateRoute([]byte("a"), []byte("m"), 1) - engine.UpdateRoute([]byte("m"), nil, 2) + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 100, Start: []byte("a"), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 101, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateActive}, + }, + })) s1 := store.NewMVCCStore() r1, stop1 := newSingleRaft(t, "kv-sampler-g1", NewKvFSMWithHLC(s1, NewHLC())) @@ -112,6 +118,60 @@ func TestShardedCoordinatorObservesEveryDispatchedMutation(t *testing.T) { } } +func TestShardedCoordinatorObservesForwardedRequests(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + require.NoError(t, engine.ApplySnapshot(distribution.CatalogSnapshot{ + Version: 1, + Routes: []distribution.RouteDescriptor{ + {RouteID: 100, Start: []byte("a"), End: []byte("m"), GroupID: 1, State: distribution.RouteStateActive}, + {RouteID: 101, Start: []byte("m"), End: nil, GroupID: 2, State: distribution.RouteStateActive}, + }, + })) + groups := map[uint64]*ShardGroup{ + 1: {}, + 2: {}, + } + rec := &recordingSampler{} + coord := NewShardedCoordinator(engine, groups, 1, NewHLC(), nil).WithSampler(rec) + + coord.ObserveForwardedRequests([]*pb.Request{ + { + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte("b"), Value: []byte("left")}, + }, + }, + { + IsTxn: true, + Phase: pb.Phase_NONE, + Mutations: []*pb.Mutation{ + {Op: pb.Op_PUT, Key: []byte(TxnMetaPrefix), Value: EncodeTxnMeta(TxnMeta{PrimaryKey: []byte("x")})}, + {Op: pb.Op_PUT, Key: []byte("x"), Value: []byte("right")}, + }, + }, + }) + + calls := rec.snapshot() + require.Len(t, calls, 2) + require.Equal(t, sampleCall{ + routeID: 100, + op: keyviz.OpWrite, + label: keyviz.LabelLegacy, + key: []byte("b"), + keyLen: 1, + valueLen: len("left"), + }, calls[0]) + require.Equal(t, sampleCall{ + routeID: 101, + op: keyviz.OpWrite, + label: keyviz.LabelLegacy, + key: []byte("x"), + keyLen: 1, + valueLen: len("right"), + }, calls[1]) +} + // TestShardedCoordinatorWithoutSamplerStaysSafe pins the nil-safe // contract: a coordinator without WithSampler (interface-nil // c.sampler) and one wired with a typed-nil *MemSampler must both diff --git a/kv/sharded_coordinator_txn_test.go b/kv/sharded_coordinator_txn_test.go index 42b65f328..94e858313 100644 --- a/kv/sharded_coordinator_txn_test.go +++ b/kv/sharded_coordinator_txn_test.go @@ -82,6 +82,22 @@ func TestShardedCoordinatorDispatchTxn_RejectsMissingPrimaryKey(t *testing.T) { require.ErrorIs(t, err, ErrTxnPrimaryKeyRequired) } +func TestNewShardedCoordinatorCopiesGroupMap(t *testing.T) { + t.Parallel() + + engine := distribution.NewEngine() + group := &ShardGroup{} + groups := map[uint64]*ShardGroup{1: group} + + coord := NewShardedCoordinator(engine, groups, 1, NewHLC(), nil) + delete(groups, 1) + groups[2] = &ShardGroup{} + + require.Same(t, group, coord.groups[1]) + _, ok := coord.groups[2] + require.False(t, ok) +} + func TestShardedCoordinatorDelPrefixBroadcast_UsesConfiguredAllShardGroups(t *testing.T) { t.Parallel() diff --git a/main.go b/main.go index c058f8651..10c1e574b 100644 --- a/main.go +++ b/main.go @@ -20,6 +20,7 @@ import ( "github.com/bootjp/elastickv/adapter" "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/distribution/autosplit" internalutil "github.com/bootjp/elastickv/internal" "github.com/bootjp/elastickv/internal/admin" "github.com/bootjp/elastickv/internal/encryption" @@ -297,6 +298,7 @@ const bytesPerMiB = 1024 * 1024 func main() { flag.Parse() + recordExplicitRuntimeFlags(flag.CommandLine) err := run() if memoryPressureExit.Load() { @@ -545,48 +547,29 @@ func run() error { // registry-read / behind-epoch failure fails the process // synchronously here, BEFORE the gRPC servers serve, so writes never // run with no registration gate installed. - distCatalog, catalogRuntime, defaultRuntime, err := setupDistributionRuntimeDependencies( - runCtx, eg, runtimes, cfg.engine, - coordinate, shardGroups[cfg.defaultGroup], cfg.defaultGroup, - encWiring, *raftId, *encryptionSidecarPath) + distStartup, err := startDistributionStartup(distributionStartupInput{ + ctx: runCtx, + eg: eg, + cancel: cancel, + cleanup: &cleanup, + runtimes: runtimes, + cfg: cfg, + coordinate: coordinate, + defaultGroup: shardGroups[cfg.defaultGroup], + encWiring: encWiring, + raftID: *raftId, + sidecarPath: *encryptionSidecarPath, + sampler: sampler, + readTracker: readTracker, + metricsRegistry: metricsRegistry, + clock: clock, + }) if err != nil { - cancel() return err } - // Seed AFTER setupDistributionCatalog so the sampler picks up the - // catalog-assigned RouteIDs. EnsureCatalogSnapshot inside - // setupDistributionCatalog applies a snapshot back into the engine - // with durable non-zero RouteIDs; seeding earlier would register - // the placeholder zero IDs from buildEngine and Observe would miss - // every dispatched mutation. - seedKeyVizRoutes(sampler, cfg.engine) - - // The local durable watcher is the 100 ms fallback required when the - // best-effort leader stream is unavailable or reconnecting. - eg.Go(func() error { - return runDistributionCatalogWatcher(runCtx, distCatalog, cfg.engine) - }) - catalogWatchConns := &kv.GRPCConnCache{} - cleanup.Add(func() { _ = catalogWatchConns.Close() }) - eg.Go(func() error { - return runDistributionCatalogStream(runCtx, catalogRuntime, catalogWatchConns, cfg.engine) - }) - startKeyVizFlusher(runCtx, eg, sampler) - startKeyVizLeaderTermPublisher(runCtx, eg, sampler, runtimes) - startMemoryWatchdog(runCtx, eg, cancel) - distServer := adapter.NewDistributionServer( - cfg.engine, - distCatalog, - adapter.WithDistributionCoordinator(coordinate), - adapter.WithDistributionActiveTimestampTracker(readTracker), - adapter.WithCatalogWatchLeaderCheck(func() bool { - engine := catalogRuntime.snapshotEngine() - return engine != nil && engine.State() == raftengine.StateLeader - }), - adapter.WithDistributionFilesystemObserver(metricsRegistry.FileSystemObserver()), - ) - startMonitoringCollectors(runCtx, metricsRegistry, runtimes, clock) - startFSMCompactorIfEnabled(runCtx, eg, runtimes, readTracker) + defaultRuntime := distStartup.defaultRuntime + distServer := distStartup.distServer + autoSplitRuntime := distStartup.autoSplitRuntime // Stage 7c §3.1: build the encryption-aware // MembershipChangeInterceptor here where the concrete @@ -620,6 +603,7 @@ func run() error { s3BlobBackfiller: s3BlobBackfiller, encWiring: encWiring, keyvizSampler: sampler, + autoSplitRuntime: autoSplitRuntime, encryptionConfChangeInterceptor: encryptionConfChangeInterceptor, }); err != nil { return err @@ -637,6 +621,88 @@ func run() error { return nil } +type distributionStartupInput struct { + ctx context.Context + eg *errgroup.Group + cancel context.CancelFunc + cleanup *internalutil.CleanupStack + runtimes []*raftGroupRuntime + cfg runtimeConfig + coordinate *kv.ShardedCoordinator + defaultGroup *kv.ShardGroup + encWiring encryptionWriteWiring + raftID string + sidecarPath string + sampler *keyviz.MemSampler + readTracker *kv.ActiveTimestampTracker + metricsRegistry *monitoring.Registry + clock *kv.HLC +} + +type distributionStartup struct { + defaultRuntime *raftGroupRuntime + distServer *adapter.DistributionServer + autoSplitRuntime adapter.AutoSplitRuntime +} + +func startDistributionStartup(in distributionStartupInput) (distributionStartup, error) { + distCatalog, catalogRuntime, defaultRuntime, err := setupDistributionRuntimeDependencies( + in.ctx, in.eg, in.runtimes, in.cfg.engine, + in.coordinate, in.defaultGroup, in.cfg.defaultGroup, + in.encWiring, in.raftID, in.sidecarPath) + if err != nil { + in.cancel() + return distributionStartup{}, err + } + // Seed AFTER setupDistributionCatalog so the sampler picks up the + // catalog-assigned RouteIDs. EnsureCatalogSnapshot inside + // setupDistributionCatalog applies a snapshot back into the engine + // with durable non-zero RouteIDs; seeding earlier would register + // the placeholder zero IDs from buildEngine and Observe would miss + // every dispatched mutation. + seedKeyVizRoutes(in.sampler, in.cfg.engine) + + catalogWatchConns := &kv.GRPCConnCache{} + in.cleanup.Add(func() { _ = catalogWatchConns.Close() }) + in.eg.Go(func() error { + return runDistributionCatalogStream(in.ctx, catalogRuntime, catalogWatchConns, in.cfg.engine) + }) + startKeyVizFlusher(in.ctx, in.eg, in.sampler) + startKeyVizLeaderTermPublisher(in.ctx, in.eg, in.sampler, in.runtimes) + startMemoryWatchdog(in.ctx, in.eg, in.cancel) + distServer := adapter.NewDistributionServer( + in.cfg.engine, + distCatalog, + adapter.WithDistributionCoordinator(in.coordinate), + adapter.WithDistributionActiveTimestampTracker(in.readTracker), + adapter.WithCatalogWatchLeaderCheck(func() bool { + engine := catalogRuntime.snapshotEngine() + return engine != nil && engine.State() == raftengine.StateLeader + }), + adapter.WithDistributionFilesystemObserver(in.metricsRegistry.FileSystemObserver()), + ) + autoSplitRuntime, err := setupDistributionWatcherAndAutoSplit( + in.ctx, + in.eg, + distCatalog, + in.cfg.engine, + distServer, + in.coordinate, + in.sampler, + autosplit.NewPrometheusObserver(in.metricsRegistry.Registerer()), + ) + if err != nil { + return distributionStartup{}, err + } + startMonitoringCollectors(in.ctx, in.metricsRegistry, in.runtimes, in.clock) + startFSMCompactorIfEnabled(in.ctx, in.eg, in.runtimes, in.readTracker) + return distributionStartup{ + defaultRuntime: defaultRuntime, + distServer: distServer, + autoSplitRuntime: autoSplitRuntime, + }, nil +} + func startRaftEngineLifecycleWatchers(ctx context.Context, eg *errgroup.Group, runtimes []*raftGroupRuntime) { for _, rt := range runtimes { if rt == nil { @@ -1806,6 +1872,9 @@ type serversInput struct { // coordinator already has its own copy from // `WithSampler(...)` higher up in run(). keyvizSampler *keyviz.MemSampler + // autoSplitRuntime is registered on the authenticated Admin gRPC service + // when automatic splitting was configured at process startup. + autoSplitRuntime adapter.AutoSplitRuntime // encryptionConfChangeInterceptor is the Stage 7c §3.1 // pre-register hook for raftadmin AddVoter/AddLearner. // Constructed in run() where concrete *kv.ShardedCoordinator and @@ -1820,7 +1889,14 @@ type serversInput struct { // to catch up, prepares the public listeners, waits for any requested startup // rotation, then starts serving public traffic. func startServersAfterStartupRotation(waitRotateOnStartup startupRotationWaiter, in serversInput) error { - adminServer, adminGRPCOpts, err := setupAdminService(*raftId, *myAddr, in.runtimes, in.bootstrapServers, in.keyvizSampler) + adminServer, adminGRPCOpts, err := setupAdminService( + *raftId, + *myAddr, + in.runtimes, + in.bootstrapServers, + in.keyvizSampler, + in.autoSplitRuntime, + ) if err != nil { return err } @@ -2092,6 +2168,7 @@ func setupAdminService( runtimes []*raftGroupRuntime, bootstrapServers []raftengine.Server, keyvizSampler *keyviz.MemSampler, + autoSplitRuntime adapter.AutoSplitRuntime, ) (*adapter.AdminServer, adminGRPCInterceptors, error) { members := adminMembersFromBootstrap(nodeID, bootstrapServers) // In multi-group mode the process does not listen on *myAddr — each group @@ -2124,6 +2201,9 @@ func setupAdminService( if keyvizSampler != nil { srv.RegisterSampler(keyvizSampler) } + if autoSplitRuntime != nil { + srv.RegisterAutoSplitRuntime(autoSplitRuntime) + } if *adminInsecureNoAuth { log.Printf("WARNING: --adminInsecureNoAuth is set; Admin gRPC service exposed without authentication") } @@ -2763,10 +2843,14 @@ func startRaftServers( } func internalTimestampOptions(coordinate kv.Coordinator) []adapter.InternalOption { + var opts []adapter.InternalOption if alloc, ok := coordinate.(kv.TimestampAllocator); ok { - return []adapter.InternalOption{adapter.WithInternalTimestampAllocator(alloc)} + opts = append(opts, adapter.WithInternalTimestampAllocator(alloc)) } - return nil + if observer, ok := coordinate.(interface{ ObserveForwardedRequests([]*pb.Request) }); ok { + opts = append(opts, adapter.WithInternalForwardWriteObserver(observer.ObserveForwardedRequests)) + } + return opts } func prepareRedisServer(ctx context.Context, lc *net.ListenConfig, redisAddr string, shardStore *kv.ShardStore, coordinate kv.Coordinator, leaderRedis map[string]string, relay *adapter.RedisPubSubRelay, metricsRegistry *monitoring.Registry, readTracker *kv.ActiveTimestampTracker, redisApplyObserver *adapter.RedisApplyObserver) (*adapter.RedisServer, *adapter.DeltaCompactor, net.Listener, error) { @@ -3003,8 +3087,23 @@ func distributionCatalogGroupID(engine *distribution.Engine) (uint64, error) { return route.GroupID, nil } -func runDistributionCatalogWatcher(ctx context.Context, catalog *distribution.CatalogStore, engine *distribution.Engine) error { - if err := distribution.RunCatalogWatcher(ctx, catalog, engine, nil); err != nil { +func runDistributionCatalogWatcher( + ctx context.Context, + catalog *distribution.CatalogStore, + engine *distribution.Engine, + observers ...distribution.CatalogSnapshotObserver, +) error { + var opts []distribution.CatalogWatcherOption + if len(observers) > 0 { + opts = append(opts, distribution.WithCatalogWatcherSnapshotObserver(func(snapshot distribution.CatalogSnapshot) { + for _, observer := range observers { + if observer != nil { + observer(snapshot) + } + } + })) + } + if err := distribution.RunCatalogWatcher(ctx, catalog, engine, nil, opts...); err != nil { return errors.Wrapf(err, "catalog watcher failed") } return nil @@ -3426,20 +3525,26 @@ func (r *runtimeServerRunner) startAdminHTTP() { } // buildKeyVizSampler constructs the in-memory keyviz sampler from -// flag-supplied options, or returns nil when --keyvizEnabled is -// false. The coordinator's WithSampler and AdminServer's +// flag-supplied options, or returns nil when neither --keyvizEnabled nor +// --autoSplit is set. The coordinator's WithSampler and AdminServer's // RegisterSampler both treat a nil receiver as "keyviz disabled," so // this is the single decision point. func buildKeyVizSampler() *keyviz.MemSampler { - if !*keyvizEnabled { + if !*keyvizEnabled && !*autoSplit { return nil } + keyBucketsPerRoute := effectiveKeyVizBucketsPerRoute( + *autoSplit, + keyvizKeyBucketsPerRouteExplicit, + *keyvizKeyBucketsPerRoute, + *autoSplitDefaultBuckets, + ) return keyviz.NewMemSampler(keyviz.MemSamplerOptions{ Step: *keyvizStep, HistoryColumns: *keyvizHistoryColumns, MaxTrackedRoutes: *keyvizMaxTrackedRoutes, MaxMemberRoutesPerSlot: *keyvizMaxMemberRoutesPerSlot, - KeyBucketsPerRoute: *keyvizKeyBucketsPerRoute, + KeyBucketsPerRoute: keyBucketsPerRoute, KeyVizLabelsEnabled: *keyvizLabelsEnabled, HotKeysEnabled: *keyvizHotKeysEnabled, HotKeysPerRoute: *keyvizHotKeysPerRoute, @@ -3461,12 +3566,9 @@ func keyVizSamplerForCoordinator(s *keyviz.MemSampler) keyviz.Sampler { return s } -// seedKeyVizRoutes copies the engine's current route catalogue into -// the sampler so the first matrix snapshots have non-empty metadata. -// No-op when the sampler is disabled. The coordinator's -// distribution.Engine handles route mutations after this point; -// route-watch propagation into the sampler is a follow-up (the -// design's Phase 3 persistence work). +// seedKeyVizRoutes copies the engine's current route catalogue into the sampler +// so the first matrix snapshots have non-empty metadata. The auto-split +// scheduler keeps membership reconciled after catalog changes. func seedKeyVizRoutes(s *keyviz.MemSampler, engine *distribution.Engine) { if s == nil || engine == nil { return diff --git a/main_autosplit.go b/main_autosplit.go new file mode 100644 index 000000000..7a1d65c33 --- /dev/null +++ b/main_autosplit.go @@ -0,0 +1,359 @@ +package main + +import ( + "context" + "flag" + "log/slog" + "math" + "strings" + + "github.com/bootjp/elastickv/adapter" + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/distribution/autosplit" + "github.com/bootjp/elastickv/keyviz" + "github.com/bootjp/elastickv/kv" + pb "github.com/bootjp/elastickv/proto" + "github.com/cockroachdb/errors" + "golang.org/x/sync/errgroup" +) + +var ( + autoSplit = flag.Bool("autoSplit", false, "Enable automatic same-group hotspot range splits on the catalog leader") + autoSplitEvalInterval = flag.Duration("autoSplitEvalInterval", autosplit.DefaultEvalInterval, "Interval between automatic hotspot split evaluations") + autoSplitSplitCooldown = flag.Duration("autoSplitCooldown", autosplit.DefaultSplitCooldown, "Minimum cooldown before a route created by SplitRange can be auto-split again") + autoSplitSplitTimeout = flag.Duration("autoSplitSplitTimeout", autosplit.DefaultSplitTimeout, "Timeout for each automatic SplitRange RPC") + autoSplitKillSwitchFile = flag.String("autoSplitKillSwitchFile", "", "If non-empty and the file exists, the auto-split scheduler observes but skips new splits") + autoSplitDefaultBuckets = flag.Int("autoSplitDefaultBuckets", autosplit.DefaultSamplerBuckets, "KeyViz buckets per route implied by --autoSplit when --keyvizKeyBucketsPerRoute was not explicitly set") + autoSplitWriteWeight = flag.Float64("autoSplitWriteWeight", autosplit.DefaultConfig().WriteWeight, "Weighted ops/min multiplier for writes in automatic hotspot detection") + autoSplitReadWeight = flag.Float64("autoSplitReadWeight", autosplit.DefaultConfig().ReadWeight, "Weighted ops/min multiplier for reads in automatic hotspot detection") + autoSplitThresholdOpsMin = flag.Float64("autoSplitThreshold", autosplit.DefaultConfig().ThresholdOpsMin, "Weighted ops/min threshold per committed KeyViz column for automatic hotspot detection") + autoSplitCandidateWindows = flag.Int("autoSplitCandidateWindows", autosplit.DefaultConfig().CandidateWindows, "Consecutive committed KeyViz columns over threshold required before an automatic split") + autoSplitMaxRoutes = flag.Int("autoSplitMaxRoutes", autosplit.DefaultConfig().MaxRoutes, "Maximum live routes allowed after automatic splits") + autoSplitMaxSplitsPerCycle = flag.Int("autoSplitMaxPerCycle", autosplit.DefaultConfig().MaxSplitsPerCycle, "Maximum automatic split decisions scheduled per evaluation cycle") + autoSplitTopKeyShare = flag.Float64("autoSplitTopKeyShare", autosplit.DefaultConfig().TopKeyShare, "Minimum lower-bound share of route writes required for hot-key isolation") + autoSplitTopKeyAbsoluteFloor = flag.Float64("autoSplitTopKeyAbsoluteFloor", 0, "Minimum lower-bound hot-key weighted ops/min; zero uses half --autoSplitThreshold") + + keyvizKeyBucketsPerRouteExplicit bool +) + +type autoSplitLeader interface { + IsLeader() bool +} + +type autoSplitKeyLeader interface { + IsLeaderForKey(key []byte) bool +} + +type autoSplitTermLeader interface { + LeadershipForKey(key []byte) (bool, uint64) + GroupLeadership(groupID uint64) (bool, uint64) +} + +func recordExplicitRuntimeFlags(fs *flag.FlagSet) { + if fs == nil { + return + } + fs.Visit(func(f *flag.Flag) { + if f.Name == "keyvizKeyBucketsPerRoute" { + keyvizKeyBucketsPerRouteExplicit = true + } + }) +} + +func effectiveKeyVizBucketsPerRoute(autoEnabled, bucketsExplicit bool, configuredBuckets, defaultBuckets int) int { + if autoEnabled && !bucketsExplicit && configuredBuckets <= keyviz.DefaultKeyBucketsPerRoute { + return defaultBuckets + } + return configuredBuckets +} + +func autoSplitConfigFromFlags(leader autoSplitLeader) (autosplit.SchedulerConfig, error) { + cfg := autosplit.SchedulerConfig{ + Enabled: *autoSplit, + EvalInterval: *autoSplitEvalInterval, + SplitCooldown: *autoSplitSplitCooldown, + SplitTimeout: *autoSplitSplitTimeout, + KillSwitchFile: strings.TrimSpace(*autoSplitKillSwitchFile), + Logger: slog.Default(), + Detector: autosplit.Config{ + WriteWeight: *autoSplitWriteWeight, + ReadWeight: *autoSplitReadWeight, + ThresholdOpsMin: *autoSplitThresholdOpsMin, + CandidateWindows: *autoSplitCandidateWindows, + MaxRoutes: *autoSplitMaxRoutes, + MaxSplitsPerCycle: *autoSplitMaxSplitsPerCycle, + TopKeyShare: *autoSplitTopKeyShare, + TopKeyAbsoluteFloor: *autoSplitTopKeyAbsoluteFloor, + }, + } + if leader != nil { + applyAutoSplitLeadership(&cfg, leader) + } + if err := validateAutoSplitConfig(cfg); err != nil { + return autosplit.SchedulerConfig{}, err + } + return cfg, nil +} + +func configureRuntimeAutoSplit(coordinate *kv.ShardedCoordinator) (autosplit.SchedulerConfig, error) { + cfg, err := autoSplitConfigFromFlags(coordinate) + if err != nil { + return autosplit.SchedulerConfig{}, err + } + return cfg, nil +} + +func setupDistributionWatcherAndAutoSplit( + ctx context.Context, + eg *errgroup.Group, + catalog *distribution.CatalogStore, + engine *distribution.Engine, + distServer *adapter.DistributionServer, + coordinate *kv.ShardedCoordinator, + sampler *keyviz.MemSampler, + observer autosplit.Observer, +) (adapter.AutoSplitRuntime, error) { + cfg, err := configureRuntimeAutoSplit(coordinate) + if err != nil { + return nil, err + } + cfg.Observer = observer + var runtime adapter.AutoSplitRuntime + if cfg.Enabled { + switchRuntime := autosplit.NewRuntimeSwitch(true) + runtime = switchRuntime + cfg.KillSwitch = switchRuntime.KillSwitch + } + var reconciler *autosplit.RouteReconciler + if sampler != nil { + reconciler = autosplit.NewRouteReconciler(sampler) + initialSnapshot, snapshotErr := catalog.Snapshot(ctx) + if snapshotErr != nil { + return nil, errors.Wrap(snapshotErr, "load initial catalog for keyviz reconciliation") + } + reconciler.Reconcile(initialSnapshot.Routes) + cfg.Reconciler = reconciler + } + eg.Go(func() error { + return runDistributionCatalogWatcher(ctx, catalog, engine, func(snapshot distribution.CatalogSnapshot) { + if reconciler != nil { + reconciler.Reconcile(snapshot.Routes) + } + }) + }) + startAutoSplitScheduler(ctx, eg, catalog, distServer, coordinate, sampler, cfg) + return runtime, nil +} + +func validateAutoSplitConfig(cfg autosplit.SchedulerConfig) error { + if !cfg.Enabled { + return nil + } + if err := validateAutoSplitSamplerConfig(cfg.Detector); err != nil { + return err + } + if err := validateAutoSplitDetectorConfig(cfg.Detector); err != nil { + return err + } + return validateAutoSplitSchedulerConfig(cfg) +} + +func validateAutoSplitSamplerConfig(cfg autosplit.Config) error { + if autoSplitUsesDefaultBuckets() { + if *autoSplitDefaultBuckets <= keyviz.DefaultKeyBucketsPerRoute { + return errors.New("--autoSplitDefaultBuckets must be greater than 1") + } + if *autoSplitDefaultBuckets > keyviz.MaxKeyBucketsPerRoute { + return errors.Errorf("--autoSplitDefaultBuckets (%d) must be <= %d", *autoSplitDefaultBuckets, keyviz.MaxKeyBucketsPerRoute) + } + } + if cfg.MaxRoutes > *keyvizMaxTrackedRoutes { + return errors.Errorf("--autoSplitMaxRoutes (%d) must be <= --keyvizMaxTrackedRoutes (%d); raise keyviz capacity or lower the auto-split route cap", cfg.MaxRoutes, *keyvizMaxTrackedRoutes) + } + return nil +} + +func autoSplitUsesDefaultBuckets() bool { + return *autoSplit && + !keyvizKeyBucketsPerRouteExplicit && + *keyvizKeyBucketsPerRoute <= keyviz.DefaultKeyBucketsPerRoute +} + +func validateAutoSplitDetectorConfig(cfg autosplit.Config) error { + if err := validateAutoSplitWeights(cfg.WriteWeight, cfg.ReadWeight); err != nil { + return err + } + if err := validateAutoSplitDetectorLimits(cfg); err != nil { + return err + } + return validateAutoSplitHotKeyThresholds(cfg) +} + +func validateAutoSplitDetectorLimits(cfg autosplit.Config) error { + if !isFiniteAutoSplitFloat(cfg.ThresholdOpsMin) || cfg.ThresholdOpsMin <= 0 { + return errors.New("--autoSplitThreshold must be positive") + } + if cfg.CandidateWindows <= 0 { + return errors.New("--autoSplitCandidateWindows must be positive") + } + if cfg.MaxRoutes <= 0 { + return errors.New("--autoSplitMaxRoutes must be positive") + } + if cfg.MaxSplitsPerCycle <= 0 { + return errors.New("--autoSplitMaxPerCycle must be positive") + } + return nil +} + +func validateAutoSplitHotKeyThresholds(cfg autosplit.Config) error { + if !isFiniteAutoSplitFloat(cfg.TopKeyShare) || cfg.TopKeyShare <= 0 || cfg.TopKeyShare > 1 { + return errors.New("--autoSplitTopKeyShare must be in (0, 1]") + } + if !isFiniteAutoSplitFloat(cfg.TopKeyAbsoluteFloor) || cfg.TopKeyAbsoluteFloor < 0 { + return errors.New("--autoSplitTopKeyAbsoluteFloor must be non-negative") + } + return nil +} + +func validateAutoSplitWeights(writeWeight, readWeight float64) error { + if !isFiniteAutoSplitFloat(writeWeight) || !isFiniteAutoSplitFloat(readWeight) || + writeWeight < 0 || readWeight < 0 || (writeWeight == 0 && readWeight == 0) { + return errors.New("--autoSplitWriteWeight and --autoSplitReadWeight must be non-negative and at least one must be positive") + } + return nil +} + +func isFiniteAutoSplitFloat(v float64) bool { + return !math.IsNaN(v) && !math.IsInf(v, 0) +} + +func validateAutoSplitSchedulerConfig(cfg autosplit.SchedulerConfig) error { + if cfg.EvalInterval <= 0 { + return errors.New("--autoSplitEvalInterval must be positive") + } + if cfg.SplitCooldown <= 0 { + return errors.New("--autoSplitCooldown must be positive") + } + if cfg.SplitTimeout <= 0 { + return errors.New("--autoSplitSplitTimeout must be positive") + } + return nil +} + +func startAutoSplitScheduler( + ctx context.Context, + eg *errgroup.Group, + catalog *distribution.CatalogStore, + distServer *adapter.DistributionServer, + leader autoSplitLeader, + sampler *keyviz.MemSampler, + cfg autosplit.SchedulerConfig, +) { + if eg == nil || !cfg.Enabled { + return + } + if sampler == nil { + cfg.Logger.Warn("autosplit: sampler not configured; scheduler disabled") + return + } + if leader != nil { + applyAutoSplitLeadership(&cfg, leader) + } + scheduler := autosplit.NewScheduler( + cfg, + catalog, + autoSplitDistributionSplitter{server: distServer}, + sampler, + sampler, + ) + eg.Go(func() error { + return scheduler.Run(ctx) + }) +} + +type autoSplitDistributionSplitter struct { + server *adapter.DistributionServer +} + +func (s autoSplitDistributionSplitter) SplitRange(ctx context.Context, req autosplit.SplitRequest) (autosplit.SplitResult, error) { + if s.server == nil { + return autosplit.SplitResult{}, errors.New("autosplit: distribution server is not configured") + } + if req.TargetGroupID != 0 { + return autosplit.SplitResult{}, errors.New("autosplit: cross-group target selection is not enabled") + } + resp, err := s.server.SplitRange(ctx, &pb.SplitRangeRequest{ + ExpectedCatalogVersion: req.ExpectedCatalogVersion, + RouteId: req.RouteID, + SplitKey: distribution.CloneBytes(req.SplitKey), + }) + if err != nil { + return autosplit.SplitResult{}, errors.Wrap(err, "autosplit: split range") + } + return autosplit.SplitResult{ + CatalogVersion: resp.GetCatalogVersion(), + Left: autoSplitRouteFromProto(resp.GetLeft()), + Right: autoSplitRouteFromProto(resp.GetRight()), + }, nil +} + +func autoSplitRouteFromProto(route *pb.RouteDescriptor) distribution.RouteDescriptor { + if route == nil { + return distribution.RouteDescriptor{} + } + return distribution.RouteDescriptor{ + RouteID: route.GetRouteId(), + Start: distribution.CloneBytes(route.GetStart()), + End: distribution.CloneBytes(route.GetEnd()), + GroupID: route.GetRaftGroupId(), + State: autoSplitRouteStateFromProto(route.GetState()), + ParentRouteID: route.GetParentRouteId(), + SplitAtHLC: route.GetSplitAtHlc(), + } +} + +func autoSplitRouteStateFromProto(state pb.RouteState) distribution.RouteState { + switch state { + case pb.RouteState_ROUTE_STATE_UNSPECIFIED: + return distribution.RouteStateWriteFenced + case pb.RouteState_ROUTE_STATE_ACTIVE: + return distribution.RouteStateActive + case pb.RouteState_ROUTE_STATE_WRITE_FENCED: + return distribution.RouteStateWriteFenced + case pb.RouteState_ROUTE_STATE_MIGRATING_SOURCE: + return distribution.RouteStateMigratingSource + case pb.RouteState_ROUTE_STATE_MIGRATING_TARGET: + return distribution.RouteStateMigratingTarget + default: + return distribution.RouteStateWriteFenced + } +} + +var _ autoSplitLeader = (*kv.ShardedCoordinator)(nil) + +func autoSplitLeaderGate(leader autoSplitLeader) func() bool { + if leader == nil { + return nil + } + if keyLeader, ok := leader.(autoSplitKeyLeader); ok { + return func() bool { + return keyLeader.IsLeaderForKey(distribution.CatalogVersionKey()) + } + } + return leader.IsLeader +} + +func applyAutoSplitLeadership(cfg *autosplit.SchedulerConfig, leader autoSplitLeader) { + if cfg == nil || leader == nil { + return + } + cfg.IsLeader = autoSplitLeaderGate(leader) + termLeader, ok := leader.(autoSplitTermLeader) + if !ok { + return + } + cfg.Leadership = func() (bool, uint64) { + return termLeader.LeadershipForKey(distribution.CatalogVersionKey()) + } + cfg.GroupLeadership = termLeader.GroupLeadership +} diff --git a/main_autosplit_e2e_test.go b/main_autosplit_e2e_test.go new file mode 100644 index 000000000..f7e995b26 --- /dev/null +++ b/main_autosplit_e2e_test.go @@ -0,0 +1,211 @@ +package main + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/distribution/autosplit" + "github.com/bootjp/elastickv/internal/raftengine" + "github.com/bootjp/elastickv/keyviz" + pb "github.com/bootjp/elastickv/proto" + "github.com/stretchr/testify/require" +) + +func TestAutoSplitE2EThreeNodeSplitKillSwitchAndLeadershipReset(t *testing.T) { + const ( + nodeCount = 3 + waitTimeout = 20 * time.Second + waitInterval = 100 * time.Millisecond + rpcTimeout = 2 * time.Second + ) + baseDir := t.TempDir() + _, endpoints, nodes := startBootstrapE2ECluster(t, baseDir, nodeCount, 5, raftEngineEtcd) + t.Cleanup(func() { closeBootstrapE2ENodes(t, nodes) }) + waitForBootstrapClusterConfig(t, nodes, bootstrapExpectedConfiguration(endpoints)) + leaderIdx := waitForSingleLeader(t, nodes) + clients, conns := rawKVClients(t, endpoints) + t.Cleanup(func() { closeGRPCConns(conns) }) + + base := time.Now().Truncate(time.Second) + leaderClock, leaderSampler, leaderRuntime, leaderScheduler := newAutoSplitE2EScheduler(t, nodes[leaderIdx], base) + first, err := leaderScheduler.Tick(context.Background(), base) + require.NoError(t, err) + require.True(t, first.Leader) + initialVersion := first.CatalogVersion + + writeAutoSplitWindow(t, clients[leaderIdx], leaderSampler, leaderClock, base.Add(time.Minute), []byte("a"), []byte("z")) + writeAutoSplitWindow(t, clients[leaderIdx], leaderSampler, leaderClock, base.Add(2*time.Minute), []byte("a"), []byte("z")) + result, err := leaderScheduler.Tick(context.Background(), base.Add(2*time.Minute)) + require.NoError(t, err) + require.Equal(t, 1, result.Scheduled) + require.Greater(t, result.CatalogVersion, initialVersion) + firstSplitVersion := currentCatalogVersion(t, nodes[leaderIdx]) + require.Greater(t, firstSplitVersion, initialVersion) + waitForCatalogVersion(t, nodes, firstSplitVersion, waitTimeout, waitInterval) + + _, err = leaderScheduler.Tick(context.Background(), base.Add(2*time.Minute+time.Second)) + require.NoError(t, err) + leaderRuntime.SetEnabled(false) + writeAutoSplitWindow(t, clients[leaderIdx], leaderSampler, leaderClock, base.Add(3*time.Minute), []byte("z"), []byte("zz")) + writeAutoSplitWindow(t, clients[leaderIdx], leaderSampler, leaderClock, base.Add(4*time.Minute), []byte("z"), []byte("zz")) + blocked, err := leaderScheduler.Tick(context.Background(), base.Add(4*time.Minute)) + require.NoError(t, err) + require.True(t, blocked.KillSwitch) + require.Equal(t, firstSplitVersion, currentCatalogVersion(t, nodes[leaderIdx])) + + leaderRuntime.SetEnabled(true) + writeAutoSplitWindow(t, clients[leaderIdx], leaderSampler, leaderClock, base.Add(5*time.Minute), []byte("z"), []byte("zz")) + reenabled, err := leaderScheduler.Tick(context.Background(), base.Add(5*time.Minute)) + require.NoError(t, err) + require.Equal(t, 1, reenabled.Scheduled) + secondSplitVersion := currentCatalogVersion(t, nodes[leaderIdx]) + require.Greater(t, secondSplitVersion, firstSplitVersion) + waitForCatalogVersion(t, nodes, secondSplitVersion, waitTimeout, waitInterval) + + targetIdx := (leaderIdx + 1) % len(nodes) + targetClock, targetSampler, _, targetScheduler := newAutoSplitE2EScheduler(t, nodes[targetIdx], base.Add(5*time.Minute)) + followerTick, err := targetScheduler.Tick(context.Background(), base.Add(5*time.Minute)) + require.NoError(t, err) + require.False(t, followerTick.Leader) + writeAutoSplitWindow(t, clients[targetIdx], targetSampler, targetClock, base.Add(6*time.Minute), []byte("a"), []byte("aa")) + writeAutoSplitWindow(t, clients[targetIdx], targetSampler, targetClock, base.Add(7*time.Minute), []byte("a"), []byte("aa")) + + admin, ok := nodes[leaderIdx].engine().(raftengine.Admin) + require.True(t, ok) + transferCtx, cancel := context.WithTimeout(context.Background(), rpcTimeout) + err = admin.TransferLeadershipToServer(transferCtx, endpoints[targetIdx].id, endpoints[targetIdx].raftAddr) + cancel() + require.NoError(t, err) + waitForGroupLeaderOnNode(t, nodes, 1, targetIdx, waitTimeout, waitInterval) + + postTransfer, err := targetScheduler.Tick(context.Background(), base.Add(7*time.Minute+30*time.Second)) + require.NoError(t, err) + require.True(t, postTransfer.Leader) + require.Empty(t, postTransfer.Detector.Decisions) + require.Equal(t, secondSplitVersion, currentCatalogVersion(t, nodes[targetIdx])) + + writeAutoSplitWindow(t, clients[targetIdx], targetSampler, targetClock, base.Add(8*time.Minute), []byte("a"), []byte("aa")) + writeAutoSplitWindow(t, clients[targetIdx], targetSampler, targetClock, base.Add(9*time.Minute), []byte("a"), []byte("aa")) + notYet, err := targetScheduler.Tick(context.Background(), base.Add(9*time.Minute)) + require.NoError(t, err) + require.Empty(t, notYet.Detector.Decisions) + writeAutoSplitWindow(t, clients[targetIdx], targetSampler, targetClock, base.Add(10*time.Minute), []byte("a"), []byte("aa")) + reearned, err := targetScheduler.Tick(context.Background(), base.Add(10*time.Minute)) + require.NoError(t, err) + require.Equal(t, 1, reearned.Scheduled) + require.Greater(t, currentCatalogVersion(t, nodes[targetIdx]), secondSplitVersion) +} + +type autoSplitE2EClock struct { + unixMilli atomic.Int64 +} + +func (c *autoSplitE2EClock) Now() time.Time { + return time.UnixMilli(c.unixMilli.Load()) +} + +func (c *autoSplitE2EClock) Set(now time.Time) { + c.unixMilli.Store(now.UnixMilli()) +} + +func newAutoSplitE2EScheduler( + t *testing.T, + node *bootstrapE2ENode, + now time.Time, +) (*autoSplitE2EClock, *keyviz.MemSampler, *autosplit.RuntimeSwitch, *autosplit.Scheduler) { + t.Helper() + clock := &autoSplitE2EClock{} + clock.Set(now) + sampler := keyviz.NewMemSampler(keyviz.MemSamplerOptions{ + Step: time.Minute, + HistoryColumns: 32, + MaxTrackedRoutes: 64, + KeyBucketsPerRoute: 16, + Now: clock.Now, + }) + snapshot, err := node.distCatalog.Snapshot(context.Background()) + require.NoError(t, err) + for _, route := range snapshot.Routes { + sampler.RegisterRoute(route.RouteID, route.Start, route.End, route.GroupID) + } + node.coordinate.WithSampler(sampler) + runtime := autosplit.NewRuntimeSwitch(true) + cfg := autosplit.SchedulerConfig{ + Enabled: true, + EvalInterval: time.Minute, + SplitCooldown: time.Millisecond, + SplitTimeout: 5 * time.Second, + KillSwitch: runtime.KillSwitch, + Detector: autosplit.Config{ + WriteWeight: 1, + ThresholdOpsMin: 1, + CandidateWindows: 2, + MaxRoutes: 64, + MaxSplitsPerCycle: 1, + }, + Reconciler: autosplit.NewRouteReconciler(sampler), + } + applyAutoSplitLeadership(&cfg, node.coordinate) + return clock, sampler, runtime, autosplit.NewScheduler( + cfg, + node.distCatalog, + autoSplitDistributionSplitter{server: node.distServer}, + sampler, + sampler, + ) +} + +func writeAutoSplitWindow( + t *testing.T, + client pb.RawKVClient, + sampler *keyviz.MemSampler, + clock *autoSplitE2EClock, + at time.Time, + keys ...[]byte, +) { + t.Helper() + for i := range 40 { + key := keys[i%len(keys)] + require.NoError(t, rawPutWithTimeout(client, key, []byte("value"))) + } + clock.Set(at) + sampler.Flush() +} + +func currentCatalogVersion(t *testing.T, node *bootstrapE2ENode) uint64 { + t.Helper() + snapshot, err := node.distCatalog.Snapshot(context.Background()) + require.NoError(t, err) + return snapshot.Version +} + +func waitForCatalogVersion( + t *testing.T, + nodes []*bootstrapE2ENode, + version uint64, + timeout time.Duration, + interval time.Duration, +) { + t.Helper() + for _, node := range nodes { + require.Eventually(t, func() bool { + snapshot, err := node.distCatalog.Snapshot(context.Background()) + return err == nil && snapshot.Version >= version && catalogRoutesActive(snapshot.Routes) + }, timeout, interval) + } +} + +func catalogRoutesActive(routes []distribution.RouteDescriptor) bool { + if len(routes) < 2 { + return false + } + for _, route := range routes { + if route.State != distribution.RouteStateActive { + return false + } + } + return true +} diff --git a/main_autosplit_test.go b/main_autosplit_test.go new file mode 100644 index 000000000..35a318f42 --- /dev/null +++ b/main_autosplit_test.go @@ -0,0 +1,158 @@ +package main + +import ( + "bytes" + "context" + "flag" + "math" + "testing" + + "github.com/bootjp/elastickv/distribution" + "github.com/bootjp/elastickv/distribution/autosplit" + "github.com/bootjp/elastickv/keyviz" + "github.com/bootjp/elastickv/store" + "github.com/stretchr/testify/require" + "golang.org/x/sync/errgroup" +) + +func TestAutoSplitCanonicalFlagsAreRegistered(t *testing.T) { + t.Parallel() + + for _, name := range []string{"autoSplitCooldown", "autoSplitThreshold", "autoSplitMaxPerCycle"} { + require.NotNil(t, flag.CommandLine.Lookup(name), name) + } + for _, name := range []string{"autoSplitSplitCooldown", "autoSplitThresholdOpsMin", "autoSplitMaxSplitsPerCycle"} { + require.Nil(t, flag.CommandLine.Lookup(name), name) + } +} + +func TestEffectiveKeyVizBucketsPerRouteAutoSplitImpliedDefault(t *testing.T) { + t.Parallel() + + require.Equal(t, 16, effectiveKeyVizBucketsPerRoute(true, false, keyviz.DefaultKeyBucketsPerRoute, 16)) + require.Equal(t, 1, effectiveKeyVizBucketsPerRoute(true, true, keyviz.DefaultKeyBucketsPerRoute, 16)) + require.Equal(t, 8, effectiveKeyVizBucketsPerRoute(true, false, 8, 16)) + require.Equal(t, 1, effectiveKeyVizBucketsPerRoute(false, false, keyviz.DefaultKeyBucketsPerRoute, 16)) +} + +func TestValidateAutoSplitSamplerConfigAllowsExplicitBucketsWithUnusedInvalidDefault(t *testing.T) { + oldAutoSplit := *autoSplit + oldExplicit := keyvizKeyBucketsPerRouteExplicit + oldKeyBuckets := *keyvizKeyBucketsPerRoute + oldDefaultBuckets := *autoSplitDefaultBuckets + t.Cleanup(func() { + *autoSplit = oldAutoSplit + keyvizKeyBucketsPerRouteExplicit = oldExplicit + *keyvizKeyBucketsPerRoute = oldKeyBuckets + *autoSplitDefaultBuckets = oldDefaultBuckets + }) + + *autoSplit = true + *keyvizKeyBucketsPerRoute = 8 + *autoSplitDefaultBuckets = keyviz.DefaultKeyBucketsPerRoute + keyvizKeyBucketsPerRouteExplicit = true + require.NoError(t, validateAutoSplitSamplerConfig(autosplit.DefaultConfig())) + + keyvizKeyBucketsPerRouteExplicit = false + *keyvizKeyBucketsPerRoute = keyviz.DefaultKeyBucketsPerRoute + require.ErrorContains(t, validateAutoSplitSamplerConfig(autosplit.DefaultConfig()), "--autoSplitDefaultBuckets") +} + +func TestSetupDistributionWatcherAndAutoSplitReturnsNilRuntimeWhenDisabled(t *testing.T) { + oldAutoSplit := *autoSplit + *autoSplit = false + t.Cleanup(func() { + *autoSplit = oldAutoSplit + }) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + catalog := distribution.NewCatalogStore(store.NewMVCCStore()) + _, err := catalog.Save(ctx, 0, []distribution.RouteDescriptor{{ + RouteID: 1, + Start: []byte(""), + End: nil, + GroupID: 1, + State: distribution.RouteStateActive, + }}) + require.NoError(t, err) + + var eg errgroup.Group + runtime, err := setupDistributionWatcherAndAutoSplit(ctx, &eg, catalog, distribution.NewEngine(), nil, nil, nil, nil) + require.NoError(t, err) + require.Nil(t, runtime) + + cancel() + require.NoError(t, eg.Wait()) +} + +func TestAutoSplitLeaderGateUsesCatalogKeyLeadership(t *testing.T) { + t.Parallel() + + leader := &fakeAutoSplitKeyLeader{catalogLeader: true} + gate := autoSplitLeaderGate(leader) + + require.True(t, gate()) + require.True(t, bytes.Equal(distribution.CatalogVersionKey(), leader.lastKey)) + require.False(t, leader.defaultLeaderCalled) +} + +func TestAutoSplitLeaderGateFallsBackToDefaultLeader(t *testing.T) { + t.Parallel() + + leader := &fakeAutoSplitLeader{leader: true} + gate := autoSplitLeaderGate(leader) + + require.True(t, gate()) + require.True(t, leader.called) +} + +func TestValidateAutoSplitDetectorConfigRejectsNonFiniteFloats(t *testing.T) { + t.Parallel() + + valid := autosplit.DefaultConfig() + tests := []struct { + name string + mutate func(*autosplit.Config) + }{ + {name: "write weight NaN", mutate: func(cfg *autosplit.Config) { cfg.WriteWeight = math.NaN() }}, + {name: "read weight positive infinity", mutate: func(cfg *autosplit.Config) { cfg.ReadWeight = math.Inf(1) }}, + {name: "threshold negative infinity", mutate: func(cfg *autosplit.Config) { cfg.ThresholdOpsMin = math.Inf(-1) }}, + {name: "top key share NaN", mutate: func(cfg *autosplit.Config) { cfg.TopKeyShare = math.NaN() }}, + {name: "absolute floor positive infinity", mutate: func(cfg *autosplit.Config) { cfg.TopKeyAbsoluteFloor = math.Inf(1) }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + cfg := valid + tc.mutate(&cfg) + require.Error(t, validateAutoSplitDetectorConfig(cfg)) + }) + } +} + +type fakeAutoSplitLeader struct { + leader bool + called bool +} + +func (f *fakeAutoSplitLeader) IsLeader() bool { + f.called = true + return f.leader +} + +type fakeAutoSplitKeyLeader struct { + catalogLeader bool + defaultLeaderCalled bool + lastKey []byte +} + +func (f *fakeAutoSplitKeyLeader) IsLeader() bool { + f.defaultLeaderCalled = true + return false +} + +func (f *fakeAutoSplitKeyLeader) IsLeaderForKey(key []byte) bool { + f.lastKey = append(f.lastKey[:0], key...) + return f.catalogLeader +} diff --git a/main_bootstrap_e2e_test.go b/main_bootstrap_e2e_test.go index d962c8105..ddbb2d9e2 100644 --- a/main_bootstrap_e2e_test.go +++ b/main_bootstrap_e2e_test.go @@ -15,6 +15,7 @@ import ( "time" "github.com/bootjp/elastickv/adapter" + "github.com/bootjp/elastickv/distribution" internalraftadmin "github.com/bootjp/elastickv/internal/raftadmin" "github.com/bootjp/elastickv/internal/raftengine" "github.com/bootjp/elastickv/kv" @@ -26,6 +27,12 @@ import ( "google.golang.org/grpc/reflection" ) +const ( + bootstrapE2EWaitTimeout = 20 * time.Second + bootstrapE2EWaitInterval = 100 * time.Millisecond + bootstrapE2ERPCRequestTimeout = 2 * time.Second +) + type bootstrapE2EEndpoint struct { id string raftAddr string @@ -56,9 +63,13 @@ type bootstrapE2ENode struct { id string runtimes []*raftGroupRuntime - shardStore *kv.ShardStore - cancel context.CancelFunc - eg *errgroup.Group + shardStore *kv.ShardStore + coordinate *kv.ShardedCoordinator + distEngine *distribution.Engine + distCatalog *distribution.CatalogStore + distServer *adapter.DistributionServer + cancel context.CancelFunc + eg *errgroup.Group } func (n *bootstrapE2ENode) engine() raftengine.Engine { @@ -97,6 +108,10 @@ func (n *bootstrapE2ENode) close() error { _ = n.shardStore.Close() n.shardStore = nil } + if n.coordinate != nil { + _ = n.coordinate.Close() + n.coordinate = nil + } for _, rt := range n.runtimes { if rt != nil { rt.Close() @@ -122,8 +137,8 @@ func TestRaftBootstrapMembers_E2E_FixedClusterWithoutAddVoter(t *testing.T) { t.Cleanup(func() { closeBootstrapE2ENodes(t, nodes) }) expected := bootstrapExpectedConfiguration(endpoints) - waitForBootstrapClusterConfig(t, nodes, expected, waitTimeout, waitInterval) - leaderIdx := waitForSingleLeader(t, nodes, waitTimeout, waitInterval) + waitForBootstrapClusterConfig(t, nodes, expected) + leaderIdx := waitForSingleLeader(t, nodes) clients, conns := rawKVClients(t, endpoints) t.Cleanup(func() { closeGRPCConns(conns) }) @@ -136,7 +151,7 @@ func TestRaftBootstrapMembers_E2E_FixedClusterWithoutAddVoter(t *testing.T) { // may not be ready immediately, causing "context canceled while waiting // for connections to become ready". require.Eventually(t, func() bool { - return rawPutWithTimeout(clients[writerIdx], key, value, rpcTimeout) == nil + return rawPutWithTimeout(clients[writerIdx], key, value) == nil }, waitTimeout, waitInterval) for i := range clients { @@ -167,8 +182,8 @@ func TestRaftBootstrapMembers_E2E_EtcdLeaderRestartRecovery(t *testing.T) { t.Cleanup(func() { closeBootstrapE2ENodes(t, nodes) }) expected := bootstrapExpectedConfiguration(endpoints) - waitForBootstrapClusterConfig(t, nodes, expected, waitTimeout, waitInterval) - leaderIdx := waitForSingleLeader(t, nodes, waitTimeout, waitInterval) + waitForBootstrapClusterConfig(t, nodes, expected) + leaderIdx := waitForSingleLeader(t, nodes) clients, conns := rawKVClients(t, endpoints) defer closeGRPCConns(conns) @@ -178,8 +193,8 @@ func TestRaftBootstrapMembers_E2E_EtcdLeaderRestartRecovery(t *testing.T) { // Retry the first Put: the gRPC connection to a freshly started node may not // be ready immediately, causing "context canceled while waiting for connections". require.Eventually(t, func() bool { - return rawPutWithTimeout(clients[(leaderIdx+1)%len(clients)], keyA, valueA, rpcTimeout) == nil - }, waitTimeout, waitInterval) + return rawPutWithTimeout(clients[(leaderIdx+1)%len(clients)], keyA, valueA) == nil + }, bootstrapE2EWaitTimeout, waitInterval) waitForValueOnAllClients(t, clients, keyA, valueA, waitTimeout, waitInterval, rpcTimeout) require.NoError(t, nodes[leaderIdx].close()) @@ -188,8 +203,8 @@ func TestRaftBootstrapMembers_E2E_EtcdLeaderRestartRecovery(t *testing.T) { require.NoError(t, err) nodes[leaderIdx] = restartedNode - waitForBootstrapClusterConfig(t, nodes, expected, waitTimeout, waitInterval) - leaderIdx = waitForSingleLeader(t, nodes, waitTimeout, waitInterval) + waitForBootstrapClusterConfig(t, nodes, expected) + leaderIdx = waitForSingleLeader(t, nodes) restartedClients, restartedConns := rawKVClients(t, endpoints) defer closeGRPCConns(restartedConns) @@ -198,8 +213,8 @@ func TestRaftBootstrapMembers_E2E_EtcdLeaderRestartRecovery(t *testing.T) { keyB := []byte("bootstrap-etcd-restart-key-b") valueB := []byte("bootstrap-etcd-restart-value-b") require.Eventually(t, func() bool { - return rawPutWithTimeout(restartedClients[(leaderIdx+1)%len(restartedClients)], keyB, valueB, rpcTimeout) == nil - }, waitTimeout, waitInterval) + return rawPutWithTimeout(restartedClients[(leaderIdx+1)%len(restartedClients)], keyB, valueB) == nil + }, bootstrapE2EWaitTimeout, waitInterval) waitForValueOnAllClients(t, restartedClients, keyB, valueB, waitTimeout, waitInterval, rpcTimeout) } @@ -589,11 +604,15 @@ func startBootstrapE2ENode( } return &bootstrapE2ENode{ - id: ep.id, - runtimes: runtimes, - shardStore: shardStore, - cancel: cancel, - eg: eg, + id: ep.id, + runtimes: runtimes, + shardStore: shardStore, + coordinate: coordinate, + distEngine: cfg.engine, + distCatalog: distCatalog, + distServer: distServer, + cancel: cancel, + eg: eg, }, nil } @@ -672,11 +691,15 @@ func startBootstrapE2EMultiGroupNode( } return &bootstrapE2ENode{ - id: ep.id, - runtimes: runtimes, - shardStore: shardStore, - cancel: cancel, - eg: eg, + id: ep.id, + runtimes: runtimes, + shardStore: shardStore, + coordinate: coordinate, + distEngine: cfg.engine, + distCatalog: distCatalog, + distServer: distServer, + cancel: cancel, + eg: eg, }, nil } @@ -871,7 +894,7 @@ func closeBootstrapE2ENodes(t *testing.T, nodes []*bootstrapE2ENode) { } } -func waitForBootstrapClusterConfig(t *testing.T, nodes []*bootstrapE2ENode, expected raftengine.Configuration, waitTimeout, waitInterval time.Duration) { +func waitForBootstrapClusterConfig(t *testing.T, nodes []*bootstrapE2ENode, expected raftengine.Configuration) { t.Helper() require.Eventually(t, func() bool { @@ -894,7 +917,7 @@ func waitForBootstrapClusterConfig(t *testing.T, nodes []*bootstrapE2ENode, expe } } return true - }, waitTimeout, waitInterval) + }, bootstrapE2EWaitTimeout, bootstrapE2EWaitInterval) } func waitForMultiGroupBootstrapClusterConfig( @@ -938,7 +961,7 @@ func waitForMultiGroupBootstrapClusterConfig( }, waitTimeout, waitInterval) } -func waitForSingleLeader(t *testing.T, nodes []*bootstrapE2ENode, waitTimeout, waitInterval time.Duration) int { +func waitForSingleLeader(t *testing.T, nodes []*bootstrapE2ENode) int { t.Helper() leaderIdx := -1 @@ -960,7 +983,7 @@ func waitForSingleLeader(t *testing.T, nodes []*bootstrapE2ENode, waitTimeout, w } leaderIdx = idx return true - }, waitTimeout, waitInterval) + }, bootstrapE2EWaitTimeout, bootstrapE2EWaitInterval) return leaderIdx } @@ -1050,8 +1073,8 @@ func rawKVClients(t *testing.T, endpoints []bootstrapE2EEndpoint) ([]pb.RawKVCli return clients, conns } -func rawPutWithTimeout(client pb.RawKVClient, key []byte, value []byte, timeout time.Duration) error { - ctx, cancel := context.WithTimeout(context.Background(), timeout) +func rawPutWithTimeout(client pb.RawKVClient, key []byte, value []byte) error { + ctx, cancel := context.WithTimeout(context.Background(), bootstrapE2ERPCRequestTimeout) defer cancel() _, err := client.RawPut(ctx, &pb.RawPutRequest{Key: key, Value: value}) return err diff --git a/proto/admin.pb.go b/proto/admin.pb.go index e6b087967..3a8908b98 100644 --- a/proto/admin.pb.go +++ b/proto/admin.pb.go @@ -128,6 +128,94 @@ func (SampleRole) EnumDescriptor() ([]byte, []int) { return file_admin_proto_rawDescGZIP(), []int{1} } +type SetAutoSplitEnabledRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Enabled bool `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SetAutoSplitEnabledRequest) Reset() { + *x = SetAutoSplitEnabledRequest{} + mi := &file_admin_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SetAutoSplitEnabledRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SetAutoSplitEnabledRequest) ProtoMessage() {} + +func (x *SetAutoSplitEnabledRequest) ProtoReflect() protoreflect.Message { + mi := &file_admin_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SetAutoSplitEnabledRequest.ProtoReflect.Descriptor instead. +func (*SetAutoSplitEnabledRequest) Descriptor() ([]byte, []int) { + return file_admin_proto_rawDescGZIP(), []int{0} +} + +func (x *SetAutoSplitEnabledRequest) GetEnabled() bool { + if x != nil { + return x.Enabled + } + return false +} + +type SetAutoSplitEnabledResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Enabled bool `protobuf:"varint,1,opt,name=enabled,proto3" json:"enabled,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SetAutoSplitEnabledResponse) Reset() { + *x = SetAutoSplitEnabledResponse{} + mi := &file_admin_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SetAutoSplitEnabledResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SetAutoSplitEnabledResponse) ProtoMessage() {} + +func (x *SetAutoSplitEnabledResponse) ProtoReflect() protoreflect.Message { + mi := &file_admin_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SetAutoSplitEnabledResponse.ProtoReflect.Descriptor instead. +func (*SetAutoSplitEnabledResponse) Descriptor() ([]byte, []int) { + return file_admin_proto_rawDescGZIP(), []int{1} +} + +func (x *SetAutoSplitEnabledResponse) GetEnabled() bool { + if x != nil { + return x.Enabled + } + return false +} + type NodeIdentity struct { state protoimpl.MessageState `protogen:"open.v1"` NodeId string `protobuf:"bytes,1,opt,name=node_id,json=nodeId,proto3" json:"node_id,omitempty"` @@ -138,7 +226,7 @@ type NodeIdentity struct { func (x *NodeIdentity) Reset() { *x = NodeIdentity{} - mi := &file_admin_proto_msgTypes[0] + mi := &file_admin_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -150,7 +238,7 @@ func (x *NodeIdentity) String() string { func (*NodeIdentity) ProtoMessage() {} func (x *NodeIdentity) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[0] + mi := &file_admin_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -163,7 +251,7 @@ func (x *NodeIdentity) ProtoReflect() protoreflect.Message { // Deprecated: Use NodeIdentity.ProtoReflect.Descriptor instead. func (*NodeIdentity) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{0} + return file_admin_proto_rawDescGZIP(), []int{2} } func (x *NodeIdentity) GetNodeId() string { @@ -191,7 +279,7 @@ type GroupLeader struct { func (x *GroupLeader) Reset() { *x = GroupLeader{} - mi := &file_admin_proto_msgTypes[1] + mi := &file_admin_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -203,7 +291,7 @@ func (x *GroupLeader) String() string { func (*GroupLeader) ProtoMessage() {} func (x *GroupLeader) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[1] + mi := &file_admin_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -216,7 +304,7 @@ func (x *GroupLeader) ProtoReflect() protoreflect.Message { // Deprecated: Use GroupLeader.ProtoReflect.Descriptor instead. func (*GroupLeader) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{1} + return file_admin_proto_rawDescGZIP(), []int{3} } func (x *GroupLeader) GetRaftGroupId() uint64 { @@ -248,7 +336,7 @@ type GetClusterOverviewRequest struct { func (x *GetClusterOverviewRequest) Reset() { *x = GetClusterOverviewRequest{} - mi := &file_admin_proto_msgTypes[2] + mi := &file_admin_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -260,7 +348,7 @@ func (x *GetClusterOverviewRequest) String() string { func (*GetClusterOverviewRequest) ProtoMessage() {} func (x *GetClusterOverviewRequest) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[2] + mi := &file_admin_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -273,7 +361,7 @@ func (x *GetClusterOverviewRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetClusterOverviewRequest.ProtoReflect.Descriptor instead. func (*GetClusterOverviewRequest) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{2} + return file_admin_proto_rawDescGZIP(), []int{4} } type GetClusterOverviewResponse struct { @@ -289,7 +377,7 @@ type GetClusterOverviewResponse struct { func (x *GetClusterOverviewResponse) Reset() { *x = GetClusterOverviewResponse{} - mi := &file_admin_proto_msgTypes[3] + mi := &file_admin_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -301,7 +389,7 @@ func (x *GetClusterOverviewResponse) String() string { func (*GetClusterOverviewResponse) ProtoMessage() {} func (x *GetClusterOverviewResponse) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[3] + mi := &file_admin_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -314,7 +402,7 @@ func (x *GetClusterOverviewResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetClusterOverviewResponse.ProtoReflect.Descriptor instead. func (*GetClusterOverviewResponse) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{3} + return file_admin_proto_rawDescGZIP(), []int{5} } func (x *GetClusterOverviewResponse) GetSelf() *NodeIdentity { @@ -370,7 +458,7 @@ type RaftGroupState struct { func (x *RaftGroupState) Reset() { *x = RaftGroupState{} - mi := &file_admin_proto_msgTypes[4] + mi := &file_admin_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -382,7 +470,7 @@ func (x *RaftGroupState) String() string { func (*RaftGroupState) ProtoMessage() {} func (x *RaftGroupState) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[4] + mi := &file_admin_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -395,7 +483,7 @@ func (x *RaftGroupState) ProtoReflect() protoreflect.Message { // Deprecated: Use RaftGroupState.ProtoReflect.Descriptor instead. func (*RaftGroupState) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{4} + return file_admin_proto_rawDescGZIP(), []int{6} } func (x *RaftGroupState) GetRaftGroupId() uint64 { @@ -448,7 +536,7 @@ type GetRaftGroupsRequest struct { func (x *GetRaftGroupsRequest) Reset() { *x = GetRaftGroupsRequest{} - mi := &file_admin_proto_msgTypes[5] + mi := &file_admin_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -460,7 +548,7 @@ func (x *GetRaftGroupsRequest) String() string { func (*GetRaftGroupsRequest) ProtoMessage() {} func (x *GetRaftGroupsRequest) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[5] + mi := &file_admin_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -473,7 +561,7 @@ func (x *GetRaftGroupsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetRaftGroupsRequest.ProtoReflect.Descriptor instead. func (*GetRaftGroupsRequest) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{5} + return file_admin_proto_rawDescGZIP(), []int{7} } type GetRaftGroupsResponse struct { @@ -485,7 +573,7 @@ type GetRaftGroupsResponse struct { func (x *GetRaftGroupsResponse) Reset() { *x = GetRaftGroupsResponse{} - mi := &file_admin_proto_msgTypes[6] + mi := &file_admin_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -497,7 +585,7 @@ func (x *GetRaftGroupsResponse) String() string { func (*GetRaftGroupsResponse) ProtoMessage() {} func (x *GetRaftGroupsResponse) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[6] + mi := &file_admin_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -510,7 +598,7 @@ func (x *GetRaftGroupsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetRaftGroupsResponse.ProtoReflect.Descriptor instead. func (*GetRaftGroupsResponse) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{6} + return file_admin_proto_rawDescGZIP(), []int{8} } func (x *GetRaftGroupsResponse) GetGroups() []*RaftGroupState { @@ -537,7 +625,7 @@ type AdapterSummary struct { func (x *AdapterSummary) Reset() { *x = AdapterSummary{} - mi := &file_admin_proto_msgTypes[7] + mi := &file_admin_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -549,7 +637,7 @@ func (x *AdapterSummary) String() string { func (*AdapterSummary) ProtoMessage() {} func (x *AdapterSummary) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[7] + mi := &file_admin_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -562,7 +650,7 @@ func (x *AdapterSummary) ProtoReflect() protoreflect.Message { // Deprecated: Use AdapterSummary.ProtoReflect.Descriptor instead. func (*AdapterSummary) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{7} + return file_admin_proto_rawDescGZIP(), []int{9} } func (x *AdapterSummary) GetAdapter() string { @@ -636,7 +724,7 @@ type GetAdapterSummaryRequest struct { func (x *GetAdapterSummaryRequest) Reset() { *x = GetAdapterSummaryRequest{} - mi := &file_admin_proto_msgTypes[8] + mi := &file_admin_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -648,7 +736,7 @@ func (x *GetAdapterSummaryRequest) String() string { func (*GetAdapterSummaryRequest) ProtoMessage() {} func (x *GetAdapterSummaryRequest) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[8] + mi := &file_admin_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -661,7 +749,7 @@ func (x *GetAdapterSummaryRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetAdapterSummaryRequest.ProtoReflect.Descriptor instead. func (*GetAdapterSummaryRequest) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{8} + return file_admin_proto_rawDescGZIP(), []int{10} } type GetAdapterSummaryResponse struct { @@ -673,7 +761,7 @@ type GetAdapterSummaryResponse struct { func (x *GetAdapterSummaryResponse) Reset() { *x = GetAdapterSummaryResponse{} - mi := &file_admin_proto_msgTypes[9] + mi := &file_admin_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -685,7 +773,7 @@ func (x *GetAdapterSummaryResponse) String() string { func (*GetAdapterSummaryResponse) ProtoMessage() {} func (x *GetAdapterSummaryResponse) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[9] + mi := &file_admin_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -698,7 +786,7 @@ func (x *GetAdapterSummaryResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetAdapterSummaryResponse.ProtoReflect.Descriptor instead. func (*GetAdapterSummaryResponse) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{9} + return file_admin_proto_rawDescGZIP(), []int{11} } func (x *GetAdapterSummaryResponse) GetSummaries() []*AdapterSummary { @@ -755,7 +843,7 @@ type KeyVizRow struct { func (x *KeyVizRow) Reset() { *x = KeyVizRow{} - mi := &file_admin_proto_msgTypes[10] + mi := &file_admin_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -767,7 +855,7 @@ func (x *KeyVizRow) String() string { func (*KeyVizRow) ProtoMessage() {} func (x *KeyVizRow) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[10] + mi := &file_admin_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -780,7 +868,7 @@ func (x *KeyVizRow) ProtoReflect() protoreflect.Message { // Deprecated: Use KeyVizRow.ProtoReflect.Descriptor instead. func (*KeyVizRow) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{10} + return file_admin_proto_rawDescGZIP(), []int{12} } func (x *KeyVizRow) GetBucketId() string { @@ -893,7 +981,7 @@ type GetKeyVizMatrixRequest struct { func (x *GetKeyVizMatrixRequest) Reset() { *x = GetKeyVizMatrixRequest{} - mi := &file_admin_proto_msgTypes[11] + mi := &file_admin_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -905,7 +993,7 @@ func (x *GetKeyVizMatrixRequest) String() string { func (*GetKeyVizMatrixRequest) ProtoMessage() {} func (x *GetKeyVizMatrixRequest) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[11] + mi := &file_admin_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -918,7 +1006,7 @@ func (x *GetKeyVizMatrixRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetKeyVizMatrixRequest.ProtoReflect.Descriptor instead. func (*GetKeyVizMatrixRequest) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{11} + return file_admin_proto_rawDescGZIP(), []int{13} } func (x *GetKeyVizMatrixRequest) GetSeries() KeyVizSeries { @@ -959,7 +1047,7 @@ type GetKeyVizMatrixResponse struct { func (x *GetKeyVizMatrixResponse) Reset() { *x = GetKeyVizMatrixResponse{} - mi := &file_admin_proto_msgTypes[12] + mi := &file_admin_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -971,7 +1059,7 @@ func (x *GetKeyVizMatrixResponse) String() string { func (*GetKeyVizMatrixResponse) ProtoMessage() {} func (x *GetKeyVizMatrixResponse) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[12] + mi := &file_admin_proto_msgTypes[14] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -984,7 +1072,7 @@ func (x *GetKeyVizMatrixResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetKeyVizMatrixResponse.ProtoReflect.Descriptor instead. func (*GetKeyVizMatrixResponse) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{12} + return file_admin_proto_rawDescGZIP(), []int{14} } func (x *GetKeyVizMatrixResponse) GetColumnUnixMs() []int64 { @@ -1014,7 +1102,7 @@ type GetRouteDetailRequest struct { func (x *GetRouteDetailRequest) Reset() { *x = GetRouteDetailRequest{} - mi := &file_admin_proto_msgTypes[13] + mi := &file_admin_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1026,7 +1114,7 @@ func (x *GetRouteDetailRequest) String() string { func (*GetRouteDetailRequest) ProtoMessage() {} func (x *GetRouteDetailRequest) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[13] + mi := &file_admin_proto_msgTypes[15] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1039,7 +1127,7 @@ func (x *GetRouteDetailRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use GetRouteDetailRequest.ProtoReflect.Descriptor instead. func (*GetRouteDetailRequest) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{13} + return file_admin_proto_rawDescGZIP(), []int{15} } func (x *GetRouteDetailRequest) GetBucketId() string { @@ -1073,7 +1161,7 @@ type GetRouteDetailResponse struct { func (x *GetRouteDetailResponse) Reset() { *x = GetRouteDetailResponse{} - mi := &file_admin_proto_msgTypes[14] + mi := &file_admin_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1085,7 +1173,7 @@ func (x *GetRouteDetailResponse) String() string { func (*GetRouteDetailResponse) ProtoMessage() {} func (x *GetRouteDetailResponse) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[14] + mi := &file_admin_proto_msgTypes[16] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1098,7 +1186,7 @@ func (x *GetRouteDetailResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use GetRouteDetailResponse.ProtoReflect.Descriptor instead. func (*GetRouteDetailResponse) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{14} + return file_admin_proto_rawDescGZIP(), []int{16} } func (x *GetRouteDetailResponse) GetRow() *KeyVizRow { @@ -1123,7 +1211,7 @@ type StreamEventsRequest struct { func (x *StreamEventsRequest) Reset() { *x = StreamEventsRequest{} - mi := &file_admin_proto_msgTypes[15] + mi := &file_admin_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1135,7 +1223,7 @@ func (x *StreamEventsRequest) String() string { func (*StreamEventsRequest) ProtoMessage() {} func (x *StreamEventsRequest) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[15] + mi := &file_admin_proto_msgTypes[17] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1148,7 +1236,7 @@ func (x *StreamEventsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use StreamEventsRequest.ProtoReflect.Descriptor instead. func (*StreamEventsRequest) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{15} + return file_admin_proto_rawDescGZIP(), []int{17} } type StreamEventsEvent struct { @@ -1164,7 +1252,7 @@ type StreamEventsEvent struct { func (x *StreamEventsEvent) Reset() { *x = StreamEventsEvent{} - mi := &file_admin_proto_msgTypes[16] + mi := &file_admin_proto_msgTypes[18] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1176,7 +1264,7 @@ func (x *StreamEventsEvent) String() string { func (*StreamEventsEvent) ProtoMessage() {} func (x *StreamEventsEvent) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[16] + mi := &file_admin_proto_msgTypes[18] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1189,7 +1277,7 @@ func (x *StreamEventsEvent) ProtoReflect() protoreflect.Message { // Deprecated: Use StreamEventsEvent.ProtoReflect.Descriptor instead. func (*StreamEventsEvent) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{16} + return file_admin_proto_rawDescGZIP(), []int{18} } func (x *StreamEventsEvent) GetEvent() isStreamEventsEvent_Event { @@ -1245,7 +1333,7 @@ type RouteTransition struct { func (x *RouteTransition) Reset() { *x = RouteTransition{} - mi := &file_admin_proto_msgTypes[17] + mi := &file_admin_proto_msgTypes[19] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1257,7 +1345,7 @@ func (x *RouteTransition) String() string { func (*RouteTransition) ProtoMessage() {} func (x *RouteTransition) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[17] + mi := &file_admin_proto_msgTypes[19] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1270,7 +1358,7 @@ func (x *RouteTransition) ProtoReflect() protoreflect.Message { // Deprecated: Use RouteTransition.ProtoReflect.Descriptor instead. func (*RouteTransition) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{17} + return file_admin_proto_rawDescGZIP(), []int{19} } func (x *RouteTransition) GetParentRouteId() uint64 { @@ -1312,7 +1400,7 @@ type KeyVizColumn struct { func (x *KeyVizColumn) Reset() { *x = KeyVizColumn{} - mi := &file_admin_proto_msgTypes[18] + mi := &file_admin_proto_msgTypes[20] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1324,7 +1412,7 @@ func (x *KeyVizColumn) String() string { func (*KeyVizColumn) ProtoMessage() {} func (x *KeyVizColumn) ProtoReflect() protoreflect.Message { - mi := &file_admin_proto_msgTypes[18] + mi := &file_admin_proto_msgTypes[20] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1337,7 +1425,7 @@ func (x *KeyVizColumn) ProtoReflect() protoreflect.Message { // Deprecated: Use KeyVizColumn.ProtoReflect.Descriptor instead. func (*KeyVizColumn) Descriptor() ([]byte, []int) { - return file_admin_proto_rawDescGZIP(), []int{18} + return file_admin_proto_rawDescGZIP(), []int{20} } func (x *KeyVizColumn) GetColumnUnixMs() int64 { @@ -1365,7 +1453,11 @@ var File_admin_proto protoreflect.FileDescriptor const file_admin_proto_rawDesc = "" + "\n" + - "\vadmin.proto\"J\n" + + "\vadmin.proto\"6\n" + + "\x1aSetAutoSplitEnabledRequest\x12\x18\n" + + "\aenabled\x18\x01 \x01(\bR\aenabled\"7\n" + + "\x1bSetAutoSplitEnabledResponse\x12\x18\n" + + "\aenabled\x18\x01 \x01(\bR\aenabled\"J\n" + "\fNodeIdentity\x12\x17\n" + "\anode_id\x18\x01 \x01(\tR\x06nodeId\x12!\n" + "\fgrpc_address\x18\x02 \x01(\tR\vgrpcAddress\"x\n" + @@ -1475,13 +1567,14 @@ const file_admin_proto_rawDesc = "" + "\x17SAMPLE_ROLE_UNSPECIFIED\x10\x00\x12\x1c\n" + "\x18SAMPLE_ROLE_LEADER_WRITE\x10\x01\x12\x1b\n" + "\x17SAMPLE_ROLE_LEADER_READ\x10\x02\x12\x1d\n" + - "\x19SAMPLE_ROLE_FOLLOWER_READ\x10\x032\xb3\x03\n" + + "\x19SAMPLE_ROLE_FOLLOWER_READ\x10\x032\x87\x04\n" + "\x05Admin\x12O\n" + "\x12GetClusterOverview\x12\x1a.GetClusterOverviewRequest\x1a\x1b.GetClusterOverviewResponse\"\x00\x12@\n" + "\rGetRaftGroups\x12\x15.GetRaftGroupsRequest\x1a\x16.GetRaftGroupsResponse\"\x00\x12L\n" + "\x11GetAdapterSummary\x12\x19.GetAdapterSummaryRequest\x1a\x1a.GetAdapterSummaryResponse\"\x00\x12F\n" + "\x0fGetKeyVizMatrix\x12\x17.GetKeyVizMatrixRequest\x1a\x18.GetKeyVizMatrixResponse\"\x00\x12C\n" + - "\x0eGetRouteDetail\x12\x16.GetRouteDetailRequest\x1a\x17.GetRouteDetailResponse\"\x00\x12<\n" + + "\x0eGetRouteDetail\x12\x16.GetRouteDetailRequest\x1a\x17.GetRouteDetailResponse\"\x00\x12R\n" + + "\x13SetAutoSplitEnabled\x12\x1b.SetAutoSplitEnabledRequest\x1a\x1c.SetAutoSplitEnabledResponse\"\x00\x12<\n" + "\fStreamEvents\x12\x14.StreamEventsRequest\x1a\x12.StreamEventsEvent\"\x000\x01B#Z!github.com/bootjp/elastickv/protob\x06proto3" var ( @@ -1497,61 +1590,65 @@ func file_admin_proto_rawDescGZIP() []byte { } var file_admin_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_admin_proto_msgTypes = make([]protoimpl.MessageInfo, 20) +var file_admin_proto_msgTypes = make([]protoimpl.MessageInfo, 22) var file_admin_proto_goTypes = []any{ - (KeyVizSeries)(0), // 0: KeyVizSeries - (SampleRole)(0), // 1: SampleRole - (*NodeIdentity)(nil), // 2: NodeIdentity - (*GroupLeader)(nil), // 3: GroupLeader - (*GetClusterOverviewRequest)(nil), // 4: GetClusterOverviewRequest - (*GetClusterOverviewResponse)(nil), // 5: GetClusterOverviewResponse - (*RaftGroupState)(nil), // 6: RaftGroupState - (*GetRaftGroupsRequest)(nil), // 7: GetRaftGroupsRequest - (*GetRaftGroupsResponse)(nil), // 8: GetRaftGroupsResponse - (*AdapterSummary)(nil), // 9: AdapterSummary - (*GetAdapterSummaryRequest)(nil), // 10: GetAdapterSummaryRequest - (*GetAdapterSummaryResponse)(nil), // 11: GetAdapterSummaryResponse - (*KeyVizRow)(nil), // 12: KeyVizRow - (*GetKeyVizMatrixRequest)(nil), // 13: GetKeyVizMatrixRequest - (*GetKeyVizMatrixResponse)(nil), // 14: GetKeyVizMatrixResponse - (*GetRouteDetailRequest)(nil), // 15: GetRouteDetailRequest - (*GetRouteDetailResponse)(nil), // 16: GetRouteDetailResponse - (*StreamEventsRequest)(nil), // 17: StreamEventsRequest - (*StreamEventsEvent)(nil), // 18: StreamEventsEvent - (*RouteTransition)(nil), // 19: RouteTransition - (*KeyVizColumn)(nil), // 20: KeyVizColumn - nil, // 21: GetClusterOverviewResponse.CapabilitiesEntry + (KeyVizSeries)(0), // 0: KeyVizSeries + (SampleRole)(0), // 1: SampleRole + (*SetAutoSplitEnabledRequest)(nil), // 2: SetAutoSplitEnabledRequest + (*SetAutoSplitEnabledResponse)(nil), // 3: SetAutoSplitEnabledResponse + (*NodeIdentity)(nil), // 4: NodeIdentity + (*GroupLeader)(nil), // 5: GroupLeader + (*GetClusterOverviewRequest)(nil), // 6: GetClusterOverviewRequest + (*GetClusterOverviewResponse)(nil), // 7: GetClusterOverviewResponse + (*RaftGroupState)(nil), // 8: RaftGroupState + (*GetRaftGroupsRequest)(nil), // 9: GetRaftGroupsRequest + (*GetRaftGroupsResponse)(nil), // 10: GetRaftGroupsResponse + (*AdapterSummary)(nil), // 11: AdapterSummary + (*GetAdapterSummaryRequest)(nil), // 12: GetAdapterSummaryRequest + (*GetAdapterSummaryResponse)(nil), // 13: GetAdapterSummaryResponse + (*KeyVizRow)(nil), // 14: KeyVizRow + (*GetKeyVizMatrixRequest)(nil), // 15: GetKeyVizMatrixRequest + (*GetKeyVizMatrixResponse)(nil), // 16: GetKeyVizMatrixResponse + (*GetRouteDetailRequest)(nil), // 17: GetRouteDetailRequest + (*GetRouteDetailResponse)(nil), // 18: GetRouteDetailResponse + (*StreamEventsRequest)(nil), // 19: StreamEventsRequest + (*StreamEventsEvent)(nil), // 20: StreamEventsEvent + (*RouteTransition)(nil), // 21: RouteTransition + (*KeyVizColumn)(nil), // 22: KeyVizColumn + nil, // 23: GetClusterOverviewResponse.CapabilitiesEntry } var file_admin_proto_depIdxs = []int32{ - 2, // 0: GetClusterOverviewResponse.self:type_name -> NodeIdentity - 2, // 1: GetClusterOverviewResponse.members:type_name -> NodeIdentity - 3, // 2: GetClusterOverviewResponse.group_leaders:type_name -> GroupLeader - 21, // 3: GetClusterOverviewResponse.capabilities:type_name -> GetClusterOverviewResponse.CapabilitiesEntry - 6, // 4: GetRaftGroupsResponse.groups:type_name -> RaftGroupState - 9, // 5: GetAdapterSummaryResponse.summaries:type_name -> AdapterSummary + 4, // 0: GetClusterOverviewResponse.self:type_name -> NodeIdentity + 4, // 1: GetClusterOverviewResponse.members:type_name -> NodeIdentity + 5, // 2: GetClusterOverviewResponse.group_leaders:type_name -> GroupLeader + 23, // 3: GetClusterOverviewResponse.capabilities:type_name -> GetClusterOverviewResponse.CapabilitiesEntry + 8, // 4: GetRaftGroupsResponse.groups:type_name -> RaftGroupState + 11, // 5: GetAdapterSummaryResponse.summaries:type_name -> AdapterSummary 1, // 6: KeyVizRow.sample_roles:type_name -> SampleRole 0, // 7: GetKeyVizMatrixRequest.series:type_name -> KeyVizSeries - 12, // 8: GetKeyVizMatrixResponse.rows:type_name -> KeyVizRow - 12, // 9: GetRouteDetailResponse.row:type_name -> KeyVizRow - 9, // 10: GetRouteDetailResponse.per_adapter:type_name -> AdapterSummary - 19, // 11: StreamEventsEvent.route_transition:type_name -> RouteTransition - 20, // 12: StreamEventsEvent.keyviz_column:type_name -> KeyVizColumn + 14, // 8: GetKeyVizMatrixResponse.rows:type_name -> KeyVizRow + 14, // 9: GetRouteDetailResponse.row:type_name -> KeyVizRow + 11, // 10: GetRouteDetailResponse.per_adapter:type_name -> AdapterSummary + 21, // 11: StreamEventsEvent.route_transition:type_name -> RouteTransition + 22, // 12: StreamEventsEvent.keyviz_column:type_name -> KeyVizColumn 0, // 13: KeyVizColumn.series:type_name -> KeyVizSeries - 12, // 14: KeyVizColumn.rows:type_name -> KeyVizRow - 4, // 15: Admin.GetClusterOverview:input_type -> GetClusterOverviewRequest - 7, // 16: Admin.GetRaftGroups:input_type -> GetRaftGroupsRequest - 10, // 17: Admin.GetAdapterSummary:input_type -> GetAdapterSummaryRequest - 13, // 18: Admin.GetKeyVizMatrix:input_type -> GetKeyVizMatrixRequest - 15, // 19: Admin.GetRouteDetail:input_type -> GetRouteDetailRequest - 17, // 20: Admin.StreamEvents:input_type -> StreamEventsRequest - 5, // 21: Admin.GetClusterOverview:output_type -> GetClusterOverviewResponse - 8, // 22: Admin.GetRaftGroups:output_type -> GetRaftGroupsResponse - 11, // 23: Admin.GetAdapterSummary:output_type -> GetAdapterSummaryResponse - 14, // 24: Admin.GetKeyVizMatrix:output_type -> GetKeyVizMatrixResponse - 16, // 25: Admin.GetRouteDetail:output_type -> GetRouteDetailResponse - 18, // 26: Admin.StreamEvents:output_type -> StreamEventsEvent - 21, // [21:27] is the sub-list for method output_type - 15, // [15:21] is the sub-list for method input_type + 14, // 14: KeyVizColumn.rows:type_name -> KeyVizRow + 6, // 15: Admin.GetClusterOverview:input_type -> GetClusterOverviewRequest + 9, // 16: Admin.GetRaftGroups:input_type -> GetRaftGroupsRequest + 12, // 17: Admin.GetAdapterSummary:input_type -> GetAdapterSummaryRequest + 15, // 18: Admin.GetKeyVizMatrix:input_type -> GetKeyVizMatrixRequest + 17, // 19: Admin.GetRouteDetail:input_type -> GetRouteDetailRequest + 2, // 20: Admin.SetAutoSplitEnabled:input_type -> SetAutoSplitEnabledRequest + 19, // 21: Admin.StreamEvents:input_type -> StreamEventsRequest + 7, // 22: Admin.GetClusterOverview:output_type -> GetClusterOverviewResponse + 10, // 23: Admin.GetRaftGroups:output_type -> GetRaftGroupsResponse + 13, // 24: Admin.GetAdapterSummary:output_type -> GetAdapterSummaryResponse + 16, // 25: Admin.GetKeyVizMatrix:output_type -> GetKeyVizMatrixResponse + 18, // 26: Admin.GetRouteDetail:output_type -> GetRouteDetailResponse + 3, // 27: Admin.SetAutoSplitEnabled:output_type -> SetAutoSplitEnabledResponse + 20, // 28: Admin.StreamEvents:output_type -> StreamEventsEvent + 22, // [22:29] is the sub-list for method output_type + 15, // [15:22] is the sub-list for method input_type 15, // [15:15] is the sub-list for extension type_name 15, // [15:15] is the sub-list for extension extendee 0, // [0:15] is the sub-list for field type_name @@ -1562,7 +1659,7 @@ func file_admin_proto_init() { if File_admin_proto != nil { return } - file_admin_proto_msgTypes[16].OneofWrappers = []any{ + file_admin_proto_msgTypes[18].OneofWrappers = []any{ (*StreamEventsEvent_RouteTransition)(nil), (*StreamEventsEvent_KeyvizColumn)(nil), } @@ -1572,7 +1669,7 @@ func file_admin_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_admin_proto_rawDesc), len(file_admin_proto_rawDesc)), NumEnums: 2, - NumMessages: 20, + NumMessages: 22, NumExtensions: 0, NumServices: 1, }, diff --git a/proto/admin.proto b/proto/admin.proto index 96d111020..673c6fe6e 100644 --- a/proto/admin.proto +++ b/proto/admin.proto @@ -2,9 +2,9 @@ syntax = "proto3"; option go_package = "github.com/bootjp/elastickv/proto"; -// Admin is the node-side read-only admin gRPC service consumed by -// cmd/elastickv-admin. Every method requires "authorization: Bearer " -// metadata unless the node was started with --adminInsecureNoAuth. +// Admin is the node-side operator gRPC service consumed by cmd/elastickv-admin. +// Every method requires "authorization: Bearer " metadata unless the +// node was started with --adminInsecureNoAuth. // See docs/admin_ui_key_visualizer_design.md §4 (Layer A). service Admin { rpc GetClusterOverview (GetClusterOverviewRequest) returns (GetClusterOverviewResponse) {} @@ -12,9 +12,18 @@ service Admin { rpc GetAdapterSummary (GetAdapterSummaryRequest) returns (GetAdapterSummaryResponse) {} rpc GetKeyVizMatrix (GetKeyVizMatrixRequest) returns (GetKeyVizMatrixResponse) {} rpc GetRouteDetail (GetRouteDetailRequest) returns (GetRouteDetailResponse) {} + rpc SetAutoSplitEnabled (SetAutoSplitEnabledRequest) returns (SetAutoSplitEnabledResponse) {} rpc StreamEvents (StreamEventsRequest) returns (stream StreamEventsEvent) {} } +message SetAutoSplitEnabledRequest { + bool enabled = 1; +} + +message SetAutoSplitEnabledResponse { + bool enabled = 1; +} + message NodeIdentity { string node_id = 1; string grpc_address = 2; diff --git a/proto/admin_grpc.pb.go b/proto/admin_grpc.pb.go index 021b1e834..0568c1595 100644 --- a/proto/admin_grpc.pb.go +++ b/proto/admin_grpc.pb.go @@ -19,21 +19,22 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - Admin_GetClusterOverview_FullMethodName = "/Admin/GetClusterOverview" - Admin_GetRaftGroups_FullMethodName = "/Admin/GetRaftGroups" - Admin_GetAdapterSummary_FullMethodName = "/Admin/GetAdapterSummary" - Admin_GetKeyVizMatrix_FullMethodName = "/Admin/GetKeyVizMatrix" - Admin_GetRouteDetail_FullMethodName = "/Admin/GetRouteDetail" - Admin_StreamEvents_FullMethodName = "/Admin/StreamEvents" + Admin_GetClusterOverview_FullMethodName = "/Admin/GetClusterOverview" + Admin_GetRaftGroups_FullMethodName = "/Admin/GetRaftGroups" + Admin_GetAdapterSummary_FullMethodName = "/Admin/GetAdapterSummary" + Admin_GetKeyVizMatrix_FullMethodName = "/Admin/GetKeyVizMatrix" + Admin_GetRouteDetail_FullMethodName = "/Admin/GetRouteDetail" + Admin_SetAutoSplitEnabled_FullMethodName = "/Admin/SetAutoSplitEnabled" + Admin_StreamEvents_FullMethodName = "/Admin/StreamEvents" ) // AdminClient is the client API for Admin service. // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. // -// Admin is the node-side read-only admin gRPC service consumed by -// cmd/elastickv-admin. Every method requires "authorization: Bearer " -// metadata unless the node was started with --adminInsecureNoAuth. +// Admin is the node-side operator gRPC service consumed by cmd/elastickv-admin. +// Every method requires "authorization: Bearer " metadata unless the +// node was started with --adminInsecureNoAuth. // See docs/admin_ui_key_visualizer_design.md §4 (Layer A). type AdminClient interface { GetClusterOverview(ctx context.Context, in *GetClusterOverviewRequest, opts ...grpc.CallOption) (*GetClusterOverviewResponse, error) @@ -41,6 +42,7 @@ type AdminClient interface { GetAdapterSummary(ctx context.Context, in *GetAdapterSummaryRequest, opts ...grpc.CallOption) (*GetAdapterSummaryResponse, error) GetKeyVizMatrix(ctx context.Context, in *GetKeyVizMatrixRequest, opts ...grpc.CallOption) (*GetKeyVizMatrixResponse, error) GetRouteDetail(ctx context.Context, in *GetRouteDetailRequest, opts ...grpc.CallOption) (*GetRouteDetailResponse, error) + SetAutoSplitEnabled(ctx context.Context, in *SetAutoSplitEnabledRequest, opts ...grpc.CallOption) (*SetAutoSplitEnabledResponse, error) StreamEvents(ctx context.Context, in *StreamEventsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[StreamEventsEvent], error) } @@ -102,6 +104,16 @@ func (c *adminClient) GetRouteDetail(ctx context.Context, in *GetRouteDetailRequ return out, nil } +func (c *adminClient) SetAutoSplitEnabled(ctx context.Context, in *SetAutoSplitEnabledRequest, opts ...grpc.CallOption) (*SetAutoSplitEnabledResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SetAutoSplitEnabledResponse) + err := c.cc.Invoke(ctx, Admin_SetAutoSplitEnabled_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *adminClient) StreamEvents(ctx context.Context, in *StreamEventsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[StreamEventsEvent], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) stream, err := c.cc.NewStream(ctx, &Admin_ServiceDesc.Streams[0], Admin_StreamEvents_FullMethodName, cOpts...) @@ -125,9 +137,9 @@ type Admin_StreamEventsClient = grpc.ServerStreamingClient[StreamEventsEvent] // All implementations must embed UnimplementedAdminServer // for forward compatibility. // -// Admin is the node-side read-only admin gRPC service consumed by -// cmd/elastickv-admin. Every method requires "authorization: Bearer " -// metadata unless the node was started with --adminInsecureNoAuth. +// Admin is the node-side operator gRPC service consumed by cmd/elastickv-admin. +// Every method requires "authorization: Bearer " metadata unless the +// node was started with --adminInsecureNoAuth. // See docs/admin_ui_key_visualizer_design.md §4 (Layer A). type AdminServer interface { GetClusterOverview(context.Context, *GetClusterOverviewRequest) (*GetClusterOverviewResponse, error) @@ -135,6 +147,7 @@ type AdminServer interface { GetAdapterSummary(context.Context, *GetAdapterSummaryRequest) (*GetAdapterSummaryResponse, error) GetKeyVizMatrix(context.Context, *GetKeyVizMatrixRequest) (*GetKeyVizMatrixResponse, error) GetRouteDetail(context.Context, *GetRouteDetailRequest) (*GetRouteDetailResponse, error) + SetAutoSplitEnabled(context.Context, *SetAutoSplitEnabledRequest) (*SetAutoSplitEnabledResponse, error) StreamEvents(*StreamEventsRequest, grpc.ServerStreamingServer[StreamEventsEvent]) error mustEmbedUnimplementedAdminServer() } @@ -161,6 +174,9 @@ func (UnimplementedAdminServer) GetKeyVizMatrix(context.Context, *GetKeyVizMatri func (UnimplementedAdminServer) GetRouteDetail(context.Context, *GetRouteDetailRequest) (*GetRouteDetailResponse, error) { return nil, status.Error(codes.Unimplemented, "method GetRouteDetail not implemented") } +func (UnimplementedAdminServer) SetAutoSplitEnabled(context.Context, *SetAutoSplitEnabledRequest) (*SetAutoSplitEnabledResponse, error) { + return nil, status.Error(codes.Unimplemented, "method SetAutoSplitEnabled not implemented") +} func (UnimplementedAdminServer) StreamEvents(*StreamEventsRequest, grpc.ServerStreamingServer[StreamEventsEvent]) error { return status.Error(codes.Unimplemented, "method StreamEvents not implemented") } @@ -275,6 +291,24 @@ func _Admin_GetRouteDetail_Handler(srv interface{}, ctx context.Context, dec fun return interceptor(ctx, in, info, handler) } +func _Admin_SetAutoSplitEnabled_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SetAutoSplitEnabledRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AdminServer).SetAutoSplitEnabled(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Admin_SetAutoSplitEnabled_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AdminServer).SetAutoSplitEnabled(ctx, req.(*SetAutoSplitEnabledRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _Admin_StreamEvents_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(StreamEventsRequest) if err := stream.RecvMsg(m); err != nil { @@ -313,6 +347,10 @@ var Admin_ServiceDesc = grpc.ServiceDesc{ MethodName: "GetRouteDetail", Handler: _Admin_GetRouteDetail_Handler, }, + { + MethodName: "SetAutoSplitEnabled", + Handler: _Admin_SetAutoSplitEnabled_Handler, + }, }, Streams: []grpc.StreamDesc{ {