Skip to content

Complete centralized TSO runtime operations #1129

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
bootjp wants to merge 2 commits into
base: design/dedicated-tso-m7
Choose a base branch
from
Open

Complete centralized TSO runtime operations #1129

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c801865
tso: complete centralized runtime semantics
bootjp Jul 19, 2026
ef59050
tso: add runtime operations and observability
bootjp Jul 19, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 26 additions & 21 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,35 +29,40 @@ TLA_LIB := ../lib

.PHONY: tla-check tla-tools

tla-tools: $(TLA_JAR)

$(TLA_JAR):
tla-tools:
@mkdir -p $(dir $(TLA_JAR))
@echo "Downloading tla2tools.jar $(TLA_VERSION)..."
@curl -fsSL -o $(TLA_JAR).tmp $(TLA_URL)
@# Prefer sha256sum (GNU coreutils, universal on Linux); fall back to
@# shasum -a 256 (default on macOS). Either yields the same hex
@# digest in the first whitespace-delimited field.
@if command -v sha256sum >/dev/null 2>&1; then \
actual=$$(sha256sum $(TLA_JAR).tmp | awk '{print $$1}'); \
elif command -v shasum >/dev/null 2>&1; then \
actual=$$(shasum -a 256 $(TLA_JAR).tmp | awk '{print $$1}'); \
else \
echo "ERROR: neither sha256sum nor shasum is available."; \
rm -f $(TLA_JAR).tmp; \
exit 1; \
@set -eu; \
sha256_file() { \
if command -v sha256sum >/dev/null 2>&1; then \
sha256sum "$$1" | awk '{print $$1}'; \
elif command -v shasum >/dev/null 2>&1; then \
shasum -a 256 "$$1" | awk '{print $$1}'; \
else \
echo "ERROR: neither sha256sum nor shasum is available." >&2; \
return 1; \
fi; \
}; \
actual=""; \
if [ -f "$(TLA_JAR)" ]; then actual=$$(sha256_file "$(TLA_JAR)"); fi; \
if [ "$$actual" = "$(TLA_SHA256)" ]; then \
echo "tla2tools.jar ready at $(TLA_JAR) (SHA-256 $(TLA_SHA256))"; \
exit 0; \
fi; \
echo "Downloading tla2tools.jar $(TLA_VERSION)..."; \
trap 'rm -f "$(TLA_JAR).tmp"' EXIT HUP INT TERM; \
curl -fsSL -o "$(TLA_JAR).tmp" "$(TLA_URL)"; \
actual=$$(sha256_file "$(TLA_JAR).tmp"); \
if [ "$$actual" != "$(TLA_SHA256)" ]; then \
echo "ERROR: tla2tools.jar SHA-256 mismatch."; \
echo " expected: $(TLA_SHA256)"; \
echo " actual: $$actual"; \
rm -f $(TLA_JAR).tmp; \
exit 1; \
fi
@mv $(TLA_JAR).tmp $(TLA_JAR)
@echo "tla2tools.jar ready at $(TLA_JAR) (SHA-256 $(TLA_SHA256))"
fi; \
mv "$(TLA_JAR).tmp" "$(TLA_JAR)"; \
trap - EXIT HUP INT TERM; \
echo "tla2tools.jar ready at $(TLA_JAR) (SHA-256 $(TLA_SHA256))"

tla-check: $(TLA_JAR)
tla-check: tla-tools
@# Per-module orchestration lives in scripts/tla-check.sh so adding
@# M3..M5 only needs an entry in that script's TLA_MODULES array
@# and a `case` line for the gap-invariant string. The script does
Expand Down
129 changes: 108 additions & 21 deletions adapter/distribution_server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,18 +147,15 @@ func (s *DistributionServer) GetTimestamp(ctx context.Context, req *pb.GetTimest
if err != nil {
return nil, err
}
activateCutover := req != nil && req.GetActivateCutover()
activateCutover, activatePhaseD, err := timestampActivationValues(req)
if err != nil {
return nil, err
}
if s.timestampAllocator == nil {
if count != 1 || minTimestamp != 0 || activateCutover {
return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "dedicated TSO allocator is not configured"))
}
if s.engine == nil {
return nil, errors.WithStack(status.Error(codes.Unavailable, "distribution engine is not configured"))
}
return &pb.GetTimestampResponse{Timestamp: s.engine.NextTimestamp()}, nil
return s.legacyTimestampResponse(count, minTimestamp, activateCutover, activatePhaseD)
}

reservation, err := s.allocateTimestampReservation(ctx, count, minTimestamp, activateCutover)
reservation, err := s.allocateTimestampReservation(ctx, count, minTimestamp, activateCutover, activatePhaseD)
if err != nil {
return nil, timestampRPCError(err)
}
Expand All @@ -171,22 +168,79 @@ func (s *DistributionServer) GetTimestamp(ctx context.Context, req *pb.GetTimest
Count: uint32(count), //nolint:gosec // count is bounded by MaxTSOBatchSize.
PreviousAllocationFloor: reservation.PreviousAllocationFloor,
CutoverActive: reservation.CutoverActive,
PhaseDActive: reservation.PhaseDActive,
PhaseDFloor: reservation.PhaseDFloor,
}, nil
}

func timestampActivationValues(req *pb.GetTimestampRequest) (bool, bool, error) {
activateCutover := req != nil && req.GetActivateCutover()
activatePhaseD := req != nil && req.GetActivatePhaseD()
if activatePhaseD && !activateCutover {
return false, false, errors.WithStack(status.Error(codes.InvalidArgument,
"phase D activation requires cutover activation"))
}
return activateCutover, activatePhaseD, nil
}

func (s *DistributionServer) legacyTimestampResponse(
count int,
minTimestamp uint64,
activateCutover bool,
activatePhaseD bool,
) (*pb.GetTimestampResponse, error) {
if count != 1 || minTimestamp != 0 || activateCutover || activatePhaseD {
return nil, errors.WithStack(status.Error(codes.FailedPrecondition, "dedicated TSO allocator is not configured"))
}
if s.engine == nil {
return nil, errors.WithStack(status.Error(codes.Unavailable, "distribution engine is not configured"))
}
return &pb.GetTimestampResponse{Timestamp: s.engine.NextTimestamp()}, nil
}

// ValidateTimestamp verifies a read/start timestamp against the durable M7
// allocation range. The local allocator performs the group-0 leader fence;
// followers reject so clients re-resolve rather than validating stale state.
func (s *DistributionServer) ValidateTimestamp(
ctx context.Context,
req *pb.ValidateTimestampRequest,
) (*pb.ValidateTimestampResponse, error) {
if req == nil || req.GetTimestamp() == 0 {
return nil, errors.WithStack(status.Error(codes.InvalidArgument, "timestamp is required"))
}
validator, ok := s.timestampAllocator.(kv.DurableTimestampValidator)
if !ok {
return nil, errors.WithStack(status.Error(codes.FailedPrecondition,
"dedicated TSO timestamp validation is not configured"))
}
if err := validator.ValidateDurableTimestamp(ctx, req.GetTimestamp()); err != nil {
return nil, timestampRPCError(err)
}
resp := &pb.ValidateTimestampResponse{Valid: true, PhaseDActive: true}
if state, ok := s.timestampAllocator.(interface {
PhaseDFloor() uint64
AllocationFloor() uint64
}); ok {
resp.PhaseDFloor = state.PhaseDFloor()
resp.AllocationFloor = state.AllocationFloor()
}
return resp, nil
}

func (s *DistributionServer) allocateTimestampReservation(
ctx context.Context,
count int,
minTimestamp uint64,
activateCutover bool,
activatePhaseD bool,
) (kv.TSOReservation, error) {
if allocator, ok := s.timestampAllocator.(kv.TSOReservationAllocator); ok {
reservation, err := allocator.ReserveBatchAfter(ctx, count, minTimestamp, activateCutover)
reservation, err := allocator.ReserveBatchAfter(ctx, count, minTimestamp, activateCutover, activatePhaseD)
return reservation, errors.Wrap(err, "reserve dedicated TSO window")
}
reservation := kv.TSOReservation{Count: count}
switch {
case activateCutover:
case activateCutover || activatePhaseD:
return reservation, errors.WithStack(status.Error(codes.FailedPrecondition,
"TSO allocator does not support durable cutover"))
case minTimestamp > 0:
Expand Down Expand Up @@ -251,6 +305,12 @@ func timestampRPCError(err error) error {
return errors.WithStack(status.Error(codes.InvalidArgument, err.Error()))
case errors.Is(err, kv.ErrTSONotLeader), errors.Is(err, kv.ErrLeaderNotFound):
return errors.WithStack(status.Error(codes.FailedPrecondition, err.Error()))
case errors.Is(err, kv.ErrTSOTimestampPrePhaseD):
return errors.WithStack(status.Error(codes.OutOfRange, err.Error()))
case errors.Is(err, kv.ErrTSOTimestampInvalid):
return errors.WithStack(status.Error(codes.InvalidArgument, err.Error()))
case errors.Is(err, kv.ErrTSOPhaseDInactive):
return errors.WithStack(status.Error(codes.FailedPrecondition, err.Error()))
default:
return errors.WithStack(status.Error(codes.Unavailable, err.Error()))
}
Expand Down Expand Up @@ -280,7 +340,16 @@ func (s *DistributionServer) SplitRange(ctx context.Context, req *pb.SplitRangeR
return nil, err
}

snapshot, err := s.loadCatalogSnapshot(ctx)
readTimestamp, err := kv.BeginReadTimestampThrough(
ctx,
s.coordinator,
s.catalog.LatestCommitTS(),
"distribution split range: begin read timestamp",
)
if err != nil {
return nil, grpcStatusErrorf(codes.Internal, "begin catalog split snapshot: %v", err)
}
snapshot, err := s.loadCatalogSnapshotAt(ctx, readTimestamp.Timestamp())
if err != nil {
return nil, err
}
Expand All @@ -290,15 +359,8 @@ func (s *DistributionServer) SplitRange(ctx context.Context, req *pb.SplitRangeR
return nil, err
}

parent, found := findRouteByID(snapshot.Routes, req.GetRouteId())
if !found {
return nil, grpcStatusError(codes.NotFound, errDistributionUnknownRoute.Error())
}

rawSplitKey := req.GetSplitKey()
splitKey := distribution.CloneBytes(fskeys.NormalizeSplitBoundary(kv.RouteKey(rawSplitKey)))
if err := validateSplitKey(parent, splitKey); err != nil {
s.observeFilePinnedHotspotIfNeeded(rawSplitKey, splitKey, err)
parent, splitKey, err := s.prepareSplitRange(snapshot.Routes, req)
if err != nil {
return nil, err
}

Expand Down Expand Up @@ -327,6 +389,20 @@ func (s *DistributionServer) SplitRange(ctx context.Context, req *pb.SplitRangeR
}, nil
}

func (s *DistributionServer) prepareSplitRange(routes []distribution.RouteDescriptor, req *pb.SplitRangeRequest) (distribution.RouteDescriptor, []byte, error) {
parent, found := findRouteByID(routes, req.GetRouteId())
if !found {
return distribution.RouteDescriptor{}, nil, grpcStatusError(codes.NotFound, errDistributionUnknownRoute.Error())
}
rawSplitKey := req.GetSplitKey()
splitKey := distribution.CloneBytes(fskeys.NormalizeSplitBoundary(kv.RouteKey(rawSplitKey)))
if err := validateSplitKey(parent, splitKey); err != nil {
s.observeFilePinnedHotspotIfNeeded(rawSplitKey, splitKey, err)
return distribution.RouteDescriptor{}, nil, err
}
return parent, splitKey, nil
}

func (s *DistributionServer) pinReadTS(ts uint64) *kv.ActiveTimestampToken {
if s == nil || s.readTracker == nil {
return nil
Expand Down Expand Up @@ -479,6 +555,17 @@ func (s *DistributionServer) loadCatalogSnapshot(ctx context.Context) (distribut
return snapshot, nil
}

func (s *DistributionServer) loadCatalogSnapshotAt(ctx context.Context, readTS uint64) (distribution.CatalogSnapshot, error) {
if s.catalog == nil {
return distribution.CatalogSnapshot{}, grpcStatusError(codes.FailedPrecondition, errDistributionCatalogNotConfigured.Error())
}
snapshot, err := s.catalog.SnapshotAt(ctx, readTS)
if err != nil {
return distribution.CatalogSnapshot{}, grpcStatusErrorf(codes.Internal, "load route catalog: %v", err)
}
return snapshot, nil
}

func (s *DistributionServer) loadCatalogSnapshotAtVersion(
ctx context.Context,
readTS uint64,
Expand Down
Loading
Loading