From 8ffb74a0c74767ae716c5e627bad64e9342dcaa0 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 21:39:21 +0900 Subject: [PATCH 1/5] raft: add physical snapshot export substrate --- .../2026_06_12_proposed_scaling_roadmap.md | 4 +- ...oposed_physical_snapshot_object_offload.md | 186 ++++++++++++++ .../etcd/external_snapshot_restore.go | 68 +++++- .../etcd/persisted_snapshot_export.go | 226 ++++++++++++++++++ .../etcd/persisted_snapshot_export_test.go | 145 +++++++++++ 5 files changed, 618 insertions(+), 11 deletions(-) create mode 100644 docs/design/2026_07_19_proposed_physical_snapshot_object_offload.md create mode 100644 internal/raftengine/etcd/persisted_snapshot_export.go create mode 100644 internal/raftengine/etcd/persisted_snapshot_export_test.go diff --git a/docs/design/2026_06_12_proposed_scaling_roadmap.md b/docs/design/2026_06_12_proposed_scaling_roadmap.md index e2567cbd8..edd7c2eef 100644 --- a/docs/design/2026_06_12_proposed_scaling_roadmap.md +++ b/docs/design/2026_06_12_proposed_scaling_roadmap.md @@ -402,8 +402,8 @@ TBD).** rows) so they do not pile up in L0 between compactor runs — reuses the existing TTL helper path. -**M4 — Disaster-recovery snapshot offload (`*_proposed_*` doc -TBD).** +**M4 — Disaster-recovery snapshot offload +([`2026_07_19_proposed_physical_snapshot_object_offload.md`](2026_07_19_proposed_physical_snapshot_object_offload.md)).** - Periodic per-shard Pebble snapshot uploaded to an S3-compatible bucket (the S3 adapter already speaks the protocol). - Restore is `s3 fetch → pebble.Ingest`; combined with M1's diff --git a/docs/design/2026_07_19_proposed_physical_snapshot_object_offload.md b/docs/design/2026_07_19_proposed_physical_snapshot_object_offload.md new file mode 100644 index 000000000..3c0cbd2fd --- /dev/null +++ b/docs/design/2026_07_19_proposed_physical_snapshot_object_offload.md @@ -0,0 +1,186 @@ +# Physical Snapshot Object Offload + +Status: Proposed +Author: bootjp +Date: 2026-07-19 + +## 1. Scope + +This design owns storage-tier milestone M4 from the scaling roadmap: periodic, +per-Raft-group physical snapshots stored in an external S3-compatible object +store, bounded retention, and restore into a fresh elastickv data directory. + +The design depends on the Pebble SST ingest snapshot transfer contract from PR +#1130. It does not copy, parse, or reimplement that format. The object layer +treats the complete FSM payload as opaque bytes. On restore, `kvFSM.Restore` +dispatches the inner store payload to the existing receiver, so `EKVSSTI1` +uses the same manifest validation and `pebble.DB.Ingest` path as routine Raft +snapshot transfer. Legacy payloads remain valid through the same receiver. + +PR #1130 is still in review. The first implementation slice therefore lands +only the independent substrate: + +- open the newest WAL-valid, persisted Raft/FSM snapshot as a single-use + stream paired with its index, term, ConfState, byte count, and CRC32C; +- pin the opened `.fsm` descriptor while it is streamed so local snapshot + retention cannot switch the artifact under an upload; +- prepare a fresh Raft data directory from a complete opaque FSM payload + without adding a second KV snapshot header; +- preserve the existing logical/external restore API and its header behavior. + +No object client or runtime scheduling is enabled by this first slice. + +## 2. Safety boundary + +The exporter consumes only snapshots already committed by the etcd engine: +the WAL-valid `.snap` metadata names an index and term, its `EKVT` token names +the matching `.fsm` file and CRC32C, and the `.fsm` payload is streamed from one +open descriptor. It must not call `StateMachine.Snapshot()` independently from +a timer. Doing so would capture FSM bytes without an atomic Raft +index/term/ConfState witness. + +For token-backed snapshots, export performs three integrity checks: + +1. token index equals `Snapshot.Metadata.Index`; +2. the footer read from the pinned descriptor equals the token CRC32C before + bytes are exposed; +3. CRC32C recomputed while streaming equals the token after the final byte. + +The object publisher additionally computes SHA-256 and exact byte count during +that same stream. A short read, local mutation, remote write error, or checksum +mismatch publishes no manifest. + +## 3. Object layout and commit protocol + +The external bucket is a disaster-recovery dependency and must not be the S3 +adapter of the cluster being backed up. Storing the cluster's own physical +snapshot inside its FSM creates circular recovery and reintroduces payload +growth into Raft. + +```text +/v1/payloads/sha256//.fsm +/v1/groups//snapshots/-.json +``` + +Payload objects are immutable and content-addressed. The per-group JSON +manifest is the commit marker and contains: + +- schema version and creation time; +- source cluster identity and Raft group ID; +- snapshot index, term, and ConfState; +- payload object key, exact length, SHA-256, and source CRC32C; +- binary version and snapshot feature capabilities used by the writer. + +Publication order is payload first, manifest last. A retry may overwrite an +identical payload key, but it must reject a different length or checksum. A +manifest is visible only after the payload upload and remote integrity check +succeed. Temporary multipart uploads are aborted on cancellation and swept by +bucket lifecycle policy. + +## 4. Scheduling + +Each process examines its local Raft groups, but only the current group leader +may publish. The scheduler uploads a persisted snapshot only when its index is +greater than the last successfully published index for that group. Leadership +is checked before opening the snapshot and again before publishing the +manifest; loss of leadership may leave an unreferenced content-addressed +payload but never a committed manifest. + +The schedule is opt-in. Initial defaults are one scan every 15 minutes and one +upload at a time per process. Jitter spreads multi-group work. Snapshot +creation cadence remains owned by the Raft engine; object offload never forces +an extra state-machine snapshot in this milestone. + +## 5. Retention and reference lifecycle + +Retention is per group and combines a minimum generation count with a time +window. GC runs in two phases: + +1. list and validate manifests, retain every manifest inside the policy plus + the newest successful manifest, then delete expired manifest objects; +2. after a grace period, rebuild the live payload SHA set from all remaining + manifests and delete only payload objects with no live reference. + +Malformed manifests fail closed: they are reported and excluded from both +automatic manifest deletion and payload reclamation. Listing failure, +pagination failure, or an incomplete group scan performs no deletes. This +keeps retryable publication or object-store inconsistency from reclaiming a +payload still referenced by a committed manifest. + +## 6. Restore + +Restore is offline and targets an absent data directory: + +1. fetch and validate the selected manifest; +2. download the payload to a same-filesystem temporary regular file while + enforcing exact length and SHA-256; +3. fsync and atomically rename the verified download; +4. call `PreparePhysicalSnapshotRestore` with manifest index/term and + operator-supplied target membership; +5. start elastickv normally. `kvFSM.Restore` consumes the complete FSM header, + and the store receiver performs SST manifest verification and ingest when + the opaque inner payload is `EKVSSTI1`. + +The target membership is explicit operator input rather than copied blindly +from the source ConfState. This supports disaster recovery onto replacement +addresses while keeping source membership in the manifest for audit. + +## 7. Configuration and security + +The runtime slice will require an external endpoint, bucket, prefix, region, +credentials provider, schedule, retention count/window, upload concurrency, +and server-side encryption mode. Static secrets must use file or environment +providers and must not appear in process arguments or manifests. + +Storage-envelope encryption protects values but not all physical keys and +metadata. The external bucket therefore requires private ACLs, TLS, and +server-side encryption (SSE-S3 or SSE-KMS). Anonymous reads and writes are a +deployment failure. Object credentials need only scoped list/get/put/delete +permissions below the configured prefix. + +## 8. Milestones + +| Milestone | Scope | Status | +|---|---|---| +| M0 | Persisted snapshot export handle, complete-payload restore preparation, focused design | Implemented in the first substrate PR | +| M1 | Object client interface, S3-compatible implementation, immutable payload/manifest publication, download verification, operator CLI | Pending; depends on stable #1130 contract | +| M2 | Leader-only per-group scheduler, metrics, jitter, concurrency bounds, cancellation and restart idempotency | Pending | +| M3 | Retention/GC, restore drills, corruption tests, multi-node acceptance, operational documentation | Pending | + +The filename and header remain `proposed` until M1-M3 complete the central +object-offload subsystem. At that point the completion PR must use `git mv` to +rename this file to `2026_07_19_implemented_physical_snapshot_object_offload.md`, +change the header status, update every reference, and verify with `rg` that the +old path is absent before requesting review. + +## 9. Acceptance criteria + +- Opaque `EKVSSTI1` bytes round-trip without object-layer parsing or header + rewriting and restore through the #1130 ingest receiver. +- Snapshot metadata and FSM payload always refer to the same Raft index. +- Corrupt local payload, footer/token mismatch, truncated upload, remote + checksum mismatch, or malformed manifest cannot publish a committed + snapshot. +- A publish interrupted before manifest creation is invisible to restore and + later reclaimed only after the GC grace period. +- Restore never mutates an existing destination and leaves no published target + directory after checksum or preparation failure. +- Retention never deletes the newest valid manifest for a group and never + deletes a payload referenced by any retained manifest. +- Restart and leadership change do not duplicate committed indexes or run + unbounded concurrent uploads. +- A multi-node drill restores each group from external objects into fresh data + directories and passes adapter reads after normal Raft startup. + +## 10. Intentional non-goals + +- Parsing or duplicating the SST ingest stream owned by #1130. +- Uploading user S3 object blobs; that is owned by the S3 Raft blob-offload + design. +- Logical, cross-adapter, vendor-independent backup; that is owned by the + logical backup design. +- A cluster-wide atomic timestamp across groups. This milestone produces + independently restorable per-group physical snapshots. +- Continuous WAL archiving, incremental backup, or cross-region authority and + fencing. + diff --git a/internal/raftengine/etcd/external_snapshot_restore.go b/internal/raftengine/etcd/external_snapshot_restore.go index e314b7092..b45173395 100644 --- a/internal/raftengine/etcd/external_snapshot_restore.go +++ b/internal/raftengine/etcd/external_snapshot_restore.go @@ -45,6 +45,18 @@ type ExternalSnapshotRestoreResult struct { Peers int } +// PhysicalSnapshotRestoreOptions describes a complete FSM payload previously +// emitted by the Raft state machine. Unlike ExternalSnapshotRestoreOptions, the +// input already contains the KV snapshot header and must not be wrapped again. +type PhysicalSnapshotRestoreOptions struct { + InputFSMPath string + DataDir string + Index uint64 + Term uint64 + Peers []Peer + ExpectedPayloadSHA256 string +} + // PrepareExternalSnapshotRestore seeds a fresh etcd-raft data directory from // an externally produced EKVPBBL1 FSM payload. The input file is the raw payload // emitted by elastickv-snapshot-encode; this helper writes the runtime @@ -54,6 +66,34 @@ func PrepareExternalSnapshotRestore(opts ExternalSnapshotRestoreOptions) (*Exter if err := validateExternalSnapshotRestoreOptions(opts); err != nil { return nil, err } + return prepareExternalSnapshotRestore(opts, writeExternalFSMSnapshotFile) +} + +// PreparePhysicalSnapshotRestore seeds a fresh etcd-raft data directory from +// a complete, opaque FSM payload. Store format dispatch remains in +// StateMachine.Restore, so legacy, SST-ingest, and future receiver-compatible +// payloads all use the same path without this package parsing their internals. +func PreparePhysicalSnapshotRestore(opts PhysicalSnapshotRestoreOptions) (*ExternalSnapshotRestoreResult, error) { + externalOpts := ExternalSnapshotRestoreOptions{ + InputFSMPath: opts.InputFSMPath, + DataDir: opts.DataDir, + Index: opts.Index, + Term: opts.Term, + Peers: opts.Peers, + ExpectedPayloadSHA256: opts.ExpectedPayloadSHA256, + } + if err := validateExternalSnapshotRestoreOptions(externalOpts); err != nil { + return nil, err + } + return prepareExternalSnapshotRestore(externalOpts, writePhysicalFSMSnapshotFile) +} + +type externalSnapshotFileWriter func(inputPath, fsmSnapDir string, index uint64, ceilingMs uint64) (uint32, int64, string, error) + +func prepareExternalSnapshotRestore( + opts ExternalSnapshotRestoreOptions, + writeSnapshot externalSnapshotFileWriter, +) (*ExternalSnapshotRestoreResult, error) { destDataDir, tempDir, err := prepareExternalSnapshotRestoreDest(opts.DataDir) if err != nil { return nil, err @@ -66,7 +106,7 @@ func PrepareExternalSnapshotRestore(opts ExternalSnapshotRestoreOptions) (*Exter }() fsmSnapDir := filepath.Join(tempDir, fsmSnapDirName) - crc, payloadBytes, payloadSHA, err := writeExternalFSMSnapshotFile(opts.InputFSMPath, fsmSnapDir, opts.Index, opts.SnapshotCeilingMs) + crc, payloadBytes, payloadSHA, err := writeSnapshot(opts.InputFSMPath, fsmSnapDir, opts.Index, opts.SnapshotCeilingMs) if err != nil { return nil, err } @@ -157,6 +197,15 @@ func ensureExternalRestorePathAbsent(path string, kind string) error { } func writeExternalFSMSnapshotFile(inputPath, fsmSnapDir string, index uint64, ceilingMs uint64) (uint32, int64, string, error) { + header := encodeExternalRestoreSnapshotHeader(ceilingMs) + return writePreparedFSMSnapshotFile(inputPath, fsmSnapDir, index, header[:]) +} + +func writePhysicalFSMSnapshotFile(inputPath, fsmSnapDir string, index uint64, _ uint64) (uint32, int64, string, error) { + return writePreparedFSMSnapshotFile(inputPath, fsmSnapDir, index, nil) +} + +func writePreparedFSMSnapshotFile(inputPath, fsmSnapDir string, index uint64, prefix []byte) (uint32, int64, string, error) { if err := os.MkdirAll(fsmSnapDir, defaultDirPerm); err != nil { return 0, 0, "", errors.WithStack(err) } @@ -180,7 +229,7 @@ func writeExternalFSMSnapshotFile(inputPath, fsmSnapDir string, index uint64, ce _ = os.Remove(tmpPath) }() - crc, bytesWritten, payloadSHA, err := copyExternalPayloadWithHeaderAndFooter(in, tmpFile, ceilingMs) + crc, bytesWritten, payloadSHA, err := copySnapshotPayloadWithPrefixAndFooter(in, tmpFile, prefix) if err != nil { return 0, 0, "", err } @@ -223,16 +272,17 @@ func requireRegularFile(f *os.File, path string) error { return nil } -func copyExternalPayloadWithHeaderAndFooter(in io.Reader, out *os.File, ceilingMs uint64) (uint32, int64, string, error) { +func copySnapshotPayloadWithPrefixAndFooter(in io.Reader, out *os.File, prefix []byte) (uint32, int64, string, error) { bw := bufio.NewWriterSize(out, fsmWriteBufSize) crcHash := crc32.New(crc32cTable) payloadHash := sha256.New() - header := encodeExternalRestoreSnapshotHeader(ceilingMs) - if _, err := bw.Write(header[:]); err != nil { - return 0, 0, "", errors.WithStack(err) - } - if _, err := crcHash.Write(header[:]); err != nil { - return 0, 0, "", errors.WithStack(err) + if len(prefix) != 0 { + if _, err := bw.Write(prefix); err != nil { + return 0, 0, "", errors.WithStack(err) + } + if _, err := crcHash.Write(prefix); err != nil { + return 0, 0, "", errors.WithStack(err) + } } n, err := io.Copy(io.MultiWriter(bw, crcHash, payloadHash), in) if err != nil { diff --git a/internal/raftengine/etcd/persisted_snapshot_export.go b/internal/raftengine/etcd/persisted_snapshot_export.go new file mode 100644 index 000000000..3046aa17f --- /dev/null +++ b/internal/raftengine/etcd/persisted_snapshot_export.go @@ -0,0 +1,226 @@ +package etcd + +import ( + "bytes" + "encoding/binary" + "hash/crc32" + "io" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/cockroachdb/errors" + etcdsnap "go.etcd.io/etcd/server/v3/etcdserver/api/snap" + "go.etcd.io/etcd/server/v3/storage/wal" + raftpb "go.etcd.io/raft/v3/raftpb" + "go.uber.org/zap" + "google.golang.org/protobuf/proto" +) + +var ( + ErrPersistedSnapshotExportInvalid = errors.New("etcd persisted snapshot export: invalid source") + ErrPersistedSnapshotExportUsed = errors.New("etcd persisted snapshot export: stream already consumed") +) + +// PersistedSnapshotExportMetadata identifies the Raft metadata paired with an +// opaque FSM payload. The payload format is deliberately not interpreted here; +// store-level receivers own compatibility and validation. +type PersistedSnapshotExportMetadata struct { + Index uint64 + Term uint64 + ConfState *raftpb.ConfState + PayloadBytes int64 + CRC32C uint32 +} + +// PersistedSnapshotExport is a single-use, read-only handle for the newest +// locally persisted FSM snapshot. WriteTo keeps an open descriptor pinned while +// streaming, so runtime snapshot retention may unlink the path without +// changing the bytes being exported. +type PersistedSnapshotExport struct { + mu sync.Mutex + metadata PersistedSnapshotExportMetadata + reader io.Reader + closer io.Closer + expectedCRC uint32 + checkCRC bool + consumed bool + closed bool +} + +// OpenPersistedSnapshotExport opens the FSM payload referenced by the newest +// WAL-valid Raft snapshot under dataDir. It performs no WAL repair and does not +// mutate the source directory. A false boolean means no persisted snapshot is +// available yet. +func OpenPersistedSnapshotExport(dataDir string) (*PersistedSnapshotExport, bool, error) { + if strings.TrimSpace(dataDir) == "" { + return nil, false, errors.Wrap(ErrPersistedSnapshotExportInvalid, "data dir is required") + } + walDir := filepath.Join(dataDir, walDirName) + if !wal.Exist(walDir) { + return nil, false, nil + } + snapshotter := etcdsnap.New(zap.NewNop(), filepath.Join(dataDir, snapDirName)) + snapshot, err := loadPersistedSnapshot(zap.NewNop(), walDir, snapshotter) + if err != nil { + return nil, false, err + } + if len(snapshot.Data) == 0 { + return nil, false, nil + } + metadata, err := persistedSnapshotExportMetadata(snapshot) + if err != nil { + return nil, false, err + } + if !isSnapshotToken(snapshot.Data) { + payload := bytes.Clone(snapshot.Data) + metadata.PayloadBytes = int64(len(payload)) + return &PersistedSnapshotExport{ + metadata: metadata, + reader: bytes.NewReader(payload), + }, true, nil + } + + tok, err := decodeSnapshotToken(snapshot.Data) + if err != nil { + return nil, false, err + } + if tok.Index != metadata.Index { + return nil, false, errors.Wrapf(ErrFSMSnapshotTokenInvalid, + "token index %d does not match snapshot metadata index %d", tok.Index, metadata.Index) + } + path := fsmSnapPath(filepath.Join(dataDir, fsmSnapDirName), tok.Index) + file, payloadBytes, err := openPersistedFSMSnapshotForExport(path, tok.CRC32C) + if err != nil { + return nil, false, err + } + metadata.PayloadBytes = payloadBytes + metadata.CRC32C = tok.CRC32C + return &PersistedSnapshotExport{ + metadata: metadata, + reader: io.LimitReader(file, payloadBytes), + closer: file, + expectedCRC: tok.CRC32C, + checkCRC: true, + }, true, nil +} + +// Metadata returns an owned copy of the metadata paired with this export. +func (e *PersistedSnapshotExport) Metadata() PersistedSnapshotExportMetadata { + if e == nil { + return PersistedSnapshotExportMetadata{} + } + e.mu.Lock() + defer e.mu.Unlock() + out := e.metadata + if e.metadata.ConfState != nil { + if cloned, ok := proto.Clone(e.metadata.ConfState).(*raftpb.ConfState); ok { + out.ConfState = cloned + } + } + return out +} + +func persistedSnapshotExportMetadata(snapshot raftpb.Snapshot) (PersistedSnapshotExportMetadata, error) { + index := snapshot.GetMetadata().GetIndex() + term := snapshot.GetMetadata().GetTerm() + confState := snapshot.GetMetadata().GetConfState() + if index == 0 || term == 0 || confState == nil { + return PersistedSnapshotExportMetadata{}, errors.Wrapf(ErrPersistedSnapshotExportInvalid, + "snapshot metadata requires non-zero index and term plus ConfState: index=%d term=%d", index, term) + } + cloned, ok := proto.Clone(confState).(*raftpb.ConfState) + if !ok || cloned == nil { + return PersistedSnapshotExportMetadata{}, errors.Wrap(ErrPersistedSnapshotExportInvalid, "clone ConfState") + } + return PersistedSnapshotExportMetadata{ + Index: index, + Term: term, + ConfState: cloned, + }, nil +} + +func openPersistedFSMSnapshotForExport(path string, tokenCRC uint32) (*os.File, int64, error) { + file, err := os.Open(path) + if err != nil { + return nil, 0, statFSMFileError(err) + } + closeOnError := func(err error) (*os.File, int64, error) { + _ = file.Close() + return nil, 0, err + } + info, err := file.Stat() + if err != nil { + return closeOnError(errors.WithStack(err)) + } + if info.Size() < fsmMinFileSize { + return closeOnError(errors.Wrapf(ErrFSMSnapshotTooSmall, + "file too small: %d bytes (minimum %d)", info.Size(), fsmMinFileSize)) + } + payloadBytes := info.Size() - fsmFooterSize + var footer [fsmFooterSize]byte + if _, err := file.ReadAt(footer[:], payloadBytes); err != nil { + return closeOnError(errors.WithStack(err)) + } + footerCRC := binary.BigEndian.Uint32(footer[:]) + if footerCRC != tokenCRC { + return closeOnError(errors.Wrapf(ErrFSMSnapshotTokenCRC, + "path=%s footer=%08x token=%08x", path, footerCRC, tokenCRC)) + } + return file, payloadBytes, nil +} + +// WriteTo streams the complete FSM payload, excluding the local .fsm CRC +// footer. Token-backed snapshots are CRC-checked during the same pass. +func (e *PersistedSnapshotExport) WriteTo(w io.Writer) (int64, error) { + if e == nil || w == nil { + return 0, errors.Wrap(ErrPersistedSnapshotExportInvalid, "export and writer are required") + } + e.mu.Lock() + if e.closed || e.consumed || e.reader == nil { + e.mu.Unlock() + return 0, errors.WithStack(ErrPersistedSnapshotExportUsed) + } + e.consumed = true + reader := e.reader + metadata := e.metadata + checkCRC := e.checkCRC + expectedCRC := e.expectedCRC + e.mu.Unlock() + + if !checkCRC { + n, err := io.Copy(w, reader) + return n, errors.WithStack(err) + } + h := crc32.New(crc32cTable) + n, err := io.Copy(io.MultiWriter(w, h), reader) + if err != nil { + return n, errors.WithStack(err) + } + if n != metadata.PayloadBytes { + return n, errors.Wrapf(io.ErrUnexpectedEOF, "persisted FSM payload: got %d bytes, expected %d", n, metadata.PayloadBytes) + } + if h.Sum32() != expectedCRC { + return n, errors.Wrapf(ErrFSMSnapshotFileCRC, + "exported payload computed=%08x token=%08x", h.Sum32(), expectedCRC) + } + return n, nil +} + +// Close releases the pinned source descriptor. It is safe to call repeatedly. +func (e *PersistedSnapshotExport) Close() error { + if e == nil { + return nil + } + e.mu.Lock() + defer e.mu.Unlock() + if e.closed { + return nil + } + e.closed = true + if e.closer == nil { + return nil + } + return errors.WithStack(e.closer.Close()) +} diff --git a/internal/raftengine/etcd/persisted_snapshot_export_test.go b/internal/raftengine/etcd/persisted_snapshot_export_test.go new file mode 100644 index 000000000..8bc88e221 --- /dev/null +++ b/internal/raftengine/etcd/persisted_snapshot_export_test.go @@ -0,0 +1,145 @@ +package etcd + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPreparePhysicalSnapshotRestoreAndExportOpaquePayload(t *testing.T) { + root := t.TempDir() + payload := append([]byte("EKVTHLC2opaque-header"), []byte("EKVSSTI1opaque-store-payload")...) + input := filepath.Join(root, "physical.fsm") + require.NoError(t, os.WriteFile(input, payload, 0o600)) + payloadSum := sha256.Sum256(payload) + payloadSHA := hex.EncodeToString(payloadSum[:]) + dataDir := filepath.Join(root, "raft") + + result, err := PreparePhysicalSnapshotRestore(PhysicalSnapshotRestoreOptions{ + InputFSMPath: input, + DataDir: dataDir, + Index: 42, + Term: 7, + ExpectedPayloadSHA256: payloadSHA, + Peers: []Peer{ + {NodeID: 1, ID: "n1", Address: "127.0.0.1:12001"}, + {NodeID: 2, ID: "n2", Address: "127.0.0.1:12002"}, + }, + }) + require.NoError(t, err) + require.Equal(t, int64(len(payload)), result.PayloadBytes) + require.Equal(t, payloadSHA, result.PayloadSHA256) + + raw, ok, err := OpenNewestFSMSnapshotPayload(dataDir) + require.NoError(t, err) + require.True(t, ok) + gotRaw, err := os.ReadFile(result.FSMPath) + require.NoError(t, err) + require.Equal(t, payload, gotRaw[:len(gotRaw)-fsmFooterSize]) + streamed, err := readAllAndClose(raw) + require.NoError(t, err) + require.Equal(t, payload, streamed) + + export, ok, err := OpenPersistedSnapshotExport(dataDir) + require.NoError(t, err) + require.True(t, ok) + metadata := export.Metadata() + require.Equal(t, uint64(42), metadata.Index) + require.Equal(t, uint64(7), metadata.Term) + require.Equal(t, []uint64{1, 2}, metadata.ConfState.GetVoters()) + require.Equal(t, int64(len(payload)), metadata.PayloadBytes) + require.Equal(t, result.CRC32C, metadata.CRC32C) + metadata.ConfState.Voters[0] = 99 + require.Equal(t, []uint64{1, 2}, export.Metadata().ConfState.GetVoters()) + + var out bytes.Buffer + n, err := export.WriteTo(&out) + require.NoError(t, err) + require.Equal(t, int64(len(payload)), n) + require.Equal(t, payload, out.Bytes()) + _, err = export.WriteTo(&bytes.Buffer{}) + require.ErrorIs(t, err, ErrPersistedSnapshotExportUsed) + require.NoError(t, export.Close()) + require.NoError(t, export.Close()) +} + +func TestPersistedSnapshotExportDetectsPayloadCorruptionDuringStream(t *testing.T) { + dataDir, result := preparePhysicalSnapshotExportFixture(t, []byte("EKVTHLC1payload-that-will-be-corrupted")) + file, err := os.OpenFile(result.FSMPath, os.O_RDWR, 0) + require.NoError(t, err) + var first [1]byte + _, err = file.ReadAt(first[:], 0) + require.NoError(t, err) + first[0] ^= 0xff + _, err = file.WriteAt(first[:], 0) + require.NoError(t, err) + require.NoError(t, file.Close()) + + export, ok, err := OpenPersistedSnapshotExport(dataDir) + require.NoError(t, err) + require.True(t, ok) + defer export.Close() + _, err = export.WriteTo(&bytes.Buffer{}) + require.ErrorIs(t, err, ErrFSMSnapshotFileCRC) +} + +func TestPersistedSnapshotExportRejectsTokenFooterMismatchBeforeStream(t *testing.T) { + dataDir, result := preparePhysicalSnapshotExportFixture(t, []byte("EKVTHLC1payload-with-footer-mismatch")) + file, err := os.OpenFile(result.FSMPath, os.O_RDWR, 0) + require.NoError(t, err) + info, err := file.Stat() + require.NoError(t, err) + var last [1]byte + _, err = file.ReadAt(last[:], info.Size()-1) + require.NoError(t, err) + last[0] ^= 0xff + _, err = file.WriteAt(last[:], info.Size()-1) + require.NoError(t, err) + require.NoError(t, file.Close()) + + _, ok, err := OpenPersistedSnapshotExport(dataDir) + require.False(t, ok) + require.ErrorIs(t, err, ErrFSMSnapshotTokenCRC) +} + +func TestOpenPersistedSnapshotExportValidation(t *testing.T) { + _, ok, err := OpenPersistedSnapshotExport("") + require.False(t, ok) + require.ErrorIs(t, err, ErrPersistedSnapshotExportInvalid) + + _, ok, err = OpenPersistedSnapshotExport(t.TempDir()) + require.NoError(t, err) + require.False(t, ok) +} + +func preparePhysicalSnapshotExportFixture(t *testing.T, payload []byte) (string, *ExternalSnapshotRestoreResult) { + t.Helper() + root := t.TempDir() + input := filepath.Join(root, "physical.fsm") + require.NoError(t, os.WriteFile(input, payload, 0o600)) + dataDir := filepath.Join(root, "raft") + result, err := PreparePhysicalSnapshotRestore(PhysicalSnapshotRestoreOptions{ + InputFSMPath: input, + DataDir: dataDir, + Index: 42, + Term: 7, + Peers: []Peer{{NodeID: 1, ID: "n1", Address: "127.0.0.1:12001"}}, + }) + require.NoError(t, err) + return dataDir, result +} + +func readAllAndClose(r interface { + Read([]byte) (int, error) + Close() error +}) ([]byte, error) { + defer r.Close() + var out bytes.Buffer + _, err := out.ReadFrom(r) + return out.Bytes(), err +} From caf1148cf4f7987795535b5cbb0e29868730b113 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 21:43:40 +0900 Subject: [PATCH 2/5] raft: preserve snapshot metadata ownership --- internal/raftengine/etcd/persisted_snapshot_export.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/raftengine/etcd/persisted_snapshot_export.go b/internal/raftengine/etcd/persisted_snapshot_export.go index 3046aa17f..ada41cb82 100644 --- a/internal/raftengine/etcd/persisted_snapshot_export.go +++ b/internal/raftengine/etcd/persisted_snapshot_export.go @@ -117,6 +117,8 @@ func (e *PersistedSnapshotExport) Metadata() PersistedSnapshotExportMetadata { if e.metadata.ConfState != nil { if cloned, ok := proto.Clone(e.metadata.ConfState).(*raftpb.ConfState); ok { out.ConfState = cloned + } else { + out.ConfState = nil } } return out From 9012ed1ed45e9693b32f99724cb58a21eb1d7df9 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 22:13:35 +0900 Subject: [PATCH 3/5] raft: normalize physical restore membership --- ...oposed_physical_snapshot_object_offload.md | 1 - .../etcd/external_snapshot_restore.go | 60 ++++++++++++++++--- .../etcd/external_snapshot_restore_test.go | 14 +++++ .../etcd/persisted_snapshot_export_test.go | 42 +++++++++++++ 4 files changed, 107 insertions(+), 10 deletions(-) diff --git a/docs/design/2026_07_19_proposed_physical_snapshot_object_offload.md b/docs/design/2026_07_19_proposed_physical_snapshot_object_offload.md index 3c0cbd2fd..e9b469692 100644 --- a/docs/design/2026_07_19_proposed_physical_snapshot_object_offload.md +++ b/docs/design/2026_07_19_proposed_physical_snapshot_object_offload.md @@ -183,4 +183,3 @@ old path is absent before requesting review. independently restorable per-group physical snapshots. - Continuous WAL archiving, incremental backup, or cross-region authority and fencing. - diff --git a/internal/raftengine/etcd/external_snapshot_restore.go b/internal/raftengine/etcd/external_snapshot_restore.go index b45173395..51932e773 100644 --- a/internal/raftengine/etcd/external_snapshot_restore.go +++ b/internal/raftengine/etcd/external_snapshot_restore.go @@ -63,10 +63,11 @@ type PhysicalSnapshotRestoreOptions struct { // fsm-snap/.fsm form by appending the CRC32C footer and persists the // matching EKVT token snapshot under snap/. func PrepareExternalSnapshotRestore(opts ExternalSnapshotRestoreOptions) (*ExternalSnapshotRestoreResult, error) { - if err := validateExternalSnapshotRestoreOptions(opts); err != nil { + normalized, err := normalizeExternalSnapshotRestoreOptions(opts) + if err != nil { return nil, err } - return prepareExternalSnapshotRestore(opts, writeExternalFSMSnapshotFile) + return prepareExternalSnapshotRestore(normalized, writeExternalFSMSnapshotFile) } // PreparePhysicalSnapshotRestore seeds a fresh etcd-raft data directory from @@ -82,10 +83,11 @@ func PreparePhysicalSnapshotRestore(opts PhysicalSnapshotRestoreOptions) (*Exter Peers: opts.Peers, ExpectedPayloadSHA256: opts.ExpectedPayloadSHA256, } - if err := validateExternalSnapshotRestoreOptions(externalOpts); err != nil { + normalized, err := normalizeExternalSnapshotRestoreOptions(externalOpts) + if err != nil { return nil, err } - return prepareExternalSnapshotRestore(externalOpts, writePhysicalFSMSnapshotFile) + return prepareExternalSnapshotRestore(normalized, writePhysicalFSMSnapshotFile) } type externalSnapshotFileWriter func(inputPath, fsmSnapDir string, index uint64, ceilingMs uint64) (uint32, int64, string, error) @@ -150,21 +152,48 @@ func validateExternalSnapshotRestoreOptions(opts ExternalSnapshotRestoreOptions) case len(opts.Peers) == 0: return errors.Wrap(ErrExternalSnapshotRestoreInvalid, "at least one peer is required") } - for i, peer := range opts.Peers { + return validateExternalSnapshotRestorePeers(opts.Peers) +} + +func validateExternalSnapshotRestorePeers(peers []Peer) error { + seenNodeIDs := make(map[uint64]struct{}, len(peers)) + voters := 0 + for i, peer := range peers { if peer.NodeID == 0 { return errors.Wrapf(ErrExternalSnapshotRestoreInvalid, "peer[%d] has zero node id", i) } - } - seenNodeIDs := make(map[uint64]struct{}, len(opts.Peers)) - for i, peer := range opts.Peers { + if strings.TrimSpace(peer.Address) == "" { + return errors.Wrapf(ErrExternalSnapshotRestoreInvalid, "peer[%d] has empty address", i) + } + if peer.Suffrage != "" && peer.Suffrage != SuffrageVoter && peer.Suffrage != SuffrageLearner { + return errors.Wrapf(ErrExternalSnapshotRestoreInvalid, "peer[%d] has invalid suffrage %q", i, peer.Suffrage) + } if _, ok := seenNodeIDs[peer.NodeID]; ok { return errors.Wrapf(ErrExternalSnapshotRestoreInvalid, "peer[%d] has duplicate node id %d", i, peer.NodeID) } seenNodeIDs[peer.NodeID] = struct{}{} + if peer.Suffrage != SuffrageLearner { + voters++ + } + } + if voters == 0 { + return errors.Wrap(ErrExternalSnapshotRestoreInvalid, "at least one voter is required") } return nil } +func normalizeExternalSnapshotRestoreOptions(opts ExternalSnapshotRestoreOptions) (ExternalSnapshotRestoreOptions, error) { + if err := validateExternalSnapshotRestoreOptions(opts); err != nil { + return ExternalSnapshotRestoreOptions{}, err + } + peers, err := normalizePersistedPeers(opts.Peers) + if err != nil { + return ExternalSnapshotRestoreOptions{}, errors.Wrap(ErrExternalSnapshotRestoreInvalid, err.Error()) + } + opts.Peers = peers + return opts, nil +} + func prepareExternalSnapshotRestoreDest(destDataDir string) (string, string, error) { destDataDir = filepath.Clean(destDataDir) if err := ensureExternalRestorePathAbsent(destDataDir, "destination"); err != nil { @@ -316,7 +345,7 @@ const ( ) func seedExternalSnapshotRestoreDir(tempDir string, opts ExternalSnapshotRestoreOptions, token []byte) error { - confState := confStateForPeers(opts.Peers) + confState := confStateForSnapshotRestorePeers(opts.Peers) state := persistedState{ HardState: raftpb.HardState{ Term: proto.Uint64(opts.Term), @@ -349,6 +378,19 @@ func seedExternalSnapshotRestoreDir(tempDir string, opts ExternalSnapshotRestore return savePersistedPeers(tempDir, opts.Index, opts.Peers) } +func confStateForSnapshotRestorePeers(peers []Peer) raftpb.ConfState { + voters := make([]uint64, 0, len(peers)) + learners := make([]uint64, 0) + for _, peer := range peers { + if peer.Suffrage == SuffrageLearner { + learners = append(learners, peer.NodeID) + continue + } + voters = append(voters, peer.NodeID) + } + return raftpb.ConfState{Voters: voters, Learners: learners} +} + func snapPath(snapDir string, term, index uint64) string { return filepath.Join(snapDir, formatSnapName(term, index)) } diff --git a/internal/raftengine/etcd/external_snapshot_restore_test.go b/internal/raftengine/etcd/external_snapshot_restore_test.go index d92c668f0..28b7b0cbf 100644 --- a/internal/raftengine/etcd/external_snapshot_restore_test.go +++ b/internal/raftengine/etcd/external_snapshot_restore_test.go @@ -110,6 +110,20 @@ func TestPrepareExternalSnapshotRestoreFailures(t *testing.T) { }, wantErr: ErrExternalSnapshotRestoreInvalid, }, + { + name: "invalid peer suffrage", + mutate: func(_ *testing.T, _ string, opts *ExternalSnapshotRestoreOptions) { + opts.Peers[0].Suffrage = "observer" + }, + wantErr: ErrExternalSnapshotRestoreInvalid, + }, + { + name: "learner only membership", + mutate: func(_ *testing.T, _ string, opts *ExternalSnapshotRestoreOptions) { + opts.Peers[0].Suffrage = SuffrageLearner + }, + wantErr: ErrExternalSnapshotRestoreInvalid, + }, { name: "copied payload sha mismatch", mutate: func(_ *testing.T, _ string, opts *ExternalSnapshotRestoreOptions) { diff --git a/internal/raftengine/etcd/persisted_snapshot_export_test.go b/internal/raftengine/etcd/persisted_snapshot_export_test.go index 8bc88e221..00ca41504 100644 --- a/internal/raftengine/etcd/persisted_snapshot_export_test.go +++ b/internal/raftengine/etcd/persisted_snapshot_export_test.go @@ -9,6 +9,10 @@ import ( "testing" "github.com/stretchr/testify/require" + etcdsnap "go.etcd.io/etcd/server/v3/etcdserver/api/snap" + "go.etcd.io/etcd/server/v3/storage/wal/walpb" + "go.uber.org/zap" + "google.golang.org/protobuf/proto" ) func TestPreparePhysicalSnapshotRestoreAndExportOpaquePayload(t *testing.T) { @@ -68,6 +72,44 @@ func TestPreparePhysicalSnapshotRestoreAndExportOpaquePayload(t *testing.T) { require.NoError(t, export.Close()) } +func TestPreparePhysicalSnapshotRestoreNormalizesLearnerMembership(t *testing.T) { + root := t.TempDir() + input := filepath.Join(root, "physical.fsm") + require.NoError(t, os.WriteFile(input, []byte("EKVTHLC1opaque"), 0o600)) + dataDir := filepath.Join(root, "raft") + inputPeers := []Peer{ + {NodeID: 3, ID: "n3", Address: "127.0.0.1:12003"}, + {NodeID: 2, ID: "n2", Address: "127.0.0.1:12002", Suffrage: SuffrageLearner}, + {NodeID: 1, ID: "n1", Address: "127.0.0.1:12001"}, + } + + _, err := PreparePhysicalSnapshotRestore(PhysicalSnapshotRestoreOptions{ + InputFSMPath: input, + DataDir: dataDir, + Index: 42, + Term: 7, + Peers: inputPeers, + }) + require.NoError(t, err) + + snapshot, err := etcdsnap.New(zap.NewNop(), filepath.Join(dataDir, snapDirName)). + LoadNewestAvailable([]*walpb.Snapshot{{Index: proto.Uint64(42), Term: proto.Uint64(7)}}) + require.NoError(t, err) + require.Equal(t, []uint64{1, 3}, snapshot.GetMetadata().GetConfState().GetVoters()) + require.Equal(t, []uint64{2}, snapshot.GetMetadata().GetConfState().GetLearners()) + + persisted, ok, err := loadPersistedPeersState(dataDir) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, []Peer{ + {NodeID: 1, ID: "n1", Address: "127.0.0.1:12001", Suffrage: SuffrageVoter}, + {NodeID: 2, ID: "n2", Address: "127.0.0.1:12002", Suffrage: SuffrageLearner}, + {NodeID: 3, ID: "n3", Address: "127.0.0.1:12003", Suffrage: SuffrageVoter}, + }, persisted.Peers) + require.NoError(t, validateOpenPeers(*snapshot, persisted.Peers, persisted, true)) + require.Equal(t, uint64(3), inputPeers[0].NodeID, "normalization must not mutate caller input") +} + func TestPersistedSnapshotExportDetectsPayloadCorruptionDuringStream(t *testing.T) { dataDir, result := preparePhysicalSnapshotExportFixture(t, []byte("EKVTHLC1payload-that-will-be-corrupted")) file, err := os.OpenFile(result.FSMPath, os.O_RDWR, 0) From e08e3b7dafdcfdde6ce8adae27e40d89b4b5aab7 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 22:35:13 +0900 Subject: [PATCH 4/5] raft: reuse restore suffrage partition --- ...19_proposed_physical_snapshot_object_offload.md | 4 ++-- .../raftengine/etcd/external_snapshot_restore.go | 14 ++++++-------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/docs/design/2026_07_19_proposed_physical_snapshot_object_offload.md b/docs/design/2026_07_19_proposed_physical_snapshot_object_offload.md index e9b469692..7980e47a2 100644 --- a/docs/design/2026_07_19_proposed_physical_snapshot_object_offload.md +++ b/docs/design/2026_07_19_proposed_physical_snapshot_object_offload.md @@ -10,8 +10,8 @@ This design owns storage-tier milestone M4 from the scaling roadmap: periodic, per-Raft-group physical snapshots stored in an external S3-compatible object store, bounded retention, and restore into a fresh elastickv data directory. -The design depends on the Pebble SST ingest snapshot transfer contract from PR -#1130. It does not copy, parse, or reimplement that format. The object layer +The design depends on the Pebble SST ingest snapshot transfer contract from +PR `#1130`. It does not copy, parse, or reimplement that format. The object layer treats the complete FSM payload as opaque bytes. On restore, `kvFSM.Restore` dispatches the inner store payload to the existing receiver, so `EKVSSTI1` uses the same manifest validation and `pebble.DB.Ingest` path as routine Raft diff --git a/internal/raftengine/etcd/external_snapshot_restore.go b/internal/raftengine/etcd/external_snapshot_restore.go index 51932e773..eeb76e04c 100644 --- a/internal/raftengine/etcd/external_snapshot_restore.go +++ b/internal/raftengine/etcd/external_snapshot_restore.go @@ -10,6 +10,7 @@ import ( "io" "os" "path/filepath" + "slices" "strings" "github.com/cockroachdb/errors" @@ -379,15 +380,12 @@ func seedExternalSnapshotRestoreDir(tempDir string, opts ExternalSnapshotRestore } func confStateForSnapshotRestorePeers(peers []Peer) raftpb.ConfState { - voters := make([]uint64, 0, len(peers)) - learners := make([]uint64, 0) - for _, peer := range peers { - if peer.Suffrage == SuffrageLearner { - learners = append(learners, peer.NodeID) - continue - } - voters = append(voters, peer.NodeID) + voters, learnerSet := splitPeersBySuffrage(peers) + learners := make([]uint64, 0, len(learnerSet)) + for nodeID := range learnerSet { + learners = append(learners, nodeID) } + slices.Sort(learners) return raftpb.ConfState{Voters: voters, Learners: learners} } From 95dab952494c1c3010a01f03ee713dc727e4ff0c Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 23:01:42 +0900 Subject: [PATCH 5/5] raft: reject duplicate restore peer identities --- .../etcd/external_snapshot_restore.go | 27 ++++++++++++++++--- .../etcd/external_snapshot_restore_test.go | 20 ++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/internal/raftengine/etcd/external_snapshot_restore.go b/internal/raftengine/etcd/external_snapshot_restore.go index eeb76e04c..71364c401 100644 --- a/internal/raftengine/etcd/external_snapshot_restore.go +++ b/internal/raftengine/etcd/external_snapshot_restore.go @@ -158,6 +158,7 @@ func validateExternalSnapshotRestoreOptions(opts ExternalSnapshotRestoreOptions) func validateExternalSnapshotRestorePeers(peers []Peer) error { seenNodeIDs := make(map[uint64]struct{}, len(peers)) + seenIDs := make(map[string]struct{}, len(peers)) voters := 0 for i, peer := range peers { if peer.NodeID == 0 { @@ -169,10 +170,9 @@ func validateExternalSnapshotRestorePeers(peers []Peer) error { if peer.Suffrage != "" && peer.Suffrage != SuffrageVoter && peer.Suffrage != SuffrageLearner { return errors.Wrapf(ErrExternalSnapshotRestoreInvalid, "peer[%d] has invalid suffrage %q", i, peer.Suffrage) } - if _, ok := seenNodeIDs[peer.NodeID]; ok { - return errors.Wrapf(ErrExternalSnapshotRestoreInvalid, "peer[%d] has duplicate node id %d", i, peer.NodeID) + if err := validateExternalSnapshotRestorePeerIdentity(i, peer, seenNodeIDs, seenIDs); err != nil { + return err } - seenNodeIDs[peer.NodeID] = struct{}{} if peer.Suffrage != SuffrageLearner { voters++ } @@ -183,6 +183,27 @@ func validateExternalSnapshotRestorePeers(peers []Peer) error { return nil } +func validateExternalSnapshotRestorePeerIdentity( + index int, + peer Peer, + seenNodeIDs map[uint64]struct{}, + seenIDs map[string]struct{}, +) error { + if _, ok := seenNodeIDs[peer.NodeID]; ok { + return errors.Wrapf(ErrExternalSnapshotRestoreInvalid, "peer[%d] has duplicate node id %d", index, peer.NodeID) + } + normalizedPeer, err := normalizePersistedPeer(peer) + if err != nil { + return errors.Wrap(ErrExternalSnapshotRestoreInvalid, err.Error()) + } + if _, ok := seenIDs[normalizedPeer.ID]; ok { + return errors.Wrapf(ErrExternalSnapshotRestoreInvalid, "peer[%d] has duplicate peer id %q", index, normalizedPeer.ID) + } + seenNodeIDs[peer.NodeID] = struct{}{} + seenIDs[normalizedPeer.ID] = struct{}{} + return nil +} + func normalizeExternalSnapshotRestoreOptions(opts ExternalSnapshotRestoreOptions) (ExternalSnapshotRestoreOptions, error) { if err := validateExternalSnapshotRestoreOptions(opts); err != nil { return ExternalSnapshotRestoreOptions{}, err diff --git a/internal/raftengine/etcd/external_snapshot_restore_test.go b/internal/raftengine/etcd/external_snapshot_restore_test.go index 28b7b0cbf..553fb392f 100644 --- a/internal/raftengine/etcd/external_snapshot_restore_test.go +++ b/internal/raftengine/etcd/external_snapshot_restore_test.go @@ -110,6 +110,26 @@ func TestPrepareExternalSnapshotRestoreFailures(t *testing.T) { }, wantErr: ErrExternalSnapshotRestoreInvalid, }, + { + name: "duplicate explicit peer id", + mutate: func(_ *testing.T, _ string, opts *ExternalSnapshotRestoreOptions) { + opts.Peers = []Peer{ + {NodeID: 1, ID: "n1", Address: "127.0.0.1:12001"}, + {NodeID: 2, ID: "n1", Address: "127.0.0.1:12002"}, + } + }, + wantErr: ErrExternalSnapshotRestoreInvalid, + }, + { + name: "duplicate normalized peer id", + mutate: func(_ *testing.T, _ string, opts *ExternalSnapshotRestoreOptions) { + opts.Peers = []Peer{ + {NodeID: 1, Address: "127.0.0.1:12001"}, + {NodeID: 2, Address: "127.0.0.1:12001"}, + } + }, + wantErr: ErrExternalSnapshotRestoreInvalid, + }, { name: "invalid peer suffrage", mutate: func(_ *testing.T, _ string, opts *ExternalSnapshotRestoreOptions) {