From a23368c469da857e8cc0a768b8b125d0214c32e0 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sat, 18 Jul 2026 23:59:54 +0900 Subject: [PATCH 1/6] Add fenced Raft member replacement --- cmd/raftadmin/main.go | 2 + cmd/raftadmin/main_test.go | 10 +- ...lemented_fenced_raft_member_replacement.md | 234 ++++ docs/raft_learner_operations.md | 21 + internal/raftadmin/server.go | 24 +- internal/raftadmin/server_test.go | 20 +- internal/raftengine/engine.go | 4 + internal/raftengine/etcd/engine.go | 25 +- internal/raftengine/etcd/engine_test.go | 4 + proto/service.pb.go | 50 +- proto/service.proto | 2 + raft_member_replace_script_test.go | 335 ++++++ scripts/raft-member-replace.sh | 1009 +++++++++++++++++ scripts/rolling-update.env.example | 15 + 14 files changed, 1705 insertions(+), 50 deletions(-) create mode 100644 docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md create mode 100644 raft_member_replace_script_test.go create mode 100755 scripts/raft-member-replace.sh diff --git a/cmd/raftadmin/main.go b/cmd/raftadmin/main.go index 4caac23f2..fc9f54f18 100644 --- a/cmd/raftadmin/main.go +++ b/cmd/raftadmin/main.go @@ -256,6 +256,8 @@ func printStatus(ctx context.Context, client pb.RaftAdminClient) error { fmt.Printf("fsm_pending: %d\n", resp.FsmPending) fmt.Printf("num_peers: %d\n", resp.NumPeers) fmt.Printf("last_contact_nanos: %d\n", resp.LastContactNanos) + fmt.Printf("configuration_index: %d\n", resp.ConfigurationIndex) + fmt.Printf("pending_conf_change: %t\n", resp.PendingConfChange) return nil } diff --git a/cmd/raftadmin/main_test.go b/cmd/raftadmin/main_test.go index 7d1f9e67c..8247c849e 100644 --- a/cmd/raftadmin/main_test.go +++ b/cmd/raftadmin/main_test.go @@ -54,9 +54,11 @@ func (fakeRaftAdminClient) TransferLeadership(context.Context, *pb.RaftAdminTran func TestExecuteCommandLeaderAndStateOutput(t *testing.T) { client := fakeRaftAdminClient{ statusResp: &pb.RaftAdminStatusResponse{ - State: pb.RaftAdminState_RAFT_ADMIN_STATE_LEADER, - LeaderId: "n1", - LeaderAddress: "127.0.0.1:50051", + State: pb.RaftAdminState_RAFT_ADMIN_STATE_LEADER, + LeaderId: "n1", + LeaderAddress: "127.0.0.1:50051", + ConfigurationIndex: 7, + PendingConfChange: true, }, } @@ -78,6 +80,8 @@ func TestExecuteCommandLeaderAndStateOutput(t *testing.T) { require.Contains(t, out, "leader_id: \"n1\"") require.Contains(t, out, "leader_address: \"127.0.0.1:50051\"") require.Contains(t, out, "commit_index: 0") + require.Contains(t, out, "configuration_index: 7") + require.Contains(t, out, "pending_conf_change: true") } func TestRPCTimeoutHonorsEnvSeconds(t *testing.T) { diff --git a/docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md b/docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md new file mode 100644 index 000000000..e49ec4f30 --- /dev/null +++ b/docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md @@ -0,0 +1,234 @@ +# Fenced Raft voter replacement operation + +## 1. Status + +Implemented by `scripts/raft-member-replace.sh` as the operational companion to +the fresh learner join mode. The command automates one same-ID voter +replacement in one Raft group while a surviving quorum remains available. + +This is not an automatic repair loop. A candidate state, failed health check, +or missing process never authorizes data deletion or membership mutation. + +## 2. Motivation + +Removing a damaged data directory and running the normal deployment command is +not a valid Raft repair. Membership survives on the other voters in committed +Raft configuration, snapshots, WAL, and peer metadata. The wiped process has +none of that authority and may campaign with an incompatible local view. + +The safe operation already had to be performed manually: + +1. fence the old ID owner; +2. commit `RemoveServer`; +3. archive the target data; +4. start a fresh process in discovery-only join mode; +5. commit `AddLearner`; +6. wait for catch-up; +7. commit `PromoteLearner`; and +8. restart without temporary join flags. + +Repeated manual execution makes ordering, stale configuration indexes, and +partial failure the dominant risks. The implementation makes those boundaries +explicit and resumable. + +## 3. Scope and non-goals + +The v1 command supports: + +- one current voter selected by exact Raft ID; +- one Raft group exposed at the configured RaftAdmin endpoint; +- same-ID replacement at the same advertised address; +- external fencing or an explicitly selected container fence; +- archive-by-default data reset; +- the existing `rolling-update.sh` fresh learner mode; and +- durable local resume state on the control host. + +The command deliberately does not support: + +- no-quorum recovery or forced configuration rewrite; +- automatic execution based on CANDIDATE state; +- simultaneous replacement of multiple members; +- changing a member ID or advertised address; +- multi-group process replacement; or +- proving that a stopped VM can never be powered on again. + +No-quorum recovery needs a separate disaster-recovery design with an explicit +choice of authoritative log and accepted data-loss boundary. + +## 4. Safety invariants + +### 4.1 Explicit authorization + +`--execute` is required and `REPLACEMENT_CONFIRM` must exactly equal +`TARGET_NODE`. `--dry-run` performs no build, RPC, SSH, fence, or deployment. + +The target must be present in `NODES`, be a current voter, and have exactly the +advertised `NODES` address. A learner, absent ID, or address mismatch aborts. + +### 4.2 Surviving quorum before fencing + +The command reads the leader's authoritative configuration and computes +`floor(voters/2)+1`. At least that many non-target voters must answer RaftAdmin +status before fencing starts. This is intentionally stricter than counting the +target: removal must still commit after the target is stopped. + +The leader must be fully applied, expose a non-zero configuration index, and +have no pending configuration change. If the target is leader, leadership is +transferred only to a non-target follower whose applied index has reached the +leader's observed commit index. + +### 4.3 One live owner of a Raft ID + +The old process is fenced before `RemoveServer`. External mode requires an +operator-supplied command that fences the VM/process from the cluster network. +Container mode is opt-in and is valid only when host lifecycle policy prevents +the stopped container from independently returning. + +After the fence command, the target RaftAdmin endpoint must remain unreachable +for the configured consecutive verification interval. It is checked again +before removal and before the fresh deployment. Membership is never removed +while the old endpoint still answers. + +### 4.4 Serialized membership changes + +RaftAdmin status now reports: + +- `configuration_index`, the current durable membership index; and +- `pending_conf_change`, whether an unapplied change is in flight. + +Every `remove_server`, `add_learner`, and `promote_learner` call passes the +current non-zero configuration index as `previous_index`. The engine rejects a +concurrent topology change instead of applying a stale plan. The command waits +for the requested suffrage and a clear pending-change flag before advancing. + +### 4.5 Catch-up before promotion + +Immediately before `AddLearner`, the command records the leader's commit index +as a durable `min_catch_up_index`. It writes that value to the resume file +before proposing the change so a control-host crash after commit cannot lose +the promotion bound. + +The learner must report `FOLLOWER`, a known leader, and +`applied_index >= min_catch_up_index`. Promotion passes the same non-zero value +to `promote_learner` and explicitly leaves `skip_min_applied_check=false`. + +### 4.6 Durable membership restart and application verification + +After promotion, the target is redeployed without `RAFT_JOIN_NODE`. This proves +it can start from its WAL, snapshot, and persisted peer metadata rather than +temporary discovery intent. + +Completion requires all original voter IDs to be present as voters, every +voter to answer as LEADER or FOLLOWER with no local apply lag, and one leader to +remain stable for the configured interval. The operator must also provide +`REPLACEMENT_VERIFY_COMMAND`; a successful application-level write/read check +is mandatory. + +## 5. Resume state machine + +The local state file is mode `0600` and is updated by write-then-rename. An +atomic directory lock prevents two control processes from operating on the +same state file. + +| Stage | Durable fact | +| --- | --- | +| 0 | Target identity, address, original voter set, operation ID, and archive path recorded | +| 1 | Preflight and any leadership transfer completed | +| 2 | Old ID owner fenced and endpoint observed down | +| 3 | Target absent from committed membership | +| 4 | Only the target data directory archived or deleted | +| 5 | Target deployed with fresh learner join intent | +| 6 | Target committed as learner; catch-up floor persisted | +| 7 | Target committed as voter | +| 8 | Target redeployed without join intent | +| 9 | Stable voter set and application write/read verification completed | + +Operations around a stage boundary are idempotent. The chosen data mode is +stored with the operation and cannot change on resume. For example, a resumed +stage 2 accepts that removal already committed, stage 5 accepts an already +committed learner only when its catch-up floor exists, and stage 6 accepts an +already promoted voter. A conflicting suffrage or voter set aborts instead of +guessing what happened. + +The state file is retained by default as operational evidence. Set +`REPLACEMENT_KEEP_STATE=false` only when another durable audit system records +the completed operation. + +## 6. Data handling + +Archive mode is the default. The deterministic archive path includes the +target ID and operation ID, making a retry distinguishable from a new +replacement. If both the live data path and archive path exist, the command +fails rather than overwriting either. + +Delete mode must be explicitly selected. Both modes execute only after the ID +is absent from committed membership and verify the target container is not +running. The command never iterates over multiple data hosts. + +## 7. Operator interface + +Validate a plan without network or filesystem changes: + +```sh +TARGET_NODE=n2 \ +ROLLING_UPDATE_ENV_FILE=~/dump/elastickv-deploy/deploy.env \ +./scripts/raft-member-replace.sh --dry-run +``` + +Execute with an external hypervisor or network fence: + +```sh +TARGET_NODE=n2 \ +REPLACEMENT_CONFIRM=n2 \ +ROLLING_UPDATE_ENV_FILE=~/dump/elastickv-deploy/deploy.env \ +REPLACEMENT_FENCE_COMMAND='fence-n2-and-block-raft' \ +REPLACEMENT_VERIFY_COMMAND='redis-cli -h 192.168.0.64 SET replacement-check ok && redis-cli -h 192.168.0.64 GET replacement-check | grep -Fx ok' \ +./scripts/raft-member-replace.sh --execute +``` + +Use container mode only when the host cannot revive the old container during +the operation: + +```sh +REPLACEMENT_FENCE_MODE=container +``` + +The replacement command sources the deployment env for inventory and then +invokes `rolling-update.sh` twice with only `TARGET_NODE` selected. Bootstrap +membership is removed from both invocations. The first deploy includes +`RAFT_JOIN_NODE`; the post-promotion deploy clears it. + +## 8. Failure handling + +| Failure | Required response | +| --- | --- | +| No leader or insufficient surviving quorum | Stop. Restore an existing voter or use a separately reviewed disaster-recovery procedure. | +| Target endpoint remains reachable after fence | Strengthen the fence. Do not remove membership. | +| `previous_index` mismatch | Inspect the concurrent topology change and reconcile the state file with the authoritative configuration. Do not retry blindly. | +| Data and deterministic archive both exist | Inspect the target host. Do not delete either path automatically. | +| Learner catch-up timeout | Leave it a learner, diagnose transport/disk/snapshot transfer, and resume with the same state file. | +| Promotion committed but normal restart fails | Keep the surviving voters running. Repair the target from its existing post-join data; do not wipe it again. | +| Final voter set differs from the recorded set | Stop and audit membership. The command does not normalize unexpected changes. | +| Application verification fails | Replacement remains incomplete even if Raft health checks pass. Diagnose the public protocol path and resume verification. | + +## 9. Tests and acceptance + +`raft_member_replace_script_test.go` runs the complete shell workflow against a +stateful fake RaftAdmin, SSH transport, and rolling deploy command. It verifies +the exact voter-to-absent-to-learner-to-voter sequence, non-zero catch-up floor, +both deploy modes, application verification, durable stage 9, and an idempotent +completed resume. + +A negative test makes one surviving voter unreachable in a three-voter group +and proves the command aborts before the fence action. Another test loses the +`AddLearner` RPC response after commit and proves the persisted catch-up floor +allows a safe resume without a duplicate add. Lock ownership is tested so a +losing control process cannot remove the winner's directory lock. RaftAdmin +server and CLI tests cover propagation and rendering of the configuration +index and pending change fields. Existing engine membership tests cover stale +previous indexes, learner catch-up enforcement, same-ID replacement, and +persisted membership restart. + +Production acceptance still requires recording the state file, fence evidence, +final `raftadmin status`/`configuration`, and the output of the configured +application verification command. diff --git a/docs/raft_learner_operations.md b/docs/raft_learner_operations.md index d5cc138a0..f4d4e354f 100644 --- a/docs/raft_learner_operations.md +++ b/docs/raft_learner_operations.md @@ -233,6 +233,27 @@ The script derives `--raftJoinMembers` from `NODES` and rejects a join rollout that contains another node. After promotion, clear `RAFT_JOIN_NODE` and run the same single-node rollout once more to remove the temporary join flags. +For recurring production replacements, use the fenced, resumable wrapper +instead of issuing those steps independently: + +```sh +TARGET_NODE=n4 \ +REPLACEMENT_CONFIRM=n4 \ +ROLLING_UPDATE_ENV_FILE=/path/to/deploy.env \ +REPLACEMENT_FENCE_COMMAND='fence-n4-and-block-raft' \ +REPLACEMENT_VERIFY_COMMAND='verify-a-real-write-and-read' \ +./scripts/raft-member-replace.sh --execute +``` + +Run the same command with `--dry-run` first. The wrapper verifies surviving +quorum before fencing, passes `configuration_index` to every membership RPC, +records a non-zero learner catch-up floor, restarts without join flags, and +requires the application verification command to succeed. It supports one +Raft group with a surviving quorum; it is not a forced no-quorum recovery tool. +See +`docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md` for the +state machine, fencing contract, and failure handling. + ## Removing a learner If a learner needs to be detached (e.g., the operator decided not to diff --git a/internal/raftadmin/server.go b/internal/raftadmin/server.go index bf6b54c9b..e07c736e8 100644 --- a/internal/raftadmin/server.go +++ b/internal/raftadmin/server.go @@ -47,17 +47,19 @@ func (s *Server) Status(context.Context, *pb.RaftAdminStatusRequest) (*pb.RaftAd } current := s.engine.Status() return &pb.RaftAdminStatusResponse{ - State: stateToProto(current.State), - LeaderId: current.Leader.ID, - LeaderAddress: current.Leader.Address, - Term: current.Term, - CommitIndex: current.CommitIndex, - AppliedIndex: current.AppliedIndex, - LastLogIndex: current.LastLogIndex, - LastSnapshotIndex: current.LastSnapshotIndex, - FsmPending: current.FSMPending, - NumPeers: current.NumPeers, - LastContactNanos: current.LastContact.Nanoseconds(), + State: stateToProto(current.State), + LeaderId: current.Leader.ID, + LeaderAddress: current.Leader.Address, + Term: current.Term, + CommitIndex: current.CommitIndex, + AppliedIndex: current.AppliedIndex, + LastLogIndex: current.LastLogIndex, + LastSnapshotIndex: current.LastSnapshotIndex, + FsmPending: current.FSMPending, + NumPeers: current.NumPeers, + LastContactNanos: current.LastContact.Nanoseconds(), + ConfigurationIndex: current.ConfigurationIndex, + PendingConfChange: current.PendingConfChange, }, nil } diff --git a/internal/raftadmin/server_test.go b/internal/raftadmin/server_test.go index 4ec02aba6..82ab27367 100644 --- a/internal/raftadmin/server_test.go +++ b/internal/raftadmin/server_test.go @@ -202,14 +202,16 @@ func TestServerMapsEngineAdminMethods(t *testing.T) { ID: "node-1", Address: "127.0.0.1:50051", }, - Term: 7, - CommitIndex: 10, - AppliedIndex: 9, - LastLogIndex: 12, - LastSnapshotIndex: 8, - FSMPending: 1, - NumPeers: 2, - LastContact: 0, + Term: 7, + CommitIndex: 10, + AppliedIndex: 9, + LastLogIndex: 12, + LastSnapshotIndex: 8, + FSMPending: 1, + NumPeers: 2, + LastContact: 0, + ConfigurationIndex: 6, + PendingConfChange: true, }, config: raftengine.Configuration{ Servers: []raftengine.Server{ @@ -225,6 +227,8 @@ func TestServerMapsEngineAdminMethods(t *testing.T) { require.Equal(t, pb.RaftAdminState_RAFT_ADMIN_STATE_LEADER, statusResp.State) require.Equal(t, "node-1", statusResp.LeaderId) require.Equal(t, uint64(10), statusResp.CommitIndex) + require.Equal(t, uint64(6), statusResp.ConfigurationIndex) + require.True(t, statusResp.PendingConfChange) cfgResp, err := server.Configuration(context.Background(), &pb.RaftAdminConfigurationRequest{}) require.NoError(t, err) diff --git a/internal/raftengine/engine.go b/internal/raftengine/engine.go index 75ceacc8c..2f68dc90d 100644 --- a/internal/raftengine/engine.go +++ b/internal/raftengine/engine.go @@ -92,6 +92,10 @@ type Status struct { FSMPending uint64 NumPeers uint64 LastContact time.Duration + // ConfigurationIndex is the highest committed membership index durably + // published by this node. Operators pass it back as previous_index on the + // next membership RPC to reject concurrent topology changes. + ConfigurationIndex uint64 // LeadTransferee is non-zero on the current leader while a leadership // transfer is in progress, and zero otherwise (including on followers). // Writers should hold new proposals while this is non-zero, since etcd/raft diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index 3fa139c8e..9390147d8 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -3564,18 +3564,19 @@ func (e *Engine) refreshStatus() { leader := e.leaderInfo(basic.Lead) status := raftengine.Status{ - State: state, - Leader: leader, - Term: basic.GetTerm(), - CommitIndex: basic.GetCommit(), - AppliedIndex: e.applied, - LastLogIndex: lastLogIndex, - LastSnapshotIndex: snapshot.GetMetadata().GetIndex(), - FSMPending: pendingEntries(basic.GetCommit(), e.applied), - NumPeers: numRemoteServers(config.Servers, e.localID), - LastContact: lastContactFor(state, basic.Lead, e.lastLeaderContactFrom, e.lastLeaderContactAt), - LeadTransferee: basic.LeadTransferee, - PendingConfChange: e.hasPendingConfChange(), + State: state, + Leader: leader, + Term: basic.GetTerm(), + CommitIndex: basic.GetCommit(), + AppliedIndex: e.applied, + LastLogIndex: lastLogIndex, + LastSnapshotIndex: snapshot.GetMetadata().GetIndex(), + FSMPending: pendingEntries(basic.GetCommit(), e.applied), + NumPeers: numRemoteServers(config.Servers, e.localID), + LastContact: lastContactFor(state, basic.Lead, e.lastLeaderContactFrom, e.lastLeaderContactAt), + ConfigurationIndex: e.currentConfigIndex(), + LeadTransferee: basic.LeadTransferee, + PendingConfChange: e.hasPendingConfChange(), } e.mu.Lock() diff --git a/internal/raftengine/etcd/engine_test.go b/internal/raftengine/etcd/engine_test.go index 7a4ec6518..7103334c9 100644 --- a/internal/raftengine/etcd/engine_test.go +++ b/internal/raftengine/etcd/engine_test.go @@ -420,6 +420,10 @@ func TestOpenInitializesAppliedIndexFromPersistedSnapshot(t *testing.T) { }) require.GreaterOrEqual(t, engine.Status().AppliedIndex, uint64(5)) + require.Eventually(t, func() bool { + return engine.Status().ConfigurationIndex == engine.currentConfigIndex() && + engine.Status().ConfigurationIndex > 0 + }, time.Second, 10*time.Millisecond) } func TestOpenRestoresPeersFromPersistedMetadata(t *testing.T) { diff --git a/proto/service.pb.go b/proto/service.pb.go index 00000ae2f..0b54bfdab 100644 --- a/proto/service.pb.go +++ b/proto/service.pb.go @@ -1537,20 +1537,22 @@ func (*RaftAdminStatusRequest) Descriptor() ([]byte, []int) { } type RaftAdminStatusResponse struct { - state protoimpl.MessageState `protogen:"open.v1"` - State RaftAdminState `protobuf:"varint,1,opt,name=state,proto3,enum=RaftAdminState" json:"state,omitempty"` - LeaderId string `protobuf:"bytes,2,opt,name=leader_id,json=leaderId,proto3" json:"leader_id,omitempty"` - LeaderAddress string `protobuf:"bytes,3,opt,name=leader_address,json=leaderAddress,proto3" json:"leader_address,omitempty"` - Term uint64 `protobuf:"varint,4,opt,name=term,proto3" json:"term,omitempty"` - CommitIndex uint64 `protobuf:"varint,5,opt,name=commit_index,json=commitIndex,proto3" json:"commit_index,omitempty"` - AppliedIndex uint64 `protobuf:"varint,6,opt,name=applied_index,json=appliedIndex,proto3" json:"applied_index,omitempty"` - LastLogIndex uint64 `protobuf:"varint,7,opt,name=last_log_index,json=lastLogIndex,proto3" json:"last_log_index,omitempty"` - LastSnapshotIndex uint64 `protobuf:"varint,8,opt,name=last_snapshot_index,json=lastSnapshotIndex,proto3" json:"last_snapshot_index,omitempty"` - FsmPending uint64 `protobuf:"varint,9,opt,name=fsm_pending,json=fsmPending,proto3" json:"fsm_pending,omitempty"` - NumPeers uint64 `protobuf:"varint,10,opt,name=num_peers,json=numPeers,proto3" json:"num_peers,omitempty"` - LastContactNanos int64 `protobuf:"varint,11,opt,name=last_contact_nanos,json=lastContactNanos,proto3" json:"last_contact_nanos,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + State RaftAdminState `protobuf:"varint,1,opt,name=state,proto3,enum=RaftAdminState" json:"state,omitempty"` + LeaderId string `protobuf:"bytes,2,opt,name=leader_id,json=leaderId,proto3" json:"leader_id,omitempty"` + LeaderAddress string `protobuf:"bytes,3,opt,name=leader_address,json=leaderAddress,proto3" json:"leader_address,omitempty"` + Term uint64 `protobuf:"varint,4,opt,name=term,proto3" json:"term,omitempty"` + CommitIndex uint64 `protobuf:"varint,5,opt,name=commit_index,json=commitIndex,proto3" json:"commit_index,omitempty"` + AppliedIndex uint64 `protobuf:"varint,6,opt,name=applied_index,json=appliedIndex,proto3" json:"applied_index,omitempty"` + LastLogIndex uint64 `protobuf:"varint,7,opt,name=last_log_index,json=lastLogIndex,proto3" json:"last_log_index,omitempty"` + LastSnapshotIndex uint64 `protobuf:"varint,8,opt,name=last_snapshot_index,json=lastSnapshotIndex,proto3" json:"last_snapshot_index,omitempty"` + FsmPending uint64 `protobuf:"varint,9,opt,name=fsm_pending,json=fsmPending,proto3" json:"fsm_pending,omitempty"` + NumPeers uint64 `protobuf:"varint,10,opt,name=num_peers,json=numPeers,proto3" json:"num_peers,omitempty"` + LastContactNanos int64 `protobuf:"varint,11,opt,name=last_contact_nanos,json=lastContactNanos,proto3" json:"last_contact_nanos,omitempty"` + ConfigurationIndex uint64 `protobuf:"varint,12,opt,name=configuration_index,json=configurationIndex,proto3" json:"configuration_index,omitempty"` + PendingConfChange bool `protobuf:"varint,13,opt,name=pending_conf_change,json=pendingConfChange,proto3" json:"pending_conf_change,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *RaftAdminStatusResponse) Reset() { @@ -1660,6 +1662,20 @@ func (x *RaftAdminStatusResponse) GetLastContactNanos() int64 { return 0 } +func (x *RaftAdminStatusResponse) GetConfigurationIndex() uint64 { + if x != nil { + return x.ConfigurationIndex + } + return 0 +} + +func (x *RaftAdminStatusResponse) GetPendingConfChange() bool { + if x != nil { + return x.PendingConfChange + } + return false +} + type RaftAdminConfigurationRequest struct { state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields @@ -2350,7 +2366,7 @@ const file_service_proto_rawDesc = "" + "\bstart_ts\x18\x01 \x01(\x04R\astartTs\",\n" + "\x10RollbackResponse\x12\x18\n" + "\asuccess\x18\x01 \x01(\bR\asuccess\"\x18\n" + - "\x16RaftAdminStatusRequest\"\xa2\x03\n" + + "\x16RaftAdminStatusRequest\"\x83\x04\n" + "\x17RaftAdminStatusResponse\x12%\n" + "\x05state\x18\x01 \x01(\x0e2\x0f.RaftAdminStateR\x05state\x12\x1b\n" + "\tleader_id\x18\x02 \x01(\tR\bleaderId\x12%\n" + @@ -2364,7 +2380,9 @@ const file_service_proto_rawDesc = "" + "fsmPending\x12\x1b\n" + "\tnum_peers\x18\n" + " \x01(\x04R\bnumPeers\x12,\n" + - "\x12last_contact_nanos\x18\v \x01(\x03R\x10lastContactNanos\"\x1f\n" + + "\x12last_contact_nanos\x18\v \x01(\x03R\x10lastContactNanos\x12/\n" + + "\x13configuration_index\x18\f \x01(\x04R\x12configurationIndex\x12.\n" + + "\x13pending_conf_change\x18\r \x01(\bR\x11pendingConfChange\"\x1f\n" + "\x1dRaftAdminConfigurationRequest\"W\n" + "\x0fRaftAdminMember\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12\x18\n" + diff --git a/proto/service.proto b/proto/service.proto index 68d12d5f7..c13e2e582 100644 --- a/proto/service.proto +++ b/proto/service.proto @@ -195,6 +195,8 @@ message RaftAdminStatusResponse { uint64 fsm_pending = 9; uint64 num_peers = 10; int64 last_contact_nanos = 11; + uint64 configuration_index = 12; + bool pending_conf_change = 13; } message RaftAdminConfigurationRequest {} diff --git a/raft_member_replace_script_test.go b/raft_member_replace_script_test.go new file mode 100644 index 000000000..2f2bd5b45 --- /dev/null +++ b/raft_member_replace_script_test.go @@ -0,0 +1,335 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestRaftMemberReplaceScriptCompletesFencedLearnerLifecycle(t *testing.T) { + repoRoot, err := os.Getwd() + require.NoError(t, err) + stateDir := t.TempDir() + fakeBin := filepath.Join(stateDir, "bin") + require.NoError(t, os.Mkdir(fakeBin, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "member"), []byte("voter\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "config-index"), []byte("10\n"), 0o600)) + + raftadmin := writeExecutableTestFile(t, fakeBin, "raftadmin", fakeRaftadminScript) + ssh := writeExecutableTestFile(t, fakeBin, "ssh", fakeReplacementSSHScript) + rolling := writeExecutableTestFile(t, fakeBin, "rolling-update", fakeReplacementRollingScript) + envFile := filepath.Join(stateDir, "deploy.env") + require.NoError(t, os.WriteFile(envFile, []byte(strings.Join([]string{ + "NODES=n1=127.0.0.1,n2=127.0.0.2,n3=127.0.0.3", + "RAFT_PORT=50051", + "SSH_USER=tester", + "CONTAINER_NAME=elastickv", + "DATA_DIR=/var/lib/elastickv", + "TARGET_NODE=n3", + "REPLACEMENT_CONFIRM=n3", + "REPLACEMENT_VERIFY_COMMAND=false", + }, "\n")+"\n"), 0o600)) + + stateFile := filepath.Join(stateDir, "replace.state") + output := runReplacementScript(t, repoRoot, []string{ + "TARGET_NODE=n2", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=container", + "REPLACEMENT_FENCE_VERIFY_SECONDS=1", + "REPLACEMENT_VERIFY_COMMAND=touch $FAKE_STATE_DIR/verified", + "REPLACEMENT_STABILITY_SECONDS=1", + "REPLACEMENT_TIMEOUT_SECONDS=5", + "REPLACEMENT_POLL_SECONDS=1", + "REPLACEMENT_STATE_FILE=" + stateFile, + "ROLLING_UPDATE_ENV_FILE=" + envFile, + "ROLLING_UPDATE_SCRIPT=" + rolling, + "RAFTADMIN_BIN=" + raftadmin, + "SSH_BIN=" + ssh, + "FAKE_STATE_DIR=" + stateDir, + }) + require.Contains(t, output, "replacement completed: target=n2") + require.FileExists(t, filepath.Join(stateDir, "verified")) + require.FileExists(t, filepath.Join(stateDir, "data-reset")) + require.Equal(t, "voter\n", mustReadTestFile(t, filepath.Join(stateDir, "member"))) + require.Equal(t, "n2:n2\nnormal:n2\n", mustReadTestFile(t, filepath.Join(stateDir, "rollouts"))) + + state := mustReadTestFile(t, stateFile) + require.Contains(t, state, "stage=9\n") + require.Contains(t, state, "expected_voters=n1,n2,n3\n") + require.Contains(t, state, "min_catch_up_index=100\n") + require.Contains(t, state, "data_mode=archive\n") + + resumeOutput := runReplacementScript(t, repoRoot, []string{ + "TARGET_NODE=n2", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=container", + "REPLACEMENT_FENCE_VERIFY_SECONDS=1", + "REPLACEMENT_VERIFY_COMMAND=true", + "REPLACEMENT_STABILITY_SECONDS=1", + "REPLACEMENT_TIMEOUT_SECONDS=5", + "REPLACEMENT_POLL_SECONDS=1", + "REPLACEMENT_STATE_FILE=" + stateFile, + "ROLLING_UPDATE_ENV_FILE=" + envFile, + "ROLLING_UPDATE_SCRIPT=" + rolling, + "RAFTADMIN_BIN=" + raftadmin, + "SSH_BIN=" + ssh, + "FAKE_STATE_DIR=" + stateDir, + }) + require.Contains(t, resumeOutput, "resuming operation") + require.Equal(t, "n2:n2\nnormal:n2\n", mustReadTestFile(t, filepath.Join(stateDir, "rollouts")), "completed resume must not repeat deployment") +} + +func TestRaftMemberReplaceScriptRejectsInsufficientSurvivingQuorum(t *testing.T) { + repoRoot, err := os.Getwd() + require.NoError(t, err) + stateDir := t.TempDir() + fakeBin := filepath.Join(stateDir, "bin") + require.NoError(t, os.Mkdir(fakeBin, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "member"), []byte("voter\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "config-index"), []byte("10\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "candidate-n3"), nil, 0o600)) + + raftadmin := writeExecutableTestFile(t, fakeBin, "raftadmin", fakeRaftadminScript) + ssh := writeExecutableTestFile(t, fakeBin, "ssh", fakeReplacementSSHScript) + rolling := writeExecutableTestFile(t, fakeBin, "rolling-update", fakeReplacementRollingScript) + envFile := filepath.Join(stateDir, "deploy.env") + require.NoError(t, os.WriteFile(envFile, []byte("NODES=n1=127.0.0.1,n2=127.0.0.2,n3=127.0.0.3\n"), 0o600)) + + cmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") + cmd.Dir = repoRoot + cmd.Env = append(os.Environ(), + "TARGET_NODE=n2", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=container", + "REPLACEMENT_VERIFY_COMMAND=true", + "REPLACEMENT_STATE_FILE="+filepath.Join(stateDir, "replace.state"), + "ROLLING_UPDATE_ENV_FILE="+envFile, + "ROLLING_UPDATE_SCRIPT="+rolling, + "RAFTADMIN_BIN="+raftadmin, + "SSH_BIN="+ssh, + "FAKE_STATE_DIR="+stateDir, + ) + output, err := cmd.CombinedOutput() + require.Error(t, err) + require.Contains(t, string(output), "only 1 non-target voters are reachable; 2 are required") + require.NoFileExists(t, filepath.Join(stateDir, "down-n2"), "fencing must not start without surviving quorum") +} + +func TestRaftMemberReplaceScriptResumesAfterLearnerCommitResponseLoss(t *testing.T) { + repoRoot, err := os.Getwd() + require.NoError(t, err) + stateDir := t.TempDir() + fakeBin := filepath.Join(stateDir, "bin") + require.NoError(t, os.Mkdir(fakeBin, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "member"), []byte("voter\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "config-index"), []byte("10\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "fail-add-once"), nil, 0o600)) + + raftadmin := writeExecutableTestFile(t, fakeBin, "raftadmin", fakeRaftadminScript) + ssh := writeExecutableTestFile(t, fakeBin, "ssh", fakeReplacementSSHScript) + rolling := writeExecutableTestFile(t, fakeBin, "rolling-update", fakeReplacementRollingScript) + envFile := filepath.Join(stateDir, "deploy.env") + require.NoError(t, os.WriteFile(envFile, []byte("NODES=n1=127.0.0.1,n2=127.0.0.2,n3=127.0.0.3\n"), 0o600)) + stateFile := filepath.Join(stateDir, "replace.state") + commandEnv := []string{ + "TARGET_NODE=n2", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=container", + "REPLACEMENT_FENCE_VERIFY_SECONDS=1", + "REPLACEMENT_VERIFY_COMMAND=true", + "REPLACEMENT_STABILITY_SECONDS=1", + "REPLACEMENT_TIMEOUT_SECONDS=5", + "REPLACEMENT_POLL_SECONDS=1", + "REPLACEMENT_STATE_FILE=" + stateFile, + "ROLLING_UPDATE_ENV_FILE=" + envFile, + "ROLLING_UPDATE_SCRIPT=" + rolling, + "RAFTADMIN_BIN=" + raftadmin, + "SSH_BIN=" + ssh, + "FAKE_STATE_DIR=" + stateDir, + } + + cmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") + cmd.Dir = repoRoot + cmd.Env = append(os.Environ(), commandEnv...) + output, err := cmd.CombinedOutput() + require.Error(t, err, "%s", output) + require.Equal(t, "learner\n", mustReadTestFile(t, filepath.Join(stateDir, "member")), "the simulated RPC response is lost after commit") + interruptedState := mustReadTestFile(t, stateFile) + require.Contains(t, interruptedState, "stage=5\n") + require.Contains(t, interruptedState, "min_catch_up_index=100\n", "catch-up floor must precede the membership proposal") + + resumeOutput := runReplacementScript(t, repoRoot, commandEnv) + require.Contains(t, resumeOutput, "target is already a learner; resuming after committed add") + require.Contains(t, resumeOutput, "replacement completed: target=n2") + require.Equal(t, "voter\n", mustReadTestFile(t, filepath.Join(stateDir, "member"))) +} + +func TestRaftMemberReplaceScriptDoesNotRemoveAnotherProcessLock(t *testing.T) { + repoRoot, err := os.Getwd() + require.NoError(t, err) + stateDir := t.TempDir() + rolling := writeExecutableTestFile(t, stateDir, "rolling-update", "#!/usr/bin/env bash\nexit 0\n") + stateFile := filepath.Join(stateDir, "replace.state") + lockDir := stateFile + ".lock" + require.NoError(t, os.Mkdir(lockDir, 0o700)) + + cmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") + cmd.Dir = repoRoot + cmd.Env = append(os.Environ(), + "TARGET_NODE=n2", + "NODES=n1=127.0.0.1,n2=127.0.0.2,n3=127.0.0.3", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=container", + "REPLACEMENT_VERIFY_COMMAND=true", + "REPLACEMENT_STATE_FILE="+stateFile, + "ROLLING_UPDATE_SCRIPT="+rolling, + ) + output, err := cmd.CombinedOutput() + require.Error(t, err) + require.Contains(t, string(output), "another replacement process holds") + info, statErr := os.Stat(lockDir) + require.NoError(t, statErr, "a process that failed to acquire the lock must not remove it") + require.True(t, info.IsDir()) +} + +func runReplacementScript(t *testing.T, repoRoot string, env []string) string { + t.Helper() + cmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") + cmd.Dir = repoRoot + cmd.Env = append(os.Environ(), env...) + output, err := cmd.CombinedOutput() + require.NoError(t, err, "%s", output) + return string(output) +} + +func writeExecutableTestFile(t *testing.T, dir, name, contents string) string { + t.Helper() + path := filepath.Join(dir, name) + require.NoError(t, os.WriteFile(path, []byte(contents), 0o600)) + require.NoError(t, os.Chmod(path, 0o700)) + return path +} + +func mustReadTestFile(t *testing.T, path string) string { + t.Helper() + contents, err := os.ReadFile(path) + require.NoError(t, err) + return string(contents) +} + +const fakeRaftadminScript = `#!/usr/bin/env bash +set -euo pipefail +addr="$1" +command="$2" +state_dir="$FAKE_STATE_DIR" + +if [[ "$command" == "status" ]]; then + if [[ "$addr" == "127.0.0.2:50051" && -f "$state_dir/down-n2" ]]; then + exit 1 + fi + if [[ "$addr" == "127.0.0.1:50051" ]]; then + raft_state=LEADER + elif [[ "$addr" == "127.0.0.3:50051" && -f "$state_dir/candidate-n3" ]]; then + raft_state=CANDIDATE + else + raft_state=FOLLOWER + fi + cat < "$state_dir/config-index" +} + +case "$command" in + remove_server) + [[ "$3" == "n2" && "$4" == "$(cat "$state_dir/config-index")" ]] + printf 'absent\n' > "$state_dir/member" + increment_index + ;; + add_learner) + [[ "$3" == "n2" && "$4" == "127.0.0.2:50051" && "$5" == "$(cat "$state_dir/config-index")" ]] + printf 'learner\n' > "$state_dir/member" + increment_index + if [[ -f "$state_dir/fail-add-once" ]]; then + rm -f "$state_dir/fail-add-once" + exit 73 + fi + ;; + promote_learner) + [[ "$3" == "n2" && "$4" == "$(cat "$state_dir/config-index")" && "$5" == "100" && "$6" == "false" ]] + printf 'voter\n' > "$state_dir/member" + increment_index + ;; + leadership_transfer_to_server) + ;; + *) + echo "unexpected raftadmin command: $*" >&2 + exit 1 + ;; +esac +printf 'index: %s\n' "$(cat "$state_dir/config-index")" +` + +const fakeReplacementSSHScript = `#!/usr/bin/env bash +set -euo pipefail +cat >/dev/null +while [[ "$#" -gt 0 && "$1" != "--" ]]; do shift; done +[[ "$#" -ge 2 ]] +action="$2" +case "$action" in + stop) touch "$FAKE_STATE_DIR/down-n2" ;; + reset-data) touch "$FAKE_STATE_DIR/data-reset" ;; + *) echo "unexpected SSH action: $action" >&2; exit 1 ;; +esac +` + +const fakeReplacementRollingScript = `#!/usr/bin/env bash +set -euo pipefail +printf '%s:%s\n' "${RAFT_JOIN_NODE:-normal}" "$ROLLING_ORDER" >> "$FAKE_STATE_DIR/rollouts" +if [[ "${RAFT_JOIN_NODE:-}" == "n2" ]]; then + rm -f "$FAKE_STATE_DIR/down-n2" +fi +` diff --git a/scripts/raft-member-replace.sh b/scripts/raft-member-replace.sh new file mode 100755 index 000000000..06f73c0a4 --- /dev/null +++ b/scripts/raft-member-replace.sh @@ -0,0 +1,1009 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +usage() { + cat <<'EOF' +Usage: + TARGET_NODE=n2 ROLLING_UPDATE_ENV_FILE=deploy.env \ + REPLACEMENT_CONFIRM=n2 ./scripts/raft-member-replace.sh --execute + TARGET_NODE=n2 ROLLING_UPDATE_ENV_FILE=deploy.env \ + ./scripts/raft-member-replace.sh --dry-run + +This command replaces one voter in one Raft group by fencing the old process, +removing its membership, archiving its data, joining a fresh learner, waiting +for catch-up, promoting it, and restarting it without join flags. + +Required environment: + TARGET_NODE + Raft ID to replace. It must be a current voter and exist in NODES. + NODES + Same ID-to-host map used by scripts/rolling-update.sh. It may be supplied + through ROLLING_UPDATE_ENV_FILE. + REPLACEMENT_CONFIRM + Must exactly match TARGET_NODE for --execute. + REPLACEMENT_VERIFY_COMMAND + Application-level write/read verification command. It runs after the + replacement is a stable voter. The command must return zero. + +Fencing: + REPLACEMENT_FENCE_MODE=external (default) + Requires REPLACEMENT_FENCE_COMMAND. The command must prevent the old VM or + process from returning with stale data. The target Raft RPC must become + unreachable before membership removal. + REPLACEMENT_FENCE_MODE=container + Stops CONTAINER_NAME over SSH. Use only when the host lifecycle guarantees + the stopped container cannot restart independently during the operation. + +Optional environment: + ROLLING_UPDATE_ENV_FILE Env file used by rolling-update.sh. + ROLLING_UPDATE_SCRIPT Default: scripts/rolling-update.sh. + RAFTADMIN_BIN Default: build ./cmd/raftadmin locally. + SSH_BIN Default: ssh. + SSH_TARGETS, SSH_USER Same meaning as rolling-update.sh. + RAFT_PORT Default: 50051. + CONTAINER_NAME Default: elastickv. + DATA_DIR Default: /var/lib/elastickv. + REPLACEMENT_STATE_FILE Durable local resume state. + REPLACEMENT_DATA_MODE archive (default) or delete. + REPLACEMENT_FENCE_COMMAND Required for external fencing. + REPLACEMENT_FENCE_VERIFY_SECONDS + Consecutive target downtime required (default: 3). + REPLACEMENT_STABILITY_SECONDS Default: 10. + REPLACEMENT_TIMEOUT_SECONDS Default: 120. + REPLACEMENT_POLL_SECONDS Default: 1. + REPLACEMENT_KEEP_STATE Keep completed state when true (default). + RAFTADMIN_ALLOW_INSECURE Default: true. + RAFTADMIN_RPC_TIMEOUT_SECONDS Default: 5. + SSH_CONNECT_TIMEOUT_SECONDS Default: 10. + SSH_STRICT_HOST_KEY_CHECKING Default: accept-new. + +The state file makes the sequence resumable. A resumed run validates the live +membership against its recorded stage and fails closed on conflicting changes. +No-quorum recovery and multi-group replacement are outside this command. +EOF +} + +MODE="" +while [[ "$#" -gt 0 ]]; do + case "$1" in + --execute) + [[ -z "$MODE" ]] || { echo "choose exactly one mode" >&2; exit 1; } + MODE="execute" + ;; + --dry-run) + [[ -z "$MODE" ]] || { echo "choose exactly one mode" >&2; exit 1; } + MODE="dry-run" + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac + shift +done + +if [[ -z "$MODE" ]]; then + echo "--execute or --dry-run is required" >&2 + usage >&2 + exit 1 +fi + +# The deployment env is inventory, while explicitly exported replacement +# controls are the operator's command for this run. Preserve the latter across +# sourcing so an old commented-in recovery value cannot retarget an operation. +INPUT_TARGET_NODE_SET="${TARGET_NODE+x}" +INPUT_TARGET_NODE="${TARGET_NODE:-}" +INPUT_REPLACEMENT_CONFIRM_SET="${REPLACEMENT_CONFIRM+x}" +INPUT_REPLACEMENT_CONFIRM="${REPLACEMENT_CONFIRM:-}" +INPUT_REPLACEMENT_STATE_FILE_SET="${REPLACEMENT_STATE_FILE+x}" +INPUT_REPLACEMENT_STATE_FILE="${REPLACEMENT_STATE_FILE:-}" +INPUT_REPLACEMENT_DATA_MODE_SET="${REPLACEMENT_DATA_MODE+x}" +INPUT_REPLACEMENT_DATA_MODE="${REPLACEMENT_DATA_MODE:-}" +INPUT_REPLACEMENT_FENCE_MODE_SET="${REPLACEMENT_FENCE_MODE+x}" +INPUT_REPLACEMENT_FENCE_MODE="${REPLACEMENT_FENCE_MODE:-}" +INPUT_REPLACEMENT_FENCE_COMMAND_SET="${REPLACEMENT_FENCE_COMMAND+x}" +INPUT_REPLACEMENT_FENCE_COMMAND="${REPLACEMENT_FENCE_COMMAND:-}" +INPUT_REPLACEMENT_VERIFY_COMMAND_SET="${REPLACEMENT_VERIFY_COMMAND+x}" +INPUT_REPLACEMENT_VERIFY_COMMAND="${REPLACEMENT_VERIFY_COMMAND:-}" + +if [[ -n "${ROLLING_UPDATE_ENV_FILE:-}" ]]; then + if [[ ! -f "$ROLLING_UPDATE_ENV_FILE" ]]; then + echo "ROLLING_UPDATE_ENV_FILE not found: $ROLLING_UPDATE_ENV_FILE" >&2 + exit 1 + fi + # shellcheck disable=SC1090 + source "$ROLLING_UPDATE_ENV_FILE" +fi + +[[ -z "$INPUT_TARGET_NODE_SET" ]] || TARGET_NODE="$INPUT_TARGET_NODE" +[[ -z "$INPUT_REPLACEMENT_CONFIRM_SET" ]] || REPLACEMENT_CONFIRM="$INPUT_REPLACEMENT_CONFIRM" +[[ -z "$INPUT_REPLACEMENT_STATE_FILE_SET" ]] || REPLACEMENT_STATE_FILE="$INPUT_REPLACEMENT_STATE_FILE" +[[ -z "$INPUT_REPLACEMENT_DATA_MODE_SET" ]] || REPLACEMENT_DATA_MODE="$INPUT_REPLACEMENT_DATA_MODE" +[[ -z "$INPUT_REPLACEMENT_FENCE_MODE_SET" ]] || REPLACEMENT_FENCE_MODE="$INPUT_REPLACEMENT_FENCE_MODE" +[[ -z "$INPUT_REPLACEMENT_FENCE_COMMAND_SET" ]] || REPLACEMENT_FENCE_COMMAND="$INPUT_REPLACEMENT_FENCE_COMMAND" +[[ -z "$INPUT_REPLACEMENT_VERIFY_COMMAND_SET" ]] || REPLACEMENT_VERIFY_COMMAND="$INPUT_REPLACEMENT_VERIFY_COMMAND" + +TARGET_NODE="${TARGET_NODE:-}" +NODES="${NODES:-}" +SSH_TARGETS="${SSH_TARGETS:-}" +SSH_USER="${SSH_USER:-${USER:-$(id -un)}}" +RAFT_PORT="${RAFT_PORT:-50051}" +CONTAINER_NAME="${CONTAINER_NAME:-elastickv}" +DATA_DIR="${DATA_DIR:-/var/lib/elastickv}" +RAFTADMIN_BIN="${RAFTADMIN_BIN:-}" +RAFTADMIN_ALLOW_INSECURE="${RAFTADMIN_ALLOW_INSECURE:-true}" +RAFTADMIN_RPC_TIMEOUT_SECONDS="${RAFTADMIN_RPC_TIMEOUT_SECONDS:-5}" +ROLLING_UPDATE_SCRIPT="${ROLLING_UPDATE_SCRIPT:-${SCRIPT_DIR}/rolling-update.sh}" +SSH_BIN="${SSH_BIN:-ssh}" +SSH_CONNECT_TIMEOUT_SECONDS="${SSH_CONNECT_TIMEOUT_SECONDS:-10}" +SSH_STRICT_HOST_KEY_CHECKING="${SSH_STRICT_HOST_KEY_CHECKING:-accept-new}" +REPLACEMENT_CONFIRM="${REPLACEMENT_CONFIRM:-}" +REPLACEMENT_FENCE_MODE="${REPLACEMENT_FENCE_MODE:-external}" +REPLACEMENT_FENCE_COMMAND="${REPLACEMENT_FENCE_COMMAND:-}" +REPLACEMENT_FENCE_VERIFY_SECONDS="${REPLACEMENT_FENCE_VERIFY_SECONDS:-3}" +REPLACEMENT_DATA_MODE="${REPLACEMENT_DATA_MODE:-archive}" +REPLACEMENT_VERIFY_COMMAND="${REPLACEMENT_VERIFY_COMMAND:-}" +REPLACEMENT_STABILITY_SECONDS="${REPLACEMENT_STABILITY_SECONDS:-10}" +REPLACEMENT_TIMEOUT_SECONDS="${REPLACEMENT_TIMEOUT_SECONDS:-120}" +REPLACEMENT_POLL_SECONDS="${REPLACEMENT_POLL_SECONDS:-1}" +REPLACEMENT_KEEP_STATE="${REPLACEMENT_KEEP_STATE:-true}" + +NODE_IDS=() +NODE_HOSTS=() +STATE_STAGE=0 +STATE_TARGET="" +STATE_TARGET_ADDRESS="" +STATE_OPERATION_ID="" +STATE_ARCHIVE_PATH="" +STATE_EXPECTED_VOTERS="" +STATE_MIN_CATCH_UP_INDEX=0 +STATE_DATA_MODE="" +RAFTADMIN_TMP_DIR="" +LOCK_DIR="" +LOCK_ACQUIRED=false + +log() { + printf '[raft-member-replace] %s\n' "$*" +} + +fail() { + printf '[raft-member-replace] error: %s\n' "$*" >&2 + exit 1 +} + +is_uint() { + [[ "$1" =~ ^[0-9]+$ ]] +} + +validate_settings() { + [[ -n "$TARGET_NODE" ]] || fail "TARGET_NODE is required" + [[ "$TARGET_NODE" =~ ^[A-Za-z0-9._-]+$ ]] || fail "TARGET_NODE contains unsupported characters" + [[ -n "$NODES" ]] || fail "NODES is required" + [[ "$DATA_DIR" == /* ]] || fail "DATA_DIR must be absolute" + state_value_valid "$DATA_DIR" || fail "DATA_DIR contains characters unsupported by the resume state" + [[ -x "$ROLLING_UPDATE_SCRIPT" ]] || fail "ROLLING_UPDATE_SCRIPT is not executable: $ROLLING_UPDATE_SCRIPT" + is_uint "$RAFT_PORT" || fail "RAFT_PORT must be an unsigned integer" + is_uint "$REPLACEMENT_STABILITY_SECONDS" || fail "REPLACEMENT_STABILITY_SECONDS must be an unsigned integer" + is_uint "$REPLACEMENT_FENCE_VERIFY_SECONDS" || fail "REPLACEMENT_FENCE_VERIFY_SECONDS must be an unsigned integer" + is_uint "$REPLACEMENT_TIMEOUT_SECONDS" || fail "REPLACEMENT_TIMEOUT_SECONDS must be an unsigned integer" + is_uint "$REPLACEMENT_POLL_SECONDS" || fail "REPLACEMENT_POLL_SECONDS must be an unsigned integer" + (( REPLACEMENT_TIMEOUT_SECONDS > 0 )) || fail "REPLACEMENT_TIMEOUT_SECONDS must be positive" + (( REPLACEMENT_POLL_SECONDS > 0 )) || fail "REPLACEMENT_POLL_SECONDS must be positive" + (( REPLACEMENT_FENCE_VERIFY_SECONDS > 0 )) || fail "REPLACEMENT_FENCE_VERIFY_SECONDS must be positive" + case "$REPLACEMENT_FENCE_MODE" in + external) + if [[ "$MODE" == "execute" && -z "$REPLACEMENT_FENCE_COMMAND" ]]; then + fail "REPLACEMENT_FENCE_COMMAND is required for external fencing" + fi + ;; + container) ;; + *) fail "REPLACEMENT_FENCE_MODE must be external or container" ;; + esac + case "$REPLACEMENT_DATA_MODE" in + archive|delete) ;; + *) fail "REPLACEMENT_DATA_MODE must be archive or delete" ;; + esac + if [[ "$MODE" == "execute" ]]; then + [[ "$REPLACEMENT_CONFIRM" == "$TARGET_NODE" ]] || fail "REPLACEMENT_CONFIRM must exactly match TARGET_NODE" + [[ -n "$REPLACEMENT_VERIFY_COMMAND" ]] || fail "REPLACEMENT_VERIFY_COMMAND is required for --execute" + fi +} + +contains_value() { + local needle="$1" + shift + local value + for value in "$@"; do + [[ "$value" == "$needle" ]] && return 0 + done + return 1 +} + +lookup_mapping() { + local key="$1" + local mapping="$2" + local pair entry_key entry_value + local -a pairs=() + + [[ -n "$mapping" ]] || return 1 + IFS=',' read -r -a pairs <<< "$mapping" + for pair in "${pairs[@]}"; do + pair="${pair//[[:space:]]/}" + [[ -n "$pair" && "$pair" == *=* ]] || continue + entry_key="${pair%%=*}" + entry_value="${pair#*=}" + if [[ "$entry_key" == "$key" ]]; then + printf '%s\n' "$entry_value" + return 0 + fi + done + return 1 +} + +parse_nodes() { + local pair node_id node_host + local -a pairs=() + + IFS=',' read -r -a pairs <<< "$NODES" + for pair in "${pairs[@]}"; do + pair="${pair//[[:space:]]/}" + [[ -n "$pair" ]] || continue + [[ "$pair" == *=* ]] || fail "invalid NODES entry: $pair" + node_id="${pair%%=*}" + node_host="${pair#*=}" + [[ -n "$node_id" && -n "$node_host" ]] || fail "invalid NODES entry: $pair" + [[ "$node_id" =~ ^[A-Za-z0-9._-]+$ ]] || fail "raft ID contains unsupported characters: $node_id" + [[ "$node_host" != *'|'* ]] || fail "raft host contains unsupported characters: $node_host" + contains_value "$node_id" "${NODE_IDS[@]}" && fail "duplicate raft ID in NODES: $node_id" + NODE_IDS+=("$node_id") + NODE_HOSTS+=("$node_host") + done + (( ${#NODE_IDS[@]} > 0 )) || fail "NODES did not contain any nodes" + contains_value "$TARGET_NODE" "${NODE_IDS[@]}" || fail "TARGET_NODE is not present in NODES: $TARGET_NODE" +} + +node_host_by_id() { + local wanted="$1" + local i + for i in "${!NODE_IDS[@]}"; do + if [[ "${NODE_IDS[$i]}" == "$wanted" ]]; then + printf '%s\n' "${NODE_HOSTS[$i]}" + return 0 + fi + done + return 1 +} + +ssh_target_by_id() { + local node_id="$1" + local target + target="$(lookup_mapping "$node_id" "$SSH_TARGETS" || true)" + [[ -n "$target" ]] || target="$(node_host_by_id "$node_id")" + if [[ "$target" == *@* ]]; then + printf '%s\n' "$target" + else + printf '%s@%s\n' "$SSH_USER" "$target" + fi +} + +cleanup() { + [[ -z "$RAFTADMIN_TMP_DIR" || ! -d "$RAFTADMIN_TMP_DIR" ]] || rm -rf "$RAFTADMIN_TMP_DIR" + [[ "$LOCK_ACQUIRED" != "true" || -z "$LOCK_DIR" || ! -d "$LOCK_DIR" ]] || rmdir "$LOCK_DIR" 2>/dev/null || true +} +trap cleanup EXIT + +ensure_raftadmin() { + if [[ -n "$RAFTADMIN_BIN" ]]; then + [[ -x "$RAFTADMIN_BIN" ]] || fail "RAFTADMIN_BIN is not executable: $RAFTADMIN_BIN" + return 0 + fi + RAFTADMIN_TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/raft-member-replace.XXXXXX")" + RAFTADMIN_BIN="${RAFTADMIN_TMP_DIR}/raftadmin" + log "building raftadmin" + (cd "$REPO_ROOT" && go build -o "$RAFTADMIN_BIN" ./cmd/raftadmin) +} + +raftadmin() { + RAFTADMIN_ALLOW_INSECURE="$RAFTADMIN_ALLOW_INSECURE" \ + RAFTADMIN_RPC_TIMEOUT_SECONDS="$RAFTADMIN_RPC_TIMEOUT_SECONDS" \ + "$RAFTADMIN_BIN" "$@" +} + +field_value() { + local field="$1" + awk -F ': ' -v field="$field" '$1 == field { value=$2; gsub(/^"|"$/, "", value); print value; exit }' +} + +configuration_rows() { + awk ' + /^servers \{/ { id=""; address=""; suffrage=""; in_server=1; next } + in_server && /^[[:space:]]+id:/ { + value=$0; sub(/^[^:]*:[[:space:]]*"/, "", value); sub(/"[[:space:]]*$/, "", value); id=value; next + } + in_server && /^[[:space:]]+address:/ { + value=$0; sub(/^[^:]*:[[:space:]]*"/, "", value); sub(/"[[:space:]]*$/, "", value); address=value; next + } + in_server && /^[[:space:]]+suffrage:/ { + value=$0; sub(/^[^:]*:[[:space:]]*"/, "", value); sub(/"[[:space:]]*$/, "", value); suffrage=value; next + } + in_server && /^}/ { print id "|" address "|" suffrage; in_server=0 } + ' +} + +status_for() { + raftadmin "$1" status 2>/dev/null +} + +discover_leader() { + local address status state leader_address leader_id + local i + for i in "${!NODE_HOSTS[@]}"; do + address="${NODE_HOSTS[$i]}:${RAFT_PORT}" + status="$(status_for "$address" || true)" + [[ -n "$status" ]] || continue + state="$(printf '%s\n' "$status" | field_value state)" + if [[ "$state" == "LEADER" ]]; then + printf '%s|%s\n' "${NODE_IDS[$i]}" "$address" + return 0 + fi + leader_id="$(printf '%s\n' "$status" | field_value leader_id)" + leader_address="$(printf '%s\n' "$status" | field_value leader_address)" + if [[ -n "$leader_id" && -n "$leader_address" ]]; then + printf '%s|%s\n' "$leader_id" "$leader_address" + return 0 + fi + done + return 1 +} + +configuration_from_leader() { + local leader leader_address + leader="$(discover_leader)" || return 1 + leader_address="${leader#*|}" + raftadmin "$leader_address" configuration +} + +member_suffrage() { + local wanted="$1" + local config="$2" + local id address suffrage + while IFS='|' read -r id address suffrage; do + if [[ "$id" == "$wanted" ]]; then + printf '%s\n' "$suffrage" + return 0 + fi + done < <(printf '%s\n' "$config" | configuration_rows) + return 1 +} + +member_address() { + local wanted="$1" + local config="$2" + local id address suffrage + while IFS='|' read -r id address suffrage; do + if [[ "$id" == "$wanted" ]]; then + printf '%s\n' "$address" + return 0 + fi + done < <(printf '%s\n' "$config" | configuration_rows) + return 1 +} + +voter_csv() { + local config="$1" + local id address suffrage result="" + while IFS='|' read -r id address suffrage; do + [[ "$suffrage" == "voter" ]] || continue + result+="${result:+,}${id}" + done < <(printf '%s\n' "$config" | configuration_rows) + printf '%s\n' "$result" +} + +same_csv_set() { + local left="$1" + local right="$2" + [[ "$(printf '%s' "$left" | tr ',' '\n' | sed '/^$/d' | sort | tr '\n' ',')" == \ + "$(printf '%s' "$right" | tr ',' '\n' | sed '/^$/d' | sort | tr '\n' ',')" ]] +} + +leader_status_ready() { + local address="$1" + local status pending config_index commit applied + status="$(status_for "$address")" || return 1 + pending="$(printf '%s\n' "$status" | field_value pending_conf_change)" + config_index="$(printf '%s\n' "$status" | field_value configuration_index)" + commit="$(printf '%s\n' "$status" | field_value commit_index)" + applied="$(printf '%s\n' "$status" | field_value applied_index)" + [[ "$pending" == "false" ]] || return 1 + is_uint "$config_index" && (( config_index > 0 )) || return 1 + is_uint "$commit" && is_uint "$applied" && (( applied >= commit )) || return 1 + printf '%s|%s|%s\n' "$config_index" "$commit" "$applied" +} + +select_transfer_candidate() { + local config="$1" + local required_index="$2" + local expected_leader="$3" + local id address suffrage status state applied observed_leader + while IFS='|' read -r id address suffrage; do + [[ "$suffrage" == "voter" && "$id" != "$TARGET_NODE" ]] || continue + status="$(status_for "$address" || true)" + [[ -n "$status" ]] || continue + state="$(printf '%s\n' "$status" | field_value state)" + applied="$(printf '%s\n' "$status" | field_value applied_index)" + observed_leader="$(printf '%s\n' "$status" | field_value leader_id)" + if [[ "$state" == "FOLLOWER" && "$observed_leader" == "$expected_leader" ]] && \ + is_uint "$applied" && (( applied >= required_index )); then + printf '%s|%s\n' "$id" "$address" + return 0 + fi + done < <(printf '%s\n' "$config" | configuration_rows) + return 1 +} + +preflight() { + local leader leader_id leader_address config target_address target_suffrage + local ready config_index commit applied voters quorum reachable=0 + local id address suffrage voter_status voter_state voter_leader + + leader="$(discover_leader)" || fail "no reachable Raft leader; no-quorum recovery is not supported" + leader_id="${leader%%|*}" + leader_address="${leader#*|}" + ready="$(leader_status_ready "$leader_address")" || fail "leader is not fully applied or has a pending configuration change" + IFS='|' read -r config_index commit applied <<< "$ready" + config="$(raftadmin "$leader_address" configuration)" || fail "cannot read leader configuration" + target_suffrage="$(member_suffrage "$TARGET_NODE" "$config" || true)" + target_address="$(member_address "$TARGET_NODE" "$config" || true)" + [[ "$target_suffrage" == "voter" ]] || fail "target must be a current voter; got ${target_suffrage:-absent}" + [[ "$target_address" == "$(node_host_by_id "$TARGET_NODE"):${RAFT_PORT}" ]] || \ + fail "target address mismatch: membership=$target_address inventory=$(node_host_by_id "$TARGET_NODE"):${RAFT_PORT}" + + voters=0 + while IFS='|' read -r id address suffrage; do + [[ "$suffrage" == "voter" ]] || continue + voters=$((voters + 1)) + [[ "$id" == "$TARGET_NODE" ]] && continue + voter_status="$(status_for "$address" || true)" + voter_state="$(printf '%s\n' "$voter_status" | field_value state)" + voter_leader="$(printf '%s\n' "$voter_status" | field_value leader_id)" + if [[ "$voter_state" == "LEADER" && "$id" == "$leader_id" ]] || \ + [[ "$voter_state" == "FOLLOWER" && "$voter_leader" == "$leader_id" ]]; then + reachable=$((reachable + 1)) + fi + done < <(printf '%s\n' "$config" | configuration_rows) + quorum=$((voters / 2 + 1)) + (( reachable >= quorum )) || fail "only $reachable non-target voters are reachable; $quorum are required to commit removal" + + if [[ "$leader_id" == "$TARGET_NODE" ]]; then + local candidate candidate_id candidate_address + candidate="$(select_transfer_candidate "$config" "$commit" "$leader_id")" || fail "target is leader and no caught-up transfer candidate exists" + candidate_id="${candidate%%|*}" + candidate_address="${candidate#*|}" + log "transferring leadership from $TARGET_NODE to $candidate_id" + raftadmin "$leader_address" leadership_transfer_to_server "$candidate_id" "$candidate_address" >/dev/null + wait_for_leader "$candidate_id" >/dev/null || fail "leadership did not transfer to $candidate_id" + fi + + STATE_EXPECTED_VOTERS="$(voter_csv "$config")" + STATE_TARGET_ADDRESS="$target_address" + log "preflight passed: leader=$leader_id leader_applied=$applied voters=$voters non_target_reachable=$reachable quorum=$quorum config_index=$config_index" +} + +wait_for_leader() { + local expected_id="${1:-}" + local elapsed=0 leader leader_id + while (( elapsed < REPLACEMENT_TIMEOUT_SECONDS )); do + leader="$(discover_leader || true)" + if [[ -n "$leader" ]]; then + leader_id="${leader%%|*}" + if [[ -z "$expected_id" || "$leader_id" == "$expected_id" ]]; then + printf '%s\n' "$leader" + return 0 + fi + fi + sleep "$REPLACEMENT_POLL_SECONDS" + elapsed=$((elapsed + REPLACEMENT_POLL_SECONDS)) + done + return 1 +} + +wait_target_down() { + local elapsed=0 down_for=0 + while (( elapsed < REPLACEMENT_TIMEOUT_SECONDS )); do + if ! status_for "$STATE_TARGET_ADDRESS" >/dev/null; then + down_for=$((down_for + REPLACEMENT_POLL_SECONDS)) + if (( down_for >= REPLACEMENT_FENCE_VERIFY_SECONDS )); then + return 0 + fi + else + down_for=0 + fi + sleep "$REPLACEMENT_POLL_SECONDS" + elapsed=$((elapsed + REPLACEMENT_POLL_SECONDS)) + done + return 1 +} + +wait_target_up() { + local elapsed=0 + while (( elapsed < REPLACEMENT_TIMEOUT_SECONDS )); do + if status_for "$STATE_TARGET_ADDRESS" >/dev/null; then + return 0 + fi + sleep "$REPLACEMENT_POLL_SECONDS" + elapsed=$((elapsed + REPLACEMENT_POLL_SECONDS)) + done + return 1 +} + +run_operator_command() { + local command="$1" + local operator_target_node="$TARGET_NODE" + local operator_target_address="$STATE_TARGET_ADDRESS" + local operator_target_ssh + operator_target_ssh="$(ssh_target_by_id "$TARGET_NODE")" + ( + if [[ -n "${ROLLING_UPDATE_ENV_FILE:-}" ]]; then + set -a + # shellcheck disable=SC1090 + source "$ROLLING_UPDATE_ENV_FILE" + set +a + fi + env \ + TARGET_NODE="$operator_target_node" \ + TARGET_ADDRESS="$operator_target_address" \ + TARGET_SSH="$operator_target_ssh" \ + bash -lc "$command" + ) +} + +remote_target_action() { + local action="$1" + local ssh_target + ssh_target="$(ssh_target_by_id "$TARGET_NODE")" + "$SSH_BIN" \ + -o BatchMode=yes \ + -o ConnectTimeout="$SSH_CONNECT_TIMEOUT_SECONDS" \ + -o StrictHostKeyChecking="$SSH_STRICT_HOST_KEY_CHECKING" \ + "$ssh_target" bash -s -- "$action" "$CONTAINER_NAME" "$DATA_DIR" "$STATE_ARCHIVE_PATH" "$REPLACEMENT_DATA_MODE" <<'REMOTE' +set -euo pipefail +action="$1" +container="$2" +data_dir="$3" +archive_path="$4" +data_mode="$5" + +sudo_cmd=() +if [[ "$(id -u)" -ne 0 ]]; then + sudo -n true + sudo_cmd=(sudo -n) +fi + +case "$action" in + stop) + docker stop "$container" >/dev/null 2>&1 || true + running="$(docker inspect --format '{{.State.Running}}' "$container" 2>/dev/null || echo false)" + [[ "$running" == "false" ]] + ;; + reset-data) + docker_running="$(docker inspect --format '{{.State.Running}}' "$container" 2>/dev/null || echo false)" + [[ "$docker_running" == "false" ]] + if [[ "$data_mode" == "archive" ]]; then + if [[ -e "$archive_path" && -e "$data_dir" ]]; then + echo "both data and archive paths exist: $data_dir $archive_path" >&2 + exit 1 + fi + if [[ -e "$data_dir" ]]; then + "${sudo_cmd[@]}" mv "$data_dir" "$archive_path" + fi + [[ -e "$archive_path" ]] || { echo "target data directory was absent and no archive exists" >&2; exit 1; } + else + if [[ -e "$data_dir" ]]; then + "${sudo_cmd[@]}" rm -rf --one-file-system "$data_dir" + fi + [[ ! -e "$data_dir" ]] + fi + ;; + *) + echo "unknown remote action: $action" >&2 + exit 1 + ;; +esac +REMOTE +} + +fence_target() { + case "$REPLACEMENT_FENCE_MODE" in + external) + log "running external fence for $TARGET_NODE" + run_operator_command "$REPLACEMENT_FENCE_COMMAND" + ;; + container) + log "stopping target container; host lifecycle must keep it fenced" + remote_target_action stop + ;; + esac + wait_target_down || fail "target Raft RPC is still reachable after fencing" + log "fence verified: $STATE_TARGET_ADDRESS is unreachable" +} + +config_index_from_leader() { + local leader address ready + leader="$(wait_for_leader)" || return 1 + address="${leader#*|}" + ready="$(leader_status_ready "$address")" || return 1 + printf '%s\n' "${ready%%|*}" +} + +wait_member_state() { + local expected="$1" + local elapsed=0 leader address status config actual pending + while (( elapsed < REPLACEMENT_TIMEOUT_SECONDS )); do + leader="$(discover_leader || true)" + if [[ -n "$leader" ]]; then + address="${leader#*|}" + status="$(status_for "$address" || true)" + pending="$(printf '%s\n' "$status" | field_value pending_conf_change)" + config="$(raftadmin "$address" configuration 2>/dev/null || true)" + actual="$(member_suffrage "$TARGET_NODE" "$config" || true)" + if [[ "$pending" == "false" ]]; then + case "$expected" in + absent) [[ -z "$actual" ]] && return 0 ;; + *) [[ "$actual" == "$expected" ]] && return 0 ;; + esac + fi + fi + sleep "$REPLACEMENT_POLL_SECONDS" + elapsed=$((elapsed + REPLACEMENT_POLL_SECONDS)) + done + return 1 +} + +wait_configuration_index() { + local minimum="$1" + local elapsed=0 leader address status current pending + while (( elapsed < REPLACEMENT_TIMEOUT_SECONDS )); do + leader="$(discover_leader || true)" + if [[ -n "$leader" ]]; then + address="${leader#*|}" + status="$(status_for "$address" || true)" + current="$(printf '%s\n' "$status" | field_value configuration_index)" + pending="$(printf '%s\n' "$status" | field_value pending_conf_change)" + if is_uint "$current" && (( current >= minimum )) && [[ "$pending" == "false" ]]; then + return 0 + fi + fi + sleep "$REPLACEMENT_POLL_SECONDS" + elapsed=$((elapsed + REPLACEMENT_POLL_SECONDS)) + done + return 1 +} + +remove_target() { + local config current leader address config_index output change_index + config="$(configuration_from_leader)" || fail "cannot read configuration before removal" + current="$(member_suffrage "$TARGET_NODE" "$config" || true)" + if [[ -z "$current" ]]; then + log "target is already absent; resuming after committed removal" + return 0 + fi + [[ "$current" == "voter" ]] || fail "target changed from voter to $current before removal" + ! status_for "$STATE_TARGET_ADDRESS" >/dev/null || fail "target returned after fencing" + leader="$(wait_for_leader)" || fail "no leader available for removal" + address="${leader#*|}" + config_index="$(config_index_from_leader)" || fail "leader is not ready for removal" + log "removing $TARGET_NODE at configuration index $config_index" + output="$(raftadmin "$address" remove_server "$TARGET_NODE" "$config_index")" + change_index="$(printf '%s\n' "$output" | field_value index)" + if ! is_uint "$change_index" || (( change_index <= config_index )); then + fail "remove_server returned an invalid configuration index" + fi + wait_member_state absent || fail "remove_server did not commit" + wait_configuration_index "$change_index" || fail "leader status did not publish committed removal index $change_index" +} + +run_rollout() { + local join_node="$1" + local rollout_target="$TARGET_NODE" + ( + if [[ -n "${ROLLING_UPDATE_ENV_FILE:-}" ]]; then + set -a + # shellcheck disable=SC1090 + source "$ROLLING_UPDATE_ENV_FILE" + set +a + fi + unset ROLLING_UPDATE_ENV_FILE RAFT_BOOTSTRAP_MEMBERS + export ROLLING_ORDER="$rollout_target" + export RAFT_JOIN_NODE="$join_node" + exec "$ROLLING_UPDATE_SCRIPT" + ) +} + +add_learner() { + local config current leader address status commit config_index output change_index + config="$(configuration_from_leader)" || fail "cannot read configuration before learner add" + current="$(member_suffrage "$TARGET_NODE" "$config" || true)" + if [[ "$current" == "learner" ]]; then + (( STATE_MIN_CATCH_UP_INDEX > 0 )) || fail "learner committed but catch-up floor is missing from replacement state" + log "target is already a learner; resuming after committed add" + return 0 + fi + [[ -z "$current" ]] || fail "target unexpectedly has suffrage $current before learner add" + wait_target_up || fail "join deployment did not expose the target RaftAdmin endpoint" + leader="$(wait_for_leader)" || fail "no leader available for learner add" + address="${leader#*|}" + status="$(status_for "$address")" || fail "cannot read leader status before learner add" + commit="$(printf '%s\n' "$status" | field_value commit_index)" + config_index="$(printf '%s\n' "$status" | field_value configuration_index)" + [[ "$(printf '%s\n' "$status" | field_value pending_conf_change)" == "false" ]] || fail "a configuration change is pending" + if ! is_uint "$commit" || ! is_uint "$config_index" || (( config_index == 0 )); then + fail "invalid leader indexes before learner add" + fi + STATE_MIN_CATCH_UP_INDEX="$commit" + # Persist the floor before proposing. If the RPC commits but the control + # process exits before advancing its stage, promotion still has a durable, + # non-zero catch-up bound on resume. + write_state + log "adding learner $TARGET_NODE at configuration index $config_index; catch-up floor=$commit" + output="$(raftadmin "$address" add_learner "$TARGET_NODE" "$STATE_TARGET_ADDRESS" "$config_index")" + change_index="$(printf '%s\n' "$output" | field_value index)" + if ! is_uint "$change_index" || (( change_index <= config_index )); then + fail "add_learner returned an invalid configuration index" + fi + wait_member_state learner || fail "add_learner did not commit" + wait_configuration_index "$change_index" || fail "leader status did not publish committed learner index $change_index" +} + +wait_for_catch_up() { + local elapsed=0 status state leader_id applied + while (( elapsed < REPLACEMENT_TIMEOUT_SECONDS )); do + status="$(status_for "$STATE_TARGET_ADDRESS" || true)" + state="$(printf '%s\n' "$status" | field_value state)" + leader_id="$(printf '%s\n' "$status" | field_value leader_id)" + applied="$(printf '%s\n' "$status" | field_value applied_index)" + if [[ "$state" == "FOLLOWER" && -n "$leader_id" ]] && is_uint "$applied" && (( applied >= STATE_MIN_CATCH_UP_INDEX )); then + log "learner caught up: applied_index=$applied floor=$STATE_MIN_CATCH_UP_INDEX" + return 0 + fi + sleep "$REPLACEMENT_POLL_SECONDS" + elapsed=$((elapsed + REPLACEMENT_POLL_SECONDS)) + done + return 1 +} + +promote_learner() { + local config current leader address config_index output change_index + config="$(configuration_from_leader)" || fail "cannot read configuration before promotion" + current="$(member_suffrage "$TARGET_NODE" "$config" || true)" + if [[ "$current" == "voter" ]]; then + log "target is already a voter; resuming after committed promotion" + return 0 + fi + [[ "$current" == "learner" ]] || fail "target must be a learner before promotion; got ${current:-absent}" + (( STATE_MIN_CATCH_UP_INDEX > 0 )) || fail "catch-up floor is missing from replacement state" + wait_for_catch_up || fail "learner did not reach catch-up floor" + leader="$(wait_for_leader)" || fail "no leader available for promotion" + address="${leader#*|}" + config_index="$(config_index_from_leader)" || fail "leader is not ready for promotion" + log "promoting $TARGET_NODE at configuration index $config_index" + output="$(raftadmin "$address" promote_learner "$TARGET_NODE" "$config_index" "$STATE_MIN_CATCH_UP_INDEX" false)" + change_index="$(printf '%s\n' "$output" | field_value index)" + if ! is_uint "$change_index" || (( change_index <= config_index )); then + fail "promote_learner returned an invalid configuration index" + fi + wait_member_state voter || fail "promote_learner did not commit" + wait_configuration_index "$change_index" || fail "leader status did not publish committed promotion index $change_index" +} + +verify_cluster_once() { + local leader leader_id address status pending config voters id member_address suffrage member_status state member_leader applied commit + leader="$(discover_leader)" || return 1 + leader_id="${leader%%|*}" + address="${leader#*|}" + status="$(status_for "$address")" || return 1 + pending="$(printf '%s\n' "$status" | field_value pending_conf_change)" + [[ "$pending" == "false" ]] || return 1 + config="$(raftadmin "$address" configuration 2>/dev/null)" || return 1 + voters="$(voter_csv "$config")" + same_csv_set "$voters" "$STATE_EXPECTED_VOTERS" || return 1 + while IFS='|' read -r id member_address suffrage; do + [[ "$suffrage" == "voter" ]] || continue + member_status="$(status_for "$member_address" || true)" + [[ -n "$member_status" ]] || return 1 + state="$(printf '%s\n' "$member_status" | field_value state)" + member_leader="$(printf '%s\n' "$member_status" | field_value leader_id)" + if [[ "$state" == "LEADER" ]]; then + [[ "$id" == "$leader_id" ]] || return 1 + else + [[ "$state" == "FOLLOWER" && "$member_leader" == "$leader_id" ]] || return 1 + fi + applied="$(printf '%s\n' "$member_status" | field_value applied_index)" + commit="$(printf '%s\n' "$member_status" | field_value commit_index)" + is_uint "$applied" && is_uint "$commit" && (( applied >= commit )) || return 1 + done < <(printf '%s\n' "$config" | configuration_rows) + printf '%s\n' "${leader%%|*}" +} + +verify_stability() { + local elapsed=0 stable=0 expected_leader="" leader + while (( elapsed < REPLACEMENT_TIMEOUT_SECONDS )); do + leader="$(verify_cluster_once || true)" + if [[ -n "$leader" && ( -z "$expected_leader" || "$leader" == "$expected_leader" ) ]]; then + expected_leader="$leader" + stable=$((stable + REPLACEMENT_POLL_SECONDS)) + if (( stable >= REPLACEMENT_STABILITY_SECONDS )); then + log "cluster stable for ${stable}s with leader=$leader" + return 0 + fi + else + expected_leader="$leader" + stable=0 + fi + sleep "$REPLACEMENT_POLL_SECONDS" + elapsed=$((elapsed + REPLACEMENT_POLL_SECONDS)) + done + return 1 +} + +state_value_valid() { + [[ "$1" =~ ^[A-Za-z0-9_./,:@-]*$ ]] +} + +write_state() { + local tmp value + for value in "$STATE_STAGE" "$STATE_TARGET" "$STATE_TARGET_ADDRESS" "$STATE_OPERATION_ID" \ + "$STATE_ARCHIVE_PATH" "$STATE_EXPECTED_VOTERS" "$STATE_MIN_CATCH_UP_INDEX" "$STATE_DATA_MODE"; do + state_value_valid "$value" || fail "replacement state contains an unsupported value" + done + mkdir -p "$(dirname "$REPLACEMENT_STATE_FILE")" + tmp="${REPLACEMENT_STATE_FILE}.tmp.$$" + { + printf 'stage=%s\n' "$STATE_STAGE" + printf 'target=%s\n' "$STATE_TARGET" + printf 'target_address=%s\n' "$STATE_TARGET_ADDRESS" + printf 'operation_id=%s\n' "$STATE_OPERATION_ID" + printf 'archive_path=%s\n' "$STATE_ARCHIVE_PATH" + printf 'expected_voters=%s\n' "$STATE_EXPECTED_VOTERS" + printf 'min_catch_up_index=%s\n' "$STATE_MIN_CATCH_UP_INDEX" + printf 'data_mode=%s\n' "$STATE_DATA_MODE" + } > "$tmp" + chmod 600 "$tmp" + mv "$tmp" "$REPLACEMENT_STATE_FILE" +} + +load_state() { + local key value + [[ -f "$REPLACEMENT_STATE_FILE" ]] || return 1 + while IFS='=' read -r key value; do + state_value_valid "$value" || fail "invalid value in state file for $key" + case "$key" in + stage) STATE_STAGE="$value" ;; + target) STATE_TARGET="$value" ;; + target_address) STATE_TARGET_ADDRESS="$value" ;; + operation_id) STATE_OPERATION_ID="$value" ;; + archive_path) STATE_ARCHIVE_PATH="$value" ;; + expected_voters) STATE_EXPECTED_VOTERS="$value" ;; + min_catch_up_index) STATE_MIN_CATCH_UP_INDEX="$value" ;; + data_mode) STATE_DATA_MODE="$value" ;; + "") ;; + *) fail "unknown key in state file: $key" ;; + esac + done < "$REPLACEMENT_STATE_FILE" + if ! is_uint "$STATE_STAGE" || ! is_uint "$STATE_MIN_CATCH_UP_INDEX"; then + fail "invalid numeric value in state file" + fi + (( STATE_STAGE <= 9 )) || fail "state stage is outside the implemented range: $STATE_STAGE" + [[ "$STATE_TARGET" == "$TARGET_NODE" ]] || fail "state file belongs to target $STATE_TARGET, not $TARGET_NODE" + [[ "$STATE_TARGET_ADDRESS" == "$(node_host_by_id "$TARGET_NODE"):${RAFT_PORT}" ]] || fail "state target address no longer matches NODES" + [[ -n "$STATE_OPERATION_ID" && -n "$STATE_ARCHIVE_PATH" && -n "$STATE_EXPECTED_VOTERS" ]] || fail "state file is incomplete" + [[ "$STATE_DATA_MODE" == "$REPLACEMENT_DATA_MODE" ]] || fail "REPLACEMENT_DATA_MODE changed from $STATE_DATA_MODE to $REPLACEMENT_DATA_MODE" + log "resuming operation $STATE_OPERATION_ID at stage $STATE_STAGE" +} + +advance() { + STATE_STAGE="$1" + write_state +} + +validate_settings +parse_nodes +TARGET_HOST="$(node_host_by_id "$TARGET_NODE")" +REPLACEMENT_STATE_FILE="${REPLACEMENT_STATE_FILE:-${REPO_ROOT}/.state/raft-member-replacement-${TARGET_NODE}.state}" + +if [[ "$MODE" == "dry-run" ]]; then + cat </dev/null || fail "another replacement process holds $LOCK_DIR" +LOCK_ACQUIRED=true +ensure_raftadmin + +if ! load_state; then + STATE_TARGET="$TARGET_NODE" + STATE_TARGET_ADDRESS="${TARGET_HOST}:${RAFT_PORT}" + STATE_OPERATION_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$" + STATE_ARCHIVE_PATH="${DATA_DIR}.replacement-${TARGET_NODE}-${STATE_OPERATION_ID}" + STATE_DATA_MODE="$REPLACEMENT_DATA_MODE" + preflight + advance 1 +fi + +if (( STATE_STAGE < 1 )); then + preflight + advance 1 +fi +if (( STATE_STAGE < 2 )); then + fence_target + advance 2 +elif (( STATE_STAGE < 5 )); then + wait_target_down || fail "fenced target is reachable again before removal" +fi +if (( STATE_STAGE < 3 )); then + remove_target + advance 3 +fi +if (( STATE_STAGE < 4 )); then + wait_member_state absent || fail "target membership returned before data reset" + log "resetting only $TARGET_NODE data at $DATA_DIR" + remote_target_action reset-data + advance 4 +fi +if (( STATE_STAGE < 5 )); then + wait_member_state absent || fail "target membership returned before join deployment" + log "deploying $TARGET_NODE in learner join mode" + run_rollout "$TARGET_NODE" + advance 5 +fi +if (( STATE_STAGE < 6 )); then + add_learner + advance 6 +fi +if (( STATE_STAGE < 7 )); then + promote_learner + advance 7 +fi +if (( STATE_STAGE < 8 )); then + log "redeploying $TARGET_NODE from durable membership without join flags" + run_rollout "" + advance 8 +fi +if (( STATE_STAGE < 9 )); then + verify_stability || fail "cluster did not reach a stable fully-applied voter configuration" + log "running application write/read verification" + run_operator_command "$REPLACEMENT_VERIFY_COMMAND" + advance 9 +fi + +log "replacement completed: target=$TARGET_NODE operation=$STATE_OPERATION_ID" +if [[ "$REPLACEMENT_KEEP_STATE" != "true" ]]; then + rm -f "$REPLACEMENT_STATE_FILE" +fi diff --git a/scripts/rolling-update.env.example b/scripts/rolling-update.env.example index a3864a649..ed91e79d8 100644 --- a/scripts/rolling-update.env.example +++ b/scripts/rolling-update.env.example @@ -27,6 +27,21 @@ RAFT_ENGINE="etcd" # ROLLING_ORDER="n2" # RAFT_JOIN_NODE="n2" +# Fenced same-ID replacement wrapper. Keep these commented during normal +# rolling updates. TARGET_NODE and REPLACEMENT_CONFIRM must match exactly. +# External fencing is the default and must prevent the old VM/process from +# returning to the Raft network. Container mode is only suitable when host +# lifecycle policy guarantees the stopped container cannot restart. +# TARGET_NODE="n2" +# REPLACEMENT_CONFIRM="n2" +# REPLACEMENT_FENCE_MODE="external" +# REPLACEMENT_FENCE_COMMAND="fence-n2-and-block-raft" +# REPLACEMENT_FENCE_VERIFY_SECONDS="3" +# REPLACEMENT_DATA_MODE="archive" +# REPLACEMENT_STABILITY_SECONDS="10" +# REPLACEMENT_TIMEOUT_SECONDS="120" +# REPLACEMENT_VERIFY_COMMAND="redis-cli -h proxy.internal.example SET replacement-check ok && redis-cli -h proxy.internal.example GET replacement-check | grep -Fx ok" + RAFT_PORT="50051" REDIS_PORT="6379" DYNAMO_PORT="8000" From 55d863c1e2e4bcf501b5aba19dd90db7706c9514 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 00:05:10 +0900 Subject: [PATCH 2/6] Allow direct target data access without sudo --- scripts/raft-member-replace.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/raft-member-replace.sh b/scripts/raft-member-replace.sh index 06f73c0a4..5398af0cc 100755 --- a/scripts/raft-member-replace.sh +++ b/scripts/raft-member-replace.sh @@ -582,8 +582,7 @@ archive_path="$4" data_mode="$5" sudo_cmd=() -if [[ "$(id -u)" -ne 0 ]]; then - sudo -n true +if [[ "$(id -u)" -ne 0 ]] && command -v sudo >/dev/null 2>&1 && sudo -n true 2>/dev/null; then sudo_cmd=(sudo -n) fi From bc84f3e587a39654db5bff67300e6ed7ad977d71 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 00:49:31 +0900 Subject: [PATCH 3/6] Harden Raft member replacement state --- ...lemented_fenced_raft_member_replacement.md | 48 ++- docs/raft_learner_operations.md | 11 +- raft_member_replace_script_test.go | 107 ++++++- scripts/raft-member-replace.sh | 279 ++++++++++++++---- scripts/rolling-update.env.example | 18 +- 5 files changed, 368 insertions(+), 95 deletions(-) diff --git a/docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md b/docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md index e49ec4f30..a7641d14c 100644 --- a/docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md +++ b/docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md @@ -61,6 +61,11 @@ choice of authoritative log and accepted data-loss boundary. `--execute` is required and `REPLACEMENT_CONFIRM` must exactly equal `TARGET_NODE`. `--dry-run` performs no build, RPC, SSH, fence, or deployment. +For `--execute`, `TARGET_NODE`, `REPLACEMENT_CONFIRM`, +`REPLACEMENT_VERIFY_COMMAND`, and the external fence command must be supplied +on the invocation. Replacement controls are never read from the deployment env; +that file is inventory only, so a stale recovery value cannot authorize a new +operation. The target must be present in `NODES`, be a current voter, and have exactly the advertised `NODES` address. A learner, absent ID, or address mismatch aborts. @@ -96,10 +101,15 @@ RaftAdmin status now reports: - `configuration_index`, the current durable membership index; and - `pending_conf_change`, whether an unapplied change is in flight. -Every `remove_server`, `add_learner`, and `promote_learner` call passes the -current non-zero configuration index as `previous_index`. The engine rejects a -concurrent topology change instead of applying a stale plan. The command waits -for the requested suffrage and a clear pending-change flag before advancing. +Preflight records both the complete member signature and its non-zero +configuration index. Every `remove_server`, `add_learner`, and +`promote_learner` call passes that recorded index as `previous_index`; it never +adopts a newer index merely because another topology change committed. After +each successful change, the command verifies the exact expected post-change +signature before persisting the returned index for the next stage. The engine +therefore rejects a concurrent topology change instead of applying a stale +plan. A resume after a lost RPC response is accepted only when the complete +member signature matches the one transition that was in flight. ### 4.5 Catch-up before promotion @@ -132,7 +142,7 @@ same state file. | Stage | Durable fact | | --- | --- | -| 0 | Target identity, address, original voter set, operation ID, and archive path recorded | +| 0 | Target identity, address, complete member signature and configuration index, original voter set, operation ID, and archive path recorded | | 1 | Preflight and any leadership transfer completed | | 2 | Old ID owner fenced and endpoint observed down | | 3 | Target absent from committed membership | @@ -150,9 +160,11 @@ committed learner only when its catch-up floor exists, and stage 6 accepts an already promoted voter. A conflicting suffrage or voter set aborts instead of guessing what happened. -The state file is retained by default as operational evidence. Set -`REPLACEMENT_KEEP_STATE=false` only when another durable audit system records -the completed operation. +The state file is retained by default as operational evidence. A retained +stage-9 file is never treated as authorization for a later incident: a new +replacement must archive that evidence or select a new +`REPLACEMENT_STATE_FILE`. Set `REPLACEMENT_KEEP_STATE=false` only when another +durable audit system records the completed operation. ## 6. Data handling @@ -161,6 +173,10 @@ target ID and operation ID, making a retry distinguishable from a new replacement. If both the live data path and archive path exist, the command fails rather than overwriting either. +Trailing slashes are removed from `DATA_DIR` before the sibling archive path is +derived, and the filesystem root is rejected. The archive can therefore never +be constructed as a child of the directory being moved. + Delete mode must be explicitly selected. Both modes execute only after the ID is absent from committed membership and verify the target container is not running. The command never iterates over multiple data hosts. @@ -196,7 +212,9 @@ REPLACEMENT_FENCE_MODE=container The replacement command sources the deployment env for inventory and then invokes `rolling-update.sh` twice with only `TARGET_NODE` selected. Bootstrap membership is removed from both invocations. The first deploy includes -`RAFT_JOIN_NODE`; the post-promotion deploy clears it. +`RAFT_JOIN_NODE`; the post-promotion deploy clears it. Both invocations force +`DRY_RUN=false`, so a stale deployment-env dry-run setting cannot advance the +replacement state without starting the target. ## 8. Failure handling @@ -204,7 +222,8 @@ membership is removed from both invocations. The first deploy includes | --- | --- | | No leader or insufficient surviving quorum | Stop. Restore an existing voter or use a separately reviewed disaster-recovery procedure. | | Target endpoint remains reachable after fence | Strengthen the fence. Do not remove membership. | -| `previous_index` mismatch | Inspect the concurrent topology change and reconcile the state file with the authoritative configuration. Do not retry blindly. | +| Recorded member signature or `previous_index` mismatch | Inspect the concurrent topology change and reconcile the state file with the authoritative configuration. Do not retry blindly. | +| Existing state is already at stage 9 | Archive the completed evidence or choose a new state-file path before authorizing a new replacement. | | Data and deterministic archive both exist | Inspect the target host. Do not delete either path automatically. | | Learner catch-up timeout | Leave it a learner, diagnose transport/disk/snapshot transfer, and resume with the same state file. | | Promotion committed but normal restart fails | Keep the surviving voters running. Repair the target from its existing post-join data; do not wipe it again. | @@ -216,8 +235,8 @@ membership is removed from both invocations. The first deploy includes `raft_member_replace_script_test.go` runs the complete shell workflow against a stateful fake RaftAdmin, SSH transport, and rolling deploy command. It verifies the exact voter-to-absent-to-learner-to-voter sequence, non-zero catch-up floor, -both deploy modes, application verification, durable stage 9, and an idempotent -completed resume. +both deploy modes, forced execute rollout, application verification, durable +stage 9, and rejection of a completed state as a new operation. A negative test makes one surviving voter unreachable in a three-voter group and proves the command aborts before the fence action. Another test loses the @@ -229,6 +248,11 @@ index and pending change fields. Existing engine membership tests cover stale previous indexes, learner catch-up enforcement, same-ID replacement, and persisted membership restart. +Additional negative tests prove that deployment-env replacement controls +cannot authorize an operation, a configuration-index change after preflight +aborts before removal, invocation-level timing controls override stale env +values, and a trailing-slash data path produces a sibling archive. + Production acceptance still requires recording the state file, fence evidence, final `raftadmin status`/`configuration`, and the output of the configured application verification command. diff --git a/docs/raft_learner_operations.md b/docs/raft_learner_operations.md index f4d4e354f..3f75c1b8f 100644 --- a/docs/raft_learner_operations.md +++ b/docs/raft_learner_operations.md @@ -246,10 +246,15 @@ REPLACEMENT_VERIFY_COMMAND='verify-a-real-write-and-read' \ ``` Run the same command with `--dry-run` first. The wrapper verifies surviving -quorum before fencing, passes `configuration_index` to every membership RPC, -records a non-zero learner catch-up floor, restarts without join flags, and -requires the application verification command to succeed. It supports one +quorum before fencing, records the complete member signature and +`configuration_index`, records a non-zero learner catch-up floor, restarts +without join flags, and requires the application verification command to +succeed. Any unrelated membership change aborts the operation. It supports one Raft group with a surviving quorum; it is not a forced no-quorum recovery tool. +Supply `TARGET_NODE` and all `REPLACEMENT_*` values on the invocation, not in +`deploy.env`; the deployment env is inventory only. A retained completed state +file is audit evidence, so archive it or choose a new state path for a later +incident. See `docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md` for the state machine, fencing contract, and failure handling. diff --git a/raft_member_replace_script_test.go b/raft_member_replace_script_test.go index 2f2bd5b45..6d977964e 100644 --- a/raft_member_replace_script_test.go +++ b/raft_member_replace_script_test.go @@ -28,10 +28,16 @@ func TestRaftMemberReplaceScriptCompletesFencedLearnerLifecycle(t *testing.T) { "RAFT_PORT=50051", "SSH_USER=tester", "CONTAINER_NAME=elastickv", - "DATA_DIR=/var/lib/elastickv", + "DATA_DIR=/var/lib/elastickv/", "TARGET_NODE=n3", "REPLACEMENT_CONFIRM=n3", "REPLACEMENT_VERIFY_COMMAND=false", + "REPLACEMENT_FENCE_VERIFY_SECONDS=999", + "REPLACEMENT_STABILITY_SECONDS=999", + "REPLACEMENT_TIMEOUT_SECONDS=999", + "REPLACEMENT_POLL_SECONDS=999", + "REPLACEMENT_KEEP_STATE=false", + "DRY_RUN=true", }, "\n")+"\n"), 0o600)) stateFile := filepath.Join(stateDir, "replace.state") @@ -60,10 +66,16 @@ func TestRaftMemberReplaceScriptCompletesFencedLearnerLifecycle(t *testing.T) { state := mustReadTestFile(t, stateFile) require.Contains(t, state, "stage=9\n") require.Contains(t, state, "expected_voters=n1,n2,n3\n") + require.Contains(t, state, "expected_members=n1|127.0.0.1:50051|voter,n2|127.0.0.2:50051|voter,n3|127.0.0.3:50051|voter\n") + require.Contains(t, state, "expected_config_index=13\n") require.Contains(t, state, "min_catch_up_index=100\n") require.Contains(t, state, "data_mode=archive\n") + require.Contains(t, state, "archive_path=/var/lib/elastickv.replacement-n2-") + require.NotContains(t, state, "archive_path=/var/lib/elastickv/.replacement") - resumeOutput := runReplacementScript(t, repoRoot, []string{ + resumeCmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") + resumeCmd.Dir = repoRoot + resumeCmd.Env = append(os.Environ(), "TARGET_NODE=n2", "REPLACEMENT_CONFIRM=n2", "REPLACEMENT_FENCE_MODE=container", @@ -72,17 +84,83 @@ func TestRaftMemberReplaceScriptCompletesFencedLearnerLifecycle(t *testing.T) { "REPLACEMENT_STABILITY_SECONDS=1", "REPLACEMENT_TIMEOUT_SECONDS=5", "REPLACEMENT_POLL_SECONDS=1", - "REPLACEMENT_STATE_FILE=" + stateFile, - "ROLLING_UPDATE_ENV_FILE=" + envFile, - "ROLLING_UPDATE_SCRIPT=" + rolling, - "RAFTADMIN_BIN=" + raftadmin, - "SSH_BIN=" + ssh, - "FAKE_STATE_DIR=" + stateDir, - }) - require.Contains(t, resumeOutput, "resuming operation") + "REPLACEMENT_STATE_FILE="+stateFile, + "ROLLING_UPDATE_ENV_FILE="+envFile, + "ROLLING_UPDATE_SCRIPT="+rolling, + "RAFTADMIN_BIN="+raftadmin, + "SSH_BIN="+ssh, + "FAKE_STATE_DIR="+stateDir, + ) + resumeOutput, resumeErr := resumeCmd.CombinedOutput() + require.Error(t, resumeErr) + require.Contains(t, string(resumeOutput), "replacement state is already complete") require.Equal(t, "n2:n2\nnormal:n2\n", mustReadTestFile(t, filepath.Join(stateDir, "rollouts")), "completed resume must not repeat deployment") } +func TestRaftMemberReplaceScriptRequiresInvocationControls(t *testing.T) { + repoRoot, err := os.Getwd() + require.NoError(t, err) + stateDir := t.TempDir() + rolling := writeExecutableTestFile(t, stateDir, "rolling-update", "#!/usr/bin/env bash\nexit 0\n") + envFile := filepath.Join(stateDir, "deploy.env") + require.NoError(t, os.WriteFile(envFile, []byte(strings.Join([]string{ + "NODES=n1=127.0.0.1,n2=127.0.0.2,n3=127.0.0.3", + "TARGET_NODE=n2", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=container", + "REPLACEMENT_VERIFY_COMMAND=true", + }, "\n")+"\n"), 0o600)) + + cmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") + cmd.Dir = repoRoot + cmd.Env = append(os.Environ(), + "ROLLING_UPDATE_ENV_FILE="+envFile, + "ROLLING_UPDATE_SCRIPT="+rolling, + ) + output, err := cmd.CombinedOutput() + require.Error(t, err) + require.Contains(t, string(output), "TARGET_NODE is required") +} + +func TestRaftMemberReplaceScriptRejectsConfigChangeAfterPreflight(t *testing.T) { + repoRoot, err := os.Getwd() + require.NoError(t, err) + stateDir := t.TempDir() + fakeBin := filepath.Join(stateDir, "bin") + require.NoError(t, os.Mkdir(fakeBin, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "member"), []byte("voter\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "config-index"), []byte("10\n"), 0o600)) + + raftadmin := writeExecutableTestFile(t, fakeBin, "raftadmin", fakeRaftadminScript) + ssh := writeExecutableTestFile(t, fakeBin, "ssh", fakeReplacementSSHScript) + rolling := writeExecutableTestFile(t, fakeBin, "rolling-update", fakeReplacementRollingScript) + envFile := filepath.Join(stateDir, "deploy.env") + require.NoError(t, os.WriteFile(envFile, []byte("NODES=n1=127.0.0.1,n2=127.0.0.2,n3=127.0.0.3\n"), 0o600)) + + cmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") + cmd.Dir = repoRoot + cmd.Env = append(os.Environ(), + "TARGET_NODE=n2", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=container", + "REPLACEMENT_FENCE_VERIFY_SECONDS=1", + "REPLACEMENT_VERIFY_COMMAND=true", + "REPLACEMENT_TIMEOUT_SECONDS=5", + "REPLACEMENT_POLL_SECONDS=1", + "REPLACEMENT_STATE_FILE="+filepath.Join(stateDir, "replace.state"), + "ROLLING_UPDATE_ENV_FILE="+envFile, + "ROLLING_UPDATE_SCRIPT="+rolling, + "RAFTADMIN_BIN="+raftadmin, + "SSH_BIN="+ssh, + "FAKE_STATE_DIR="+stateDir, + "FAKE_BUMP_CONFIG_ON_FENCE=true", + ) + output, err := cmd.CombinedOutput() + require.Error(t, err) + require.Contains(t, string(output), "remove_server configuration index changed: expected=10 actual=11") + require.Equal(t, "voter\n", mustReadTestFile(t, filepath.Join(stateDir, "member"))) +} + func TestRaftMemberReplaceScriptRejectsInsufficientSurvivingQuorum(t *testing.T) { repoRoot, err := os.Getwd() require.NoError(t, err) @@ -320,7 +398,13 @@ while [[ "$#" -gt 0 && "$1" != "--" ]]; do shift; done [[ "$#" -ge 2 ]] action="$2" case "$action" in - stop) touch "$FAKE_STATE_DIR/down-n2" ;; + stop) + touch "$FAKE_STATE_DIR/down-n2" + if [[ "${FAKE_BUMP_CONFIG_ON_FENCE:-false}" == "true" ]]; then + current="$(cat "$FAKE_STATE_DIR/config-index")" + printf '%s\n' "$((current + 1))" > "$FAKE_STATE_DIR/config-index" + fi + ;; reset-data) touch "$FAKE_STATE_DIR/data-reset" ;; *) echo "unexpected SSH action: $action" >&2; exit 1 ;; esac @@ -328,6 +412,7 @@ esac const fakeReplacementRollingScript = `#!/usr/bin/env bash set -euo pipefail +[[ "${DRY_RUN:-}" == "false" ]] printf '%s:%s\n' "${RAFT_JOIN_NODE:-normal}" "$ROLLING_ORDER" >> "$FAKE_STATE_DIR/rollouts" if [[ "${RAFT_JOIN_NODE:-}" == "n2" ]]; then rm -f "$FAKE_STATE_DIR/down-n2" diff --git a/scripts/raft-member-replace.sh b/scripts/raft-member-replace.sh index 5398af0cc..fbe9918b0 100755 --- a/scripts/raft-member-replace.sh +++ b/scripts/raft-member-replace.sh @@ -39,6 +39,8 @@ Fencing: Optional environment: ROLLING_UPDATE_ENV_FILE Env file used by rolling-update.sh. + It supplies inventory only; replacement + controls from this file are ignored. ROLLING_UPDATE_SCRIPT Default: scripts/rolling-update.sh. RAFTADMIN_BIN Default: build ./cmd/raftadmin locally. SSH_BIN Default: ssh. @@ -62,6 +64,8 @@ Optional environment: The state file makes the sequence resumable. A resumed run validates the live membership against its recorded stage and fails closed on conflicting changes. +An existing completed state is rejected; archive it or choose a new state path +before authorizing a later replacement of the same node. No-quorum recovery and multi-group replacement are outside this command. EOF } @@ -113,6 +117,16 @@ INPUT_REPLACEMENT_FENCE_COMMAND_SET="${REPLACEMENT_FENCE_COMMAND+x}" INPUT_REPLACEMENT_FENCE_COMMAND="${REPLACEMENT_FENCE_COMMAND:-}" INPUT_REPLACEMENT_VERIFY_COMMAND_SET="${REPLACEMENT_VERIFY_COMMAND+x}" INPUT_REPLACEMENT_VERIFY_COMMAND="${REPLACEMENT_VERIFY_COMMAND:-}" +INPUT_REPLACEMENT_FENCE_VERIFY_SECONDS_SET="${REPLACEMENT_FENCE_VERIFY_SECONDS+x}" +INPUT_REPLACEMENT_FENCE_VERIFY_SECONDS="${REPLACEMENT_FENCE_VERIFY_SECONDS:-}" +INPUT_REPLACEMENT_STABILITY_SECONDS_SET="${REPLACEMENT_STABILITY_SECONDS+x}" +INPUT_REPLACEMENT_STABILITY_SECONDS="${REPLACEMENT_STABILITY_SECONDS:-}" +INPUT_REPLACEMENT_TIMEOUT_SECONDS_SET="${REPLACEMENT_TIMEOUT_SECONDS+x}" +INPUT_REPLACEMENT_TIMEOUT_SECONDS="${REPLACEMENT_TIMEOUT_SECONDS:-}" +INPUT_REPLACEMENT_POLL_SECONDS_SET="${REPLACEMENT_POLL_SECONDS+x}" +INPUT_REPLACEMENT_POLL_SECONDS="${REPLACEMENT_POLL_SECONDS:-}" +INPUT_REPLACEMENT_KEEP_STATE_SET="${REPLACEMENT_KEEP_STATE+x}" +INPUT_REPLACEMENT_KEEP_STATE="${REPLACEMENT_KEEP_STATE:-}" if [[ -n "${ROLLING_UPDATE_ENV_FILE:-}" ]]; then if [[ ! -f "$ROLLING_UPDATE_ENV_FILE" ]]; then @@ -123,13 +137,31 @@ if [[ -n "${ROLLING_UPDATE_ENV_FILE:-}" ]]; then source "$ROLLING_UPDATE_ENV_FILE" fi -[[ -z "$INPUT_TARGET_NODE_SET" ]] || TARGET_NODE="$INPUT_TARGET_NODE" -[[ -z "$INPUT_REPLACEMENT_CONFIRM_SET" ]] || REPLACEMENT_CONFIRM="$INPUT_REPLACEMENT_CONFIRM" -[[ -z "$INPUT_REPLACEMENT_STATE_FILE_SET" ]] || REPLACEMENT_STATE_FILE="$INPUT_REPLACEMENT_STATE_FILE" -[[ -z "$INPUT_REPLACEMENT_DATA_MODE_SET" ]] || REPLACEMENT_DATA_MODE="$INPUT_REPLACEMENT_DATA_MODE" -[[ -z "$INPUT_REPLACEMENT_FENCE_MODE_SET" ]] || REPLACEMENT_FENCE_MODE="$INPUT_REPLACEMENT_FENCE_MODE" -[[ -z "$INPUT_REPLACEMENT_FENCE_COMMAND_SET" ]] || REPLACEMENT_FENCE_COMMAND="$INPUT_REPLACEMENT_FENCE_COMMAND" -[[ -z "$INPUT_REPLACEMENT_VERIFY_COMMAND_SET" ]] || REPLACEMENT_VERIFY_COMMAND="$INPUT_REPLACEMENT_VERIFY_COMMAND" +restore_invocation_control() { + local name="$1" + local was_set="$2" + local value="$3" + if [[ -n "$was_set" ]]; then + printf -v "$name" '%s' "$value" + else + unset "$name" + fi +} + +# Replacement controls are never inherited from the deployment inventory. +# An unset invocation value therefore resolves to the script default below. +restore_invocation_control TARGET_NODE "$INPUT_TARGET_NODE_SET" "$INPUT_TARGET_NODE" +restore_invocation_control REPLACEMENT_CONFIRM "$INPUT_REPLACEMENT_CONFIRM_SET" "$INPUT_REPLACEMENT_CONFIRM" +restore_invocation_control REPLACEMENT_STATE_FILE "$INPUT_REPLACEMENT_STATE_FILE_SET" "$INPUT_REPLACEMENT_STATE_FILE" +restore_invocation_control REPLACEMENT_DATA_MODE "$INPUT_REPLACEMENT_DATA_MODE_SET" "$INPUT_REPLACEMENT_DATA_MODE" +restore_invocation_control REPLACEMENT_FENCE_MODE "$INPUT_REPLACEMENT_FENCE_MODE_SET" "$INPUT_REPLACEMENT_FENCE_MODE" +restore_invocation_control REPLACEMENT_FENCE_COMMAND "$INPUT_REPLACEMENT_FENCE_COMMAND_SET" "$INPUT_REPLACEMENT_FENCE_COMMAND" +restore_invocation_control REPLACEMENT_VERIFY_COMMAND "$INPUT_REPLACEMENT_VERIFY_COMMAND_SET" "$INPUT_REPLACEMENT_VERIFY_COMMAND" +restore_invocation_control REPLACEMENT_FENCE_VERIFY_SECONDS "$INPUT_REPLACEMENT_FENCE_VERIFY_SECONDS_SET" "$INPUT_REPLACEMENT_FENCE_VERIFY_SECONDS" +restore_invocation_control REPLACEMENT_STABILITY_SECONDS "$INPUT_REPLACEMENT_STABILITY_SECONDS_SET" "$INPUT_REPLACEMENT_STABILITY_SECONDS" +restore_invocation_control REPLACEMENT_TIMEOUT_SECONDS "$INPUT_REPLACEMENT_TIMEOUT_SECONDS_SET" "$INPUT_REPLACEMENT_TIMEOUT_SECONDS" +restore_invocation_control REPLACEMENT_POLL_SECONDS "$INPUT_REPLACEMENT_POLL_SECONDS_SET" "$INPUT_REPLACEMENT_POLL_SECONDS" +restore_invocation_control REPLACEMENT_KEEP_STATE "$INPUT_REPLACEMENT_KEEP_STATE_SET" "$INPUT_REPLACEMENT_KEEP_STATE" TARGET_NODE="${TARGET_NODE:-}" NODES="${NODES:-}" @@ -138,6 +170,9 @@ SSH_USER="${SSH_USER:-${USER:-$(id -un)}}" RAFT_PORT="${RAFT_PORT:-50051}" CONTAINER_NAME="${CONTAINER_NAME:-elastickv}" DATA_DIR="${DATA_DIR:-/var/lib/elastickv}" +while [[ "$DATA_DIR" != "/" && "$DATA_DIR" == */ ]]; do + DATA_DIR="${DATA_DIR%/}" +done RAFTADMIN_BIN="${RAFTADMIN_BIN:-}" RAFTADMIN_ALLOW_INSECURE="${RAFTADMIN_ALLOW_INSECURE:-true}" RAFTADMIN_RPC_TIMEOUT_SECONDS="${RAFTADMIN_RPC_TIMEOUT_SECONDS:-5}" @@ -164,8 +199,16 @@ STATE_TARGET_ADDRESS="" STATE_OPERATION_ID="" STATE_ARCHIVE_PATH="" STATE_EXPECTED_VOTERS="" +STATE_EXPECTED_MEMBERS="" +STATE_EXPECTED_CONFIG_INDEX=0 STATE_MIN_CATCH_UP_INDEX=0 STATE_DATA_MODE="" +VIEW_LEADER_ID="" +VIEW_LEADER_ADDRESS="" +VIEW_CONFIG_INDEX=0 +VIEW_COMMIT_INDEX=0 +VIEW_APPLIED_INDEX=0 +VIEW_CONFIGURATION="" RAFTADMIN_TMP_DIR="" LOCK_DIR="" LOCK_ACQUIRED=false @@ -188,6 +231,7 @@ validate_settings() { [[ "$TARGET_NODE" =~ ^[A-Za-z0-9._-]+$ ]] || fail "TARGET_NODE contains unsupported characters" [[ -n "$NODES" ]] || fail "NODES is required" [[ "$DATA_DIR" == /* ]] || fail "DATA_DIR must be absolute" + [[ "$DATA_DIR" != "/" ]] || fail "DATA_DIR must not be the filesystem root" state_value_valid "$DATA_DIR" || fail "DATA_DIR contains characters unsupported by the resume state" [[ -x "$ROLLING_UPDATE_SCRIPT" ]] || fail "ROLLING_UPDATE_SCRIPT is not executable: $ROLLING_UPDATE_SCRIPT" is_uint "$RAFT_PORT" || fail "RAFT_PORT must be an unsigned integer" @@ -212,8 +256,16 @@ validate_settings() { *) fail "REPLACEMENT_DATA_MODE must be archive or delete" ;; esac if [[ "$MODE" == "execute" ]]; then + [[ -n "$INPUT_TARGET_NODE_SET" && -n "$INPUT_TARGET_NODE" ]] || fail "TARGET_NODE must be supplied explicitly for --execute" + [[ -n "$INPUT_REPLACEMENT_CONFIRM_SET" ]] || fail "REPLACEMENT_CONFIRM must be supplied explicitly for --execute" + [[ -n "$INPUT_REPLACEMENT_VERIFY_COMMAND_SET" && -n "$INPUT_REPLACEMENT_VERIFY_COMMAND" ]] || \ + fail "REPLACEMENT_VERIFY_COMMAND must be supplied explicitly for --execute" [[ "$REPLACEMENT_CONFIRM" == "$TARGET_NODE" ]] || fail "REPLACEMENT_CONFIRM must exactly match TARGET_NODE" [[ -n "$REPLACEMENT_VERIFY_COMMAND" ]] || fail "REPLACEMENT_VERIFY_COMMAND is required for --execute" + if [[ "$REPLACEMENT_FENCE_MODE" == "external" ]]; then + [[ -n "$INPUT_REPLACEMENT_FENCE_COMMAND_SET" && -n "$INPUT_REPLACEMENT_FENCE_COMMAND" ]] || \ + fail "REPLACEMENT_FENCE_COMMAND must be supplied explicitly for external fencing" + fi fi } @@ -338,10 +390,99 @@ configuration_rows() { ' } +configuration_signature() { + local config="$1" + printf '%s\n' "$config" | configuration_rows | LC_ALL=C sort | \ + awk 'NF { if (result != "") result=result ","; result=result $0 } END { print result }' +} + +configuration_signature_with_target() { + local signature="$1" + local desired="$2" + local entry id + local -a entries=() + { + IFS=',' read -r -a entries <<< "$signature" + for entry in "${entries[@]}"; do + [[ -n "$entry" ]] || continue + id="${entry%%|*}" + [[ "$id" == "$TARGET_NODE" ]] || printf '%s\n' "$entry" + done + if [[ "$desired" != "absent" ]]; then + printf '%s|%s|%s\n' "$TARGET_NODE" "$STATE_TARGET_ADDRESS" "$desired" + fi + } | LC_ALL=C sort | awk 'NF { if (result != "") result=result ","; result=result $0 } END { print result }' +} + status_for() { raftadmin "$1" status 2>/dev/null } +load_leader_view() { + local leader before_ready after_ready before_index after_index + local after_commit after_applied + local before_status after_status + + leader="$(wait_for_leader)" || return 1 + VIEW_LEADER_ID="${leader%%|*}" + VIEW_LEADER_ADDRESS="${leader#*|}" + before_status="$(status_for "$VIEW_LEADER_ADDRESS")" || return 1 + [[ "$(printf '%s\n' "$before_status" | field_value state)" == "LEADER" ]] || return 1 + before_ready="$(status_ready_fields "$before_status")" || return 1 + before_index="${before_ready%%|*}" + VIEW_CONFIGURATION="$(raftadmin "$VIEW_LEADER_ADDRESS" configuration)" || return 1 + after_status="$(status_for "$VIEW_LEADER_ADDRESS")" || return 1 + [[ "$(printf '%s\n' "$after_status" | field_value state)" == "LEADER" ]] || return 1 + after_ready="$(status_ready_fields "$after_status")" || return 1 + IFS='|' read -r after_index after_commit after_applied <<< "$after_ready" + [[ "$before_index" == "$after_index" ]] || return 1 + VIEW_CONFIG_INDEX="$after_index" + VIEW_COMMIT_INDEX="$after_commit" + VIEW_APPLIED_INDEX="$after_applied" +} + +assert_expected_configuration() { + local context="$1" + local actual_signature + actual_signature="$(configuration_signature "$VIEW_CONFIGURATION")" + [[ "$VIEW_CONFIG_INDEX" == "$STATE_EXPECTED_CONFIG_INDEX" ]] || \ + fail "$context configuration index changed: expected=$STATE_EXPECTED_CONFIG_INDEX actual=$VIEW_CONFIG_INDEX" + [[ "$actual_signature" == "$STATE_EXPECTED_MEMBERS" ]] || \ + fail "$context membership changed outside this operation" +} + +adopt_committed_target_change() { + local desired="$1" + local context="$2" + local expected_signature actual_signature + expected_signature="$(configuration_signature_with_target "$STATE_EXPECTED_MEMBERS" "$desired")" + actual_signature="$(configuration_signature "$VIEW_CONFIGURATION")" + [[ "$actual_signature" == "$expected_signature" ]] || \ + fail "$context observed an unexpected membership after a lost response" + (( VIEW_CONFIG_INDEX > STATE_EXPECTED_CONFIG_INDEX )) || \ + fail "$context did not advance the configuration index" + STATE_EXPECTED_MEMBERS="$actual_signature" + STATE_EXPECTED_CONFIG_INDEX="$VIEW_CONFIG_INDEX" + write_state +} + +record_committed_target_change() { + local desired="$1" + local change_index="$2" + local context="$3" + local expected_signature actual_signature + load_leader_view || fail "cannot read stable leader view after $context" + expected_signature="$(configuration_signature_with_target "$STATE_EXPECTED_MEMBERS" "$desired")" + actual_signature="$(configuration_signature "$VIEW_CONFIGURATION")" + [[ "$VIEW_CONFIG_INDEX" == "$change_index" ]] || \ + fail "$context configuration index was superseded: committed=$change_index actual=$VIEW_CONFIG_INDEX" + [[ "$actual_signature" == "$expected_signature" ]] || \ + fail "$context committed an unexpected membership" + STATE_EXPECTED_MEMBERS="$actual_signature" + STATE_EXPECTED_CONFIG_INDEX="$VIEW_CONFIG_INDEX" + write_state +} + discover_leader() { local address status state leader_address leader_id local i @@ -414,10 +555,9 @@ same_csv_set() { "$(printf '%s' "$right" | tr ',' '\n' | sed '/^$/d' | sort | tr '\n' ',')" ]] } -leader_status_ready() { - local address="$1" - local status pending config_index commit applied - status="$(status_for "$address")" || return 1 +status_ready_fields() { + local status="$1" + local pending config_index commit applied pending="$(printf '%s\n' "$status" | field_value pending_conf_change)" config_index="$(printf '%s\n' "$status" | field_value configuration_index)" commit="$(printf '%s\n' "$status" | field_value commit_index)" @@ -428,6 +568,13 @@ leader_status_ready() { printf '%s|%s|%s\n' "$config_index" "$commit" "$applied" } +leader_status_ready() { + local address="$1" + local status + status="$(status_for "$address")" || return 1 + status_ready_fields "$status" +} + select_transfer_candidate() { local config="$1" local required_index="$2" @@ -450,16 +597,19 @@ select_transfer_candidate() { } preflight() { - local leader leader_id leader_address config target_address target_suffrage - local ready config_index commit applied voters quorum reachable=0 + local leader_id leader_address config target_address target_suffrage + local config_index commit applied voters quorum reachable=0 local id address suffrage voter_status voter_state voter_leader - - leader="$(discover_leader)" || fail "no reachable Raft leader; no-quorum recovery is not supported" - leader_id="${leader%%|*}" - leader_address="${leader#*|}" - ready="$(leader_status_ready "$leader_address")" || fail "leader is not fully applied or has a pending configuration change" - IFS='|' read -r config_index commit applied <<< "$ready" - config="$(raftadmin "$leader_address" configuration)" || fail "cannot read leader configuration" + local initial_signature + + load_leader_view || fail "no stable fully-applied Raft leader; no-quorum recovery is not supported" + leader_id="$VIEW_LEADER_ID" + leader_address="$VIEW_LEADER_ADDRESS" + config_index="$VIEW_CONFIG_INDEX" + commit="$VIEW_COMMIT_INDEX" + applied="$VIEW_APPLIED_INDEX" + config="$VIEW_CONFIGURATION" + initial_signature="$(configuration_signature "$config")" target_suffrage="$(member_suffrage "$TARGET_NODE" "$config" || true)" target_address="$(member_address "$TARGET_NODE" "$config" || true)" [[ "$target_suffrage" == "voter" ]] || fail "target must be a current voter; got ${target_suffrage:-absent}" @@ -490,9 +640,15 @@ preflight() { log "transferring leadership from $TARGET_NODE to $candidate_id" raftadmin "$leader_address" leadership_transfer_to_server "$candidate_id" "$candidate_address" >/dev/null wait_for_leader "$candidate_id" >/dev/null || fail "leadership did not transfer to $candidate_id" + load_leader_view || fail "cannot read stable leader view after leadership transfer" + [[ "$VIEW_CONFIG_INDEX" == "$config_index" ]] || fail "membership changed during leadership transfer" + [[ "$(configuration_signature "$VIEW_CONFIGURATION")" == "$initial_signature" ]] || \ + fail "membership changed during leadership transfer" fi STATE_EXPECTED_VOTERS="$(voter_csv "$config")" + STATE_EXPECTED_MEMBERS="$initial_signature" + STATE_EXPECTED_CONFIG_INDEX="$config_index" STATE_TARGET_ADDRESS="$target_address" log "preflight passed: leader=$leader_id leader_applied=$applied voters=$voters non_target_reachable=$reachable quorum=$quorum config_index=$config_index" } @@ -687,26 +843,27 @@ wait_configuration_index() { } remove_target() { - local config current leader address config_index output change_index - config="$(configuration_from_leader)" || fail "cannot read configuration before removal" + local config current output change_index + load_leader_view || fail "cannot read stable leader view before removal" + config="$VIEW_CONFIGURATION" current="$(member_suffrage "$TARGET_NODE" "$config" || true)" if [[ -z "$current" ]]; then + adopt_committed_target_change absent "remove_server" log "target is already absent; resuming after committed removal" return 0 fi [[ "$current" == "voter" ]] || fail "target changed from voter to $current before removal" + assert_expected_configuration "remove_server" ! status_for "$STATE_TARGET_ADDRESS" >/dev/null || fail "target returned after fencing" - leader="$(wait_for_leader)" || fail "no leader available for removal" - address="${leader#*|}" - config_index="$(config_index_from_leader)" || fail "leader is not ready for removal" - log "removing $TARGET_NODE at configuration index $config_index" - output="$(raftadmin "$address" remove_server "$TARGET_NODE" "$config_index")" + log "removing $TARGET_NODE at configuration index $STATE_EXPECTED_CONFIG_INDEX" + output="$(raftadmin "$VIEW_LEADER_ADDRESS" remove_server "$TARGET_NODE" "$STATE_EXPECTED_CONFIG_INDEX")" change_index="$(printf '%s\n' "$output" | field_value index)" - if ! is_uint "$change_index" || (( change_index <= config_index )); then + if ! is_uint "$change_index" || (( change_index <= STATE_EXPECTED_CONFIG_INDEX )); then fail "remove_server returned an invalid configuration index" fi wait_member_state absent || fail "remove_server did not commit" wait_configuration_index "$change_index" || fail "leader status did not publish committed removal index $change_index" + record_committed_target_change absent "$change_index" "remove_server" } run_rollout() { @@ -722,43 +879,41 @@ run_rollout() { unset ROLLING_UPDATE_ENV_FILE RAFT_BOOTSTRAP_MEMBERS export ROLLING_ORDER="$rollout_target" export RAFT_JOIN_NODE="$join_node" + export DRY_RUN=false exec "$ROLLING_UPDATE_SCRIPT" ) } add_learner() { - local config current leader address status commit config_index output change_index - config="$(configuration_from_leader)" || fail "cannot read configuration before learner add" + local config current commit output change_index + load_leader_view || fail "cannot read stable leader view before learner add" + config="$VIEW_CONFIGURATION" current="$(member_suffrage "$TARGET_NODE" "$config" || true)" if [[ "$current" == "learner" ]]; then (( STATE_MIN_CATCH_UP_INDEX > 0 )) || fail "learner committed but catch-up floor is missing from replacement state" + adopt_committed_target_change learner "add_learner" log "target is already a learner; resuming after committed add" return 0 fi [[ -z "$current" ]] || fail "target unexpectedly has suffrage $current before learner add" wait_target_up || fail "join deployment did not expose the target RaftAdmin endpoint" - leader="$(wait_for_leader)" || fail "no leader available for learner add" - address="${leader#*|}" - status="$(status_for "$address")" || fail "cannot read leader status before learner add" - commit="$(printf '%s\n' "$status" | field_value commit_index)" - config_index="$(printf '%s\n' "$status" | field_value configuration_index)" - [[ "$(printf '%s\n' "$status" | field_value pending_conf_change)" == "false" ]] || fail "a configuration change is pending" - if ! is_uint "$commit" || ! is_uint "$config_index" || (( config_index == 0 )); then - fail "invalid leader indexes before learner add" - fi + load_leader_view || fail "cannot read stable leader view before learner add" + assert_expected_configuration "add_learner" + commit="$VIEW_COMMIT_INDEX" STATE_MIN_CATCH_UP_INDEX="$commit" # Persist the floor before proposing. If the RPC commits but the control # process exits before advancing its stage, promotion still has a durable, # non-zero catch-up bound on resume. write_state - log "adding learner $TARGET_NODE at configuration index $config_index; catch-up floor=$commit" - output="$(raftadmin "$address" add_learner "$TARGET_NODE" "$STATE_TARGET_ADDRESS" "$config_index")" + log "adding learner $TARGET_NODE at configuration index $STATE_EXPECTED_CONFIG_INDEX; catch-up floor=$commit" + output="$(raftadmin "$VIEW_LEADER_ADDRESS" add_learner "$TARGET_NODE" "$STATE_TARGET_ADDRESS" "$STATE_EXPECTED_CONFIG_INDEX")" change_index="$(printf '%s\n' "$output" | field_value index)" - if ! is_uint "$change_index" || (( change_index <= config_index )); then + if ! is_uint "$change_index" || (( change_index <= STATE_EXPECTED_CONFIG_INDEX )); then fail "add_learner returned an invalid configuration index" fi wait_member_state learner || fail "add_learner did not commit" wait_configuration_index "$change_index" || fail "leader status did not publish committed learner index $change_index" + record_committed_target_change learner "$change_index" "add_learner" } wait_for_catch_up() { @@ -779,31 +934,34 @@ wait_for_catch_up() { } promote_learner() { - local config current leader address config_index output change_index - config="$(configuration_from_leader)" || fail "cannot read configuration before promotion" + local config current output change_index + load_leader_view || fail "cannot read stable leader view before promotion" + config="$VIEW_CONFIGURATION" current="$(member_suffrage "$TARGET_NODE" "$config" || true)" if [[ "$current" == "voter" ]]; then + adopt_committed_target_change voter "promote_learner" log "target is already a voter; resuming after committed promotion" return 0 fi [[ "$current" == "learner" ]] || fail "target must be a learner before promotion; got ${current:-absent}" (( STATE_MIN_CATCH_UP_INDEX > 0 )) || fail "catch-up floor is missing from replacement state" + assert_expected_configuration "promote_learner" wait_for_catch_up || fail "learner did not reach catch-up floor" - leader="$(wait_for_leader)" || fail "no leader available for promotion" - address="${leader#*|}" - config_index="$(config_index_from_leader)" || fail "leader is not ready for promotion" - log "promoting $TARGET_NODE at configuration index $config_index" - output="$(raftadmin "$address" promote_learner "$TARGET_NODE" "$config_index" "$STATE_MIN_CATCH_UP_INDEX" false)" + load_leader_view || fail "cannot read stable leader view after learner catch-up" + assert_expected_configuration "promote_learner" + log "promoting $TARGET_NODE at configuration index $STATE_EXPECTED_CONFIG_INDEX" + output="$(raftadmin "$VIEW_LEADER_ADDRESS" promote_learner "$TARGET_NODE" "$STATE_EXPECTED_CONFIG_INDEX" "$STATE_MIN_CATCH_UP_INDEX" false)" change_index="$(printf '%s\n' "$output" | field_value index)" - if ! is_uint "$change_index" || (( change_index <= config_index )); then + if ! is_uint "$change_index" || (( change_index <= STATE_EXPECTED_CONFIG_INDEX )); then fail "promote_learner returned an invalid configuration index" fi wait_member_state voter || fail "promote_learner did not commit" wait_configuration_index "$change_index" || fail "leader status did not publish committed promotion index $change_index" + record_committed_target_change voter "$change_index" "promote_learner" } verify_cluster_once() { - local leader leader_id address status pending config voters id member_address suffrage member_status state member_leader applied commit + local leader leader_id address status pending config voters signature id member_address suffrage member_status state member_leader applied commit leader="$(discover_leader)" || return 1 leader_id="${leader%%|*}" address="${leader#*|}" @@ -811,6 +969,8 @@ verify_cluster_once() { pending="$(printf '%s\n' "$status" | field_value pending_conf_change)" [[ "$pending" == "false" ]] || return 1 config="$(raftadmin "$address" configuration 2>/dev/null)" || return 1 + signature="$(configuration_signature "$config")" + [[ "$signature" == "$STATE_EXPECTED_MEMBERS" ]] || return 1 voters="$(voter_csv "$config")" same_csv_set "$voters" "$STATE_EXPECTED_VOTERS" || return 1 while IFS='|' read -r id member_address suffrage; do @@ -853,13 +1013,14 @@ verify_stability() { } state_value_valid() { - [[ "$1" =~ ^[A-Za-z0-9_./,:@-]*$ ]] + [[ "$1" =~ ^[A-Za-z0-9_./,:@|-]*$ ]] } write_state() { local tmp value for value in "$STATE_STAGE" "$STATE_TARGET" "$STATE_TARGET_ADDRESS" "$STATE_OPERATION_ID" \ - "$STATE_ARCHIVE_PATH" "$STATE_EXPECTED_VOTERS" "$STATE_MIN_CATCH_UP_INDEX" "$STATE_DATA_MODE"; do + "$STATE_ARCHIVE_PATH" "$STATE_EXPECTED_VOTERS" "$STATE_EXPECTED_MEMBERS" \ + "$STATE_EXPECTED_CONFIG_INDEX" "$STATE_MIN_CATCH_UP_INDEX" "$STATE_DATA_MODE"; do state_value_valid "$value" || fail "replacement state contains an unsupported value" done mkdir -p "$(dirname "$REPLACEMENT_STATE_FILE")" @@ -871,6 +1032,8 @@ write_state() { printf 'operation_id=%s\n' "$STATE_OPERATION_ID" printf 'archive_path=%s\n' "$STATE_ARCHIVE_PATH" printf 'expected_voters=%s\n' "$STATE_EXPECTED_VOTERS" + printf 'expected_members=%s\n' "$STATE_EXPECTED_MEMBERS" + printf 'expected_config_index=%s\n' "$STATE_EXPECTED_CONFIG_INDEX" printf 'min_catch_up_index=%s\n' "$STATE_MIN_CATCH_UP_INDEX" printf 'data_mode=%s\n' "$STATE_DATA_MODE" } > "$tmp" @@ -890,20 +1053,26 @@ load_state() { operation_id) STATE_OPERATION_ID="$value" ;; archive_path) STATE_ARCHIVE_PATH="$value" ;; expected_voters) STATE_EXPECTED_VOTERS="$value" ;; + expected_members) STATE_EXPECTED_MEMBERS="$value" ;; + expected_config_index) STATE_EXPECTED_CONFIG_INDEX="$value" ;; min_catch_up_index) STATE_MIN_CATCH_UP_INDEX="$value" ;; data_mode) STATE_DATA_MODE="$value" ;; "") ;; *) fail "unknown key in state file: $key" ;; esac done < "$REPLACEMENT_STATE_FILE" - if ! is_uint "$STATE_STAGE" || ! is_uint "$STATE_MIN_CATCH_UP_INDEX"; then + if ! is_uint "$STATE_STAGE" || ! is_uint "$STATE_EXPECTED_CONFIG_INDEX" || \ + ! is_uint "$STATE_MIN_CATCH_UP_INDEX"; then fail "invalid numeric value in state file" fi (( STATE_STAGE <= 9 )) || fail "state stage is outside the implemented range: $STATE_STAGE" [[ "$STATE_TARGET" == "$TARGET_NODE" ]] || fail "state file belongs to target $STATE_TARGET, not $TARGET_NODE" [[ "$STATE_TARGET_ADDRESS" == "$(node_host_by_id "$TARGET_NODE"):${RAFT_PORT}" ]] || fail "state target address no longer matches NODES" - [[ -n "$STATE_OPERATION_ID" && -n "$STATE_ARCHIVE_PATH" && -n "$STATE_EXPECTED_VOTERS" ]] || fail "state file is incomplete" + [[ -n "$STATE_OPERATION_ID" && -n "$STATE_ARCHIVE_PATH" && -n "$STATE_EXPECTED_VOTERS" && \ + -n "$STATE_EXPECTED_MEMBERS" ]] || fail "state file is incomplete" + (( STATE_EXPECTED_CONFIG_INDEX > 0 )) || fail "state file has no expected configuration index" [[ "$STATE_DATA_MODE" == "$REPLACEMENT_DATA_MODE" ]] || fail "REPLACEMENT_DATA_MODE changed from $STATE_DATA_MODE to $REPLACEMENT_DATA_MODE" + (( STATE_STAGE < 9 )) || fail "replacement state is already complete; archive it or choose a new REPLACEMENT_STATE_FILE" log "resuming operation $STATE_OPERATION_ID at stage $STATE_STAGE" } diff --git a/scripts/rolling-update.env.example b/scripts/rolling-update.env.example index ed91e79d8..2261987fd 100644 --- a/scripts/rolling-update.env.example +++ b/scripts/rolling-update.env.example @@ -27,20 +27,10 @@ RAFT_ENGINE="etcd" # ROLLING_ORDER="n2" # RAFT_JOIN_NODE="n2" -# Fenced same-ID replacement wrapper. Keep these commented during normal -# rolling updates. TARGET_NODE and REPLACEMENT_CONFIRM must match exactly. -# External fencing is the default and must prevent the old VM/process from -# returning to the Raft network. Container mode is only suitable when host -# lifecycle policy guarantees the stopped container cannot restart. -# TARGET_NODE="n2" -# REPLACEMENT_CONFIRM="n2" -# REPLACEMENT_FENCE_MODE="external" -# REPLACEMENT_FENCE_COMMAND="fence-n2-and-block-raft" -# REPLACEMENT_FENCE_VERIFY_SECONDS="3" -# REPLACEMENT_DATA_MODE="archive" -# REPLACEMENT_STABILITY_SECONDS="10" -# REPLACEMENT_TIMEOUT_SECONDS="120" -# REPLACEMENT_VERIFY_COMMAND="redis-cli -h proxy.internal.example SET replacement-check ok && redis-cli -h proxy.internal.example GET replacement-check | grep -Fx ok" +# Do not store fenced replacement controls in this inventory file. Pass +# TARGET_NODE and every REPLACEMENT_* authorization/tuning value directly to +# scripts/raft-member-replace.sh. The wrapper intentionally ignores replacement +# controls sourced from deployment inventory. RAFT_PORT="50051" REDIS_PORT="6379" From df27d7089ebed79fd5f6fc34dbacf20aa1731683 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 17:56:34 +0900 Subject: [PATCH 4/6] Harden Raft replacement recovery --- ...lemented_fenced_raft_member_replacement.md | 41 ++++--- internal/raftengine/etcd/engine.go | 19 +--- raft_member_replace_script_test.go | 102 +++++++++++++++++- scripts/raft-member-replace.sh | 51 +++++++-- 4 files changed, 171 insertions(+), 42 deletions(-) diff --git a/docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md b/docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md index a7641d14c..555d1002e 100644 --- a/docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md +++ b/docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md @@ -94,6 +94,11 @@ for the configured consecutive verification interval. It is checked again before removal and before the fresh deployment. Membership is never removed while the old endpoint still answers. +External mode also stops and verifies the target container over the management +SSH path after the Raft fence is observed. If that management action is not +available, the operation stops before `RemoveServer` rather than shrinking the +group with a still-running process. + ### 4.4 Serialized membership changes RaftAdmin status now reports: @@ -142,7 +147,7 @@ same state file. | Stage | Durable fact | | --- | --- | -| 0 | Target identity, address, complete member signature and configuration index, original voter set, operation ID, and archive path recorded | +| 0 | Target identity, address, complete member signature and configuration index, original voter set, operation ID, canonical data directory, and archive path recorded | | 1 | Preflight and any leadership transfer completed | | 2 | Old ID owner fenced and endpoint observed down | | 3 | Target absent from committed membership | @@ -153,12 +158,14 @@ same state file. | 8 | Target redeployed without join intent | | 9 | Stable voter set and application write/read verification completed | -Operations around a stage boundary are idempotent. The chosen data mode is -stored with the operation and cannot change on resume. For example, a resumed -stage 2 accepts that removal already committed, stage 5 accepts an already -committed learner only when its catch-up floor exists, and stage 6 accepts an -already promoted voter. A conflicting suffrage or voter set aborts instead of -guessing what happened. +Operations around a stage boundary are idempotent. The chosen data mode and +canonical data directory are stored with the operation and cannot change on +resume. For example, a resumed stage 2 accepts that removal already committed, +stage 4 safely reruns the target-only learner deployment after a lost rollout +response, stage 5 accepts an already committed learner only when its catch-up +floor exists, and stage 6 accepts an already promoted voter. A conflicting +suffrage, voter set, or data directory aborts instead of guessing what +happened. The state file is retained by default as operational evidence. A retained stage-9 file is never treated as authorization for a later incident: a new @@ -236,22 +243,26 @@ replacement state without starting the target. stateful fake RaftAdmin, SSH transport, and rolling deploy command. It verifies the exact voter-to-absent-to-learner-to-voter sequence, non-zero catch-up floor, both deploy modes, forced execute rollout, application verification, durable -stage 9, and rejection of a completed state as a new operation. +stage 9, canonical data-directory persistence, external-fence process stop, +and rejection of a completed state as a new operation. A negative test makes one surviving voter unreachable in a three-voter group and proves the command aborts before the fence action. Another test loses the `AddLearner` RPC response after commit and proves the persisted catch-up floor -allows a safe resume without a duplicate add. Lock ownership is tested so a -losing control process cannot remove the winner's directory lock. RaftAdmin -server and CLI tests cover propagation and rendering of the configuration -index and pending change fields. Existing engine membership tests cover stale -previous indexes, learner catch-up enforcement, same-ID replacement, and -persisted membership restart. +allows a safe resume without a duplicate add. A separate lost-response test +proves a completed learner rollout is safely repeated from stage 4 rather than +leaving the group shrunken. Lock ownership is tested so a losing control +process cannot remove the winner's directory lock. RaftAdmin server and CLI +tests cover propagation and rendering of the configuration index and pending +change fields. Existing engine membership tests cover stale previous indexes, +learner catch-up enforcement, same-ID replacement, and persisted membership +restart. Additional negative tests prove that deployment-env replacement controls cannot authorize an operation, a configuration-index change after preflight aborts before removal, invocation-level timing controls override stale env -values, and a trailing-slash data path produces a sibling archive. +values, dot-segment data paths canonicalize to a sibling archive, and a changed +data directory is rejected on resume. Production acceptance still requires recording the state file, fence evidence, final `raftadmin status`/`configuration`, and the output of the configured diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index 9390147d8..bc4281d9e 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -2062,24 +2062,7 @@ func (e *Engine) handleRemoveServer(req adminRequest) { req.done <- adminResult{err: errors.Wrapf(errTransportPeerUnknown, "id=%q", req.peer.ID)} return } - contextBytes, err := encodeConfChangeContext(req.id, peer) - if err != nil { - req.done <- adminResult{err: err} - return - } - cc := raftpb.ConfChange{ - Type: confChangeTypePtr(raftpb.ConfChangeRemoveNode), - NodeId: uint64Ptr(peer.NodeID), - Context: contextBytes, - } - if err := e.storePendingConfig(req); err != nil { - req.done <- adminResult{err: err} - return - } - if err := e.rawNode.ProposeConfChange(&cc); err != nil { - e.cancelPendingConfig(req.id) - req.done <- adminResult{err: errors.WithStack(err)} - } + e.proposeMembershipChange(req, raftpb.ConfChangeRemoveNode, peer) } func (e *Engine) handleTransferLeadership(req adminRequest) { diff --git a/raft_member_replace_script_test.go b/raft_member_replace_script_test.go index 6d977964e..affb9e09a 100644 --- a/raft_member_replace_script_test.go +++ b/raft_member_replace_script_test.go @@ -28,7 +28,7 @@ func TestRaftMemberReplaceScriptCompletesFencedLearnerLifecycle(t *testing.T) { "RAFT_PORT=50051", "SSH_USER=tester", "CONTAINER_NAME=elastickv", - "DATA_DIR=/var/lib/elastickv/", + "DATA_DIR=/var/lib/./elastickv/.", "TARGET_NODE=n3", "REPLACEMENT_CONFIRM=n3", "REPLACEMENT_VERIFY_COMMAND=false", @@ -70,8 +70,10 @@ func TestRaftMemberReplaceScriptCompletesFencedLearnerLifecycle(t *testing.T) { require.Contains(t, state, "expected_config_index=13\n") require.Contains(t, state, "min_catch_up_index=100\n") require.Contains(t, state, "data_mode=archive\n") + require.Contains(t, state, "data_dir=/var/lib/elastickv\n") require.Contains(t, state, "archive_path=/var/lib/elastickv.replacement-n2-") require.NotContains(t, state, "archive_path=/var/lib/elastickv/.replacement") + require.Equal(t, "stop\nreset-data\n", mustReadTestFile(t, filepath.Join(stateDir, "remote-actions"))) resumeCmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") resumeCmd.Dir = repoRoot @@ -97,6 +99,43 @@ func TestRaftMemberReplaceScriptCompletesFencedLearnerLifecycle(t *testing.T) { require.Equal(t, "n2:n2\nnormal:n2\n", mustReadTestFile(t, filepath.Join(stateDir, "rollouts")), "completed resume must not repeat deployment") } +func TestRaftMemberReplaceScriptStopsProcessAfterExternalFence(t *testing.T) { + repoRoot, err := os.Getwd() + require.NoError(t, err) + stateDir := t.TempDir() + fakeBin := filepath.Join(stateDir, "bin") + require.NoError(t, os.Mkdir(fakeBin, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "member"), []byte("voter\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "config-index"), []byte("10\n"), 0o600)) + + raftadmin := writeExecutableTestFile(t, fakeBin, "raftadmin", fakeRaftadminScript) + ssh := writeExecutableTestFile(t, fakeBin, "ssh", fakeReplacementSSHScript) + rolling := writeExecutableTestFile(t, fakeBin, "rolling-update", fakeReplacementRollingScript) + envFile := filepath.Join(stateDir, "deploy.env") + require.NoError(t, os.WriteFile(envFile, []byte("NODES=n1=127.0.0.1,n2=127.0.0.2,n3=127.0.0.3\n"), 0o600)) + + output := runReplacementScript(t, repoRoot, []string{ + "TARGET_NODE=n2", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=external", + `REPLACEMENT_FENCE_COMMAND=touch "$FAKE_STATE_DIR/external-fenced"; touch "$FAKE_STATE_DIR/down-n2"`, + "REPLACEMENT_FENCE_VERIFY_SECONDS=1", + "REPLACEMENT_VERIFY_COMMAND=true", + "REPLACEMENT_STABILITY_SECONDS=1", + "REPLACEMENT_TIMEOUT_SECONDS=5", + "REPLACEMENT_POLL_SECONDS=1", + "REPLACEMENT_STATE_FILE=" + filepath.Join(stateDir, "replace.state"), + "ROLLING_UPDATE_ENV_FILE=" + envFile, + "ROLLING_UPDATE_SCRIPT=" + rolling, + "RAFTADMIN_BIN=" + raftadmin, + "SSH_BIN=" + ssh, + "FAKE_STATE_DIR=" + stateDir, + }) + require.Contains(t, output, "external fence verified; stopping the target process") + require.FileExists(t, filepath.Join(stateDir, "external-fenced")) + require.Equal(t, "stop\nreset-data\n", mustReadTestFile(t, filepath.Join(stateDir, "remote-actions"))) +} + func TestRaftMemberReplaceScriptRequiresInvocationControls(t *testing.T) { repoRoot, err := os.Getwd() require.NoError(t, err) @@ -239,6 +278,15 @@ func TestRaftMemberReplaceScriptResumesAfterLearnerCommitResponseLoss(t *testing interruptedState := mustReadTestFile(t, stateFile) require.Contains(t, interruptedState, "stage=5\n") require.Contains(t, interruptedState, "min_catch_up_index=100\n", "catch-up floor must precede the membership proposal") + require.Contains(t, interruptedState, "data_dir=/var/lib/elastickv\n") + + changedDataDirCmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") + changedDataDirCmd.Dir = repoRoot + changedDataDirEnv := append(append([]string{}, commandEnv...), "DATA_DIR=/srv/elastickv") + changedDataDirCmd.Env = append(os.Environ(), changedDataDirEnv...) + changedDataDirOutput, changedDataDirErr := changedDataDirCmd.CombinedOutput() + require.Error(t, changedDataDirErr) + require.Contains(t, string(changedDataDirOutput), "DATA_DIR changed from /var/lib/elastickv to /srv/elastickv") resumeOutput := runReplacementScript(t, repoRoot, commandEnv) require.Contains(t, resumeOutput, "target is already a learner; resuming after committed add") @@ -246,6 +294,53 @@ func TestRaftMemberReplaceScriptResumesAfterLearnerCommitResponseLoss(t *testing require.Equal(t, "voter\n", mustReadTestFile(t, filepath.Join(stateDir, "member"))) } +func TestRaftMemberReplaceScriptResumesAfterJoinRolloutResponseLoss(t *testing.T) { + repoRoot, err := os.Getwd() + require.NoError(t, err) + stateDir := t.TempDir() + fakeBin := filepath.Join(stateDir, "bin") + require.NoError(t, os.Mkdir(fakeBin, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "member"), []byte("voter\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "config-index"), []byte("10\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "fail-join-rollout-once"), nil, 0o600)) + + raftadmin := writeExecutableTestFile(t, fakeBin, "raftadmin", fakeRaftadminScript) + ssh := writeExecutableTestFile(t, fakeBin, "ssh", fakeReplacementSSHScript) + rolling := writeExecutableTestFile(t, fakeBin, "rolling-update", fakeReplacementRollingScript) + envFile := filepath.Join(stateDir, "deploy.env") + require.NoError(t, os.WriteFile(envFile, []byte("NODES=n1=127.0.0.1,n2=127.0.0.2,n3=127.0.0.3\n"), 0o600)) + stateFile := filepath.Join(stateDir, "replace.state") + commandEnv := []string{ + "TARGET_NODE=n2", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=container", + "REPLACEMENT_FENCE_VERIFY_SECONDS=1", + "REPLACEMENT_VERIFY_COMMAND=true", + "REPLACEMENT_STABILITY_SECONDS=1", + "REPLACEMENT_TIMEOUT_SECONDS=5", + "REPLACEMENT_POLL_SECONDS=1", + "REPLACEMENT_STATE_FILE=" + stateFile, + "ROLLING_UPDATE_ENV_FILE=" + envFile, + "ROLLING_UPDATE_SCRIPT=" + rolling, + "RAFTADMIN_BIN=" + raftadmin, + "SSH_BIN=" + ssh, + "FAKE_STATE_DIR=" + stateDir, + } + + cmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") + cmd.Dir = repoRoot + cmd.Env = append(os.Environ(), commandEnv...) + output, err := cmd.CombinedOutput() + require.Error(t, err, "%s", output) + require.Contains(t, mustReadTestFile(t, stateFile), "stage=4\n") + require.Equal(t, "absent\n", mustReadTestFile(t, filepath.Join(stateDir, "member"))) + require.Equal(t, "n2:n2\n", mustReadTestFile(t, filepath.Join(stateDir, "rollouts"))) + + resumeOutput := runReplacementScript(t, repoRoot, commandEnv) + require.Contains(t, resumeOutput, "replacement completed: target=n2") + require.Equal(t, "n2:n2\nn2:n2\nnormal:n2\n", mustReadTestFile(t, filepath.Join(stateDir, "rollouts"))) +} + func TestRaftMemberReplaceScriptDoesNotRemoveAnotherProcessLock(t *testing.T) { repoRoot, err := os.Getwd() require.NoError(t, err) @@ -397,6 +492,7 @@ cat >/dev/null while [[ "$#" -gt 0 && "$1" != "--" ]]; do shift; done [[ "$#" -ge 2 ]] action="$2" +printf '%s\n' "$action" >> "$FAKE_STATE_DIR/remote-actions" case "$action" in stop) touch "$FAKE_STATE_DIR/down-n2" @@ -416,5 +512,9 @@ set -euo pipefail printf '%s:%s\n' "${RAFT_JOIN_NODE:-normal}" "$ROLLING_ORDER" >> "$FAKE_STATE_DIR/rollouts" if [[ "${RAFT_JOIN_NODE:-}" == "n2" ]]; then rm -f "$FAKE_STATE_DIR/down-n2" + if [[ -f "$FAKE_STATE_DIR/fail-join-rollout-once" ]]; then + rm -f "$FAKE_STATE_DIR/fail-join-rollout-once" + exit 74 + fi fi ` diff --git a/scripts/raft-member-replace.sh b/scripts/raft-member-replace.sh index fbe9918b0..fa343a470 100755 --- a/scripts/raft-member-replace.sh +++ b/scripts/raft-member-replace.sh @@ -32,7 +32,8 @@ Fencing: REPLACEMENT_FENCE_MODE=external (default) Requires REPLACEMENT_FENCE_COMMAND. The command must prevent the old VM or process from returning with stale data. The target Raft RPC must become - unreachable before membership removal. + unreachable before membership removal. Management SSH must remain available + so the command can stop and verify CONTAINER_NAME before removal. REPLACEMENT_FENCE_MODE=container Stops CONTAINER_NAME over SSH. Use only when the host lifecycle guarantees the stopped container cannot restart independently during the operation. @@ -170,9 +171,6 @@ SSH_USER="${SSH_USER:-${USER:-$(id -un)}}" RAFT_PORT="${RAFT_PORT:-50051}" CONTAINER_NAME="${CONTAINER_NAME:-elastickv}" DATA_DIR="${DATA_DIR:-/var/lib/elastickv}" -while [[ "$DATA_DIR" != "/" && "$DATA_DIR" == */ ]]; do - DATA_DIR="${DATA_DIR%/}" -done RAFTADMIN_BIN="${RAFTADMIN_BIN:-}" RAFTADMIN_ALLOW_INSECURE="${RAFTADMIN_ALLOW_INSECURE:-true}" RAFTADMIN_RPC_TIMEOUT_SECONDS="${RAFTADMIN_RPC_TIMEOUT_SECONDS:-5}" @@ -203,6 +201,7 @@ STATE_EXPECTED_MEMBERS="" STATE_EXPECTED_CONFIG_INDEX=0 STATE_MIN_CATCH_UP_INDEX=0 STATE_DATA_MODE="" +STATE_DATA_DIR="" VIEW_LEADER_ID="" VIEW_LEADER_ADDRESS="" VIEW_CONFIG_INDEX=0 @@ -226,11 +225,40 @@ is_uint() { [[ "$1" =~ ^[0-9]+$ ]] } +canonicalize_absolute_path() { + local path="$1" + local component last + local -a components=() + local -a normalized=() + [[ "$path" == /* ]] || return 1 + IFS='/' read -r -a components <<< "${path#/}" + for component in "${components[@]}"; do + case "$component" in + ""|.) ;; + ..) + ((${#normalized[@]} > 0)) || return 1 + last=$((${#normalized[@]} - 1)) + unset "normalized[$last]" + ;; + *) normalized+=("$component") ;; + esac + done + if ((${#normalized[@]} == 0)); then + printf '/\n' + return 0 + fi + local IFS=/ + printf '/%s\n' "${normalized[*]}" +} + validate_settings() { + local canonical_data_dir [[ -n "$TARGET_NODE" ]] || fail "TARGET_NODE is required" [[ "$TARGET_NODE" =~ ^[A-Za-z0-9._-]+$ ]] || fail "TARGET_NODE contains unsupported characters" [[ -n "$NODES" ]] || fail "NODES is required" [[ "$DATA_DIR" == /* ]] || fail "DATA_DIR must be absolute" + canonical_data_dir="$(canonicalize_absolute_path "$DATA_DIR")" || fail "DATA_DIR escapes the filesystem root" + DATA_DIR="$canonical_data_dir" [[ "$DATA_DIR" != "/" ]] || fail "DATA_DIR must not be the filesystem root" state_value_valid "$DATA_DIR" || fail "DATA_DIR contains characters unsupported by the resume state" [[ -x "$ROLLING_UPDATE_SCRIPT" ]] || fail "ROLLING_UPDATE_SCRIPT is not executable: $ROLLING_UPDATE_SCRIPT" @@ -780,6 +808,9 @@ fence_target() { external) log "running external fence for $TARGET_NODE" run_operator_command "$REPLACEMENT_FENCE_COMMAND" + wait_target_down || fail "target Raft RPC is still reachable after external fencing" + log "external fence verified; stopping the target process before membership removal" + remote_target_action stop ;; container) log "stopping target container; host lifecycle must keep it fenced" @@ -1020,7 +1051,7 @@ write_state() { local tmp value for value in "$STATE_STAGE" "$STATE_TARGET" "$STATE_TARGET_ADDRESS" "$STATE_OPERATION_ID" \ "$STATE_ARCHIVE_PATH" "$STATE_EXPECTED_VOTERS" "$STATE_EXPECTED_MEMBERS" \ - "$STATE_EXPECTED_CONFIG_INDEX" "$STATE_MIN_CATCH_UP_INDEX" "$STATE_DATA_MODE"; do + "$STATE_EXPECTED_CONFIG_INDEX" "$STATE_MIN_CATCH_UP_INDEX" "$STATE_DATA_MODE" "$STATE_DATA_DIR"; do state_value_valid "$value" || fail "replacement state contains an unsupported value" done mkdir -p "$(dirname "$REPLACEMENT_STATE_FILE")" @@ -1036,6 +1067,7 @@ write_state() { printf 'expected_config_index=%s\n' "$STATE_EXPECTED_CONFIG_INDEX" printf 'min_catch_up_index=%s\n' "$STATE_MIN_CATCH_UP_INDEX" printf 'data_mode=%s\n' "$STATE_DATA_MODE" + printf 'data_dir=%s\n' "$STATE_DATA_DIR" } > "$tmp" chmod 600 "$tmp" mv "$tmp" "$REPLACEMENT_STATE_FILE" @@ -1057,6 +1089,7 @@ load_state() { expected_config_index) STATE_EXPECTED_CONFIG_INDEX="$value" ;; min_catch_up_index) STATE_MIN_CATCH_UP_INDEX="$value" ;; data_mode) STATE_DATA_MODE="$value" ;; + data_dir) STATE_DATA_DIR="$value" ;; "") ;; *) fail "unknown key in state file: $key" ;; esac @@ -1069,9 +1102,10 @@ load_state() { [[ "$STATE_TARGET" == "$TARGET_NODE" ]] || fail "state file belongs to target $STATE_TARGET, not $TARGET_NODE" [[ "$STATE_TARGET_ADDRESS" == "$(node_host_by_id "$TARGET_NODE"):${RAFT_PORT}" ]] || fail "state target address no longer matches NODES" [[ -n "$STATE_OPERATION_ID" && -n "$STATE_ARCHIVE_PATH" && -n "$STATE_EXPECTED_VOTERS" && \ - -n "$STATE_EXPECTED_MEMBERS" ]] || fail "state file is incomplete" + -n "$STATE_EXPECTED_MEMBERS" && -n "$STATE_DATA_DIR" ]] || fail "state file is incomplete" (( STATE_EXPECTED_CONFIG_INDEX > 0 )) || fail "state file has no expected configuration index" [[ "$STATE_DATA_MODE" == "$REPLACEMENT_DATA_MODE" ]] || fail "REPLACEMENT_DATA_MODE changed from $STATE_DATA_MODE to $REPLACEMENT_DATA_MODE" + [[ "$STATE_DATA_DIR" == "$DATA_DIR" ]] || fail "DATA_DIR changed from $STATE_DATA_DIR to $DATA_DIR" (( STATE_STAGE < 9 )) || fail "replacement state is already complete; archive it or choose a new REPLACEMENT_STATE_FILE" log "resuming operation $STATE_OPERATION_ID at stage $STATE_STAGE" } @@ -1119,7 +1153,8 @@ if ! load_state; then STATE_TARGET="$TARGET_NODE" STATE_TARGET_ADDRESS="${TARGET_HOST}:${RAFT_PORT}" STATE_OPERATION_ID="$(date -u +%Y%m%dT%H%M%SZ)-$$" - STATE_ARCHIVE_PATH="${DATA_DIR}.replacement-${TARGET_NODE}-${STATE_OPERATION_ID}" + STATE_DATA_DIR="$DATA_DIR" + STATE_ARCHIVE_PATH="${STATE_DATA_DIR}.replacement-${TARGET_NODE}-${STATE_OPERATION_ID}" STATE_DATA_MODE="$REPLACEMENT_DATA_MODE" preflight advance 1 @@ -1132,7 +1167,7 @@ fi if (( STATE_STAGE < 2 )); then fence_target advance 2 -elif (( STATE_STAGE < 5 )); then +elif (( STATE_STAGE < 4 )); then wait_target_down || fail "fenced target is reachable again before removal" fi if (( STATE_STAGE < 3 )); then From b1e6e5a40e76692f6df11e356b56ddf96a0ea2d0 Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 18:26:55 +0900 Subject: [PATCH 5/6] Harden replacement stage recovery --- ...lemented_fenced_raft_member_replacement.md | 17 +++++-- internal/raftengine/etcd/engine.go | 7 ++- internal/raftengine/etcd/engine_test.go | 13 +++++ raft_member_replace_script_test.go | 12 +++++ scripts/raft-member-replace.sh | 50 ++++++++++++++----- 5 files changed, 81 insertions(+), 18 deletions(-) diff --git a/docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md b/docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md index 555d1002e..ddc257ea0 100644 --- a/docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md +++ b/docs/design/2026_07_18_implemented_fenced_raft_member_replacement.md @@ -97,7 +97,9 @@ while the old endpoint still answers. External mode also stops and verifies the target container over the management SSH path after the Raft fence is observed. If that management action is not available, the operation stops before `RemoveServer` rather than shrinking the -group with a still-running process. +group with a still-running process. Docker inspection must itself succeed; +missing containers, daemon failures, and permission errors are not interpreted +as proof that the process is stopped. ### 4.4 Serialized membership changes @@ -112,9 +114,12 @@ configuration index. Every `remove_server`, `add_learner`, and adopts a newer index merely because another topology change committed. After each successful change, the command verifies the exact expected post-change signature before persisting the returned index for the next stage. The engine -therefore rejects a concurrent topology change instead of applying a stale -plan. A resume after a lost RPC response is accepted only when the complete -member signature matches the one transition that was in flight. +rejects every new add, promote, or remove proposal while an earlier +configuration entry remains unapplied, so etcd/raft cannot silently rewrite a +racing proposal to a no-op. A resume after a lost RPC response is accepted only +when the complete member signature matches the one transition that was in +flight. It also accepts an equal saved/live configuration index when that +signature already proves the metadata write completed before the stage write. ### 4.5 Catch-up before promotion @@ -262,7 +267,9 @@ Additional negative tests prove that deployment-env replacement controls cannot authorize an operation, a configuration-index change after preflight aborts before removal, invocation-level timing controls override stale env values, dot-segment data paths canonicalize to a sibling archive, and a changed -data directory is rejected on resume. +data directory is rejected on resume. Engine coverage also proves a membership +proposal is rejected while the pending configuration fence is active; the +shell resume test covers the equal-index metadata-before-stage crash window. Production acceptance still requires recording the state file, fence evidence, final `raftadmin status`/`configuration`, and the output of the configured diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index bc4281d9e..c7aa71b84 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -143,6 +143,7 @@ var ( errLeadershipTransferNoHealthyTarget = errors.Mark(errors.New("etcd raft leadership transfer has no healthy target"), raftengine.ErrLeadershipTransferNoHealthyTarget) errLeadershipTransferTargetNotCaughtUp = errors.Mark(errors.New("etcd raft leadership transfer target is not caught up"), raftengine.ErrLeadershipTransferTargetNotCaughtUp) errLeadershipTransferConfChangePending = errors.Mark(errors.New("etcd raft leadership transfer blocked by pending config change"), raftengine.ErrLeadershipTransferConfChangePending) + errMembershipConfChangePending = errors.New("etcd raft membership change blocked by pending config change") errTooManyPendingConfigs = errors.New("etcd raft engine has too many pending config changes") errPromoteLearnerNotLearner = errors.New("etcd raft promote-learner target is not a learner") errPromoteLearnerNoProgress = errors.New("etcd raft promote-learner target has no leader-side progress entry") @@ -2030,9 +2031,13 @@ func (e *Engine) handlePromoteLearner(req adminRequest) { // proposeMembershipChange wraps the encode + storePendingConfig + // ProposeConfChange dance shared by AddVoter / AddLearner / -// PromoteLearner. The caller has already validated the leader and +// PromoteLearner / RemoveServer. The caller has already validated the leader and // prevIndex preconditions in handleAdmin. func (e *Engine) proposeMembershipChange(req adminRequest, changeType raftpb.ConfChangeType, peer Peer) { + if e.hasPendingConfChange() { + req.done <- adminResult{err: errors.WithStack(errMembershipConfChangePending)} + return + } contextBytes, err := encodeConfChangeContext(req.id, peer) if err != nil { req.done <- adminResult{err: err} diff --git a/internal/raftengine/etcd/engine_test.go b/internal/raftengine/etcd/engine_test.go index 7103334c9..022b48ed7 100644 --- a/internal/raftengine/etcd/engine_test.go +++ b/internal/raftengine/etcd/engine_test.go @@ -1637,6 +1637,19 @@ func TestPendingConfChangeFenceTracksUnappliedConfig(t *testing.T) { require.False(t, engine.hasPendingConfChange()) } +func TestProposeMembershipChangeRejectsPendingConfig(t *testing.T) { + engine := &Engine{} + engine.markPendingConfChange(12) + done := make(chan adminResult, 1) + req := adminRequest{id: 17, done: done} + + engine.proposeMembershipChange(req, raftpb.ConfChangeRemoveNode, Peer{NodeID: 2, ID: "n2"}) + + result := <-done + require.ErrorIs(t, result.err, errMembershipConfChangePending) + require.Empty(t, engine.pendingConfigs) +} + func TestRestorePendingConfChangeFenceFromStorage(t *testing.T) { storage := committedTailStorageWithEntries(t, 100, 150, map[uint64]raftpb.Entry{ 130: { diff --git a/raft_member_replace_script_test.go b/raft_member_replace_script_test.go index affb9e09a..0b1352150 100644 --- a/raft_member_replace_script_test.go +++ b/raft_member_replace_script_test.go @@ -280,6 +280,18 @@ func TestRaftMemberReplaceScriptResumesAfterLearnerCommitResponseLoss(t *testing require.Contains(t, interruptedState, "min_catch_up_index=100\n", "catch-up floor must precede the membership proposal") require.Contains(t, interruptedState, "data_dir=/var/lib/elastickv\n") + // Simulate the old two-write window where the membership metadata reached + // disk but the stage advance did not. Equal index plus the expected + // signature must be accepted so an interrupted replacement remains resumable. + interruptedState = strings.Replace(interruptedState, "expected_config_index=11\n", "expected_config_index=12\n", 1) + interruptedState = strings.Replace( + interruptedState, + "expected_members=n1|127.0.0.1:50051|voter,n3|127.0.0.3:50051|voter\n", + "expected_members=n1|127.0.0.1:50051|voter,n2|127.0.0.2:50051|learner,n3|127.0.0.3:50051|voter\n", + 1, + ) + require.NoError(t, os.WriteFile(stateFile, []byte(interruptedState), 0o600)) + changedDataDirCmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") changedDataDirCmd.Dir = repoRoot changedDataDirEnv := append(append([]string{}, commandEnv...), "DATA_DIR=/srv/elastickv") diff --git a/scripts/raft-member-replace.sh b/scripts/raft-member-replace.sh index fa343a470..6b5daee61 100755 --- a/scripts/raft-member-replace.sh +++ b/scripts/raft-member-replace.sh @@ -342,7 +342,9 @@ parse_nodes() { [[ -n "$node_id" && -n "$node_host" ]] || fail "invalid NODES entry: $pair" [[ "$node_id" =~ ^[A-Za-z0-9._-]+$ ]] || fail "raft ID contains unsupported characters: $node_id" [[ "$node_host" != *'|'* ]] || fail "raft host contains unsupported characters: $node_host" - contains_value "$node_id" "${NODE_IDS[@]}" && fail "duplicate raft ID in NODES: $node_id" + if (( ${#NODE_IDS[@]} > 0 )) && contains_value "$node_id" "${NODE_IDS[@]}"; then + fail "duplicate raft ID in NODES: $node_id" + fi NODE_IDS+=("$node_id") NODE_HOSTS+=("$node_host") done @@ -487,8 +489,8 @@ adopt_committed_target_change() { actual_signature="$(configuration_signature "$VIEW_CONFIGURATION")" [[ "$actual_signature" == "$expected_signature" ]] || \ fail "$context observed an unexpected membership after a lost response" - (( VIEW_CONFIG_INDEX > STATE_EXPECTED_CONFIG_INDEX )) || \ - fail "$context did not advance the configuration index" + (( VIEW_CONFIG_INDEX >= STATE_EXPECTED_CONFIG_INDEX )) || \ + fail "$context regressed the configuration index" STATE_EXPECTED_MEMBERS="$actual_signature" STATE_EXPECTED_CONFIG_INDEX="$VIEW_CONFIG_INDEX" write_state @@ -700,15 +702,16 @@ wait_for_leader() { } wait_target_down() { - local elapsed=0 down_for=0 + local elapsed=0 down_since=-1 while (( elapsed < REPLACEMENT_TIMEOUT_SECONDS )); do if ! status_for "$STATE_TARGET_ADDRESS" >/dev/null; then - down_for=$((down_for + REPLACEMENT_POLL_SECONDS)) - if (( down_for >= REPLACEMENT_FENCE_VERIFY_SECONDS )); then + if (( down_since < 0 )); then + down_since=$elapsed + elif (( elapsed - down_since >= REPLACEMENT_FENCE_VERIFY_SECONDS )); then return 0 fi else - down_for=0 + down_since=-1 fi sleep "$REPLACEMENT_POLL_SECONDS" elapsed=$((elapsed + REPLACEMENT_POLL_SECONDS)) @@ -770,14 +773,37 @@ if [[ "$(id -u)" -ne 0 ]] && command -v sudo >/dev/null 2>&1 && sudo -n true 2>/ sudo_cmd=(sudo -n) fi +run_privileged() { + if (( ${#sudo_cmd[@]} > 0 )); then + "${sudo_cmd[@]}" "$@" + else + "$@" + fi +} + +container_running() { + local running + if ! running="$(docker inspect --format '{{.State.Running}}' "$container" 2>/dev/null)"; then + echo "cannot inspect container state: $container" >&2 + return 1 + fi + case "$running" in + true|false) printf '%s\n' "$running" ;; + *) echo "invalid container running state: $running" >&2; return 1 ;; + esac +} + case "$action" in stop) - docker stop "$container" >/dev/null 2>&1 || true - running="$(docker inspect --format '{{.State.Running}}' "$container" 2>/dev/null || echo false)" + running="$(container_running)" + if [[ "$running" == "true" ]]; then + docker stop "$container" >/dev/null + fi + running="$(container_running)" [[ "$running" == "false" ]] ;; reset-data) - docker_running="$(docker inspect --format '{{.State.Running}}' "$container" 2>/dev/null || echo false)" + docker_running="$(container_running)" [[ "$docker_running" == "false" ]] if [[ "$data_mode" == "archive" ]]; then if [[ -e "$archive_path" && -e "$data_dir" ]]; then @@ -785,12 +811,12 @@ case "$action" in exit 1 fi if [[ -e "$data_dir" ]]; then - "${sudo_cmd[@]}" mv "$data_dir" "$archive_path" + run_privileged mv "$data_dir" "$archive_path" fi [[ -e "$archive_path" ]] || { echo "target data directory was absent and no archive exists" >&2; exit 1; } else if [[ -e "$data_dir" ]]; then - "${sudo_cmd[@]}" rm -rf --one-file-system "$data_dir" + run_privileged rm -rf --one-file-system "$data_dir" fi [[ ! -e "$data_dir" ]] fi From 4d54f118ff9274b303aabfb8c7fef0fdbe1f2fde Mon Sep 17 00:00:00 2001 From: bootjp Date: Sun, 19 Jul 2026 19:19:39 +0900 Subject: [PATCH 6/6] Close membership replacement race windows --- internal/raftadmin/server.go | 53 +++++++---- internal/raftadmin/server_test.go | 79 +++++++++++++++++ internal/raftengine/engine.go | 3 + internal/raftengine/etcd/engine.go | 69 ++++++++++++++- internal/raftengine/etcd/engine_test.go | 25 ++++++ raft_member_replace_script_test.go | 113 +++++++++++++++++++++++- scripts/raft-member-replace.sh | 39 ++++++-- 7 files changed, 351 insertions(+), 30 deletions(-) diff --git a/internal/raftadmin/server.go b/internal/raftadmin/server.go index e07c736e8..a2f893db0 100644 --- a/internal/raftadmin/server.go +++ b/internal/raftadmin/server.go @@ -3,6 +3,7 @@ package raftadmin import ( "context" stderrors "errors" + "sync" "github.com/bootjp/elastickv/internal/raftengine" pb "github.com/bootjp/elastickv/proto" @@ -19,6 +20,11 @@ type Server struct { // aware adapter via NewServerWithInterceptor; encryption-unaware // builds keep this nil. interceptor MembershipChangeInterceptor + // membershipMu serializes the pre-add interceptor and the configuration + // proposal through completion. This closes the window where a concurrent + // membership RPC could mutate encryption metadata before the engine exposes + // the first request's pending configuration change in Status. + membershipMu sync.Mutex pb.UnimplementedRaftAdminServer } @@ -91,20 +97,7 @@ func (s *Server) AddVoter(ctx context.Context, req *pb.RaftAdminAddVoterRequest) if req == nil || req.Id == "" || req.Address == "" { return nil, grpcStatus(codes.InvalidArgument, "id and address are required") } - // Stage 7c §3.1 pre-step: run the encryption-aware adapter (if any) - // before the conf-change proposal so the new node's writer-registry - // row exists at apply time and any §6.1 uint16 collision halts here - // rather than after the conf-change is durable. - if s.interceptor != nil { - if err := s.interceptor.PreAddMember(ctx, req.Id); err != nil { - return nil, adminError(err) - } - } - index, err := s.admin.AddVoter(ctx, req.Id, req.Address, req.PreviousIndex) - if err != nil { - return nil, adminError(err) - } - return &pb.RaftAdminConfigurationChangeResponse{Index: index}, nil + return s.addMember(ctx, req.Id, req.Address, req.PreviousIndex, s.admin.AddVoter) } func (s *Server) AddLearner(ctx context.Context, req *pb.RaftAdminAddLearnerRequest) (*pb.RaftAdminConfigurationChangeResponse, error) { @@ -114,18 +107,42 @@ func (s *Server) AddLearner(ctx context.Context, req *pb.RaftAdminAddLearnerRequ if req == nil || req.Id == "" || req.Address == "" { return nil, grpcStatus(codes.InvalidArgument, "id and address are required") } + return s.addMember(ctx, req.Id, req.Address, req.PreviousIndex, s.admin.AddLearner) +} + +func (s *Server) addMember( + ctx context.Context, + id string, + address string, + previousIndex uint64, + propose func(context.Context, string, string, uint64) (uint64, error), +) (*pb.RaftAdminConfigurationChangeResponse, error) { + s.membershipMu.Lock() + defer s.membershipMu.Unlock() + if err := s.requireMembershipChangeReady(); err != nil { + return nil, err + } + // Stage 7c pre-registers encryption metadata before proposing membership. + // Keep that pre-step and the proposal inside the same serialized region. if s.interceptor != nil { - if err := s.interceptor.PreAddMember(ctx, req.Id); err != nil { + if err := s.interceptor.PreAddMember(ctx, id); err != nil { return nil, adminError(err) } } - index, err := s.admin.AddLearner(ctx, req.Id, req.Address, req.PreviousIndex) + index, err := propose(ctx, id, address, previousIndex) if err != nil { return nil, adminError(err) } return &pb.RaftAdminConfigurationChangeResponse{Index: index}, nil } +func (s *Server) requireMembershipChangeReady() error { + if s.engine.Status().PendingConfChange { + return adminError(raftengine.ErrMembershipChangePending) + } + return nil +} + func (s *Server) PromoteLearner(ctx context.Context, req *pb.RaftAdminPromoteLearnerRequest) (*pb.RaftAdminConfigurationChangeResponse, error) { if s == nil || s.admin == nil { return nil, grpcStatus(codes.Unimplemented, "promote learner is not supported by this raft engine") @@ -133,6 +150,8 @@ func (s *Server) PromoteLearner(ctx context.Context, req *pb.RaftAdminPromoteLea if req == nil || req.Id == "" { return nil, grpcStatus(codes.InvalidArgument, "id is required") } + s.membershipMu.Lock() + defer s.membershipMu.Unlock() index, err := s.admin.PromoteLearner(ctx, req.Id, req.PreviousIndex, req.MinAppliedIndex, req.SkipMinAppliedCheck) if err != nil { return nil, adminError(err) @@ -147,6 +166,8 @@ func (s *Server) RemoveServer(ctx context.Context, req *pb.RaftAdminRemoveServer if req == nil || req.Id == "" { return nil, grpcStatus(codes.InvalidArgument, "id is required") } + s.membershipMu.Lock() + defer s.membershipMu.Unlock() index, err := s.admin.RemoveServer(ctx, req.Id, req.PreviousIndex) if err != nil { return nil, adminError(err) diff --git a/internal/raftadmin/server_test.go b/internal/raftadmin/server_test.go index 82ab27367..ac576f91b 100644 --- a/internal/raftadmin/server_test.go +++ b/internal/raftadmin/server_test.go @@ -229,6 +229,9 @@ func TestServerMapsEngineAdminMethods(t *testing.T) { require.Equal(t, uint64(10), statusResp.CommitIndex) require.Equal(t, uint64(6), statusResp.ConfigurationIndex) require.True(t, statusResp.PendingConfChange) + engine.mu.Lock() + engine.status.PendingConfChange = false + engine.mu.Unlock() cfgResp, err := server.Configuration(context.Background(), &pb.RaftAdminConfigurationRequest{}) require.NoError(t, err) @@ -499,6 +502,82 @@ func TestServer_AddVoter_NilInterceptorSkipsPreStep(t *testing.T) { require.Equal(t, 1, len(engine.addVoterCalls)) } +func TestServer_AddMemberRejectsPendingConfigBeforeInterceptor(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + call func(*Server) error + }{ + { + name: "voter", + call: func(server *Server) error { + _, err := server.AddVoter(context.Background(), &pb.RaftAdminAddVoterRequest{Id: "n42", Address: "127.0.0.1:9999"}) + return err + }, + }, + { + name: "learner", + call: func(server *Server) error { + _, err := server.AddLearner(context.Background(), &pb.RaftAdminAddLearnerRequest{Id: "n42", Address: "127.0.0.1:9999"}) + return err + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + engine := &fakeEngine{status: raftengine.Status{PendingConfChange: true}} + interceptor := &recordingInterceptor{} + server := NewServerWithInterceptor(engine, interceptor) + + err := test.call(server) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) + require.ErrorContains(t, err, raftengine.ErrMembershipChangePending.Error()) + require.Empty(t, interceptor.calls) + require.Empty(t, engine.addVoterCalls) + require.Empty(t, engine.addLearnerCalls) + }) + } +} + +func TestServerSerializesMembershipInterceptorAndProposal(t *testing.T) { + t.Parallel() + + firstProposalEntered := make(chan struct{}) + releaseFirstProposal := make(chan struct{}) + interceptorCalls := make(chan string, 2) + engine := &fakeEngine{addVoterHook: func() { + close(firstProposalEntered) + <-releaseFirstProposal + }} + interceptor := &recordingInterceptor{preHook: func(id string) { interceptorCalls <- id }} + server := NewServerWithInterceptor(engine, interceptor) + + firstDone := make(chan error, 1) + go func() { + _, err := server.AddVoter(context.Background(), &pb.RaftAdminAddVoterRequest{Id: "n1", Address: "127.0.0.1:9001"}) + firstDone <- err + }() + require.Equal(t, "n1", <-interceptorCalls) + <-firstProposalEntered + + secondDone := make(chan error, 1) + go func() { + _, err := server.AddLearner(context.Background(), &pb.RaftAdminAddLearnerRequest{Id: "n2", Address: "127.0.0.1:9002"}) + secondDone <- err + }() + select { + case id := <-interceptorCalls: + require.Failf(t, "concurrent interceptor ran", "interceptor for %s ran before the first membership proposal completed", id) + case <-time.After(100 * time.Millisecond): + } + + close(releaseFirstProposal) + require.NoError(t, <-firstDone) + require.Equal(t, "n2", <-interceptorCalls) + require.NoError(t, <-secondDone) +} + // TestServer_AddLearner_InterceptorContract is the symmetric set for // AddLearner — combined into one test since the behavior mirrors // AddVoter exactly. diff --git a/internal/raftengine/engine.go b/internal/raftengine/engine.go index 2f68dc90d..7b010a3b4 100644 --- a/internal/raftengine/engine.go +++ b/internal/raftengine/engine.go @@ -35,6 +35,9 @@ var ( // unapplied configuration change, so transfer is rejected until the // membership transition settles. ErrLeadershipTransferConfChangePending = errors.New("raft engine: leadership transfer blocked by pending config change") + // ErrMembershipChangePending indicates that a membership proposal is + // already present in the local Raft log and has not settled yet. + ErrMembershipChangePending = errors.New("raft engine: membership change blocked by pending config change") // ErrEnvelopeCutoverInProgress indicates the §7.1 raft-envelope // cutover barrier is open on this leader and rejecting fresh // USER proposals on the Propose path. The error is the step-1 diff --git a/internal/raftengine/etcd/engine.go b/internal/raftengine/etcd/engine.go index c7aa71b84..15a2bfc26 100644 --- a/internal/raftengine/etcd/engine.go +++ b/internal/raftengine/etcd/engine.go @@ -143,7 +143,7 @@ var ( errLeadershipTransferNoHealthyTarget = errors.Mark(errors.New("etcd raft leadership transfer has no healthy target"), raftengine.ErrLeadershipTransferNoHealthyTarget) errLeadershipTransferTargetNotCaughtUp = errors.Mark(errors.New("etcd raft leadership transfer target is not caught up"), raftengine.ErrLeadershipTransferTargetNotCaughtUp) errLeadershipTransferConfChangePending = errors.Mark(errors.New("etcd raft leadership transfer blocked by pending config change"), raftengine.ErrLeadershipTransferConfChangePending) - errMembershipConfChangePending = errors.New("etcd raft membership change blocked by pending config change") + errMembershipConfChangePending = errors.Mark(errors.New("etcd raft membership change blocked by pending config change"), raftengine.ErrMembershipChangePending) errTooManyPendingConfigs = errors.New("etcd raft engine has too many pending config changes") errPromoteLearnerNotLearner = errors.New("etcd raft promote-learner target is not a learner") errPromoteLearnerNoProgress = errors.New("etcd raft promote-learner target has no leader-side progress entry") @@ -2034,6 +2034,10 @@ func (e *Engine) handlePromoteLearner(req adminRequest) { // PromoteLearner / RemoveServer. The caller has already validated the leader and // prevIndex preconditions in handleAdmin. func (e *Engine) proposeMembershipChange(req adminRequest, changeType raftpb.ConfChangeType, peer Peer) { + if err := e.reconcilePendingConfChangeFence(); err != nil { + req.done <- adminResult{err: err} + return + } if e.hasPendingConfChange() { req.done <- adminResult{err: errors.WithStack(errMembershipConfChangePending)} return @@ -2540,12 +2544,15 @@ func (e *Engine) applyReadyEntries(entries []*raftpb.Entry) error { if len(entries) == 0 { return nil } + if err := e.storage.Append(entries); err != nil { + return errors.WithStack(err) + } for _, entry := range entries { if entry.GetType() == raftpb.EntryConfChange || entry.GetType() == raftpb.EntryConfChangeV2 { e.markPendingConfChange(entry.GetIndex()) } } - return errors.WithStack(e.storage.Append(entries)) + return e.reconcilePendingConfChangeFence() } func (e *Engine) restorePendingConfChangeFenceFromStorage(applied uint64) error { @@ -3916,6 +3923,64 @@ func (e *Engine) hasPendingConfChange() bool { return e.pendingConfChangeIndex.Load() != 0 } +// reconcilePendingConfChangeFence drops a volatile fence when a later Ready +// proves that the proposal at that index was overwritten after leadership +// loss. A missing index is not enough: the proposal may still live only in +// raft's unstable log, so the fence remains until storage covers the index. +func (e *Engine) reconcilePendingConfChangeFence() error { + pending := e.pendingConfChangeIndex.Load() + if pending == 0 { + return nil + } + if e.storage == nil { + return nil + } + if e.appliedIndex.Load() >= pending { + e.clearPendingConfChange(pending) + return nil + } + covered, isConfChange, err := e.persistedPendingConfChange(pending) + if err != nil { + return err + } + if !covered { + return nil + } + if !isConfChange { + e.pendingConfChangeIndex.CompareAndSwap(pending, 0) + } + return nil +} + +func (e *Engine) persistedPendingConfChange(index uint64) (bool, bool, error) { + firstIndex, err := e.storage.FirstIndex() + if err != nil { + return false, false, errors.Wrap(err, "reconcile pending conf-change fence: first index") + } + lastIndex, err := e.storage.LastIndex() + if err != nil { + return false, false, errors.Wrap(err, "reconcile pending conf-change fence: last index") + } + if lastIndex < index { + return false, false, nil + } + if index < firstIndex { + return true, false, nil + } + if index == math.MaxUint64 { + return false, false, errors.New("reconcile pending conf-change fence: index overflows half-open range") + } + entries, err := e.storage.Entries(index, index+1, math.MaxUint64) + if err != nil { + return false, false, errors.Wrapf(err, "reconcile pending conf-change fence: entry %d", index) + } + if len(entries) != 1 { + return false, false, errors.Errorf("reconcile pending conf-change fence: expected one entry at %d, got %d", index, len(entries)) + } + entryType := entries[0].GetType() + return true, entryType == raftpb.EntryConfChange || entryType == raftpb.EntryConfChangeV2, nil +} + func (e *Engine) shouldCampaignSingleNode() bool { // Count VOTERS in the current configuration, not all servers -- // a learner does not vote and must not block the single-voter diff --git a/internal/raftengine/etcd/engine_test.go b/internal/raftengine/etcd/engine_test.go index 022b48ed7..22beaf800 100644 --- a/internal/raftengine/etcd/engine_test.go +++ b/internal/raftengine/etcd/engine_test.go @@ -1650,6 +1650,31 @@ func TestProposeMembershipChangeRejectsPendingConfig(t *testing.T) { require.Empty(t, engine.pendingConfigs) } +func TestReconcilePendingConfChangeFenceClearsOverwrittenProposal(t *testing.T) { + storage := committedTailStorageWithEntries(t, 10, 12, map[uint64]raftpb.Entry{ + 12: { + Type: entryTypePtr(raftpb.EntryNormal), + Data: []byte("replacement leader entry"), + }, + }) + engine := &Engine{storage: storage} + engine.appliedIndex.Store(10) + engine.markPendingConfChange(12) + + require.NoError(t, engine.reconcilePendingConfChangeFence()) + require.False(t, engine.hasPendingConfChange()) +} + +func TestReconcilePendingConfChangeFenceKeepsUnstableProposal(t *testing.T) { + storage := committedTailStorageWithEntries(t, 10, 11, nil) + engine := &Engine{storage: storage} + engine.appliedIndex.Store(10) + engine.markPendingConfChange(12) + + require.NoError(t, engine.reconcilePendingConfChangeFence()) + require.True(t, engine.hasPendingConfChange()) +} + func TestRestorePendingConfChangeFenceFromStorage(t *testing.T) { storage := committedTailStorageWithEntries(t, 100, 150, map[uint64]raftpb.Entry{ 130: { diff --git a/raft_member_replace_script_test.go b/raft_member_replace_script_test.go index 0b1352150..685e4943e 100644 --- a/raft_member_replace_script_test.go +++ b/raft_member_replace_script_test.go @@ -58,6 +58,8 @@ func TestRaftMemberReplaceScriptCompletesFencedLearnerLifecycle(t *testing.T) { "FAKE_STATE_DIR=" + stateDir, }) require.Contains(t, output, "replacement completed: target=n2") + require.GreaterOrEqual(t, strings.Count(mustReadTestFile(t, filepath.Join(stateDir, "stability-status-calls")), "\n"), 10, + "a one-second stability window with one-second polling requires two complete healthy observations") require.FileExists(t, filepath.Join(stateDir, "verified")) require.FileExists(t, filepath.Join(stateDir, "data-reset")) require.Equal(t, "voter\n", mustReadTestFile(t, filepath.Join(stateDir, "member"))) @@ -136,6 +138,80 @@ func TestRaftMemberReplaceScriptStopsProcessAfterExternalFence(t *testing.T) { require.Equal(t, "stop\nreset-data\n", mustReadTestFile(t, filepath.Join(stateDir, "remote-actions"))) } +func TestRaftMemberReplaceScriptValidatesReportedLeader(t *testing.T) { + repoRoot, err := os.Getwd() + require.NoError(t, err) + stateDir := t.TempDir() + fakeBin := filepath.Join(stateDir, "bin") + require.NoError(t, os.Mkdir(fakeBin, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "member"), []byte("voter\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "config-index"), []byte("10\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "stale-leader-once"), nil, 0o600)) + + raftadmin := writeExecutableTestFile(t, fakeBin, "raftadmin", fakeRaftadminScript) + ssh := writeExecutableTestFile(t, fakeBin, "ssh", fakeReplacementSSHScript) + rolling := writeExecutableTestFile(t, fakeBin, "rolling-update", fakeReplacementRollingScript) + envFile := filepath.Join(stateDir, "deploy.env") + require.NoError(t, os.WriteFile(envFile, []byte("NODES=n3=127.0.0.3,n1=127.0.0.1,n2=127.0.0.2\n"), 0o600)) + + output := runReplacementScript(t, repoRoot, []string{ + "TARGET_NODE=n2", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=container", + "REPLACEMENT_FENCE_VERIFY_SECONDS=1", + "REPLACEMENT_VERIFY_COMMAND=true", + "REPLACEMENT_STABILITY_SECONDS=1", + "REPLACEMENT_TIMEOUT_SECONDS=5", + "REPLACEMENT_POLL_SECONDS=1", + "REPLACEMENT_STATE_FILE=" + filepath.Join(stateDir, "replace.state"), + "ROLLING_UPDATE_ENV_FILE=" + envFile, + "ROLLING_UPDATE_SCRIPT=" + rolling, + "RAFTADMIN_BIN=" + raftadmin, + "SSH_BIN=" + ssh, + "FAKE_STATE_DIR=" + stateDir, + }) + require.Contains(t, output, "replacement completed: target=n2") +} + +func TestRaftMemberReplaceScriptAdoptsConfigIndexAfterLeadershipTransfer(t *testing.T) { + repoRoot, err := os.Getwd() + require.NoError(t, err) + stateDir := t.TempDir() + fakeBin := filepath.Join(stateDir, "bin") + require.NoError(t, os.Mkdir(fakeBin, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "member"), []byte("voter\n"), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(stateDir, "config-index"), []byte("10\n"), 0o600)) + + raftadmin := writeExecutableTestFile(t, fakeBin, "raftadmin", fakeRaftadminScript) + ssh := writeExecutableTestFile(t, fakeBin, "ssh", fakeReplacementSSHScript) + rolling := writeExecutableTestFile(t, fakeBin, "rolling-update", fakeReplacementRollingScript) + envFile := filepath.Join(stateDir, "deploy.env") + require.NoError(t, os.WriteFile(envFile, []byte("NODES=n1=127.0.0.1,n2=127.0.0.2,n3=127.0.0.3\n"), 0o600)) + stateFile := filepath.Join(stateDir, "replace.state") + + output := runReplacementScript(t, repoRoot, []string{ + "TARGET_NODE=n2", + "REPLACEMENT_CONFIRM=n2", + "REPLACEMENT_FENCE_MODE=container", + "REPLACEMENT_FENCE_VERIFY_SECONDS=1", + "REPLACEMENT_VERIFY_COMMAND=true", + "REPLACEMENT_STABILITY_SECONDS=1", + "REPLACEMENT_TIMEOUT_SECONDS=5", + "REPLACEMENT_POLL_SECONDS=1", + "REPLACEMENT_STATE_FILE=" + stateFile, + "ROLLING_UPDATE_ENV_FILE=" + envFile, + "ROLLING_UPDATE_SCRIPT=" + rolling, + "RAFTADMIN_BIN=" + raftadmin, + "SSH_BIN=" + ssh, + "FAKE_STATE_DIR=" + stateDir, + "FAKE_TARGET_STARTS_LEADER=true", + "FAKE_BUMP_CONFIG_ON_TRANSFER=true", + }) + require.Contains(t, output, "transferring leadership from n2 to n1") + require.Contains(t, output, "replacement completed: target=n2") + require.Contains(t, mustReadTestFile(t, stateFile), "expected_config_index=14\n") +} + func TestRaftMemberReplaceScriptRequiresInvocationControls(t *testing.T) { repoRoot, err := os.Getwd() require.NoError(t, err) @@ -292,6 +368,16 @@ func TestRaftMemberReplaceScriptResumesAfterLearnerCommitResponseLoss(t *testing ) require.NoError(t, os.WriteFile(stateFile, []byte(interruptedState), 0o600)) + staleMetadataState := strings.Replace(interruptedState, "expected_config_index=12\n", "expected_config_index=11\n", 1) + require.NoError(t, os.WriteFile(stateFile, []byte(staleMetadataState), 0o600)) + staleMetadataCmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") + staleMetadataCmd.Dir = repoRoot + staleMetadataCmd.Env = append(os.Environ(), commandEnv...) + staleMetadataOutput, staleMetadataErr := staleMetadataCmd.CombinedOutput() + require.Error(t, staleMetadataErr) + require.Contains(t, string(staleMetadataOutput), "saved post-change configuration index changed: expected=11 actual=12") + require.NoError(t, os.WriteFile(stateFile, []byte(interruptedState), 0o600)) + changedDataDirCmd := exec.CommandContext(t.Context(), "bash", "scripts/raft-member-replace.sh", "--execute") changedDataDirCmd.Dir = repoRoot changedDataDirEnv := append(append([]string{}, commandEnv...), "DATA_DIR=/srv/elastickv") @@ -413,20 +499,35 @@ command="$2" state_dir="$FAKE_STATE_DIR" if [[ "$command" == "status" ]]; then + if [[ -f "$state_dir/stability-phase" ]]; then + printf '%s\n' "$addr" >> "$state_dir/stability-status-calls" + fi if [[ "$addr" == "127.0.0.2:50051" && -f "$state_dir/down-n2" ]]; then exit 1 fi - if [[ "$addr" == "127.0.0.1:50051" ]]; then + leader_id=n1 + leader_address=127.0.0.1:50051 + if [[ "${FAKE_TARGET_STARTS_LEADER:-false}" == "true" && ! -f "$state_dir/transferred" ]]; then + leader_id=n2 + leader_address=127.0.0.2:50051 + fi + if [[ "$addr" == "$leader_address" ]]; then raft_state=LEADER elif [[ "$addr" == "127.0.0.3:50051" && -f "$state_dir/candidate-n3" ]]; then raft_state=CANDIDATE else raft_state=FOLLOWER fi + if [[ "$addr" == "127.0.0.3:50051" && -f "$state_dir/stale-leader-once" ]]; then + rm -f "$state_dir/stale-leader-once" + raft_state=FOLLOWER + leader_id=n2 + leader_address=127.0.0.2:50051 + fi cat <&2 @@ -528,5 +633,7 @@ if [[ "${RAFT_JOIN_NODE:-}" == "n2" ]]; then rm -f "$FAKE_STATE_DIR/fail-join-rollout-once" exit 74 fi +else + touch "$FAKE_STATE_DIR/stability-phase" fi ` diff --git a/scripts/raft-member-replace.sh b/scripts/raft-member-replace.sh index 6b5daee61..e15970f52 100755 --- a/scripts/raft-member-replace.sh +++ b/scripts/raft-member-replace.sh @@ -489,8 +489,13 @@ adopt_committed_target_change() { actual_signature="$(configuration_signature "$VIEW_CONFIGURATION")" [[ "$actual_signature" == "$expected_signature" ]] || \ fail "$context observed an unexpected membership after a lost response" - (( VIEW_CONFIG_INDEX >= STATE_EXPECTED_CONFIG_INDEX )) || \ - fail "$context regressed the configuration index" + if [[ "$STATE_EXPECTED_MEMBERS" == "$actual_signature" ]]; then + [[ "$VIEW_CONFIG_INDEX" == "$STATE_EXPECTED_CONFIG_INDEX" ]] || \ + fail "$context saved post-change configuration index changed: expected=$STATE_EXPECTED_CONFIG_INDEX actual=$VIEW_CONFIG_INDEX" + else + (( VIEW_CONFIG_INDEX > STATE_EXPECTED_CONFIG_INDEX )) || \ + fail "$context did not advance the configuration index after the lost response" + fi STATE_EXPECTED_MEMBERS="$actual_signature" STATE_EXPECTED_CONFIG_INDEX="$VIEW_CONFIG_INDEX" write_state @@ -514,7 +519,8 @@ record_committed_target_change() { } discover_leader() { - local address status state leader_address leader_id + local address status state leader_address leader_id leader_status + local verified_state verified_id verified_address local i for i in "${!NODE_HOSTS[@]}"; do address="${NODE_HOSTS[$i]}:${RAFT_PORT}" @@ -528,8 +534,15 @@ discover_leader() { leader_id="$(printf '%s\n' "$status" | field_value leader_id)" leader_address="$(printf '%s\n' "$status" | field_value leader_address)" if [[ -n "$leader_id" && -n "$leader_address" ]]; then - printf '%s|%s\n' "$leader_id" "$leader_address" - return 0 + leader_status="$(status_for "$leader_address" || true)" + [[ -n "$leader_status" ]] || continue + verified_state="$(printf '%s\n' "$leader_status" | field_value state)" + verified_id="$(printf '%s\n' "$leader_status" | field_value leader_id)" + verified_address="$(printf '%s\n' "$leader_status" | field_value leader_address)" + if [[ "$verified_state" == "LEADER" && "$verified_id" == "$leader_id" && "$verified_address" == "$leader_address" ]]; then + printf '%s|%s\n' "$leader_id" "$leader_address" + return 0 + fi fi done return 1 @@ -671,9 +684,10 @@ preflight() { raftadmin "$leader_address" leadership_transfer_to_server "$candidate_id" "$candidate_address" >/dev/null wait_for_leader "$candidate_id" >/dev/null || fail "leadership did not transfer to $candidate_id" load_leader_view || fail "cannot read stable leader view after leadership transfer" - [[ "$VIEW_CONFIG_INDEX" == "$config_index" ]] || fail "membership changed during leadership transfer" [[ "$(configuration_signature "$VIEW_CONFIGURATION")" == "$initial_signature" ]] || \ fail "membership changed during leadership transfer" + config_index="$VIEW_CONFIG_INDEX" + config="$VIEW_CONFIGURATION" fi STATE_EXPECTED_VOTERS="$(voter_csv "$config")" @@ -1049,19 +1063,26 @@ verify_cluster_once() { } verify_stability() { - local elapsed=0 stable=0 expected_leader="" leader + local elapsed=0 stable_since=-1 expected_leader="" leader stable while (( elapsed < REPLACEMENT_TIMEOUT_SECONDS )); do leader="$(verify_cluster_once || true)" if [[ -n "$leader" && ( -z "$expected_leader" || "$leader" == "$expected_leader" ) ]]; then + if [[ -z "$expected_leader" ]]; then + stable_since=$elapsed + fi expected_leader="$leader" - stable=$((stable + REPLACEMENT_POLL_SECONDS)) + stable=$((elapsed - stable_since)) if (( stable >= REPLACEMENT_STABILITY_SECONDS )); then log "cluster stable for ${stable}s with leader=$leader" return 0 fi else expected_leader="$leader" - stable=0 + if [[ -n "$leader" ]]; then + stable_since=$elapsed + else + stable_since=-1 + fi fi sleep "$REPLACEMENT_POLL_SECONDS" elapsed=$((elapsed + REPLACEMENT_POLL_SECONDS))