From fd9cc13a99a1b9d9aeadb3d97f2695685d099824 Mon Sep 17 00:00:00 2001 From: Ilan Zuckerman Date: Wed, 29 Jul 2026 14:55:05 +0300 Subject: [PATCH 1/2] OCPBUGS-98719: retry GetVotingMemberNames on transient etcd client failures `GetVotingMemberNames` previously failed immediately when `NewEtcdClient` or `MemberList` errored, which is flaky during master replacement while etcd is restarting and port-forwards drop. Poll and retry those transient errors instead of failing the CPMS OnDelete scaling test on the first attempt. --- test/extended/etcd/helpers/helpers.go | 50 +++++++++++++++++---------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/test/extended/etcd/helpers/helpers.go b/test/extended/etcd/helpers/helpers.go index 129475f09b17..665b1045dcab 100644 --- a/test/extended/etcd/helpers/helpers.go +++ b/test/extended/etcd/helpers/helpers.go @@ -442,29 +442,43 @@ func EnsureUpdatedReplicasOnCPMS(ctx context.Context, t TestingT, expectedCount }) } -// GetVotingMemberNames returns the list of current voting etcd member names +// GetVotingMemberNames returns the list of current voting etcd member names. +// It retries transient etcd client failures (e.g. port-forward errors while etcd is restarting). func GetVotingMemberNames(ctx context.Context, t TestingT, etcdClientFactory EtcdClientCreator) ([]string, error) { - etcdClient, closeFn, err := etcdClientFactory.NewEtcdClient() - if err != nil { - return nil, fmt.Errorf("failed to get etcd client: %w", err) - } - defer closeFn() + waitPollInterval := 15 * time.Second + waitPollTimeout := 10 * time.Minute + var votingMemberNames []string - memberCtx, cancel := context.WithTimeout(ctx, 15*time.Second) - defer cancel() - memberList, err := etcdClient.MemberList(memberCtx) - if err != nil { - return nil, fmt.Errorf("failed to get the member list: %w", err) - } + framework.Logf("Waiting up to %s to get current voting etcd member names", waitPollTimeout.String()) + err := wait.PollUntilContextTimeout(ctx, waitPollInterval, waitPollTimeout, true, func(ctx context.Context) (done bool, err error) { + etcdClient, closeFn, err := etcdClientFactory.NewEtcdClient() + if err != nil { + framework.Logf("failed to get etcd client, will retry, err: %v", err) + return false, nil + } + defer closeFn() - var votingMemberNames []string - for _, member := range memberList.Members { - if !member.IsLearner { - votingMemberNames = append(votingMemberNames, member.Name) + memberCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + memberList, err := etcdClient.MemberList(memberCtx) + if err != nil { + framework.Logf("failed to get the member list, will retry, err: %v", err) + return false, nil + } + + votingMemberNames = nil + for _, member := range memberList.Members { + if !member.IsLearner { + votingMemberNames = append(votingMemberNames, member.Name) + } } - } - framework.Logf("Current voting etcd members: %v", votingMemberNames) + framework.Logf("Current voting etcd members: %v", votingMemberNames) + return true, nil + }) + if err != nil { + return nil, fmt.Errorf("failed to get voting member names: %w", err) + } return votingMemberNames, nil } From 4800f55c3bb03ba85c4d45c763e4fd1f6f7a34d4 Mon Sep 17 00:00:00 2001 From: Ilan Zuckerman Date: Mon, 3 Aug 2026 10:10:45 +0300 Subject: [PATCH 2/2] Introducing new var `memberListTimeout` for consistency --- test/extended/etcd/helpers/helpers.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/extended/etcd/helpers/helpers.go b/test/extended/etcd/helpers/helpers.go index 665b1045dcab..63f60f694030 100644 --- a/test/extended/etcd/helpers/helpers.go +++ b/test/extended/etcd/helpers/helpers.go @@ -447,6 +447,7 @@ func EnsureUpdatedReplicasOnCPMS(ctx context.Context, t TestingT, expectedCount func GetVotingMemberNames(ctx context.Context, t TestingT, etcdClientFactory EtcdClientCreator) ([]string, error) { waitPollInterval := 15 * time.Second waitPollTimeout := 10 * time.Minute + memberListTimeout := 15 * time.Second var votingMemberNames []string framework.Logf("Waiting up to %s to get current voting etcd member names", waitPollTimeout.String()) @@ -458,7 +459,7 @@ func GetVotingMemberNames(ctx context.Context, t TestingT, etcdClientFactory Etc } defer closeFn() - memberCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + memberCtx, cancel := context.WithTimeout(ctx, memberListTimeout) defer cancel() memberList, err := etcdClient.MemberList(memberCtx) if err != nil {