Skip to content

Add whitelist to retries #39

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
YuryLysov wants to merge 3 commits into
base: main
Choose a base branch
from
Draft

Add whitelist to retries #39

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
70565de
Add whitelist to retries
YuryLysov Jul 13, 2026
4aa6302
Fix gomod
YuryLysov Jul 13, 2026
23efcf2
Bump lib-dhctl
YuryLysov Jul 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion examples/cobra/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.25.8

require (
github.com/deckhouse/lib-connection v0.0.0-00010101000000-000000000000
github.com/deckhouse/lib-dhctl v0.20.0
github.com/deckhouse/lib-dhctl v0.21.1-0.20260714100121-89ca3451ee2b
github.com/name212/govalue v1.1.0
github.com/spf13/cobra v1.10.2
k8s.io/apimachinery v0.33.12
Expand Down
4 changes: 2 additions & 2 deletions examples/cobra/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,8 +55,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deckhouse/deckhouse/pkg/log v0.1.1-0.20251230144142-2bad7c3d1edf h1:4HrDzRZcLpREJ+2cSGNmxHVQlxXRcH2r5TGmTcoTZiU=
github.com/deckhouse/deckhouse/pkg/log v0.1.1-0.20251230144142-2bad7c3d1edf/go.mod h1:pbAxTSDcPmwyl3wwKDcEB3qdxHnRxqTV+J0K+sha8bw=
github.com/deckhouse/lib-dhctl v0.20.0 h1:ahubdw6pdgg/9GGKCgU68d+6VSv04ZEhzt6+xQIDO48=
github.com/deckhouse/lib-dhctl v0.20.0/go.mod h1:hICPf+k3u8V73ndNRNVWUdye/mzo6adLZkXM6vfcmVc=
github.com/deckhouse/lib-dhctl v0.21.1-0.20260714100121-89ca3451ee2b h1:x+uNJ0D7Q18UFgLDVji/Q1R7H5w/jwzvA9VYuxcdrKI=
github.com/deckhouse/lib-dhctl v0.21.1-0.20260714100121-89ca3451ee2b/go.mod h1:hICPf+k3u8V73ndNRNVWUdye/mzo6adLZkXM6vfcmVc=
github.com/deckhouse/lib-gossh v0.3.0 h1:FUAlF8+fLnBCII9hXSNx+arZ4PH3H/6fzp5LBlnmlps=
github.com/deckhouse/lib-gossh v0.3.0/go.mod h1:6bT8jf2fkBPEhYBU35+vMBr5YscliTiS+Vr8v06C+70=
github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.25.8
require (
al.essio.dev/pkg/shellescape v1.6.0
github.com/bramvdbogaerde/go-scp v1.6.0
github.com/deckhouse/lib-dhctl v0.20.0
github.com/deckhouse/lib-dhctl v0.21.1-0.20260714100121-89ca3451ee2b
github.com/deckhouse/lib-gossh v0.3.0
github.com/flant/kube-client v1.6.0
github.com/go-openapi/spec v0.19.8
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deckhouse/deckhouse/pkg/log v0.1.1-0.20251230144142-2bad7c3d1edf h1:4HrDzRZcLpREJ+2cSGNmxHVQlxXRcH2r5TGmTcoTZiU=
github.com/deckhouse/deckhouse/pkg/log v0.1.1-0.20251230144142-2bad7c3d1edf/go.mod h1:pbAxTSDcPmwyl3wwKDcEB3qdxHnRxqTV+J0K+sha8bw=
github.com/deckhouse/lib-dhctl v0.20.0 h1:ahubdw6pdgg/9GGKCgU68d+6VSv04ZEhzt6+xQIDO48=
github.com/deckhouse/lib-dhctl v0.20.0/go.mod h1:hICPf+k3u8V73ndNRNVWUdye/mzo6adLZkXM6vfcmVc=
github.com/deckhouse/lib-dhctl v0.21.1-0.20260714100121-89ca3451ee2b h1:x+uNJ0D7Q18UFgLDVji/Q1R7H5w/jwzvA9VYuxcdrKI=
github.com/deckhouse/lib-dhctl v0.21.1-0.20260714100121-89ca3451ee2b/go.mod h1:hICPf+k3u8V73ndNRNVWUdye/mzo6adLZkXM6vfcmVc=
github.com/deckhouse/lib-gossh v0.3.0 h1:FUAlF8+fLnBCII9hXSNx+arZ4PH3H/6fzp5LBlnmlps=
github.com/deckhouse/lib-gossh v0.3.0/go.mod h1:6bT8jf2fkBPEhYBU35+vMBr5YscliTiS+Vr8v06C+70=
github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
Expand Down
10 changes: 8 additions & 2 deletions pkg/kube/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,12 @@ var (
_ connection.KubeClient = &KubernetesClient{}

ErrStoppedKubeClient = fmt.Errorf("use stopped kube client")

// ErrKubeClientTransient marks InitContext failures that may succeed on retry
// (e.g. flaky kube-proxy startup over SSH), as opposed to permanent configuration
// errors (bad kubeconfig path/context, malformed rest config) which do not involve
// any network call and fail identically on every attempt.
ErrKubeClientTransient = fmt.Errorf("kube client: transient error, may succeed on retry")
)

type ClientLoopParams struct {
Expand Down Expand Up @@ -182,14 +188,14 @@ func (k *KubernetesClient) initContext(ctx context.Context, params *Config, opts
if !options.NoStartKubeProxy {
_, err := k.StartKubernetesProxy(ctx, options)
if err != nil {
return err
return fmt.Errorf("%w: %w", ErrKubeClientTransient, err)
}
}
default:
if !options.NoStartKubeProxy {
port, err := k.StartKubernetesProxy(ctx, options)
if err != nil {
return err
return fmt.Errorf("%w: %w", ErrKubeClientTransient, err)
}
kubeClient.WithServer("http://localhost:" + port)
}
Expand Down
4 changes: 4 additions & 0 deletions pkg/provider/kube.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,6 +213,10 @@ func (p *DefaultKubeProvider) connectToKubernetesAPI(ctx context.Context, client
initClientLoopParams := retry.SafeCloneOrNewParams(p.loopsParams.InitClient, defaultInitClientParamsOpts...).Clone(
retry.WithName("Get Kubernetes API client"),
retry.WithLogger(logger),
// Permanent configuration errors (bad kubeconfig path/context, malformed rest
// config) fail identically on every attempt, so only retry the transient
// failures raised while starting the kube-proxy tunnel over SSH.
retry.WithWhitelist(kube.ErrKubeClientTransient),
)

err := retry.NewLoopWithParams(initClientLoopParams).RunContext(ctx, func() error {
Expand Down
37 changes: 29 additions & 8 deletions pkg/ssh/gossh/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,11 @@ var (
_ connection.Interface = &Client{}
)

// ErrDialTransient marks dial failures that may succeed on retry (network-level
// connect/handshake errors), as opposed to permanent local socket errors (wrong
// conn type, failed to set socket options) that fail identically on every attempt.
var ErrDialTransient = errors.New("dial: transient error, may succeed on retry")

func NewClient(ctx context.Context, sett settings.Settings, session *session.Session, privKeys []session.AgentPrivateKey) *Client {
return &Client{
sessionClient: session,
Expand Down Expand Up @@ -473,7 +478,13 @@ func (s *Client) directConnectToTarget(ctx context.Context) (*gossh.Client, erro
}

hostLoopParams := retry.SafeCloneOrNewParams(s.loopsParams.ConnectToHostDirectly, defaultClientDirectlyLoopParamsOps...).
Clone(retry.WithName("Get SSH client"))
Clone(
retry.WithName("Get SSH client"),
// Permanent local socket errors (wrong conn type, failed to set socket
// options) fail identically on every attempt, so only retry dial/handshake
// failures that may clear up on their own.
retry.WithWhitelist(ErrDialTransient),
)

if err := s.runInLoop(ctx, hostLoopParams, connectToHost); err != nil {
lastHost := fmt.Sprintf("'%s:%s' with user '%s'", s.sessionClient.Host(), s.sessionClient.Port, s.sessionClient.User)
Expand Down Expand Up @@ -516,7 +527,13 @@ func (s *Client) connectToBastion(ctx context.Context) (*gossh.Client, error) {
}

bastionLoopParams := retry.SafeCloneOrNewParams(s.loopsParams.ConnectToBastion, defaultClientViaBastionLoopParamsOps...).
Clone(retry.WithName("Get bastion SSH client"))
Clone(
retry.WithName("Get bastion SSH client"),
// Permanent local socket errors (wrong conn type, failed to set socket
// options) fail identically on every attempt, so only retry dial/handshake
// failures that may clear up on their own.
retry.WithWhitelist(ErrDialTransient),
)

if err := s.runInLoop(ctx, bastionLoopParams, connectToBastion); err != nil {
return nil, fmt.Errorf("Could not connect to %s: %w", fullHost, err)
Expand Down Expand Up @@ -631,8 +648,11 @@ func (s *Client) createSSHConnection(c net.Conn, addr string, config *gossh.Clie
}

func (s *Client) dialContext(ctx context.Context, network, addr string, config *gossh.ClientConfig) (*gossh.Client, error) {
closeConnectionAndReturnErr := func(msg string, err error, conn net.Conn) (*gossh.Client, error) {
closeConnectionAndReturnErr := func(msg string, err error, conn net.Conn, retryable bool) (*gossh.Client, error) {
err = fmt.Errorf("Cannot Dial to '%s' %s: %w", addr, msg, err)
if retryable {
err = fmt.Errorf("%w: %w", err, ErrDialTransient)
}

if closeErr := utils.SafeClose(conn); closeErr != nil {
err = fmt.Errorf("%w and cannot close connection %w", err, closeErr)
Expand All @@ -643,17 +663,17 @@ func (s *Client) dialContext(ctx context.Context, network, addr string, config *
d := net.Dialer{Timeout: config.Timeout}
conn, err := d.DialContext(ctx, network, addr)
if err != nil {
return closeConnectionAndReturnErr("connect", err, conn)
return closeConnectionAndReturnErr("connect", err, conn, true)
}

tcpConn, ok := conn.(*net.TCPConn)
if !ok {
return closeConnectionAndReturnErr("is not tcp", err, conn)
return closeConnectionAndReturnErr("is not tcp", err, conn, false)
}

err = tcpConn.SetKeepAlive(true)
if err != nil {
return closeConnectionAndReturnErr("cannot set keepalive", err, tcpConn)
return closeConnectionAndReturnErr("cannot set keepalive", err, tcpConn, false)
}

timeFactor := time.Duration(3)
Expand All @@ -664,17 +684,18 @@ func (s *Client) dialContext(ctx context.Context, network, addr string, config *
fmt.Sprintf("cannot set deadline %s", deadline.String()),
err,
tcpConn,
false,
)
}

sshConn, err := s.createSSHConnection(tcpConn, addr, config)
if err != nil {
return closeConnectionAndReturnErr("cannot create ssh connection", err, tcpConn)
return closeConnectionAndReturnErr("cannot create ssh connection", err, tcpConn, true)
}

err = tcpConn.SetDeadline(time.Time{})
if err != nil {
return closeConnectionAndReturnErr("cannot reset deadline", err, tcpConn)
return closeConnectionAndReturnErr("cannot reset deadline", err, tcpConn, false)
}

return sshConn.createGoClient(), nil
Expand Down
2 changes: 1 addition & 1 deletion tests/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ require (
github.com/clipperhouse/uax29/v2 v2.7.0 // indirect
github.com/containerd/console v1.0.5 // indirect
github.com/deckhouse/deckhouse/pkg/log v0.1.1-0.20251230144142-2bad7c3d1edf // indirect
github.com/deckhouse/lib-dhctl v0.20.0 // indirect
github.com/deckhouse/lib-dhctl v0.21.1-0.20260714100121-89ca3451ee2b // indirect
github.com/deckhouse/lib-gossh v0.3.0 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
Expand Down
4 changes: 2 additions & 2 deletions tests/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/deckhouse/deckhouse/pkg/log v0.1.1-0.20251230144142-2bad7c3d1edf h1:4HrDzRZcLpREJ+2cSGNmxHVQlxXRcH2r5TGmTcoTZiU=
github.com/deckhouse/deckhouse/pkg/log v0.1.1-0.20251230144142-2bad7c3d1edf/go.mod h1:pbAxTSDcPmwyl3wwKDcEB3qdxHnRxqTV+J0K+sha8bw=
github.com/deckhouse/lib-dhctl v0.20.0 h1:ahubdw6pdgg/9GGKCgU68d+6VSv04ZEhzt6+xQIDO48=
github.com/deckhouse/lib-dhctl v0.20.0/go.mod h1:hICPf+k3u8V73ndNRNVWUdye/mzo6adLZkXM6vfcmVc=
github.com/deckhouse/lib-dhctl v0.21.1-0.20260714100121-89ca3451ee2b h1:x+uNJ0D7Q18UFgLDVji/Q1R7H5w/jwzvA9VYuxcdrKI=
github.com/deckhouse/lib-dhctl v0.21.1-0.20260714100121-89ca3451ee2b/go.mod h1:hICPf+k3u8V73ndNRNVWUdye/mzo6adLZkXM6vfcmVc=
github.com/deckhouse/lib-gossh v0.3.0 h1:FUAlF8+fLnBCII9hXSNx+arZ4PH3H/6fzp5LBlnmlps=
github.com/deckhouse/lib-gossh v0.3.0/go.mod h1:6bT8jf2fkBPEhYBU35+vMBr5YscliTiS+Vr8v06C+70=
github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
Expand Down
Loading