diff --git a/examples/cobra/go.mod b/examples/cobra/go.mod index cd31011..ccc48d5 100644 --- a/examples/cobra/go.mod +++ b/examples/cobra/go.mod @@ -4,7 +4,7 @@ go 1.25.8 require ( github.com/deckhouse/lib-connection v0.0.0-00010101000000-000000000000 - github.com/deckhouse/lib-dhctl v0.20.0 + github.com/deckhouse/lib-dhctl v0.21.1-0.20260714100121-89ca3451ee2b github.com/name212/govalue v1.1.0 github.com/spf13/cobra v1.10.2 k8s.io/apimachinery v0.33.12 diff --git a/examples/cobra/go.sum b/examples/cobra/go.sum index 93bd83d..46603d5 100644 --- a/examples/cobra/go.sum +++ b/examples/cobra/go.sum @@ -55,8 +55,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckhouse/deckhouse/pkg/log v0.1.1-0.20251230144142-2bad7c3d1edf h1:4HrDzRZcLpREJ+2cSGNmxHVQlxXRcH2r5TGmTcoTZiU= github.com/deckhouse/deckhouse/pkg/log v0.1.1-0.20251230144142-2bad7c3d1edf/go.mod h1:pbAxTSDcPmwyl3wwKDcEB3qdxHnRxqTV+J0K+sha8bw= -github.com/deckhouse/lib-dhctl v0.20.0 h1:ahubdw6pdgg/9GGKCgU68d+6VSv04ZEhzt6+xQIDO48= -github.com/deckhouse/lib-dhctl v0.20.0/go.mod h1:hICPf+k3u8V73ndNRNVWUdye/mzo6adLZkXM6vfcmVc= +github.com/deckhouse/lib-dhctl v0.21.1-0.20260714100121-89ca3451ee2b h1:x+uNJ0D7Q18UFgLDVji/Q1R7H5w/jwzvA9VYuxcdrKI= +github.com/deckhouse/lib-dhctl v0.21.1-0.20260714100121-89ca3451ee2b/go.mod h1:hICPf+k3u8V73ndNRNVWUdye/mzo6adLZkXM6vfcmVc= github.com/deckhouse/lib-gossh v0.3.0 h1:FUAlF8+fLnBCII9hXSNx+arZ4PH3H/6fzp5LBlnmlps= github.com/deckhouse/lib-gossh v0.3.0/go.mod h1:6bT8jf2fkBPEhYBU35+vMBr5YscliTiS+Vr8v06C+70= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= diff --git a/go.mod b/go.mod index 0592a72..65d8611 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.25.8 require ( al.essio.dev/pkg/shellescape v1.6.0 github.com/bramvdbogaerde/go-scp v1.6.0 - github.com/deckhouse/lib-dhctl v0.20.0 + github.com/deckhouse/lib-dhctl v0.21.1-0.20260714100121-89ca3451ee2b github.com/deckhouse/lib-gossh v0.3.0 github.com/flant/kube-client v1.6.0 github.com/go-openapi/spec v0.19.8 diff --git a/go.sum b/go.sum index 53020ab..3cdba54 100644 --- a/go.sum +++ b/go.sum @@ -54,8 +54,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckhouse/deckhouse/pkg/log v0.1.1-0.20251230144142-2bad7c3d1edf h1:4HrDzRZcLpREJ+2cSGNmxHVQlxXRcH2r5TGmTcoTZiU= github.com/deckhouse/deckhouse/pkg/log v0.1.1-0.20251230144142-2bad7c3d1edf/go.mod h1:pbAxTSDcPmwyl3wwKDcEB3qdxHnRxqTV+J0K+sha8bw= -github.com/deckhouse/lib-dhctl v0.20.0 h1:ahubdw6pdgg/9GGKCgU68d+6VSv04ZEhzt6+xQIDO48= -github.com/deckhouse/lib-dhctl v0.20.0/go.mod h1:hICPf+k3u8V73ndNRNVWUdye/mzo6adLZkXM6vfcmVc= +github.com/deckhouse/lib-dhctl v0.21.1-0.20260714100121-89ca3451ee2b h1:x+uNJ0D7Q18UFgLDVji/Q1R7H5w/jwzvA9VYuxcdrKI= +github.com/deckhouse/lib-dhctl v0.21.1-0.20260714100121-89ca3451ee2b/go.mod h1:hICPf+k3u8V73ndNRNVWUdye/mzo6adLZkXM6vfcmVc= github.com/deckhouse/lib-gossh v0.3.0 h1:FUAlF8+fLnBCII9hXSNx+arZ4PH3H/6fzp5LBlnmlps= github.com/deckhouse/lib-gossh v0.3.0/go.mod h1:6bT8jf2fkBPEhYBU35+vMBr5YscliTiS+Vr8v06C+70= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= diff --git a/pkg/kube/client.go b/pkg/kube/client.go index a00f96f..546ea65 100644 --- a/pkg/kube/client.go +++ b/pkg/kube/client.go @@ -42,6 +42,12 @@ var ( _ connection.KubeClient = &KubernetesClient{} ErrStoppedKubeClient = fmt.Errorf("use stopped kube client") + + // ErrKubeClientTransient marks InitContext failures that may succeed on retry + // (e.g. flaky kube-proxy startup over SSH), as opposed to permanent configuration + // errors (bad kubeconfig path/context, malformed rest config) which do not involve + // any network call and fail identically on every attempt. + ErrKubeClientTransient = fmt.Errorf("kube client: transient error, may succeed on retry") ) type ClientLoopParams struct { @@ -182,14 +188,14 @@ func (k *KubernetesClient) initContext(ctx context.Context, params *Config, opts if !options.NoStartKubeProxy { _, err := k.StartKubernetesProxy(ctx, options) if err != nil { - return err + return fmt.Errorf("%w: %w", ErrKubeClientTransient, err) } } default: if !options.NoStartKubeProxy { port, err := k.StartKubernetesProxy(ctx, options) if err != nil { - return err + return fmt.Errorf("%w: %w", ErrKubeClientTransient, err) } kubeClient.WithServer("http://localhost:" + port) } diff --git a/pkg/provider/kube.go b/pkg/provider/kube.go index d33015d..e477e70 100644 --- a/pkg/provider/kube.go +++ b/pkg/provider/kube.go @@ -213,6 +213,10 @@ func (p *DefaultKubeProvider) connectToKubernetesAPI(ctx context.Context, client initClientLoopParams := retry.SafeCloneOrNewParams(p.loopsParams.InitClient, defaultInitClientParamsOpts...).Clone( retry.WithName("Get Kubernetes API client"), retry.WithLogger(logger), + // Permanent configuration errors (bad kubeconfig path/context, malformed rest + // config) fail identically on every attempt, so only retry the transient + // failures raised while starting the kube-proxy tunnel over SSH. + retry.WithWhitelist(kube.ErrKubeClientTransient), ) err := retry.NewLoopWithParams(initClientLoopParams).RunContext(ctx, func() error { diff --git a/pkg/ssh/gossh/client.go b/pkg/ssh/gossh/client.go index dadba16..1a3a960 100644 --- a/pkg/ssh/gossh/client.go +++ b/pkg/ssh/gossh/client.go @@ -40,6 +40,11 @@ var ( _ connection.Interface = &Client{} ) +// ErrDialTransient marks dial failures that may succeed on retry (network-level +// connect/handshake errors), as opposed to permanent local socket errors (wrong +// conn type, failed to set socket options) that fail identically on every attempt. +var ErrDialTransient = errors.New("dial: transient error, may succeed on retry") + func NewClient(ctx context.Context, sett settings.Settings, session *session.Session, privKeys []session.AgentPrivateKey) *Client { return &Client{ sessionClient: session, @@ -473,7 +478,13 @@ func (s *Client) directConnectToTarget(ctx context.Context) (*gossh.Client, erro } hostLoopParams := retry.SafeCloneOrNewParams(s.loopsParams.ConnectToHostDirectly, defaultClientDirectlyLoopParamsOps...). - Clone(retry.WithName("Get SSH client")) + Clone( + retry.WithName("Get SSH client"), + // Permanent local socket errors (wrong conn type, failed to set socket + // options) fail identically on every attempt, so only retry dial/handshake + // failures that may clear up on their own. + retry.WithWhitelist(ErrDialTransient), + ) if err := s.runInLoop(ctx, hostLoopParams, connectToHost); err != nil { lastHost := fmt.Sprintf("'%s:%s' with user '%s'", s.sessionClient.Host(), s.sessionClient.Port, s.sessionClient.User) @@ -516,7 +527,13 @@ func (s *Client) connectToBastion(ctx context.Context) (*gossh.Client, error) { } bastionLoopParams := retry.SafeCloneOrNewParams(s.loopsParams.ConnectToBastion, defaultClientViaBastionLoopParamsOps...). - Clone(retry.WithName("Get bastion SSH client")) + Clone( + retry.WithName("Get bastion SSH client"), + // Permanent local socket errors (wrong conn type, failed to set socket + // options) fail identically on every attempt, so only retry dial/handshake + // failures that may clear up on their own. + retry.WithWhitelist(ErrDialTransient), + ) if err := s.runInLoop(ctx, bastionLoopParams, connectToBastion); err != nil { return nil, fmt.Errorf("Could not connect to %s: %w", fullHost, err) @@ -631,8 +648,11 @@ func (s *Client) createSSHConnection(c net.Conn, addr string, config *gossh.Clie } func (s *Client) dialContext(ctx context.Context, network, addr string, config *gossh.ClientConfig) (*gossh.Client, error) { - closeConnectionAndReturnErr := func(msg string, err error, conn net.Conn) (*gossh.Client, error) { + closeConnectionAndReturnErr := func(msg string, err error, conn net.Conn, retryable bool) (*gossh.Client, error) { err = fmt.Errorf("Cannot Dial to '%s' %s: %w", addr, msg, err) + if retryable { + err = fmt.Errorf("%w: %w", err, ErrDialTransient) + } if closeErr := utils.SafeClose(conn); closeErr != nil { err = fmt.Errorf("%w and cannot close connection %w", err, closeErr) @@ -643,17 +663,17 @@ func (s *Client) dialContext(ctx context.Context, network, addr string, config * d := net.Dialer{Timeout: config.Timeout} conn, err := d.DialContext(ctx, network, addr) if err != nil { - return closeConnectionAndReturnErr("connect", err, conn) + return closeConnectionAndReturnErr("connect", err, conn, true) } tcpConn, ok := conn.(*net.TCPConn) if !ok { - return closeConnectionAndReturnErr("is not tcp", err, conn) + return closeConnectionAndReturnErr("is not tcp", err, conn, false) } err = tcpConn.SetKeepAlive(true) if err != nil { - return closeConnectionAndReturnErr("cannot set keepalive", err, tcpConn) + return closeConnectionAndReturnErr("cannot set keepalive", err, tcpConn, false) } timeFactor := time.Duration(3) @@ -664,17 +684,18 @@ func (s *Client) dialContext(ctx context.Context, network, addr string, config * fmt.Sprintf("cannot set deadline %s", deadline.String()), err, tcpConn, + false, ) } sshConn, err := s.createSSHConnection(tcpConn, addr, config) if err != nil { - return closeConnectionAndReturnErr("cannot create ssh connection", err, tcpConn) + return closeConnectionAndReturnErr("cannot create ssh connection", err, tcpConn, true) } err = tcpConn.SetDeadline(time.Time{}) if err != nil { - return closeConnectionAndReturnErr("cannot reset deadline", err, tcpConn) + return closeConnectionAndReturnErr("cannot reset deadline", err, tcpConn, false) } return sshConn.createGoClient(), nil diff --git a/tests/go.mod b/tests/go.mod index 4ecb3dc..5904bf2 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -13,7 +13,7 @@ require ( github.com/clipperhouse/uax29/v2 v2.7.0 // indirect github.com/containerd/console v1.0.5 // indirect github.com/deckhouse/deckhouse/pkg/log v0.1.1-0.20251230144142-2bad7c3d1edf // indirect - github.com/deckhouse/lib-dhctl v0.20.0 // indirect + github.com/deckhouse/lib-dhctl v0.21.1-0.20260714100121-89ca3451ee2b // indirect github.com/deckhouse/lib-gossh v0.3.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect diff --git a/tests/go.sum b/tests/go.sum index 70a6fbd..949ec85 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -42,8 +42,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckhouse/deckhouse/pkg/log v0.1.1-0.20251230144142-2bad7c3d1edf h1:4HrDzRZcLpREJ+2cSGNmxHVQlxXRcH2r5TGmTcoTZiU= github.com/deckhouse/deckhouse/pkg/log v0.1.1-0.20251230144142-2bad7c3d1edf/go.mod h1:pbAxTSDcPmwyl3wwKDcEB3qdxHnRxqTV+J0K+sha8bw= -github.com/deckhouse/lib-dhctl v0.20.0 h1:ahubdw6pdgg/9GGKCgU68d+6VSv04ZEhzt6+xQIDO48= -github.com/deckhouse/lib-dhctl v0.20.0/go.mod h1:hICPf+k3u8V73ndNRNVWUdye/mzo6adLZkXM6vfcmVc= +github.com/deckhouse/lib-dhctl v0.21.1-0.20260714100121-89ca3451ee2b h1:x+uNJ0D7Q18UFgLDVji/Q1R7H5w/jwzvA9VYuxcdrKI= +github.com/deckhouse/lib-dhctl v0.21.1-0.20260714100121-89ca3451ee2b/go.mod h1:hICPf+k3u8V73ndNRNVWUdye/mzo6adLZkXM6vfcmVc= github.com/deckhouse/lib-gossh v0.3.0 h1:FUAlF8+fLnBCII9hXSNx+arZ4PH3H/6fzp5LBlnmlps= github.com/deckhouse/lib-gossh v0.3.0/go.mod h1:6bT8jf2fkBPEhYBU35+vMBr5YscliTiS+Vr8v06C+70= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=