Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 9 additions & 7 deletions kubernetes/client/configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -388,13 +388,15 @@ def __init__(
"""Options to pass down to the underlying urllib3 socket
"""
self.keep_alive = False
"""Enable TCP keepalive on the underlying urllib3 sockets.

Long lived requests such as watches are otherwise dropped
silently by an idle proxy or load balancer. When enabled, the
client asks the kernel for the same keepalive timings client-go
uses. Ignored if ``socket_options`` is set, which takes
precedence.
"""Enable TCP keepalive on the underlying urllib3 sockets and periodic
WebSocket ping frames for synchronous exec streams. WebSocket pings
are sent every 30 seconds to help prevent idle intermediary timeouts.
They only help when proxies forward WebSocket control frames; servers
and proxies can still close idle streams.

For TCP keepalive, the client asks the kernel for the same timings
client-go uses. ``socket_options`` takes precedence over those TCP
options.
"""

self.datetime_format = datetime_format
Expand Down
100 changes: 69 additions & 31 deletions kubernetes/stream/ws_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@

V4_CHANNEL_PROTOCOL = "v4.channel.k8s.io"
V5_CHANNEL_PROTOCOL = "v5.channel.k8s.io"
WEBSOCKET_PING_INTERVAL = 30

class _IgnoredIO:
def write(self, _x):
Expand Down Expand Up @@ -69,6 +70,16 @@ def __init__(self, configuration, url, headers, capture_all, binary=False):
self._all = _IgnoredIO()
self.sock = create_websocket(configuration, url, headers)
self.subprotocol = getattr(self.sock, 'subprotocol', None)
self.ping_interval = (
WEBSOCKET_PING_INTERVAL
if getattr(configuration, 'keep_alive', False) is True
else None
)
self._next_ping = (
time.monotonic() + self.ping_interval
if self.ping_interval is not None
else None
)
if not self.subprotocol and self.sock:
headers_dict = self.sock.getheaders()
if headers_dict:
Expand Down Expand Up @@ -220,39 +231,66 @@ def update(self, timeout=0):
self._connected = False
return

# SSL sockets decrypt an entire TLS record at a time, so a previous
# recv_data_frame() call may have pulled the bytes of the next frames
# off the socket already: those frames sit decrypted in the
# SSLSocket's internal buffer, invisible to select()/poll() on the
# underlying socket. Without this check the buffered frames would
# only be delivered once new data arrives on the socket, and would be
# lost if it never does.
if (isinstance(self.sock.sock, ssl.SSLSocket)
and self.sock.sock.pending()):
r = True
# The options here are:
# select.select() - this will work on most OS, however, it has a
# limitation of only able to read fd numbers up to 1024.
# i.e. does not scale well. This was the original
# implementation.
# select.poll() - this will work on most unix based OS, but not as
# efficient as epoll. Will work for fd numbers above 1024.
# select.epoll() - newest and most efficient way of polling.
# However, only works on linux.
elif hasattr(select, "poll"):
poll = select.poll()
poll.register(self.sock.sock, select.POLLIN)
deadline = None
if self.ping_interval is not None:
if timeout is not None and timeout != float("inf"):
timeout *= 1_000 # poll method uses milliseconds as the time unit
deadline = time.monotonic() + max(timeout, 0)

while True:
wait_timeout = timeout
if self.ping_interval is not None:
now = time.monotonic()
wait_timeout = max(self._next_ping - now, 0)
if deadline is not None:
wait_timeout = min(wait_timeout, max(deadline - now, 0))

# SSL sockets decrypt an entire TLS record at a time, so a previous
# recv_data_frame() call may have pulled the bytes of the next frames
# off the socket already: those frames sit decrypted in the
# SSLSocket's internal buffer, invisible to select()/poll() on the
# underlying socket. Without this check the buffered frames would
# only be delivered once new data arrives on the socket, and would be
# lost if it never does.
if (isinstance(self.sock.sock, ssl.SSLSocket)
and self.sock.sock.pending()):
r = True
# The options here are:
# select.select() - this will work on most OS, however, it has a
# limitation of only able to read fd numbers up to 1024.
# i.e. does not scale well. This was the original
# implementation.
# select.poll() - this will work on most unix based OS, but not as
# efficient as epoll. Will work for fd numbers above 1024.
# select.epoll() - newest and most efficient way of polling.
# However, only works on linux.
elif hasattr(select, "poll"):
poll = select.poll()
poll.register(self.sock.sock, select.POLLIN)
if wait_timeout is not None and wait_timeout != float("inf"):
wait_timeout *= 1_000 # poll method uses milliseconds as the time unit
else:
wait_timeout = None
r = poll.poll(wait_timeout)
poll.unregister(self.sock.sock)
else:
timeout = None
r = poll.poll(timeout)
poll.unregister(self.sock.sock)
else:
if timeout == float("inf"):
timeout = None
r, _, _ = select.select(
(self.sock.sock, ), (), (), timeout)
if wait_timeout == float("inf"):
wait_timeout = None
r, _, _ = select.select(
(self.sock.sock, ), (), (), wait_timeout)

if r:
break

if self.ping_interval is None:
return

now = time.monotonic()
if now >= self._next_ping:
self.sock.ping()
self._next_ping = now + self.ping_interval

if deadline is not None and now >= deadline:
return

while r:
op_code, frame = self.sock.recv_data_frame(True)
Expand Down
45 changes: 45 additions & 0 deletions kubernetes/stream/ws_client_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -540,13 +540,58 @@ def setUp(self):
self.config_mock.cert_file = None
self.config_mock.key_file = None
self.config_mock.verify_ssl = True
self.config_mock.keep_alive = False

def _make_client(self, mock_ws):
with patch.object(ws_client_module, 'create_websocket') as mock_create:
mock_create.return_value = mock_ws
return WSClient(self.config_mock, "wss://test", headers=None,
capture_all=True, binary=True)

def test_update_pings_before_controlled_proxy_idle_timeout(self):
now = [0]
last_activity = [0]
idle_timeout = 3
remote_process_running = True

class ControlledProxyPoll:
def register(self, *_args):
pass

def unregister(self, *_args):
pass

def poll(self, timeout):
now[0] += timeout / 1000
if now[0] - last_activity[0] >= idle_timeout:
mock_ws.connected = False
return [(10, select.POLLHUP)]
return []

mock_ws = MagicMock()
mock_ws.subprotocol = V4_CHANNEL_PROTOCOL
mock_ws.connected = True
mock_ws.sock.connected = True
mock_ws.sock.fileno.return_value = 10
mock_ws.recv_data_frame.return_value = (
websocket.ABNF.OPCODE_CLOSE,
MagicMock(),
)
mock_ws.ping.side_effect = (
lambda *_args: last_activity.__setitem__(0, now[0])
)
self.config_mock.keep_alive = True

with patch('select.poll', ControlledProxyPoll), \
patch('time.monotonic', side_effect=lambda: now[0]), \
patch.object(ws_client_module, 'WEBSOCKET_PING_INTERVAL', 1):
client = self._make_client(mock_ws)
client.update(timeout=5)

self.assertTrue(remote_process_running)
self.assertTrue(client.is_open())
self.assertGreater(mock_ws.ping.call_count, 0)

def test_update_reads_frames_pending_in_ssl_buffer(self):
"""Verify update reads a frame buffered inside the SSL socket even
when the underlying socket does not report as readable.
Expand Down