Fix pane launcher and channel capacity diagnostics

app/paramiko_channels.py

@@ -1,5 +1,7 @@
 """Bound Paramiko channel handshakes and long-lived channel operations."""
 
+import logging
+import re
 import socket
 import struct
 import time
@@ -11,6 +13,79 @@
 import config
 
 
+SSH_OPEN_FAILED_RESOURCE_SHORTAGE = 4
+_PARAMIKO_TRANSPORT_LOGGER = 'paramiko.transport'
+_CHANNEL_OPEN_FAILURE_LOG = re.compile(
+    r'\ASecsh channel (?P<channel_id>[0-9]+) open FAILED: .*: '
+    r'(?P<reason>Administratively prohibited|Connect failed|'
+    r'Unknown channel type|Resource shortage|\(unknown code\))\Z',
+    re.DOTALL,
+)
+_CHANNEL_OPEN_FAILURE_FILTER_MARKER = (
+    '_webssh_paramiko_channel_open_failure_filter'
+)
+
+
+class _ChannelOpenFailureLogFilter(logging.Filter):
+    """Remove the SSH server's description from Paramiko failure records."""
+
+    _webssh_paramiko_channel_open_failure_filter = True
+
+    def filter(self, record):
+        if (
+            record.name != _PARAMIKO_TRANSPORT_LOGGER
+            or record.levelno != logging.ERROR
+            or record.args
+            or not isinstance(record.msg, str)
+        ):
+            return True
+        match = _CHANNEL_OPEN_FAILURE_LOG.fullmatch(record.msg)
+        if match:
+            record.msg = (
+                f"Secsh channel {match.group('channel_id')} open FAILED: "
+                f"{match.group('reason')} (server description omitted)"
+            )
+            record.args = ()
+        return True
+
+
+def _install_channel_open_failure_log_filter():
+    logger = logging.getLogger(_PARAMIKO_TRANSPORT_LOGGER)
+    for existing_filter in logger.filters:
+        if getattr(
+            existing_filter,
+            _CHANNEL_OPEN_FAILURE_FILTER_MARKER,
+            False,
+        ):
+            return existing_filter
+    channel_filter = _ChannelOpenFailureLogFilter()
+    logger.addFilter(channel_filter)
+    return channel_filter
+
+
+_install_channel_open_failure_log_filter()
+
+
+def optional_channel_rejection_fields(error):
+    """Describe a remote capacity rejection for an optional SSH channel.
+
+    RFC 4254 reason code 4 is supplied by the remote SSH server.  Keep the
+    server-provided text out of application logs because it is untrusted, and
+    leave retry policy to the caller because this condition can be temporary.
+    Do not classify other channel failures here: callers may need to retry or
+    fail a primary SSH operation for those errors.
+    """
+    if not isinstance(error, paramiko.ChannelException):
+        return None
+    code = getattr(error, 'code', None)
+    if type(code) is not int or code != SSH_OPEN_FAILED_RESOURCE_SHORTAGE:
+        return None
+    return {
+        'ssh_channel_code': SSH_OPEN_FAILED_RESOURCE_SHORTAGE,
+        'ssh_channel_reason': 'remote_resource_shortage',
+    }
+
+
 class BoundedSFTPClient(paramiko.SFTPClient):
     """Reject attacker-declared SFTP packets before allocating their body."""
 

app/session_insights.py

@@ -8,6 +8,8 @@
 from paramiko import SSHException
 
 from . import ssh_manager
+from .audit_logger import log_info
+from .paramiko_channels import optional_channel_rejection_fields
 
 
 DEFAULT_MAX_BYTES = 16 * 1024
@@ -443,7 +445,17 @@ def parse_output():
             return None, 'unsupported'
 
         return parse_output()
-    except SSHException:
+    except SSHException as error:
+        rejection = optional_channel_rejection_fields(error)
+        if rejection:
+            log_info(
+                'Diagnostics temporarily unavailable because the remote SSH '
+                'server reported insufficient capacity for an additional '
+                'channel',
+                session_id=session_id,
+                **rejection,
+            )
+            return None, 'resource_shortage'
         return None, 'transient'
     except (OSError, socket.timeout):
         return None, 'transient'

app/sftp_handler.py

@@ -23,8 +23,11 @@
 from paramiko.sftp_attr import SFTPAttributes
 import config
 from . import ssh_manager
-from .paramiko_channels import open_sftp_client
-from .audit_logger import log_warning, log_error
+from .paramiko_channels import (
+    open_sftp_client,
+    optional_channel_rejection_fields,
+)
+from .audit_logger import log_info, log_warning, log_error
 from .file_backend import FileReaderLease, FileWriteOutcome
 
 _sftp_cache = {}
@@ -34,6 +37,7 @@
 _sftp_session_locks_lock = Lock()
 CAPABILITY_RATE_LIMIT = '10 per minute'
 CAPABILITY_TIMEOUT = 3.0
+CAPABILITY_RESOURCE_SHORTAGE = 'resource_shortage'
 
 _capability_probe_locks = {}
 _capability_probe_locks_guard = Lock()
@@ -237,6 +241,19 @@ def public_sftp_error(error, fallback=_PUBLIC_SFTP_ERROR):
     return fallback
 
 
+def _log_sftp_channel_rejection(identifier, error):
+    rejection = optional_channel_rejection_fields(error)
+    if not rejection:
+        return False
+    log_info(
+        'SFTP temporarily unavailable because the remote SSH server reported '
+        'insufficient capacity for an additional channel',
+        session_id=identifier,
+        **rejection,
+    )
+    return True
+
+
 class UploadConflict(SFTPOperationError):
     """The upload destination exists and replacement was not approved."""
 
@@ -897,6 +914,7 @@ def get_sftp_client(session_id):
 
         return sftp, None
     except Exception as e:
+        _log_sftp_channel_rejection(session_id, e)
         return None, public_sftp_error(e, 'Failed to open SFTP channel')
 
 def get_sftp_client_fresh(session_id):
@@ -914,6 +932,7 @@ def get_sftp_client_fresh(session_id):
 
         return sftp, None
     except Exception as e:
+        _log_sftp_channel_rejection(session_id, e)
         return None, public_sftp_error(e, 'Failed to open SFTP channel')
 
 
@@ -1017,8 +1036,10 @@ def probe_sftp_capability(session_id):
     Opening the subsystem alone is insufficient for some appliances, so the
     probe performs one bounded directory read through a fresh short-lived
     channel. It never waits behind cached SFTP operations and concurrent probes
-    for the same session are deduplicated. ``None`` is retryable busy/timeout.
-    Remote exception details intentionally stay server-side.
+    for the same session are deduplicated. ``None`` is a generic retryable busy
+    or timeout result; ``CAPABILITY_RESOURCE_SHORTAGE`` preserves the distinct
+    retry signal for temporary remote channel exhaustion. Remote exception
+    details intentionally stay server-side.
     """
     probe_lock = _acquire_capability_probe(session_id)
     if probe_lock is None:
@@ -1071,11 +1092,13 @@ def expire_probe():
         return True
     except (socket.timeout, TimeoutError):
         return None
-    except Exception:
+    except Exception as error:
         if deadline_expired.is_set() or (
             deadline is not None and time.monotonic() >= deadline
         ):
             return None
+        if _log_sftp_channel_rejection(session_id, error):
+            return CAPABILITY_RESOURCE_SHORTAGE
         return False
     finally:
         if deadline_guard is not None: